def _set_cluster_info(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") info = {} if nn: address = utils.get_config_value_or_default( 'HDFS', 'dfs.http.address', cluster) port = address[address.rfind(':') + 1:] info['HDFS'] = { 'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), port) } info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname() if sp_master: port = utils.get_config_value_or_default('Spark', 'Master webui port', cluster) if port is not None: info['Spark'] = { 'Web UI': 'http://%s:%s' % (sp_master.get_ip_or_dns_name(), port) } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def _update_cluster_info(self, cluster_context): LOG.debug('Updating UI information.') info = {'Admin user credentials': {'Username': '******', 'Password': pu.get_mapr_password (cluster_context.cluster)}} for service in cluster_context.cluster_services: for title, node_process, ui_info in ( service.get_ui_info(cluster_context)): removed = cluster_context.removed_instances(node_process) instances = cluster_context.get_instances(node_process) instances = [i for i in instances if i not in removed] if len(instances) == 1: display_name_template = "%(title)s" else: display_name_template = "%(title)s %(index)s" for index, instance in enumerate(instances, start=1): args = {"title": title, "index": index} display_name = display_name_template % args data = ui_info.copy() data[srvc.SERVICE_UI] = (data[srvc.SERVICE_UI] % instance.get_ip_or_dns_name()) info.update({display_name: data}) ctx = context.ctx() conductor.cluster_update(ctx, cluster_context.cluster, {'info': info})
def get_hive_password(cluster): cluster = conductor.cluster_get(context.ctx(), cluster) extra = cluster.extra.to_dict() if 'hive_pass_id' not in extra: extra['hive_pass_id'] = u.generate_random_password() conductor.cluster_update(context.ctx(), cluster, {'extra': extra}) return castellan.get_secret(extra['hive_pass_id'])
def _prepare_ranger(cluster): ranger = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN) if not ranger: return ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) with ambari.remote() as r: sudo = functools.partial(r.execute_command, run_as_root=True) sudo("ambari-server setup --jdbc-db=mysql " "--jdbc-driver=/usr/share/java/mysql-connector-java.jar") init_db_template = ( "create user 'root'@'%' identified by '{password}';\n" "set password for 'root'@'localhost' = password('{password}');") password = uuidutils.generate_uuid() extra = cluster.extra.to_dict() if cluster.extra else {} extra["ranger_db_password"] = password ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"extra": extra}) with ranger.remote() as r: sudo = functools.partial(r.execute_command, run_as_root=True) # TODO(sreshetnyak): add ubuntu support sudo("yum install -y mysql-server") sudo("service mysqld start") r.write_file_to("/tmp/init.sql", init_db_template.format(password=password)) sudo("mysql < /tmp/init.sql") sudo("rm /tmp/init.sql")
def _set_cluster_info(self, cluster): info = self.cloudera_utils.get_cloudera_manager_info(cluster) hue = self.cloudera_utils.pu.get_hue(cluster) if hue: info['Hue Dashboard'] = { 'Web UI': 'http://%s:8888' % hue.get_ip_or_dns_name() } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def update_default_ambari_password(cluster): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) new_password = uuidutils.generate_uuid() with ambari_client.AmbariClient(ambari) as client: client.update_user_password("admin", "admin", new_password) extra = cluster.extra.to_dict() if cluster.extra else {} extra["ambari_password"] = new_password ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"extra": extra}) cluster = conductor.cluster_get(ctx, cluster.id)
def _set_cluster_info(self, cluster): st_master = utils.get_instance(cluster, "nimbus") info = {} if st_master: port = "8080" info['Strom'] = { 'Web UI': 'http://%s:%s' % (st_master.get_ip_or_dns_name(), port) } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster): nn = vu.get_namenode(cluster) rm = vu.get_resourcemanager(cluster) hs = vu.get_historyserver(cluster) oo = vu.get_oozie(cluster) sp = vu.get_spark_history_server(cluster) info = {} if rm: info['YARN'] = { 'Web UI': 'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'), 'ResourceManager': 'http://%s:%s' % (rm.get_ip_or_dns_name(), '8032') } if nn: info['HDFS'] = { 'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'), 'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000') } if oo: info['JobFlow'] = { 'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000') } if hs: info['MapReduce JobHistory Server'] = { 'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888') } if sp: info['Apache Spark'] = { 'Spark UI': 'http://%s:%s' % (sp.management_ip, '4040'), 'Spark History Server UI': 'http://%s:%s' % (sp.management_ip, '18080') } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def get_password_from_db(cluster, pwname): """return a password for the named entry This function will return, or create and return, a password for the named entry. It will store the password in the key manager and use the ID in the database entry. :param cluster: The cluster record containing the password :param pwname: The entry name associated with the password :returns: The cleartext password """ ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster.id) passwd = cluster.extra.get(pwname) if cluster.extra else None if passwd: return key_manager.get_secret(passwd, ctx) passwd = uuidutils.generate_uuid() extra = cluster.extra.to_dict() if cluster.extra else {} extra[pwname] = key_manager.store_secret(passwd, ctx) conductor.cluster_update(ctx, cluster, {'extra': extra}) return passwd
def provision_keypairs(cluster, instances=None): extra = cluster.extra.to_dict() if cluster.extra else {} # use same keypair for scaling keypair = extra.get('vanilla_keypair') if not instances: instances = utils.get_instances(cluster) else: # scaling if not keypair: # cluster created before mitaka, skipping provisioning return if not keypair: private, public = utils.generate_key_pair() keypair = {'public': public, 'private': private} extra['vanilla_keypair'] = keypair extra['vanilla_keypair']['private'] = _store_secret(keypair['private']) conductor.cluster_update(context.ctx(), cluster, {'extra': extra}) else: keypair['private'] = _get_secret(keypair['private']) with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('provision-key-%s' % instance.instance_name, _provision_key, instance, keypair)
def _set_cluster_info(self, cluster): ambari_ip = plugin_utils.get_instance( cluster, p_common.AMBARI_SERVER).get_ip_or_dns_name() ambari_port = "8080" info = { p_common.AMBARI_SERVER: { "Web UI": "http://{host}:{port}".format(host=ambari_ip, port=ambari_port), "Username": "******", "Password": cluster.extra["ambari_password"] } } nns = plugin_utils.get_instances(cluster, p_common.NAMENODE) info[p_common.NAMENODE] = {} for idx, namenode in enumerate(nns): info[p_common.NAMENODE][ "Web UI %s" % (idx + 1)] = ( "http://%s:50070" % namenode.get_ip_or_dns_name()) rms = plugin_utils.get_instances(cluster, p_common.RESOURCEMANAGER) info[p_common.RESOURCEMANAGER] = {} for idx, resourcemanager in enumerate(rms): info[p_common.RESOURCEMANAGER][ "Web UI %s" % (idx + 1)] = ( "http://%s:8088" % resourcemanager.get_ip_or_dns_name()) historyserver = plugin_utils.get_instance(cluster, p_common.HISTORYSERVER) if historyserver: info[p_common.HISTORYSERVER] = { "Web UI": "http://%s:19888" % historyserver.get_ip_or_dns_name() } atlserver = plugin_utils.get_instance(cluster, p_common.APP_TIMELINE_SERVER) if atlserver: info[p_common.APP_TIMELINE_SERVER] = { "Web UI": "http://%s:8188" % atlserver.get_ip_or_dns_name() } oozie = plugin_utils.get_instance(cluster, p_common.OOZIE_SERVER) if oozie: info[p_common.OOZIE_SERVER] = { "Web UI": "http://%s:11000/oozie" % oozie.get_ip_or_dns_name() } hbase_master = plugin_utils.get_instance(cluster, p_common.HBASE_MASTER) if hbase_master: info[p_common.HBASE_MASTER] = { "Web UI": "http://%s:16010" % hbase_master.get_ip_or_dns_name() } falcon = plugin_utils.get_instance(cluster, p_common.FALCON_SERVER) if falcon: info[p_common.FALCON_SERVER] = { "Web UI": "http://%s:15000" % falcon.get_ip_or_dns_name() } storm_ui = plugin_utils.get_instance(cluster, p_common.STORM_UI_SERVER) if storm_ui: info[p_common.STORM_UI_SERVER] = { "Web UI": "http://%s:8744" % storm_ui.get_ip_or_dns_name() } ranger_admin = plugin_utils.get_instance(cluster, p_common.RANGER_ADMIN) if ranger_admin: info[p_common.RANGER_ADMIN] = { "Web UI": "http://%s:6080" % ranger_admin.get_ip_or_dns_name(), "Username": "******", "Password": "******" } spark_hs = plugin_utils.get_instance(cluster, p_common.SPARK_JOBHISTORYSERVER) if spark_hs: info[p_common.SPARK_JOBHISTORYSERVER] = { "Web UI": "http://%s:18080" % spark_hs.get_ip_or_dns_name() } info.update(cluster.info.to_dict()) ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"info": info}) cluster = conductor.cluster_get(ctx, cluster.id)
def configure_cluster(self, cluster): self.deploy.configure_cluster(cluster) conductor.cluster_update( context.ctx(), cluster, { 'info': self.cloudera_utils.get_cloudera_manager_info(cluster)})