def start_spark_history_server(master): sp_home = config_helper.get_spark_home(master.cluster) with context.set_current_instance_id(master.instance_id): with master.remote() as r: r.execute_command( 'sudo su - -c "bash %s" hadoop' % os.path.join(sp_home, 'sbin/start-history-server.sh'))
def start_zk_server(instances): utils.add_provisioning_step(instances[0].cluster_id, utils.start_process_event_message("ZooKeeper"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-start-processes-%s' % instance.instance_name, _start_zk_processes, instance, 'start')
def configure_instances(pctx, instances): if len(instances) == 0: return utils.add_provisioning_step(instances[0].cluster_id, _("Configure instances"), len(instances)) for instance in instances: with context.set_current_instance_id(instance.instance_id): _configure_instance(pctx, instance)
def _disable_repos_on_inst(instance): with context.set_current_instance_id(instance_id=instance.instance_id): with instance.remote() as r: sudo = functools.partial(r.execute_command, run_as_root=True) if r.get_os_distrib() == "ubuntu": sudo("mv /etc/apt/sources.list /etc/apt/sources.list.tmp") else: tmp_name = "/tmp/yum.repos.d-%s" % instance.instance_id[:8] # moving to other folder sudo("mv /etc/yum.repos.d/ {fold_name}".format( fold_name=tmp_name)) sudo("mkdir /etc/yum.repos.d")
def refresh_zk_servers(cluster, to_delete_instances=None): instances = vu.get_zk_servers(cluster) if to_delete_instances: for instance in to_delete_instances: if instance in instances: instances.remove(instance) utils.add_provisioning_step(cluster.id, utils.start_process_event_message("ZooKeeper"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-restart-processes-%s' % instance.instance_name, _start_zk_processes, instance, 'restart')
def _provision_key(instance, keypair): def append_to(remote, file, *args, **kwargs): kwargs['run_as_root'] = True path = "/home/hadoop/.ssh/%s" % file remote.append_to_file(path, *args, **kwargs) public, private = keypair['public'], keypair['private'] folder = '/home/hadoop/.ssh' with context.set_current_instance_id(instance_id=instance.instance_id): with instance.remote() as r: r.execute_command('sudo mkdir -p %s' % folder) append_to(r, 'authorized_keys', public) append_to(r, 'id_rsa', private) append_to(r, 'id_rsa.pub', public) r.execute_command('sudo chown -R hadoop %s' % folder) r.execute_command("sudo chmod 600 %s/id_rsa" % folder) LOG.debug("Passwordless ssh enabled")
def start_dn_nm_processes(instances): filternames = ['datanode', 'nodemanager'] instances = utils.instances_with_services(instances, filternames) if len(instances) == 0: return utils.add_provisioning_step( instances[0].cluster_id, utils.start_process_event_message("DataNodes, NodeManagers"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): processes = set(instance.node_group.node_processes) processes = processes.intersection(filternames) tg.spawn('vanilla-start-processes-%s' % instance.instance_name, _start_processes, instance, list(processes))
def start_oozie_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: if config_helper.is_mysql_enabled(pctx, instance.cluster): _start_mysql(r) LOG.debug("Creating Oozie DB Schema") sql_script = utils.get_file_text( 'plugins/vanilla/hadoop2/resources/create_oozie_db.sql', 'sahara_plugin_vanilla') password = oozie_helper.get_oozie_mysql_configs( instance.cluster)['oozie.service.JPAService.jdbc.password'] sql_script = sql_script.replace("password", password) script_location = "create_oozie_db.sql" r.write_file_to(script_location, sql_script) r.execute_command('mysql -u root < %(script_location)s && ' 'rm %(script_location)s' % {"script_location": script_location}) _oozie_share_lib(r) _start_oozie(r)
def start_hiveserver_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: _hive_create_warehouse_dir(r) _hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop')) if config_helper.is_mysql_enabled(pctx, instance.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) version = instance.cluster.hadoop_version sql_script = utils.get_file_text( 'plugins/vanilla/v{}/resources/create_hive_db.sql'.format( version.replace('.', '_')), 'sahara_plugin_vanilla') sql_script = sql_script.replace( '{{password}}', u.get_hive_password(instance.cluster)) r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) LOG.info("Hive Metastore server at {host} has been " "started".format(host=instance.hostname()))