def start_hiveserver_process(pctx, instance, backup=None): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: if backup is None: _hive_create_warehouse_dir(r) _hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop'), backup) if c_helper.is_mysql_enabled(pctx, instance.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) if backup is None: sql_script = files.get_file_text( 'plugins/sandbox/hadoop2/resources/create_hive_db.sql') sql_script = sql_script.replace( '{{password}}', u.get_hive_password(instance.cluster)) r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) _hive_hiveserver2_start(r) LOG.info( _LI("Hive Metastore server at {host} has been " "started").format(host=instance.hostname()))
def test_start_hiveserver_process( self, add_provisioning_step, check_cluster_exists, set_current_instance_id, get_oozie, _hive_create_warehouse_dir, _hive_copy_shared_conf, _start_mysql, _hive_create_db, _hive_metastore_start, is_mysql_enabled, get_hive_password): pctx = mock.Mock() path = edp.get_hive_shared_conf_path('hadoop') is_mysql_enabled.return_value = True cluster = self.instance.cluster ng_cluster = self.instance.node_group.cluster get_oozie.return_value = None sql_script = files.get_file_text( 'plugins/vanilla/hadoop2/resources/create_hive_db.sql') get_hive_password.return_value = '123' pwd_script = sql_script.replace('{{password}}', '123') rs.start_hiveserver_process(pctx, self.instance) set_current_instance_id.assert_called_once_with( self.instance.instance_id) _hive_create_warehouse_dir.assert_called_once_with(self.r) _hive_copy_shared_conf.assert_called_once_with(self.r, path) is_mysql_enabled.assert_called_once_with(pctx, cluster) get_oozie.assert_called_once_with(ng_cluster) _start_mysql.assert_called_once_with(self.r) get_hive_password.assert_called_once_with(cluster) self.r.write_file_to.assert_called_once_with('/tmp/create_hive_db.sql', pwd_script) _hive_create_db.assert_called_once_with(self.r) _hive_metastore_start.assert_called_once_with(self.r)
def put_hive_hdfs_xml(self, cluster): servers = self.get_hive_servers(cluster) with servers[0].remote() as r: conf_path = edp_u.get_hive_shared_conf_path('hdfs') r.execute_command('sudo su - -c "hadoop fs -mkdir -p %s" hdfs' % os.path.dirname(conf_path)) r.execute_command( 'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml ' '%s" hdfs' % conf_path)
def put_hive_hdfs_xml(self, cluster): servers = self.get_hive_servers(cluster) with servers[0].remote() as r: conf_path = edp_u.get_hive_shared_conf_path('hdfs') r.execute_command( 'sudo su - -c "hadoop fs -mkdir -p %s" hdfs' % os.path.dirname(conf_path)) r.execute_command( 'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml ' '%s" hdfs' % conf_path)
def get_workflow_xml(self, cluster, execution, input_data, output_data, hdfs_user): job_dict = {'configs': self.get_configs(input_data, output_data), 'params': self.get_params(input_data, output_data)} self.update_job_dict(job_dict, execution.job_configs) creator = hive_workflow.HiveWorkflowCreator() creator.build_workflow_xml(self.name, edp.get_hive_shared_conf_path(hdfs_user), configuration=job_dict['configs'], params=job_dict['params']) return creator.get_built_workflow_xml()
def _start_hiveserver(self, cluster, hive_server): oozie = vu.get_oozie(cluster) with remote.get_remote(hive_server) as r: run.hive_create_warehouse_dir(r) run.hive_copy_shared_conf(r, edp.get_hive_shared_conf_path("hadoop")) if c_helper.is_mysql_enable(cluster): if not oozie or hive_server.hostname() != oozie.hostname(): run.mysql_start(r, hive_server) run.hive_create_db(r, cluster.extra["hive_mysql_passwd"]) run.hive_metastore_start(r) LOG.info(_LI("Hive Metastore server at {host} has been " "started").format(host=hive_server.hostname()))
def get_workflow_xml(self, cluster, execution, input_data, output_data): job_dict = {'configs': self.get_configs(input_data, output_data), 'params': self.get_params(input_data, output_data)} self.update_job_dict(job_dict, execution.job_configs) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) hdfs_user = plugin.get_hdfs_user() creator = hive_workflow.HiveWorkflowCreator() creator.build_workflow_xml(self.name, edp.get_hive_shared_conf_path(hdfs_user), configuration=job_dict['configs'], params=job_dict['params']) return creator.get_built_workflow_xml()
def get_workflow_xml(self, cluster, job_configs, input_data, output_data, hdfs_user): proxy_configs = job_configs.get('proxy_configs') job_dict = {'configs': self.get_configs(input_data, output_data, proxy_configs), 'params': self.get_params(input_data, output_data)} self.update_job_dict(job_dict, job_configs) creator = hive_workflow.HiveWorkflowCreator() creator.build_workflow_xml(self.name, edp.get_hive_shared_conf_path(hdfs_user), configuration=job_dict['configs'], params=job_dict['params']) return creator.get_built_workflow_xml()
def start_cluster(self, cluster): nn_instance = vu.get_namenode(cluster) with remote.get_remote(nn_instance) as r: run.format_namenode(r) run.start_processes(r, "namenode") for snn in vu.get_secondarynamenodes(cluster): run.start_processes(remote.get_remote(snn), "secondarynamenode") jt_instance = vu.get_jobtracker(cluster) if jt_instance: run.start_processes(remote.get_remote(jt_instance), "jobtracker") self._start_tt_dn_processes(utils.get_instances(cluster)) self._await_datanodes(cluster) LOG.info(_LI("Hadoop services in cluster %s have been started"), cluster.name) oozie = vu.get_oozie(cluster) if oozie: with remote.get_remote(oozie) as r: if c_helper.is_mysql_enable(cluster): run.mysql_start(r, oozie) run.oozie_create_db(r) run.oozie_share_lib(r, nn_instance.hostname()) run.start_oozie(r) LOG.info(_LI("Oozie service at '%s' has been started"), nn_instance.hostname()) hive_server = vu.get_hiveserver(cluster) if hive_server: with remote.get_remote(hive_server) as r: run.hive_create_warehouse_dir(r) run.hive_copy_shared_conf( r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enable(cluster): if not oozie or hive_server.hostname() != oozie.hostname(): run.mysql_start(r, hive_server) run.hive_create_db(r) run.hive_metastore_start(r) LOG.info(_LI("Hive Metastore server at %s has been " "started"), hive_server.hostname()) LOG.info(_LI('Cluster %s has been started successfully'), cluster.name) self._set_cluster_info(cluster)
def start_cluster(self, cluster): nn_instance = vu.get_namenode(cluster) with remote.get_remote(nn_instance) as r: run.format_namenode(r) run.start_processes(r, "namenode") for snn in vu.get_secondarynamenodes(cluster): run.start_processes(remote.get_remote(snn), "secondarynamenode") jt_instance = vu.get_jobtracker(cluster) if jt_instance: run.start_processes(remote.get_remote(jt_instance), "jobtracker") self._start_tt_dn_processes(utils.get_instances(cluster)) self._await_datanodes(cluster) LOG.info(_LI("Hadoop services in cluster %s have been started"), cluster.name) oozie = vu.get_oozie(cluster) if oozie: with remote.get_remote(oozie) as r: if c_helper.is_mysql_enable(cluster): run.mysql_start(r, oozie) run.oozie_create_db(r) run.oozie_share_lib(r, nn_instance.hostname()) run.start_oozie(r) LOG.info(_LI("Oozie service at '%s' has been started"), nn_instance.hostname()) hive_server = vu.get_hiveserver(cluster) if hive_server: with remote.get_remote(hive_server) as r: run.hive_create_warehouse_dir(r) run.hive_copy_shared_conf( r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enable(cluster): if not oozie or hive_server.hostname() != oozie.hostname(): run.mysql_start(r, hive_server) run.hive_create_db(r, cluster.extra['hive_mysql_passwd']) run.hive_metastore_start(r) LOG.info( _LI("Hive Metastore server at %s has been " "started"), hive_server.hostname()) LOG.info(_LI('Cluster %s has been started successfully'), cluster.name) self._set_cluster_info(cluster)
def get_workflow_xml(self, cluster, job_configs, input_data, output_data, hdfs_user, data_source_urls): proxy_configs = job_configs.get('proxy_configs') job_dict = {'configs': self.get_configs(input_data, output_data, proxy_configs), 'params': self.get_params(input_data, output_data, data_source_urls)} self.update_job_dict(job_dict, job_configs) creator = hive_workflow.HiveWorkflowCreator() creator.build_workflow_xml(self.name, edp.get_hive_shared_conf_path(hdfs_user), configuration=job_dict['configs'], params=job_dict['params']) return creator.get_built_workflow_xml()
def _start_hiveserver(self, cluster, hive_server): oozie = vu.get_oozie(cluster) with remote.get_remote(hive_server) as r: with context.set_current_instance_id(hive_server.instance_id): run.hive_create_warehouse_dir(r) run.hive_copy_shared_conf( r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enable(cluster): if not oozie or hive_server.hostname() != oozie.hostname(): run.mysql_start(r) run.hive_create_db(r, cluster.extra['hive_mysql_passwd']) run.hive_metastore_start(r) LOG.info(_LI("Hive Metastore server has been started"))
def _start_hiveserver(self, cluster, hive_server): oozie = vu.get_oozie(cluster) with remote.get_remote(hive_server) as r: run.hive_create_warehouse_dir(r) run.hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enable(cluster): if not oozie or hive_server.hostname() != oozie.hostname(): run.mysql_start(r, hive_server) run.hive_create_db(r, cluster.extra['hive_mysql_passwd']) run.hive_metastore_start(r) LOG.info( _LI("Hive Metastore server at %s has been " "started"), hive_server.hostname())
def get_workflow_xml(self, cluster, execution, input_data, output_data): job_dict = { "configs": self.get_configs(input_data, output_data), "params": self.get_params(input_data, output_data), } self.update_job_dict(job_dict, execution.job_configs) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) hdfs_user = plugin.get_hdfs_user() creator = hive_workflow.HiveWorkflowCreator() creator.build_workflow_xml( self.name, edp.get_hive_shared_conf_path(hdfs_user), configuration=job_dict["configs"], params=job_dict["params"], ) return creator.get_built_workflow_xml()
def start_hiveserver_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: _hive_create_warehouse_dir(r) _hive_copy_shared_conf(r, edp.get_hive_shared_conf_path("hadoop")) if c_helper.is_mysql_enabled(pctx, instance.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) sql_script = files.get_file_text("plugins/vanilla/hadoop2/resources/create_hive_db.sql") r.write_file_to("/tmp/create_hive_db.sql", sql_script) _hive_create_db(r) _hive_metastore_start(r) LOG.info(_LI("Hive Metastore server at {host} has been " "started").format(host=instance.hostname()))
def start_hiveserver_process(pctx, instance): with instance.remote() as r: _hive_create_warehouse_dir(r) _hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enabled(pctx, instance.node_group.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) sql_script = files.get_file_text( 'plugins/vanilla/hadoop2/resources/create_hive_db.sql') r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) LOG.info(_LI("Hive Metastore server at %s has been " "started"), instance.hostname())
def start_hiveserver_process(pctx, instance): with instance.remote() as r: _hive_create_warehouse_dir(r) _hive_copy_shared_conf( r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enabled(pctx, instance.node_group.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) sql_script = files.get_file_text( 'plugins/vanilla/hadoop2/resources/create_hive_db.sql' ) r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) LOG.info(_LI("Hive Metastore server at %s has been " "started"), instance.hostname())
def start_hiveserver_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: _hive_create_warehouse_dir(r) _hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop')) if c_helper.is_mysql_enabled(pctx, instance.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) version = instance.cluster.hadoop_version sql_script = files.get_file_text( 'plugins/vanilla/v{}/resources/create_hive_db.sql'.format( version.replace('.', '_'))) sql_script = sql_script.replace( '{{password}}', u.get_hive_password(instance.cluster)) r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) LOG.info("Hive Metastore server at {host} has been " "started".format(host=instance.hostname()))
def get_hive_shared_conf_path(hdfs_user, **kwargs): return edp.get_hive_shared_conf_path(hdfs_user)
def get_hive_shared_conf_path(hdfs_user, **kwargs): return edp.get_hive_shared_conf_path(hdfs_user)