def start_hiveserver_process(pctx, instance, backup=None):
with context.set_current_instance_id(instance.instance_id):
with instance.remote() as r:
if backup is None:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop'),
backup)
if c_helper.is_mysql_enabled(pctx, instance.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
if backup is None:
sql_script = files.get_file_text(
'plugins/sandbox/hadoop2/resources/create_hive_db.sql')
sql_script = sql_script.replace(
'{{password}}', u.get_hive_password(instance.cluster))
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
_hive_hiveserver2_start(r)
LOG.info(
_LI("Hive Metastore server at {host} has been "
"started").format(host=instance.hostname()))
def test_start_hiveserver_process(
self, add_provisioning_step, check_cluster_exists,
set_current_instance_id, get_oozie, _hive_create_warehouse_dir,
_hive_copy_shared_conf, _start_mysql, _hive_create_db,
_hive_metastore_start, is_mysql_enabled, get_hive_password):
pctx = mock.Mock()
path = edp.get_hive_shared_conf_path('hadoop')
is_mysql_enabled.return_value = True
cluster = self.instance.cluster
ng_cluster = self.instance.node_group.cluster
get_oozie.return_value = None
sql_script = files.get_file_text(
'plugins/vanilla/hadoop2/resources/create_hive_db.sql')
get_hive_password.return_value = '123'
pwd_script = sql_script.replace('{{password}}', '123')
rs.start_hiveserver_process(pctx, self.instance)
set_current_instance_id.assert_called_once_with(
self.instance.instance_id)
_hive_create_warehouse_dir.assert_called_once_with(self.r)
_hive_copy_shared_conf.assert_called_once_with(self.r, path)
is_mysql_enabled.assert_called_once_with(pctx, cluster)
get_oozie.assert_called_once_with(ng_cluster)
_start_mysql.assert_called_once_with(self.r)
get_hive_password.assert_called_once_with(cluster)
self.r.write_file_to.assert_called_once_with('/tmp/create_hive_db.sql',
pwd_script)
_hive_create_db.assert_called_once_with(self.r)
_hive_metastore_start.assert_called_once_with(self.r)
def put_hive_hdfs_xml(self, cluster):
servers = self.get_hive_servers(cluster)
with servers[0].remote() as r:
conf_path = edp_u.get_hive_shared_conf_path('hdfs')
r.execute_command('sudo su - -c "hadoop fs -mkdir -p %s" hdfs' %
os.path.dirname(conf_path))
r.execute_command(
'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml '
'%s" hdfs' % conf_path)
def put_hive_hdfs_xml(self, cluster):
servers = self.get_hive_servers(cluster)
with servers[0].remote() as r:
conf_path = edp_u.get_hive_shared_conf_path('hdfs')
r.execute_command(
'sudo su - -c "hadoop fs -mkdir -p %s" hdfs'
% os.path.dirname(conf_path))
r.execute_command(
'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml '
'%s" hdfs' % conf_path)
def get_workflow_xml(self, cluster, execution, input_data, output_data,
hdfs_user):
job_dict = {'configs': self.get_configs(input_data, output_data),
'params': self.get_params(input_data, output_data)}
self.update_job_dict(job_dict, execution.job_configs)
creator = hive_workflow.HiveWorkflowCreator()
creator.build_workflow_xml(self.name,
edp.get_hive_shared_conf_path(hdfs_user),
configuration=job_dict['configs'],
params=job_dict['params'])
return creator.get_built_workflow_xml()
def _start_hiveserver(self, cluster, hive_server):
oozie = vu.get_oozie(cluster)
with remote.get_remote(hive_server) as r:
run.hive_create_warehouse_dir(r)
run.hive_copy_shared_conf(r, edp.get_hive_shared_conf_path("hadoop"))
if c_helper.is_mysql_enable(cluster):
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(r, hive_server)
run.hive_create_db(r, cluster.extra["hive_mysql_passwd"])
run.hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server at {host} has been " "started").format(host=hive_server.hostname()))
def get_workflow_xml(self, cluster, execution, input_data, output_data):
job_dict = {'configs': self.get_configs(input_data, output_data),
'params': self.get_params(input_data, output_data)}
self.update_job_dict(job_dict, execution.job_configs)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
hdfs_user = plugin.get_hdfs_user()
creator = hive_workflow.HiveWorkflowCreator()
creator.build_workflow_xml(self.name,
edp.get_hive_shared_conf_path(hdfs_user),
configuration=job_dict['configs'],
params=job_dict['params'])
return creator.get_built_workflow_xml()
def get_workflow_xml(self, cluster, job_configs, input_data, output_data,
hdfs_user):
proxy_configs = job_configs.get('proxy_configs')
job_dict = {'configs': self.get_configs(input_data, output_data,
proxy_configs),
'params': self.get_params(input_data, output_data)}
self.update_job_dict(job_dict, job_configs)
creator = hive_workflow.HiveWorkflowCreator()
creator.build_workflow_xml(self.name,
edp.get_hive_shared_conf_path(hdfs_user),
configuration=job_dict['configs'],
params=job_dict['params'])
return creator.get_built_workflow_xml()
def start_cluster(self, cluster):
nn_instance = vu.get_namenode(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_processes(r, "namenode")
for snn in vu.get_secondarynamenodes(cluster):
run.start_processes(remote.get_remote(snn), "secondarynamenode")
jt_instance = vu.get_jobtracker(cluster)
if jt_instance:
run.start_processes(remote.get_remote(jt_instance), "jobtracker")
self._start_tt_dn_processes(utils.get_instances(cluster))
self._await_datanodes(cluster)
LOG.info(_LI("Hadoop services in cluster %s have been started"),
cluster.name)
oozie = vu.get_oozie(cluster)
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname())
run.start_oozie(r)
LOG.info(_LI("Oozie service at '%s' has been started"),
nn_instance.hostname())
hive_server = vu.get_hiveserver(cluster)
if hive_server:
with remote.get_remote(hive_server) as r:
run.hive_create_warehouse_dir(r)
run.hive_copy_shared_conf(
r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enable(cluster):
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(r, hive_server)
run.hive_create_db(r)
run.hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server at %s has been "
"started"),
hive_server.hostname())
LOG.info(_LI('Cluster %s has been started successfully'), cluster.name)
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
nn_instance = vu.get_namenode(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_processes(r, "namenode")
for snn in vu.get_secondarynamenodes(cluster):
run.start_processes(remote.get_remote(snn), "secondarynamenode")
jt_instance = vu.get_jobtracker(cluster)
if jt_instance:
run.start_processes(remote.get_remote(jt_instance), "jobtracker")
self._start_tt_dn_processes(utils.get_instances(cluster))
self._await_datanodes(cluster)
LOG.info(_LI("Hadoop services in cluster %s have been started"),
cluster.name)
oozie = vu.get_oozie(cluster)
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname())
run.start_oozie(r)
LOG.info(_LI("Oozie service at '%s' has been started"),
nn_instance.hostname())
hive_server = vu.get_hiveserver(cluster)
if hive_server:
with remote.get_remote(hive_server) as r:
run.hive_create_warehouse_dir(r)
run.hive_copy_shared_conf(
r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enable(cluster):
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(r, hive_server)
run.hive_create_db(r, cluster.extra['hive_mysql_passwd'])
run.hive_metastore_start(r)
LOG.info(
_LI("Hive Metastore server at %s has been "
"started"), hive_server.hostname())
LOG.info(_LI('Cluster %s has been started successfully'), cluster.name)
self._set_cluster_info(cluster)
def get_workflow_xml(self, cluster, job_configs, input_data, output_data,
hdfs_user, data_source_urls):
proxy_configs = job_configs.get('proxy_configs')
job_dict = {'configs': self.get_configs(input_data, output_data,
proxy_configs),
'params': self.get_params(input_data, output_data,
data_source_urls)}
self.update_job_dict(job_dict, job_configs)
creator = hive_workflow.HiveWorkflowCreator()
creator.build_workflow_xml(self.name,
edp.get_hive_shared_conf_path(hdfs_user),
configuration=job_dict['configs'],
params=job_dict['params'])
return creator.get_built_workflow_xml()
def _start_hiveserver(self, cluster, hive_server):
oozie = vu.get_oozie(cluster)
with remote.get_remote(hive_server) as r:
with context.set_current_instance_id(hive_server.instance_id):
run.hive_create_warehouse_dir(r)
run.hive_copy_shared_conf(
r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enable(cluster):
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(r)
run.hive_create_db(r, cluster.extra['hive_mysql_passwd'])
run.hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server has been started"))
def _start_hiveserver(self, cluster, hive_server):
oozie = vu.get_oozie(cluster)
with remote.get_remote(hive_server) as r:
run.hive_create_warehouse_dir(r)
run.hive_copy_shared_conf(r,
edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enable(cluster):
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(r, hive_server)
run.hive_create_db(r, cluster.extra['hive_mysql_passwd'])
run.hive_metastore_start(r)
LOG.info(
_LI("Hive Metastore server at %s has been "
"started"), hive_server.hostname())
def get_workflow_xml(self, cluster, execution, input_data, output_data):
job_dict = {
"configs": self.get_configs(input_data, output_data),
"params": self.get_params(input_data, output_data),
}
self.update_job_dict(job_dict, execution.job_configs)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
hdfs_user = plugin.get_hdfs_user()
creator = hive_workflow.HiveWorkflowCreator()
creator.build_workflow_xml(
self.name,
edp.get_hive_shared_conf_path(hdfs_user),
configuration=job_dict["configs"],
params=job_dict["params"],
)
return creator.get_built_workflow_xml()
def start_hiveserver_process(pctx, instance):
with context.set_current_instance_id(instance.instance_id):
with instance.remote() as r:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(r, edp.get_hive_shared_conf_path("hadoop"))
if c_helper.is_mysql_enabled(pctx, instance.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
sql_script = files.get_file_text("plugins/vanilla/hadoop2/resources/create_hive_db.sql")
r.write_file_to("/tmp/create_hive_db.sql", sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server at {host} has been " "started").format(host=instance.hostname()))
def start_hiveserver_process(pctx, instance):
with instance.remote() as r:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enabled(pctx, instance.node_group.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
sql_script = files.get_file_text(
'plugins/vanilla/hadoop2/resources/create_hive_db.sql')
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server at %s has been "
"started"), instance.hostname())
def start_hiveserver_process(pctx, instance):
with instance.remote() as r:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(
r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enabled(pctx, instance.node_group.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
sql_script = files.get_file_text(
'plugins/vanilla/hadoop2/resources/create_hive_db.sql'
)
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
LOG.info(_LI("Hive Metastore server at %s has been "
"started"),
instance.hostname())
def start_hiveserver_process(pctx, instance):
with context.set_current_instance_id(instance.instance_id):
with instance.remote() as r:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop'))
if c_helper.is_mysql_enabled(pctx, instance.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
version = instance.cluster.hadoop_version
sql_script = files.get_file_text(
'plugins/vanilla/v{}/resources/create_hive_db.sql'.format(
version.replace('.', '_')))
sql_script = sql_script.replace(
'{{password}}', u.get_hive_password(instance.cluster))
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
LOG.info("Hive Metastore server at {host} has been "
"started".format(host=instance.hostname()))
def get_hive_shared_conf_path(hdfs_user, **kwargs):
return edp.get_hive_shared_conf_path(hdfs_user)
def get_hive_shared_conf_path(hdfs_user, **kwargs):
return edp.get_hive_shared_conf_path(hdfs_user)
