def start_oozie_process(pctx, instance):
with context.set_current_instance_id(instance.instance_id):
with instance.remote() as r:
if config_helper.is_mysql_enabled(pctx, instance.cluster):
_start_mysql(r)
LOG.debug("Creating Oozie DB Schema")
sql_script = utils.get_file_text(
'plugins/vanilla/hadoop2/resources/create_oozie_db.sql',
'sahara_plugin_vanilla')
password = oozie_helper.get_oozie_mysql_configs(
instance.cluster)['oozie.service.JPAService.jdbc.password']
sql_script = sql_script.replace("password", password)
script_location = "create_oozie_db.sql"
r.write_file_to(script_location, sql_script)
r.execute_command('mysql -u root < %(script_location)s && '
'rm %(script_location)s' %
{"script_location": script_location})
_oozie_share_lib(r)
_start_oozie(r)
def start_hiveserver_process(pctx, instance):
with context.set_current_instance_id(instance.instance_id):
with instance.remote() as r:
_hive_create_warehouse_dir(r)
_hive_copy_shared_conf(r, edp.get_hive_shared_conf_path('hadoop'))
if config_helper.is_mysql_enabled(pctx, instance.cluster):
oozie = vu.get_oozie(instance.node_group.cluster)
if not oozie or instance.hostname() != oozie.hostname():
_start_mysql(r)
version = instance.cluster.hadoop_version
sql_script = utils.get_file_text(
'plugins/vanilla/v{}/resources/create_hive_db.sql'.format(
version.replace('.', '_')), 'sahara_plugin_vanilla')
sql_script = sql_script.replace(
'{{password}}', u.get_hive_password(instance.cluster))
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
_hive_create_db(r)
_hive_metastore_start(r)
LOG.info("Hive Metastore server at {host} has been "
"started".format(host=instance.hostname()))
def _get_hadoop_configs(pctx, instance):
cluster = instance.node_group.cluster
nn_hostname = vu.get_instance_hostname(vu.get_namenode(cluster))
dirs = _get_hadoop_dirs(instance)
confs = {
'Hadoop': {
'fs.defaultFS': 'hdfs://%s:9000' % nn_hostname
},
'HDFS': {
'dfs.namenode.name.dir': ','.join(dirs['hadoop_name_dirs']),
'dfs.datanode.data.dir': ','.join(dirs['hadoop_data_dirs']),
'dfs.hosts': '%s/dn-include' % HADOOP_CONF_DIR,
'dfs.hosts.exclude': '%s/dn-exclude' % HADOOP_CONF_DIR
}
}
res_hostname = vu.get_instance_hostname(vu.get_resourcemanager(cluster))
if res_hostname:
confs['YARN'] = {
'yarn.nodemanager.aux-services':
'mapreduce_shuffle',
'yarn.resourcemanager.hostname':
'%s' % res_hostname,
'yarn.resourcemanager.nodes.include-path':
'%s/nm-include' % (HADOOP_CONF_DIR),
'yarn.resourcemanager.nodes.exclude-path':
'%s/nm-exclude' % (HADOOP_CONF_DIR)
}
confs['MapReduce'] = {'mapreduce.framework.name': 'yarn'}
hs_hostname = vu.get_instance_hostname(vu.get_historyserver(cluster))
if hs_hostname:
confs['MapReduce']['mapreduce.jobhistory.address'] = ("%s:10020" %
hs_hostname)
oozie = vu.get_oozie(cluster)
if oozie:
hadoop_cfg = {
'hadoop.proxyuser.hadoop.hosts': '*',
'hadoop.proxyuser.hadoop.groups': 'hadoop'
}
confs['Hadoop'].update(hadoop_cfg)
oozie_cfg = oozie_helper.get_oozie_required_xml_configs(
HADOOP_CONF_DIR)
if config_helper.is_mysql_enabled(pctx, cluster):
oozie_cfg.update(oozie_helper.get_oozie_mysql_configs(cluster))
confs['JobFlow'] = oozie_cfg
if config_helper.is_swift_enabled(pctx, cluster):
swift_configs = {}
for config in swift.get_swift_configs():
swift_configs[config['name']] = config['value']
confs['Hadoop'].update(swift_configs)
if config_helper.is_data_locality_enabled(pctx, cluster):
confs['Hadoop'].update(th.TOPOLOGY_CONFIG)
confs['Hadoop'].update(
{"topology.script.file.name": HADOOP_CONF_DIR + "/topology.sh"})
hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster))
if hive_hostname:
hive_pass = u.get_hive_password(cluster)
hive_cfg = {
'hive.warehouse.subdir.inherit.perms':
True,
'javax.jdo.option.ConnectionURL':
'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
}
if config_helper.is_mysql_enabled(pctx, cluster):
hive_cfg.update({
'javax.jdo.option.ConnectionURL':
'jdbc:mysql://%s/metastore' % hive_hostname,
'javax.jdo.option.ConnectionDriverName':
'com.mysql.jdbc.Driver',
'javax.jdo.option.ConnectionUserName':
'******',
'javax.jdo.option.ConnectionPassword':
hive_pass,
'datanucleus.autoCreateSchema':
'false',
'datanucleus.fixedDatastore':
'true',
'hive.metastore.uris':
'thrift://%s:9083' % hive_hostname,
})
proxy_configs = cluster.cluster_configs.get('proxy_configs')
if proxy_configs and config_helper.is_swift_enabled(pctx, cluster):
hive_cfg.update({
swift.HADOOP_SWIFT_USERNAME:
proxy_configs['proxy_username'],
swift.HADOOP_SWIFT_PASSWORD:
key_manager.get_secret(proxy_configs['proxy_password']),
swift.HADOOP_SWIFT_TRUST_ID:
proxy_configs['proxy_trust_id'],
swift.HADOOP_SWIFT_DOMAIN_NAME:
CONF.proxy_user_domain_name
})
confs['Hive'] = hive_cfg
return confs
def test_is_mysql_enabled(self, get_config_value):
target = config_helper.ENABLE_MYSQL.applicable_target
name = config_helper.ENABLE_MYSQL.name
config_helper.is_mysql_enabled(self.pctx, self.cluster)
get_config_value.assert_called_once_with(self.pctx, target, name,
self.cluster)