def test_cleanup_configs(self, get_config_value):
getter = lambda plugin, key, cluster: plugin_configs[key]
get_config_value.side_effect = getter
plugin_configs = {
"Minimum cleanup megabytes": 4096,
"Minimum cleanup seconds": 86400,
"Maximum cleanup seconds": 1209600,
}
configs = c_helper.generate_job_cleanup_config(None)
self.assertTrue(configs["valid"])
expected = [
"MINIMUM_CLEANUP_MEGABYTES=4096",
"MINIMUM_CLEANUP_SECONDS=86400",
"MAXIMUM_CLEANUP_SECONDS=1209600",
]
for config_value in expected:
self.assertIn(config_value, configs["script"])
self.assertIn("0 * * * * root /etc/hadoop/tmp-cleanup.sh", configs["cron"][0])
plugin_configs["Maximum cleanup seconds"] = 0
configs = c_helper.generate_job_cleanup_config(None)
self.assertFalse(configs["valid"])
self.assertNotIn(configs, "script")
self.assertNotIn(configs, "cron")
plugin_configs = {
"Minimum cleanup megabytes": 0,
"Minimum cleanup seconds": 0,
"Maximum cleanup seconds": 1209600,
}
configs = c_helper.generate_job_cleanup_config(None)
self.assertFalse(configs["valid"])
self.assertNotIn(configs, "script")
self.assertNotIn(configs, "cron")
def test_cleanup_configs(self, get_config_value):
getter = lambda plugin, key, cluster: plugin_configs[key]
get_config_value.side_effect = getter
plugin_configs = {
"Minimum cleanup megabytes": 4096,
"Minimum cleanup seconds": 86400,
"Maximum cleanup seconds": 1209600
}
configs = c_helper.generate_job_cleanup_config(None)
self.assertTrue(configs['valid'])
expected = [
"MINIMUM_CLEANUP_MEGABYTES=4096", "MINIMUM_CLEANUP_SECONDS=86400",
"MAXIMUM_CLEANUP_SECONDS=1209600"
]
for config_value in expected:
self.assertIn(config_value, configs['script'])
self.assertIn("0 * * * * root /etc/hadoop/tmp-cleanup.sh",
configs['cron'][0])
plugin_configs['Maximum cleanup seconds'] = 0
configs = c_helper.generate_job_cleanup_config(None)
self.assertFalse(configs['valid'])
self.assertNotIn(configs, 'script')
self.assertNotIn(configs, 'cron')
plugin_configs = {
"Minimum cleanup megabytes": 0,
"Minimum cleanup seconds": 0,
"Maximum cleanup seconds": 1209600
}
configs = c_helper.generate_job_cleanup_config(None)
self.assertFalse(configs['valid'])
self.assertNotIn(configs, 'script')
self.assertNotIn(configs, 'cron')
def _extract_configs_to_extra(self, cluster):
sp_master = utils.get_instance(cluster, "master")
sp_slaves = utils.get_instances(cluster, "slave")
extra = dict()
config_master = config_slaves = ''
if sp_master is not None:
config_master = c_helper.generate_spark_env_configs(cluster)
if sp_slaves is not None:
slavenames = []
for slave in sp_slaves:
slavenames.append(slave.hostname())
config_slaves = c_helper.generate_spark_slaves_configs(slavenames)
else:
config_slaves = "\n"
# Any node that might be used to run spark-submit will need
# these libs for swift integration
config_defaults = c_helper.generate_spark_executor_classpath(cluster)
extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster)
extra['sp_master'] = config_master
extra['sp_slaves'] = config_slaves
extra['sp_defaults'] = config_defaults
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
sp_master = utils.get_instance(cluster, "master")
sp_slaves = utils.get_instances(cluster, "slave")
extra = dict()
config_master = config_slaves = ''
if sp_master is not None:
config_master = c_helper.generate_spark_env_configs(cluster)
if sp_slaves is not None:
slavenames = []
for slave in sp_slaves:
slavenames.append(slave.hostname())
config_slaves = c_helper.generate_spark_slaves_configs(slavenames)
else:
config_slaves = "\n"
# Any node that might be used to run spark-submit will need
# these libs for swift integration
config_defaults = c_helper.generate_spark_executor_classpath(cluster)
extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster)
extra['sp_master'] = config_master
extra['sp_slaves'] = config_slaves
extra['sp_defaults'] = config_defaults
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
sp_slaves = utils.get_instances(cluster, "slave")
extra = dict()
config_master = config_slaves = ''
if sp_master is not None:
config_master = c_helper.generate_spark_env_configs(cluster)
if sp_slaves is not None:
slavenames = []
for slave in sp_slaves:
slavenames.append(slave.hostname())
config_slaves = c_helper.generate_spark_slaves_configs(slavenames)
else:
config_slaves = "\n"
# Any node that might be used to run spark-submit will need
# these libs for swift integration
config_defaults = c_helper.generate_spark_executor_classpath(cluster)
extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster)
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
ng.configuration(),
ng.storage_paths(),
nn.hostname(), None
),
'setup_script': c_helper.generate_hadoop_setup_script(
ng.storage_paths(),
c_helper.extract_hadoop_environment_confs(
ng.configuration())
),
'sp_master': config_master,
'sp_slaves': config_slaves,
'sp_defaults': config_defaults
}
if "zeppelin" in ng.node_processes:
extra[ng.id].update({
"zeppelin_setup_script":
c_helper.generate_zeppelin_setup_script(sp_master)})
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _extract_configs_to_extra(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
sp_slaves = utils.get_instances(cluster, "slave")
extra = dict()
config_master = config_slaves = ''
if sp_master is not None:
config_master = c_helper.generate_spark_env_configs(cluster)
if sp_slaves is not None:
slavenames = []
for slave in sp_slaves:
slavenames.append(slave.hostname())
config_slaves = c_helper.generate_spark_slaves_configs(slavenames)
else:
config_slaves = "\n"
# Any node that might be used to run spark-submit will need
# these libs for swift integration
config_defaults = c_helper.generate_spark_executor_classpath(cluster)
extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster)
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(ng.configuration(),
ng.storage_paths(),
nn.hostname(), None),
'setup_script':
c_helper.generate_hadoop_setup_script(
ng.storage_paths(),
c_helper.extract_hadoop_environment_confs(
ng.configuration())),
'sp_master':
config_master,
'sp_slaves':
config_slaves,
'sp_defaults':
config_defaults
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
