def test_cleanup_configs(self, get_config_value): getter = lambda plugin, key, cluster: plugin_configs[key] get_config_value.side_effect = getter plugin_configs = { "Minimum cleanup megabytes": 4096, "Minimum cleanup seconds": 86400, "Maximum cleanup seconds": 1209600, } configs = c_helper.generate_job_cleanup_config(None) self.assertTrue(configs["valid"]) expected = [ "MINIMUM_CLEANUP_MEGABYTES=4096", "MINIMUM_CLEANUP_SECONDS=86400", "MAXIMUM_CLEANUP_SECONDS=1209600", ] for config_value in expected: self.assertIn(config_value, configs["script"]) self.assertIn("0 * * * * root /etc/hadoop/tmp-cleanup.sh", configs["cron"][0]) plugin_configs["Maximum cleanup seconds"] = 0 configs = c_helper.generate_job_cleanup_config(None) self.assertFalse(configs["valid"]) self.assertNotIn(configs, "script") self.assertNotIn(configs, "cron") plugin_configs = { "Minimum cleanup megabytes": 0, "Minimum cleanup seconds": 0, "Maximum cleanup seconds": 1209600, } configs = c_helper.generate_job_cleanup_config(None) self.assertFalse(configs["valid"]) self.assertNotIn(configs, "script") self.assertNotIn(configs, "cron")
def test_cleanup_configs(self, get_config_value): getter = lambda plugin, key, cluster: plugin_configs[key] get_config_value.side_effect = getter plugin_configs = { "Minimum cleanup megabytes": 4096, "Minimum cleanup seconds": 86400, "Maximum cleanup seconds": 1209600 } configs = c_helper.generate_job_cleanup_config(None) self.assertTrue(configs['valid']) expected = [ "MINIMUM_CLEANUP_MEGABYTES=4096", "MINIMUM_CLEANUP_SECONDS=86400", "MAXIMUM_CLEANUP_SECONDS=1209600" ] for config_value in expected: self.assertIn(config_value, configs['script']) self.assertIn("0 * * * * root /etc/hadoop/tmp-cleanup.sh", configs['cron'][0]) plugin_configs['Maximum cleanup seconds'] = 0 configs = c_helper.generate_job_cleanup_config(None) self.assertFalse(configs['valid']) self.assertNotIn(configs, 'script') self.assertNotIn(configs, 'cron') plugin_configs = { "Minimum cleanup megabytes": 0, "Minimum cleanup seconds": 0, "Maximum cleanup seconds": 1209600 } configs = c_helper.generate_job_cleanup_config(None) self.assertFalse(configs['valid']) self.assertNotIn(configs, 'script') self.assertNotIn(configs, 'cron')
def _extract_configs_to_extra(self, cluster): sp_master = utils.get_instance(cluster, "master") sp_slaves = utils.get_instances(cluster, "slave") extra = dict() config_master = config_slaves = '' if sp_master is not None: config_master = c_helper.generate_spark_env_configs(cluster) if sp_slaves is not None: slavenames = [] for slave in sp_slaves: slavenames.append(slave.hostname()) config_slaves = c_helper.generate_spark_slaves_configs(slavenames) else: config_slaves = "\n" # Any node that might be used to run spark-submit will need # these libs for swift integration config_defaults = c_helper.generate_spark_executor_classpath(cluster) extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster) extra['sp_master'] = config_master extra['sp_slaves'] = config_slaves extra['sp_defaults'] = config_defaults if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") sp_slaves = utils.get_instances(cluster, "slave") extra = dict() config_master = config_slaves = '' if sp_master is not None: config_master = c_helper.generate_spark_env_configs(cluster) if sp_slaves is not None: slavenames = [] for slave in sp_slaves: slavenames.append(slave.hostname()) config_slaves = c_helper.generate_spark_slaves_configs(slavenames) else: config_slaves = "\n" # Any node that might be used to run spark-submit will need # these libs for swift integration config_defaults = c_helper.generate_spark_executor_classpath(cluster) extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration(), ng.storage_paths(), nn.hostname(), None ), 'setup_script': c_helper.generate_hadoop_setup_script( ng.storage_paths(), c_helper.extract_hadoop_environment_confs( ng.configuration()) ), 'sp_master': config_master, 'sp_slaves': config_slaves, 'sp_defaults': config_defaults } if "zeppelin" in ng.node_processes: extra[ng.id].update({ "zeppelin_setup_script": c_helper.generate_zeppelin_setup_script(sp_master)}) if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _extract_configs_to_extra(self, cluster): nn = utils.get_instance(cluster, "namenode") sp_master = utils.get_instance(cluster, "master") sp_slaves = utils.get_instances(cluster, "slave") extra = dict() config_master = config_slaves = '' if sp_master is not None: config_master = c_helper.generate_spark_env_configs(cluster) if sp_slaves is not None: slavenames = [] for slave in sp_slaves: slavenames.append(slave.hostname()) config_slaves = c_helper.generate_spark_slaves_configs(slavenames) else: config_slaves = "\n" # Any node that might be used to run spark-submit will need # these libs for swift integration config_defaults = c_helper.generate_spark_executor_classpath(cluster) extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs(ng.configuration(), ng.storage_paths(), nn.hostname(), None), 'setup_script': c_helper.generate_hadoop_setup_script( ng.storage_paths(), c_helper.extract_hadoop_environment_confs( ng.configuration())), 'sp_master': config_master, 'sp_slaves': config_slaves, 'sp_defaults': config_defaults } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra