def test_resolve_config_path(self):
with patch('hod.config.config.resource_filename', side_effect=lambda pks, *args: os.path.join('/path/to/python/pkgs/', *args)):
with patch('os.path.exists', return_value=True):
self.assertEqual(hcc.resolve_config_paths('', 'Program-1.2.3'), '/path/to/python/pkgs/etc/hod/Program-1.2.3/hod.conf')
self.assertEqual(hcc.resolve_config_paths('/path/to/python/pkgs/etc/hod/Program-1.2.3/hod.conf', ''),
'/path/to/python/pkgs/etc/hod/Program-1.2.3/hod.conf')
self.assertRaises(RuntimeError, hcc.resolve_config_paths, '', '')
def validate_hodconf_or_dist(hodconf, dist):
"""
Returns true if either the hodconf or the dist can be resolved successfully.
If it cannot be resolved, an error is logged and false is returned..
"""
try:
resolve_config_paths(hodconf, dist)
except ValueError as e:
_log.error(e)
return False
return True
def __init__(self, options):
super(PbsHodJob, self).__init__(options)
self.modules = [options.options.hod_module]
config_filenames = resolve_config_paths(options.options.hodconf,
options.options.dist)
self.log.debug('Manifest config paths resolved to: %s',
config_filenames)
config_filenames = parse_comma_delim_list(config_filenames)
self.log.info('Loading "%s" manifest config', config_filenames)
# If the user mistypes the --dist argument (e.g. Haddoop-...) then this will
# raise; TODO: cleanup the error reporting.
precfg = PreServiceConfigOpts.from_file_list(
config_filenames,
workdir=options.options.workdir,
modulepaths=options.options.modulepaths,
modules=options.options.modules)
for modulepath in precfg.modulepaths:
self.log.debug("Adding extra module path '%s' to startup script",
modulepath)
self.modulepaths.append(modulepath)
for module in precfg.modules:
self.log.debug("Adding '%s' module to startup script.", module)
self.modules.append(module)
def distribution(self, *master_template_args, **kwargs):
"""
Master makes the distribution
This only needs to run if there are more than 1 node (self.size>1)
"""
config_path = resolve_config_paths(self.options.hodconf, self.options.dist)
m_config = load_hod_config(config_path, self.options.workdir, self.options.modulepaths, self.options.modules)
m_config.autogen_configs()
resolver = _setup_template_resolver(m_config, master_template_args)
_setup_config_paths(m_config, resolver)
def gen_cluster_info(label, options):
"""Generate cluster info as a dict, intended to use as template values for CLUSTER_ENV_TEMPLATE."""
# list of modules that should be loaded: modules for selected service + extra modules specified via --modules
config_path = resolve_config_paths(options.hodconf, options.dist)
hodconf = hc.load_hod_config(config_path, options.workdir, options.modules)
cluster_info = {
'hadoop_conf_dir': hodconf.configdir,
'hod_localworkdir': hodconf.localworkdir,
'label': label,
'modules': ' '.join(hodconf.modules),
'workdir': options.workdir
}
return cluster_info
def gen_cluster_info(label, options):
"""Generate cluster info as a dict, intended to use as template values for CLUSTER_ENV_TEMPLATE."""
# list of modules that should be loaded: modules for selected service + extra modules specified via --modules
config_path = resolve_config_paths(options.hodconf, options.dist)
hodconf = hc.load_hod_config(config_path, options.workdir, options.modulepaths, options.modules)
cluster_info = {
'hadoop_conf_dir': hodconf.configdir,
'hod_localworkdir': hodconf.localworkdir,
'label': label,
'modpaths': ' '.join(hodconf.modulepaths),
'modules': ' '.join(hodconf.modules),
'workdir': options.workdir
}
return cluster_info
def __init__(self, options):
super(PbsHodJob, self).__init__(options)
self.modules = [options.options.hod_module]
config_filenames = resolve_config_paths(options.options.hodconf, options.options.dist)
self.log.debug('Manifest config paths resolved to: %s', config_filenames)
config_filenames = parse_comma_delim_list(config_filenames)
self.log.info('Loading "%s" manifest config', config_filenames)
# If the user mistypes the --dist argument (e.g. Haddoop-...) then this will
# raise; TODO: cleanup the error reporting.
precfg = PreServiceConfigOpts.from_file_list(config_filenames, workdir=options.options.workdir)
for module in precfg.modules:
self.log.debug("Adding '%s' module to startup script.", module)
self.modules.append(module)
def distribution(self, *master_template_args, **kwargs):
"""Master makes the distribution"""
self.tasks = []
config_path = resolve_config_paths(self.options.hodconf, self.options.dist)
m_config = load_hod_config(config_path, self.options.workdir, self.options.modulepaths, self.options.modules)
m_config.autogen_configs()
resolver = _setup_template_resolver(m_config, master_template_args)
_setup_config_paths(m_config, resolver)
master_env = dict([(v, os.getenv(v)) for v in m_config.master_env])
# There may be scripts in the hod.conf dir so add it to the PATH
master_env['PATH'] = master_env.get('PATH', os.getenv('PATH')) + os.pathsep + m_config.hodconfdir
self.log.debug('MasterEnv is: %s', env2str(master_env))
svc_cfgs = m_config.service_files
self.log.info('Loading %d service configs.', len(svc_cfgs))
for config_filename in svc_cfgs:
self.log.info('Loading "%s" service config', config_filename)
config = ConfigOpts.from_file(open(config_filename, 'r'), resolver)
ranks_to_run = config.runs_on(MASTERRANK, range(self.size))
self.log.debug('Adding ConfiguredService Task to work with config: %s', str(config))
cfg_opts = config.to_params(m_config.workdir, m_config.modulepaths, m_config.modules, master_template_args)
self.tasks.append(Task(ConfiguredService, config.name, ranks_to_run, cfg_opts, master_env))
if hasattr(self.options, 'script') and self.options.script is not None:
label = self.options.label
env_script = 'source ' + hc.cluster_env_file(label)
script = self.options.script
script_stdout, script_stderr = _script_output_paths(script, label)
redirection = ' > %s 2> %s' % (script_stdout, script_stderr)
start_script = env_script + ' && ' + script + redirection + '; qdel $PBS_JOBID'
self.log.debug('Adding script Task: %s', start_script)
# TODO: How can we test this?
config = ConfigOpts(script, RUNS_ON_MASTER, '', start_script, '', master_env, resolver, timeout=NO_TIMEOUT)
ranks_to_run = config.runs_on(MASTERRANK, range(self.size))
cfg_opts = config.to_params(m_config.workdir, m_config.modulepaths, m_config.modules, master_template_args)
self.tasks.append(Task(ConfiguredService, config.name, ranks_to_run, cfg_opts, master_env))
