def _load_config(conffile, modif=None, starttime=None):
"""
Gets the conf file and dumps it to the
working copy
:conffile: file to be loaded
:modif: dictionnary with modifications
returns the content dictionnary
"""
starttime = starttime or datetime.now()
try:
conf = _open_yaml(conffile, modif)
except yaml.parser.ParserError:
logging.warning('Problem parsing config. Using loaded one.')
conf = _open_yaml(LOADED_CONF)
default = dict(DEFAULT_PNN_CONF, **conf.pop('default'))
main = dict(DEFAULT_MAIN_CONF, **conf.pop('main'))
loaded = dict({'main': main}, **{
pnn: dict(default, **dict({'rse': pnn}, **sec))
for pnn, sec in conf.items()
})
loaded = {
name: _run_status(sec, starttime)
for name, sec in loaded.items()
}
logging.my_lvl(loaded['main']['verbosity'])
logging.debug('Loaded conf %s from %s with modif %s', loaded, conffile,
modif)
with open(LOADED_CONF, 'w') as outfile:
yaml.dump(loaded, outfile, default_flow_style=False)
return loaded
def _poll_workers(workers, pnns):
"""
poll for finished pnn workers
"""
logging.debug('polling workers: %s; pnns %s', workers, pnns)
done = []
for pnn, work in workers.items():
logging.debug("Checking worker %s", pnn)
if work.ready():
logging.debug("Worker %s is ready", pnn)
work.get()
done.append(pnn)
pnns.append(pnn)
logging.summary("Got worker %s and re-queued", pnn)
for pnn in done:
workers.pop(pnn)