def run(self, local=False, verbose=False):
"""
Run a node of the pipeline
If the pipeline is completed return False otherwise True
The 'local' switch turns on the configuration for local running (no server)
"""
# Setup the logger
logger.add_file(os.path.join(self.work_dir, 'pipeline.log'))
if local or verbose:
logger.set_stdout_level(logger.DEBUG if verbose else logger.INFO)
self.log = logger.get_log()
set_scheduler(self.schedname)
# Check and create lock file
self.lock = os.path.join(self.work_dir, LOCK_FILE)
if os.path.exists(self.lock):
raise Exception('Directory already in use: remove %s first' %
self.lock)
else:
with open(self.lock, 'w') as fh:
pass
# Store config file and log some information
with open(os.path.join(self.work_dir, 'pipeline.cfg'), 'w') as fh:
self.log.info('Saving configuration to file %s' % fh.name)
json.dump(self.cfg, fh, indent=4, sort_keys=True)
#self.log.info('Environment:\n %s' % os.environ)
self.all_ok = True
while self.all_ok:
self.update_status()
if self.all_ok: # because update_status takes time...
if (len(self.dag.nodes()) - len(self.completed)) > 0:
for step_name in self.get_next_steps():
self.run_step(step_name)
else:
if self.work_dir != self.output_dir:
self.db.update_output_dir(self.work_dir,
self.output_dir)
self.log.info('')
self.log.info('')
self.log.info('+++ Pipeline %s completed +++' % self.name)
self.status = JOB_STATUS.SUCCEEDED
self.db.update_pipeline_status(self.status)
return 0
else:
self.stop()
def run(self, local=False, verbose=False):
"""
Run a node of the pipeline
If the pipeline is completed return False otherwise True
The 'local' switch turns on the configuration for local running (no server)
"""
# Setup the logger
logger.add_file(os.path.join(self.work_dir, 'pipeline.log'))
if local or verbose:
logger.set_stdout_level(logger.DEBUG if verbose else logger.INFO)
self.log = logger.get_log()
set_scheduler(self.schedname)
# Check and create lock file
self.lock = os.path.join(self.work_dir, LOCK_FILE)
if os.path.exists(self.lock):
raise Exception('Directory already in use: remove %s first' % self.lock)
else:
with open(self.lock, 'w') as fh:
pass
# Store config file and log some information
with open(os.path.join(self.work_dir, 'pipeline.cfg'), 'w') as fh:
self.log.info('Saving configuration to file %s' % fh.name)
json.dump(self.cfg, fh, indent=4, sort_keys=True)
#self.log.info('Environment:\n %s' % os.environ)
self.all_ok = True
while self.all_ok:
self.update_status()
if self.all_ok: # because update_status takes time...
if (len(self.dag.nodes()) - len(self.completed))>0:
for step_name in self.get_next_steps():
self.run_step(step_name)
else:
if self.work_dir != self.output_dir:
self.db.update_output_dir(self.work_dir, self.output_dir)
self.log.info('')
self.log.info('')
self.log.info('+++ Pipeline %s completed +++' % self.name)
self.status = JOB_STATUS.SUCCEEDED
self.db.update_pipeline_status(self.status)
return 0
else:
self.stop()
def __init__(self):
self.bootstrap = STARTUP_CYCLE
self.status = JOB_STATUS.QUEUED
self.meta = {'pipeline': {}, 'step': {}, 'job': {}}
self.requirements = {'memory': '1', 'cpus': '1'}
self.output_dir = '.'
self.jobs = OrderedDict()
self.cmd_count = 0
logger.set_stdout_level(logger.DEBUG)
self.log = logger.get_log()
# parse specs and create keys
self.spec["name"] = self.__module__.replace('nespipe.steps.',
'').split('.')[-1]
self.name = self.spec["name"]
self.__version__ = self.spec['version']
self.local_step = self.spec.get('local', False)
global scheduler
if self.local_step:
self.scheduler = get_scheduler("SCHED_LOCAL")
else:
self.scheduler = scheduler
for k, v in self.spec["args"].iteritems():
for param in v:
if param.get('name', None):
setattr(self, param['name'], param.get('value', []))
ut.dict_update(
self.requirements,
self.spec.get('requirements', {
'memory': '1',
'cpus': '1'
}))
for k, v in self.requirements.iteritems():
setattr(self, k, int(v))
#set the jvm memory
if 'memory' in self.requirements:
self.jvm_memory = int(int(self.requirements['memory']) * 0.9)
if not self.jvm_memory:
self.jvm_memory = 1
def __init__(self):
self.bootstrap = STARTUP_CYCLE
self.status = JOB_STATUS.QUEUED
self.meta = { 'pipeline':{}, 'step':{}, 'job':{}}
self.requirements = {'memory' : '1', 'cpus' : '1'}
self.output_dir = '.'
self.jobs = OrderedDict()
self.cmd_count = 0
logger.set_stdout_level(logger.DEBUG)
self.log = logger.get_log()
# parse specs and create keys
self.spec["name"] = self.__module__.replace('nespipe.steps.','').split('.')[-1]
self.name = self.spec["name"]
self.__version__ = self.spec['version']
self.local_step = self.spec.get('local', False)
global scheduler
if self.local_step:
self.scheduler = get_scheduler("SCHED_LOCAL")
else:
self.scheduler = scheduler
for k, v in self.spec["args"].iteritems():
for param in v:
if param.get('name', None):
setattr(self, param['name'], param.get('value', []))
ut.dict_update(self.requirements, self.spec.get('requirements', {'memory' : '1', 'cpus' : '1'}))
for k, v in self.requirements.iteritems():
setattr(self, k, int(v))
#set the jvm memory
if 'memory' in self.requirements:
self.jvm_memory = int(int(self.requirements['memory']) * 0.9)
if not self.jvm_memory:
self.jvm_memory = 1