def run(self, local=False, verbose=False): """ Run a node of the pipeline If the pipeline is completed return False otherwise True The 'local' switch turns on the configuration for local running (no server) """ # Setup the logger logger.add_file(os.path.join(self.work_dir, 'pipeline.log')) if local or verbose: logger.set_stdout_level(logger.DEBUG if verbose else logger.INFO) self.log = logger.get_log() set_scheduler(self.schedname) # Check and create lock file self.lock = os.path.join(self.work_dir, LOCK_FILE) if os.path.exists(self.lock): raise Exception('Directory already in use: remove %s first' % self.lock) else: with open(self.lock, 'w') as fh: pass # Store config file and log some information with open(os.path.join(self.work_dir, 'pipeline.cfg'), 'w') as fh: self.log.info('Saving configuration to file %s' % fh.name) json.dump(self.cfg, fh, indent=4, sort_keys=True) #self.log.info('Environment:\n %s' % os.environ) self.all_ok = True while self.all_ok: self.update_status() if self.all_ok: # because update_status takes time... if (len(self.dag.nodes()) - len(self.completed)) > 0: for step_name in self.get_next_steps(): self.run_step(step_name) else: if self.work_dir != self.output_dir: self.db.update_output_dir(self.work_dir, self.output_dir) self.log.info('') self.log.info('') self.log.info('+++ Pipeline %s completed +++' % self.name) self.status = JOB_STATUS.SUCCEEDED self.db.update_pipeline_status(self.status) return 0 else: self.stop()
def run(self, local=False, verbose=False): """ Run a node of the pipeline If the pipeline is completed return False otherwise True The 'local' switch turns on the configuration for local running (no server) """ # Setup the logger logger.add_file(os.path.join(self.work_dir, 'pipeline.log')) if local or verbose: logger.set_stdout_level(logger.DEBUG if verbose else logger.INFO) self.log = logger.get_log() set_scheduler(self.schedname) # Check and create lock file self.lock = os.path.join(self.work_dir, LOCK_FILE) if os.path.exists(self.lock): raise Exception('Directory already in use: remove %s first' % self.lock) else: with open(self.lock, 'w') as fh: pass # Store config file and log some information with open(os.path.join(self.work_dir, 'pipeline.cfg'), 'w') as fh: self.log.info('Saving configuration to file %s' % fh.name) json.dump(self.cfg, fh, indent=4, sort_keys=True) #self.log.info('Environment:\n %s' % os.environ) self.all_ok = True while self.all_ok: self.update_status() if self.all_ok: # because update_status takes time... if (len(self.dag.nodes()) - len(self.completed))>0: for step_name in self.get_next_steps(): self.run_step(step_name) else: if self.work_dir != self.output_dir: self.db.update_output_dir(self.work_dir, self.output_dir) self.log.info('') self.log.info('') self.log.info('+++ Pipeline %s completed +++' % self.name) self.status = JOB_STATUS.SUCCEEDED self.db.update_pipeline_status(self.status) return 0 else: self.stop()
def __init__(self): self.bootstrap = STARTUP_CYCLE self.status = JOB_STATUS.QUEUED self.meta = {'pipeline': {}, 'step': {}, 'job': {}} self.requirements = {'memory': '1', 'cpus': '1'} self.output_dir = '.' self.jobs = OrderedDict() self.cmd_count = 0 logger.set_stdout_level(logger.DEBUG) self.log = logger.get_log() # parse specs and create keys self.spec["name"] = self.__module__.replace('nespipe.steps.', '').split('.')[-1] self.name = self.spec["name"] self.__version__ = self.spec['version'] self.local_step = self.spec.get('local', False) global scheduler if self.local_step: self.scheduler = get_scheduler("SCHED_LOCAL") else: self.scheduler = scheduler for k, v in self.spec["args"].iteritems(): for param in v: if param.get('name', None): setattr(self, param['name'], param.get('value', [])) ut.dict_update( self.requirements, self.spec.get('requirements', { 'memory': '1', 'cpus': '1' })) for k, v in self.requirements.iteritems(): setattr(self, k, int(v)) #set the jvm memory if 'memory' in self.requirements: self.jvm_memory = int(int(self.requirements['memory']) * 0.9) if not self.jvm_memory: self.jvm_memory = 1
def __init__(self): self.bootstrap = STARTUP_CYCLE self.status = JOB_STATUS.QUEUED self.meta = { 'pipeline':{}, 'step':{}, 'job':{}} self.requirements = {'memory' : '1', 'cpus' : '1'} self.output_dir = '.' self.jobs = OrderedDict() self.cmd_count = 0 logger.set_stdout_level(logger.DEBUG) self.log = logger.get_log() # parse specs and create keys self.spec["name"] = self.__module__.replace('nespipe.steps.','').split('.')[-1] self.name = self.spec["name"] self.__version__ = self.spec['version'] self.local_step = self.spec.get('local', False) global scheduler if self.local_step: self.scheduler = get_scheduler("SCHED_LOCAL") else: self.scheduler = scheduler for k, v in self.spec["args"].iteritems(): for param in v: if param.get('name', None): setattr(self, param['name'], param.get('value', [])) ut.dict_update(self.requirements, self.spec.get('requirements', {'memory' : '1', 'cpus' : '1'})) for k, v in self.requirements.iteritems(): setattr(self, k, int(v)) #set the jvm memory if 'memory' in self.requirements: self.jvm_memory = int(int(self.requirements['memory']) * 0.9) if not self.jvm_memory: self.jvm_memory = 1
def __init__(self): self.log = logger.get_log()