def execute(self, runid): """Invoke each of the extractors in the chain. Keyword arguments: runid -- A UTC ID string which identifies the run. An empty run node is created as an instance of Extract.Node and passed to the handlerunnode() method of each extractor in the chain. An empty job node is created, as a subnode of the run node, for each job in the jobtree directory named by the runid, e.g. /2007-06-25_09.18.46, and passed to the handlejobnode() method of each extractor in the chain. The run node is saved as XML to the configurable BaseExtractor_XmlFile, replacing ${runid} with the current run id. e.g. ~/gangadir/robot/extract/${runid}.xml """ logger.info("Extracting data for run '%s'.", runid) runnode = Node('run') for extractor in self.chain: extractor.handlerunnode(runnode, runid) path = Utility.jobtreepath(runid) ids = jobtree.listjobs(path) ids.sort() for id in ids: jobnode = Node('job') job = jobs(id) for extractor in self.chain: extractor.handlejobnode(jobnode, job) runnode.nodes.append(jobnode) self._saveextract(runnode, runid) logger.info('Extract data saved.')
def test_submitter(self): """Test submitter submits 3 jobs and adds them to the jobtree.""" #execute action self.submitter.execute(self.runid) #check jobs are added to jobtree path = Utility.jobtreepath(self.runid) js = jobtree.getjobs(path) assert len(js) == 3, 'number of jobs added to jobtree path is not 3' for j in js: assert j.status != 'new', 'job status is new indicating that it may not have been submitted'
def test_finisher(self): """Test finisher waits for jobs to finish.""" #submit jobs self.submitter.execute(self.runid) #execute action self.finisher.execute(self.runid) #get jobs from jobtree path = Utility.jobtreepath(self.runid) js = jobtree.getjobs(path) #check jobs are finished for j in js: assert j.status == 'completed', 'job status is not completed indicating that it may not have finished'
def execute(self, runid): """Invoke each of the finishers in the chain. Keyword arguments: runid -- A UTC ID string which identifies the run. Loops until the run is finished or the elapsed time exceeds the configurable BaseFinisher_Timeout (seconds). Each job in the jobtree directory named by the runid, e.g. /2007-06-25_09.18.46, is passed to the handleisfinished() method of each finisher in the chain. If any finisher returns False, for any job, then the run is considered unfinished. The sleep period in the loop is 1/10th the timeout, constrained to the range [10, 60] seconds. """ logger.info("Finishing jobs for run '%s'.", runid) startsecs = time.time() timeoutsecs = self.getoption('BaseFinisher_Timeout') sleepsecs = timeoutsecs / 10 if sleepsecs < 10: sleepsecs = 10 if sleepsecs > 60: sleepsecs = 60 logger.info('Waiting for jobs to finish. Timeout %d seconds.' ' Sleep period %d seconds', timeoutsecs, sleepsecs) path = Utility.jobtreepath(runid) jobs = jobtree.getjobs(path) while True: #suppose finished is True finished = True for j in jobs: #if one job not finished then finished is False for finisher in self.chain: if not finisher.handleisfinished(j): finished = False break elapsedsecs = time.time() - startsecs # test break condition here to avoid sleep after satisfied condition if finished or timeoutsecs < elapsedsecs: break # break condition on satisfied, so sleep time.sleep(sleepsecs) if finished: logger.info('All jobs finished after %d seconds.', elapsedsecs) else: logger.info('Timeout after %d seconds.', elapsedsecs)
def execute(self, runid): """Invoke each of the submitters in the chain. Keyword arguments: runid -- A UTC ID string which identifies the run. An empty list of jobids is created and passed to the handlesubmit() method of each the submitter in the chain. Any job, whose id is added to the argument jobids, is added to the jobtree in the directory named by the runid, e.g. /2007-06-25_09.18.46. """ logger.info("Submitting jobs for run '%s'.", runid) jobids = [] for submitter in self.chain: submitter.handlesubmit(jobids, runid) path = Utility.jobtreepath(runid) jobtree.mkdir(path) for id in jobids: jobtree.add(jobs(id), path) logger.info("%d submitted jobs added to jobtree path '%s'.", len(jobids), path)