def execute(self, runid):
"""Invoke each of the extractors in the chain.
Keyword arguments:
runid -- A UTC ID string which identifies the run.
An empty run node is created as an instance of Extract.Node and passed
to the handlerunnode() method of each extractor in the chain.
An empty job node is created, as a subnode of the run node, for each job
in the jobtree directory named by the runid, e.g. /2007-06-25_09.18.46,
and passed to the handlejobnode() method of each extractor in the chain.
The run node is saved as XML to the configurable BaseExtractor_XmlFile,
replacing ${runid} with the current run id.
e.g. ~/gangadir/robot/extract/${runid}.xml
"""
logger.info("Extracting data for run '%s'.", runid)
runnode = Node('run')
for extractor in self.chain:
extractor.handlerunnode(runnode, runid)
path = Utility.jobtreepath(runid)
ids = jobtree.listjobs(path)
ids.sort()
for id in ids:
jobnode = Node('job')
job = jobs(id)
for extractor in self.chain:
extractor.handlejobnode(jobnode, job)
runnode.nodes.append(jobnode)
self._saveextract(runnode, runid)
logger.info('Extract data saved.')
def test_submitter(self):
"""Test submitter submits 3 jobs and adds them to the jobtree."""
#execute action
self.submitter.execute(self.runid)
#check jobs are added to jobtree
path = Utility.jobtreepath(self.runid)
js = jobtree.getjobs(path)
assert len(js) == 3, 'number of jobs added to jobtree path is not 3'
for j in js:
assert j.status != 'new', 'job status is new indicating that it may not have been submitted'
def test_submitter(self):
"""Test submitter submits 3 jobs and adds them to the jobtree."""
#execute action
self.submitter.execute(self.runid)
#check jobs are added to jobtree
path = Utility.jobtreepath(self.runid)
js = jobtree.getjobs(path)
assert len(js) == 3, 'number of jobs added to jobtree path is not 3'
for j in js:
assert j.status != 'new', 'job status is new indicating that it may not have been submitted'
def test_finisher(self):
"""Test finisher waits for jobs to finish."""
#submit jobs
self.submitter.execute(self.runid)
#execute action
self.finisher.execute(self.runid)
#get jobs from jobtree
path = Utility.jobtreepath(self.runid)
js = jobtree.getjobs(path)
#check jobs are finished
for j in js:
assert j.status == 'completed', 'job status is not completed indicating that it may not have finished'
def test_finisher(self):
"""Test finisher waits for jobs to finish."""
#submit jobs
self.submitter.execute(self.runid)
#execute action
self.finisher.execute(self.runid)
#get jobs from jobtree
path = Utility.jobtreepath(self.runid)
js = jobtree.getjobs(path)
#check jobs are finished
for j in js:
assert j.status == 'completed', 'job status is not completed indicating that it may not have finished'
def execute(self, runid):
"""Invoke each of the finishers in the chain.
Keyword arguments:
runid -- A UTC ID string which identifies the run.
Loops until the run is finished or the elapsed time exceeds the
configurable BaseFinisher_Timeout (seconds).
Each job in the jobtree directory named by the runid, e.g.
/2007-06-25_09.18.46, is passed to the handleisfinished() method of each
finisher in the chain. If any finisher returns False, for any job,
then the run is considered unfinished.
The sleep period in the loop is 1/10th the timeout, constrained to the
range [10, 60] seconds.
"""
logger.info("Finishing jobs for run '%s'.", runid)
startsecs = time.time()
timeoutsecs = self.getoption('BaseFinisher_Timeout')
sleepsecs = timeoutsecs / 10
if sleepsecs < 10: sleepsecs = 10
if sleepsecs > 60: sleepsecs = 60
logger.info('Waiting for jobs to finish. Timeout %d seconds.'
' Sleep period %d seconds', timeoutsecs, sleepsecs)
path = Utility.jobtreepath(runid)
jobs = jobtree.getjobs(path)
while True:
#suppose finished is True
finished = True
for j in jobs:
#if one job not finished then finished is False
for finisher in self.chain:
if not finisher.handleisfinished(j):
finished = False
break
elapsedsecs = time.time() - startsecs
# test break condition here to avoid sleep after satisfied condition
if finished or timeoutsecs < elapsedsecs:
break
# break condition on satisfied, so sleep
time.sleep(sleepsecs)
if finished:
logger.info('All jobs finished after %d seconds.', elapsedsecs)
else:
logger.info('Timeout after %d seconds.', elapsedsecs)
def execute(self, runid):
"""Invoke each of the submitters in the chain.
Keyword arguments:
runid -- A UTC ID string which identifies the run.
An empty list of jobids is created and passed to the handlesubmit()
method of each the submitter in the chain.
Any job, whose id is added to the argument jobids, is added to the
jobtree in the directory named by the runid, e.g. /2007-06-25_09.18.46.
"""
logger.info("Submitting jobs for run '%s'.", runid)
jobids = []
for submitter in self.chain:
submitter.handlesubmit(jobids, runid)
path = Utility.jobtreepath(runid)
jobtree.mkdir(path)
for id in jobids:
jobtree.add(jobs(id), path)
logger.info("%d submitted jobs added to jobtree path '%s'.", len(jobids), path)
def execute(self, runid):
"""Invoke each of the submitters in the chain.
Keyword arguments:
runid -- A UTC ID string which identifies the run.
An empty list of jobids is created and passed to the handlesubmit()
method of each the submitter in the chain.
Any job, whose id is added to the argument jobids, is added to the
jobtree in the directory named by the runid, e.g. /2007-06-25_09.18.46.
"""
logger.info("Submitting jobs for run '%s'.", runid)
jobids = []
for submitter in self.chain:
submitter.handlesubmit(jobids, runid)
path = Utility.jobtreepath(runid)
jobtree.mkdir(path)
for id in jobids:
jobtree.add(jobs(id), path)
logger.info("%d submitted jobs added to jobtree path '%s'.",
len(jobids), path)
