def check_results(self, jobname, start_time, timeout, poll_interval):
try:
status, results = self.results(jobname, timeout=poll_interval)
except CommError as e:
status = 'active'
if status == 'ready':
return results
if status != 'active':
raise JobError(Job(name=jobname, master=self),
"Status {0}".format(status))
if timeout and time.time() - start_time > timeout:
raise JobError(Job(name=jobname, master=self), "Timeout")
raise Continue()
def profile_stats(self, jobname, mode='', stream=sys.stdout):
"""
Returns results of job profiling.
:ref:`jobdict` must have had the ``profile`` flag enabled.
:type mode: 'map' or 'reduce' or ''
:param mode: restricts results to the map or reduce phase, or not.
:type stream: file-like object
:param stream: alternate output stream.
See the `pstats.Stats constructor <http://docs.python.org/library/profile.html#pstats.Stats>`_.
The function returns a `pstats.Stats object <http://docs.python.org/library/profile.html#the-stats-class>`_.
For instance, you can print out results as follows::
job.profile_stats().sort_stats('cumulative').print_stats()
.. versionadded:: 0.2.1
"""
prefix = 'profile-%s' % mode
f = [s for s in self.oob_list(jobname) if s.startswith(prefix)]
if not f:
raise JobError(Job(name=jobname, master=self), "No profile data")
import pstats
stats = pstats.Stats(Stats(self.oob_get(jobname, f[0])), stream=stream)
for s in f[1:]:
stats.add(Stats(self.oob_get(jobname, s)))
stats.strip_dirs()
stats.sort_stats('cumulative')
return stats
def profile_stats(self, name, mode=''):
"""
Returns results of profiling of the given job *name*.
The job must have been run with the ``profile`` flag enabled.
You can restrict results specifically to the map or reduce task
by setting *mode* either to ``"map"`` or ``"reduce"``. By default
results include both the map and the reduce phases. Results are
accumulated from all nodes.
The function returns a `pstats.Stats object <http://docs.python.org/library/profile.html#the-stats-class>`_.
You can print out results as follows::
job.profile_stats().print_stats()
(*Added in version 0.2.1*)
"""
prefix = 'profile-%s' % mode
f = [s for s in self.oob_list(name) if s.startswith(prefix)]
if not f:
raise JobError("No profile data", self.master, name)
import pstats
stats = pstats.Stats(Stats(self.oob_get(name, f[0])))
for s in f[1:]:
stats.add(Stats(self.oob_get(name, s)))
return stats
def inputs(self, job):
for input in util.iterify(self[job]):
if isinstance(input, Job):
status, results = input.results()
if status in ('unknown job', 'active'):
yield [None]
elif status == 'ready':
yield results
else:
raise JobError(input, "Status %s" % status)
else:
yield [input]
def walk(self):
for job in self:
status, results = job.results()
if status == 'unknown job':
input = util.chainify(self.inputs(job))
if not any(i is None for i in input):
job.run(input=input)
elif status == 'active':
pass
elif status == 'ready':
yield 1
else:
raise JobError(job, "Status %s" % status)
return others, active
def jobinfo(self, jobname):
"""Returns a dict containing information about the job."""
return json.loads(self.request('/disco/ctrl/jobinfo?name=%s' %
jobname))
def check_results(self, jobname, start_time, timeout, poll_interval):
try:
status, results = self.results(jobname, timeout=poll_interval)
except CommError, e:
status = 'active'
if status == 'ready':
return results
if status != 'active':
raise JobError(Job(name=jobname, master=self),
"Status %s" % status)
if timeout and time.time() - start_time > timeout:
raise JobError(Job(name=jobname, master=self), "Timeout")
raise Continue()
def wait(self,
jobname,
poll_interval=2,
timeout=None,
clean=False,
show=None):
"""
Block until the job has finished.
Returns a list of the result urls.
:type poll_interval: int
others.append(result)
return others, active
def jobinfo(self, name):
"""Returns a dictionary containing information about the job *name*."""
return json.loads(self.request('/disco/ctrl/jobinfo?name=%s' % name))
def check_results(self, name, start_time, timeout, poll_interval):
try:
status, results = self.results(name, timeout=poll_interval)
except CommError, e:
status = 'active'
if status == 'ready':
return results
if status != 'active':
raise JobError("Job status %s" % status, self.master, name)
if timeout and time.time() - start_time > timeout:
raise JobError("Timeout", self.master, name)
raise Continue()
def wait(self,
name,
poll_interval=2,
timeout=None,
clean=False,
show=DiscoSettings()['DISCO_EVENTS']):
"""
Block until the job *name* has finished. Returns a list URLs to the
results files which is typically processed with :func:`result_iterator`.
:meth:`Disco.wait` polls the server for the job status every