def contents(self, offset=HEADER_SIZE):
for field in (json.dumps(self.jobdict),
json.dumps(self.jobenvs),
self.jobhome,
self.jobdata):
yield offset, field
offset += len(field)
def tag(self, tag, urls, token=None, **kwargs):
"""Append the list of ``urls`` to the ``tag``."""
defaults = {'delayed': False, 'update': False}
defaults.update(kwargs)
url = '%s?%s' % (canonizetag(tag), '&'.join(
'%s=%s' % (k, '1' if v else '') for k, v in defaults.items()))
return self._download(url, json.dumps(urls), token=token)
def tag(self, tag, urls, token=None, **kwargs):
"""Append the list of ``urls`` to the ``tag``."""
defaults = {'delayed': False, 'update': False}
defaults.update(kwargs)
url = '%s?%s' % (canonizetag(tag),
'&'.join('%s=%s' % (k, '1' if v else '')
for k, v in defaults.items()))
return self._download(url, json.dumps(urls), token=token)
def send(cls, type, payload=''):
from disco.json import dumps, loads
body = dumps(payload)
cls.stderr.write('%s %d %s\n' % (type, len(body), body))
spent, rtype = sys.stdin.t_read_until(' ')
spent, rsize = sys.stdin.t_read_until(' ', spent=spent)
spent, rbody = sys.stdin.t_read(int(rsize) + 1, spent=spent)
if type == 'ERROR':
raise ValueError(loads(rbody[:-1]))
return loads(rbody[:-1])
def send(cls, type, payload=''):
from disco.json import dumps, loads
body = dumps(payload)
sys.stderr.write('%s %d %s\n' % (type, len(body), body))
spent, rtype = sys.stdin.t_read_until(' ')
spent, rsize = sys.stdin.t_read_until(' ', spent=spent)
spent, rbody = sys.stdin.t_read(int(rsize) + 1, spent=spent)
if type == 'ERROR':
raise ValueError(loads(rbody[:-1]))
return loads(rbody[:-1])
def put(self, tag, urls, token=None):
"""Put the list of ``urls`` to the tag ``tag``.
.. warning::
Generally speaking, concurrent applications should use
:meth:`DDFS.tag` instead.
"""
return self._upload(canonizetag(tag),
StringIO(json.dumps(urls)),
token=token)
def set_config(self, config):
response = json.loads(
self.request('/disco/ctrl/save_config_table', json.dumps(config)))
if response != 'table saved!':
raise DiscoError(response)
def results(self, jobspec, timeout=2000):
"""
Returns a list of results for a single job or for many
concurrently running jobs, depending on the type of *jobspec*.
:type jobspec: :class:`disco.job.Job`, string, or list
:param jobspec: If a job or job name is provided,
return a tuple which looks like::
status, results
If a list is provided,
return two lists: inactive jobs and active jobs.
Both the lists contain elements of the following type::
jobname, (status, results)
where status is one of:
``'unknown job'``,
``'dead'``,
``'active'``, or
``'ready'``.
:type timeout: int
:param timeout: wait at most this many milliseconds,
for at least one on the jobs to finish.
Using a list of jobs is a more efficient way to wait
for multiple jobs to finish.
Consider the following example that prints out results
as soon as the jobs (initially ``active``) finish::
while active:
inactive, active = disco.results(jobs)
for jobname, (status, results) in inactive:
if status == 'ready':
for k, v in result_iterator(results):
print k, v
disco.purge(jobname)
Note how the list of active jobs, ``active``,
returned by :meth:`Disco.results`,
can be used as the input to this function as well.
"""
def jobname(job):
if isinstance(job, Job):
return job.name
elif isinstance(job, basestring):
return job
return job[0]
jobnames = [jobname(job) for job in util.iterify(jobspec)]
results = json.loads(
self.request('/disco/ctrl/get_results',
json.dumps([timeout, jobnames])))
others, active = [], []
for jobname, (status, result) in results:
if isinstance(jobspec, (Job, basestring)):
return status, result
elif status == 'active':
active.append((jobname, (status, result)))
else:
others.append((jobname, (status, result)))
return others, active
def setattr(self, tag, attr, val, token=None):
"""Set the value of the attribute ``attr`` of the tag ``tag``."""
return self._upload(self._tagattr(tag, attr),
StringIO(json.dumps(val)),
token=token)
def set_config(self, config):
response = json.loads(self.request('/disco/ctrl/save_config_table', json.dumps(config)))
if response != 'table saved!':
raise DiscoError(response)
def results(self, jobspec, timeout=2000):
"""
Returns a list of results for a single job or for many
concurrently running jobs, depending on the type of *jobspec*.
:type jobspec: :class:`disco.job.Job`, string, or list
:param jobspec: If a job or job name is provided,
return a tuple which looks like::
status, results
If a list is provided,
return two lists: inactive jobs and active jobs.
Both the lists contain elements of the following type::
jobname, (status, results)
where status is one of:
``'unknown job'``,
``'dead'``,
``'active'``, or
``'ready'``.
:type timeout: int
:param timeout: wait at most this many milliseconds,
for at least one on the jobs to finish.
Using a list of jobs is a more efficient way to wait
for multiple jobs to finish.
Consider the following example that prints out results
as soon as the jobs (initially ``active``) finish::
while active:
inactive, active = disco.results(jobs)
for jobname, (status, results) in inactive:
if status == 'ready':
for k, v in result_iterator(results):
print k, v
disco.purge(jobname)
Note how the list of active jobs, ``active``,
returned by :meth:`Disco.results`,
can be used as the input to this function as well.
"""
def jobname(job):
if isinstance(job, Job):
return job.name
elif isinstance(job, basestring):
return job
return job[0]
jobnames = [jobname(job) for job in util.iterify(jobspec)]
results = json.loads(self.request('/disco/ctrl/get_results',
json.dumps([timeout, jobnames])))
others, active = [], []
for jobname, (status, result) in results:
if isinstance(jobspec, (Job, basestring)):
return status, result
elif status == 'active':
active.append((jobname, (status, result)))
else:
others.append((jobname, (status, result)))
return others, active
def contents(self, offset=HEADER_SIZE):
for field in (json.dumps(self.jobdict), json.dumps(self.jobenvs),
self.jobhome, self.jobdata):
yield offset, field
offset += len(field)
def write(self, status=None, timestamp=None, host=None, message=None):
if timestamp:
print json.dumps([timestamp, host, message])
