def unpack(cls, jobpack, globals={}):
"""Unpack the previously packed :class:`JobDict`."""
jobdict = cls.defaults.copy()
jobdict.update(**decode_netstring_fd(jobpack))
for key in cls.defaults:
if key == 'input':
jobdict['input'] = [i.split()
for i in jobdict['input'].split(' ')]
elif key == 'nr_reduces':
jobdict[key] = int(jobdict[key])
elif key == 'scheduler':
for key in cls.scheduler_keys:
if 'sched_%s' % key in jobdict:
jobdict['scheduler'][key] = jobdict.pop('sched_%s' % key)
elif key == 'prefix':
pass
elif jobdict[key] is None:
pass
elif key in cls.stacks:
jobdict[key] = util.unpack_stack(jobdict[key], globals=globals)
else:
jobdict[key] = util.unpack(jobdict[key], globals=globals)
# map readers and writers to streams
for oldio, stream, wrapper in cls.io_mappings:
if jobdict[oldio]:
jobdict[stream].append(wrapper(jobdict[oldio]))
return cls(**jobdict)
def unpack(cls, jobpack, globals={}):
"""Unpack the previously packed :class:`JobDict`."""
jobdict = cls.defaults.copy()
jobdict.update(**decode_netstring_fd(jobpack))
for key in cls.defaults:
if key == 'input':
jobdict['input'] = [
i.split() for i in jobdict['input'].split(' ')
]
elif key == 'username':
pass
elif key == 'nr_reduces':
jobdict[key] = int(jobdict[key])
elif key == 'scheduler':
for key in cls.scheduler_keys:
if 'sched_%s' % key in jobdict:
jobdict['scheduler'][key] = jobdict.pop('sched_%s' %
key)
elif key == 'prefix':
pass
elif jobdict[key] is None:
pass
elif key in cls.stacks:
jobdict[key] = util.unpack_stack(jobdict[key], globals=globals)
else:
jobdict[key] = util.unpack(jobdict[key], globals=globals)
return cls(**jobdict)
def new_job(self, *args, **kwargs):
job = Job(DummyDisco(), **kwargs)
req = decode_netstring_fd(cStringIO.StringIO(job.msg))
argv_backup = sys.argv[:]
out_backup = sys.stderr
sys.argv = ["", "", job.name, "localhost",
"http://nohost", self.partition]
sys.argv += kwargs["input"]
from disconode import disco_worker
sys.stderr = out = MsgStream()
try:
if self.mode == "map":
disco_worker.op_map(req)
elif self.mode == "reduce":
disco_worker.op_reduce(req)
else:
raise "Unknown mode: %s "\
"(must be 'map' or 'reduce')"\
% self.mode
finally:
sys.argv = argv_backup
sys.stderr = out_backup
return out.out
def unpack(cls, jobpack, globals={}):
"""Unpack the previously packed :class:`JobDict`."""
jobdict = cls.defaults.copy()
jobdict.update(**decode_netstring_fd(jobpack))
for key in cls.defaults:
if key == 'input':
jobdict['input'] = [i.split()
for i in jobdict['input'].split(' ')]
elif key == 'username':
pass
elif key == 'nr_reduces':
jobdict[key] = int(jobdict[key])
elif key == 'scheduler':
for key in cls.scheduler_keys:
if 'sched_%s' % key in jobdict:
jobdict['scheduler'][key] = jobdict.pop('sched_%s' % key)
elif key == 'prefix':
pass
elif jobdict[key] is None:
pass
elif key in cls.stacks:
jobdict[key] = util.unpack_stack(jobdict[key], globals=globals)
else:
jobdict[key] = util.unpack(jobdict[key], globals=globals)
return cls(**jobdict)
def jobspec(self, name):
r = self.request("/disco/ctrl/parameters?name=%s" % name, redir=True)
return decode_netstring_fd(cStringIO.StringIO(r))
