def jobzip(self, job, **jobargs):
from disco.util import iskv
from disco.worker.classic.modutil import find_modules
jobzip = super(Worker, self).jobzip(job, **jobargs)
def get(key):
return self.getitem(key, job, jobargs)
if isinstance(get('required_files'), dict):
for path, bytes in get('required_files').iteritems():
jobzip.writestr(path, bytes)
else:
for path in get('required_files'):
jobzip.write(path, os.path.join('lib', os.path.basename(path)))
if get('required_modules') is None:
self['required_modules'] = find_modules([obj
for key in self
for obj in util.iterify(get(key))
if callable(obj)],
exclude=['Task'])
for mod in get('required_modules'):
if iskv(mod):
jobzip.writepath(mod[1])
for func in ('map', 'reduce'):
if isinstance(get(func), dict):
for path, bytes in get(func).iteritems():
jobzip.writestr(os.path.join('ext.%s' % func, path), bytes)
return jobzip
def jobzip(self, job, **jobargs):
from disco.util import iskv
from disco.worker.classic.modutil import find_modules
jobzip = super(Worker, self).jobzip(job, **jobargs)
def get(key):
return self.getitem(key, job, jobargs)
if isinstance(get('required_files'), dict):
for path, bytes in get('required_files').items():
jobzip.writestr(path, bytes)
else:
for path in get('required_files'):
jobzip.write(path, os.path.join('lib', os.path.basename(path)))
if get('required_modules') is None:
self['required_modules'] = find_modules([
obj for key in self
for obj in util.iterify(get(key)) if callable(obj)
],
exclude=['Task'])
for mod in get('required_modules'):
if iskv(mod):
jobzip.writepath(mod[1])
for func in ('map', 'reduce'):
if isinstance(get(func), dict):
for path, bytes in get(func).items():
jobzip.writestr(os.path.join('ext.{0}'.format(func), path),
bytes)
return jobzip
def assertFindsModules(self,
functions,
modules,
send_modules=False,
recurse=True):
self.assertEquals(
sorted(
find_modules(functions,
send_modules=send_modules,
recurse=recurse)), sorted(modules))
def test_find_modules(self):
self.job = ModUtilJob()
self.job.run(input=self.test_server.urls(['0.5|1.2']),
required_modules=find_modules([self.job.map]))
self.assertResults(self.job, [(4.0, '')])
def test_missing(self):
self.assertRaises(ModUtilImportError,
lambda: find_modules([missing_module]))
def test_find_modules(self):
self.job = ModUtilJob()
self.job.run(input=self.test_server.urls(['0.5|1.2']),
required_modules=find_modules([self.job.map]))
self.assertResults(self.job, [(4.0, '')])
def test_missing(self):
self.assertRaises(ModUtilImportError, lambda: find_modules([missing_module]))
def assertFindsModules(self, functions, modules, send_modules=False, recurse=True):
self.assertEquals(sorted(find_modules(functions,
send_modules=send_modules,
recurse=recurse)),
sorted(modules))
