def __init__(self,
netlocstr='',
id=-1,
inputs=None,
jobdict=None,
jobname='',
settings=DiscoSettings()):
self.netloc = util.netloc.parse(netlocstr)
self.id = int(id)
self.inputs = inputs
self.jobdict = jobdict
self.jobname = jobname
self.settings = settings
self.blobs = []
self.mode = self.__class__.__name__.lower()
self.run_id = "%s:%d-%x-%x" % (self.mode,
self.id,
int(time.time() * 1000),
os.getpid())
set_mem_limit(self.settings['DISCO_WORKER_MAX_MEM'])
if not jobdict:
self.jobdict = JobDict.unpack(open(self.jobpack),
globals=worker.__dict__)
self.insert_globals(self.functions)
def start(self, task, job, **jobargs):
from disco.sysutil import set_mem_limit
set_mem_limit(job.settings['DISCO_WORKER_MAX_MEM'])
task.makedirs()
if self.getitem('profile', job, jobargs):
from cProfile import runctx
name = 'profile-{0}'.format(task.uid)
path = task.path(name)
runctx('self.run(task, job, **jobargs)', globals(), locals(), path)
task.put(name, open(path, 'rb').read())
else:
self.run(task, job, **jobargs)
self.end(task, job, **jobargs)
def start(self, task, job, **jobargs):
from disco.sysutil import set_mem_limit
set_mem_limit(job.settings['DISCO_WORKER_MAX_MEM'])
task.makedirs()
if self.getitem('profile', job, jobargs):
from cProfile import runctx
name = 'profile-%s' % task.uid
path = task.path(name)
runctx('self.run(task, job, **jobargs)', globals(), locals(), path)
task.put(name, open(path).read())
else:
self.run(task, job, **jobargs)
self.end(task, job, **jobargs)