def process(self):
"""
Waits for a job from the streams queue then reads the jobs context from the
data-store. The jobs method will then be de-serialized and executed.
The results, including any execution errors, are then returned to the
data-store. If no jobs appear on the streams queue within the designated
timeout then the Worker will die.
"""
sig= None
try:
#print "processing stream", self.pid, self.stream_id, getppid()
job_id= None
job= {}
try:
(priority, job_id)= self.queue.get(block= True, timeout= self.properties.get("timeout", 15)).pop()
except Empty:
#print "stream idle", self.pid, self.stream_id
self.alive= False
return
sig= signal(SIGINT, SIG_IGN)
job= self.store.get(job_id)
job.update([("status", "processing"), ("node", self.id), ("worker", getpid())])
self.store.update([(job.get("id"), job)])
print "processing job: %s, %s, %s, %s" % (self.pid, job.get("id"), job.get("name"), job.get("status"))
sleep(float(job.get("delay", 0.0)))
method= FunctionType(mloads(decompress(decode(job.get("code")))), globals(), job.get("name"))
result= method(job.get("args"))
job.update([("result", result), ("status", "ready")])
self.store.update([(job.get("id"), job)])
print "completed job: %s, %s, %s, %s" % (self.pid, job.get("id"), job.get("name"), job.get("status"))
except KeyboardInterrupt, e:
print "worker shutting down", self.pid
self.alive= False
def load_binint2(self):
self.append(Int(mloads('i' + self.read(2) + '\000\000')))
def load_binint(self):
self.append(Int(mloads('i' + self.read(4))))
def load_long_binput(self):
i = mloads('i' + self.read(4))
last = self.stack[-1]
if getattr(last, 'id', last) is last:
last.id = self.idprefix + repr(i)
def load_long_binget(self):
i = mloads('i' + self.read(4))
self.append(Get(self.idprefix + repr(i)))
def load_binunicode(self):
len = mloads('i' + self.read(4))
self.append(Unicode(self.read(len),'utf-8'))
def load_binint(self):
self.append(Int(mloads('i' + self.read(4)), self.id_mapping))
def load_long_binget(self):
i = mloads('i' + self.read(4))
self.append(Get(self.idprefix + repr(i)))
def load_binint(self):
self.append(Int(mloads('i' + self.read(4))))
def load_long_binput(self):
i = mloads('i' + self.read(4))
self.stack[-1].id=self.idprefix+repr(i)
def load_binput(self):
i = mloads('i' + self.read(1) + '\000\000\000')
#LOG('load_binput', 0, 'self.stack = %r, self.idprefix+`i` = %r' % (self.stack, self.idprefix+`i`))
self.stack[-1].id=self.idprefix+repr(i)
def load_binget(self):
i = mloads('i' + self.read(1) + '\000\000\000')
self.append(Get(self.idprefix+repr(i), self.id_mapping))
def load_short_binstring(self):
len = mloads('i' + self.read(1) + '\000\000\000')
self.append(String(self.read(len), self.id_mapping))
def load_binunicode(self):
len = mloads('i' + self.read(4))
self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
def load_binint2(self):
self.append(Int(mloads('i' + self.read(2) + '\000\000'), self.id_mapping))
def load_binstring(self):
len = mloads('i' + self.read(4))
self.append(String(self.read(len)))
def load_binunicode(self):
len = mloads('i' + self.read(4))
self.append(Unicode(self.read(len), 'utf-8'))
def load_binint2(self):
self.append(Int(mloads('i' + self.read(2) + '\000\000')))
def load_long_binput(self):
i = mloads('i' + self.read(4))
last = self.stack[-1]
if getattr(last, 'id', last) is last:
last.id = self.idprefix + repr(i)
def load_binstring(self):
len = mloads('i' + self.read(4))
self.append(String(self.read(len)))
