def process(self): """ Waits for a job from the streams queue then reads the jobs context from the data-store. The jobs method will then be de-serialized and executed. The results, including any execution errors, are then returned to the data-store. If no jobs appear on the streams queue within the designated timeout then the Worker will die. """ sig= None try: #print "processing stream", self.pid, self.stream_id, getppid() job_id= None job= {} try: (priority, job_id)= self.queue.get(block= True, timeout= self.properties.get("timeout", 15)).pop() except Empty: #print "stream idle", self.pid, self.stream_id self.alive= False return sig= signal(SIGINT, SIG_IGN) job= self.store.get(job_id) job.update([("status", "processing"), ("node", self.id), ("worker", getpid())]) self.store.update([(job.get("id"), job)]) print "processing job: %s, %s, %s, %s" % (self.pid, job.get("id"), job.get("name"), job.get("status")) sleep(float(job.get("delay", 0.0))) method= FunctionType(mloads(decompress(decode(job.get("code")))), globals(), job.get("name")) result= method(job.get("args")) job.update([("result", result), ("status", "ready")]) self.store.update([(job.get("id"), job)]) print "completed job: %s, %s, %s, %s" % (self.pid, job.get("id"), job.get("name"), job.get("status")) except KeyboardInterrupt, e: print "worker shutting down", self.pid self.alive= False
def load_binint2(self): self.append(Int(mloads('i' + self.read(2) + '\000\000')))
def load_binint(self): self.append(Int(mloads('i' + self.read(4))))
def load_long_binput(self): i = mloads('i' + self.read(4)) last = self.stack[-1] if getattr(last, 'id', last) is last: last.id = self.idprefix + repr(i)
def load_long_binget(self): i = mloads('i' + self.read(4)) self.append(Get(self.idprefix + repr(i)))
def load_binunicode(self): len = mloads('i' + self.read(4)) self.append(Unicode(self.read(len),'utf-8'))
def load_binint(self): self.append(Int(mloads('i' + self.read(4)), self.id_mapping))
def load_long_binput(self): i = mloads('i' + self.read(4)) self.stack[-1].id=self.idprefix+repr(i)
def load_binput(self): i = mloads('i' + self.read(1) + '\000\000\000') #LOG('load_binput', 0, 'self.stack = %r, self.idprefix+`i` = %r' % (self.stack, self.idprefix+`i`)) self.stack[-1].id=self.idprefix+repr(i)
def load_binget(self): i = mloads('i' + self.read(1) + '\000\000\000') self.append(Get(self.idprefix+repr(i), self.id_mapping))
def load_short_binstring(self): len = mloads('i' + self.read(1) + '\000\000\000') self.append(String(self.read(len), self.id_mapping))
def load_binunicode(self): len = mloads('i' + self.read(4)) self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
def load_binint2(self): self.append(Int(mloads('i' + self.read(2) + '\000\000'), self.id_mapping))
def load_binstring(self): len = mloads('i' + self.read(4)) self.append(String(self.read(len)))
def load_binunicode(self): len = mloads('i' + self.read(4)) self.append(Unicode(self.read(len), 'utf-8'))