def _dispatcher(self): http = httplib2.Http() while True: try: task = Task.unserialize(self.queue.get()) timeout = gevent.Timeout(constants.WORKER_TIMEOUT) timeout.start() self.scheduler.send('start:%s\n' % task.id) if task.url.startswith('http'): headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"} resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params) else: zmq_remotes = frozenset(task.url.split(',')) if not zmq_remotes in self.zmq_sockets: sock = self.zmq.socket(zmq.REQ) lock = gevent.coros.Semaphore() for remote in zmq_remotes: sock.connect(remote) self.zmq_sockets[zmq_remotes] = (sock, lock) else: sock, lock = self.zmq_sockets[zmq_remotes] try: lock.acquire() # Because send/recv have to be done together sock.send(task.url) resp = sock.recv() except zmq.ZMQError: raise finally: lock.release() self.scheduler.send('success:%s\n' % task.id) except (gevent.Timeout, zmq.ZMQError, TaskFailure), e: self.scheduler.send('failure:%s:%s\n' % (task.id, str(e))) finally:
def _dispatcher(self): http = httplib2.Http() while True: try: task = Task.unserialize(self.queue.get()) timeout = gevent.Timeout(constants.WORKER_TIMEOUT) timeout.start() self.scheduler.send('start:%s\n' % task.id) if task.url.startswith('http'): headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"} resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params) if task.result_callback_url and content: result_content_type = resp.get_header('Content-type') result_data = content if util.walks_like_json(result_content_type): result_data = json.loads(content) elif util.talks_like_form(result_content_type): result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ]) else: try: result_data = json.loads(content) except Exception, e: try: result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ]) except Exception, e: pass result_data.update({'status': resp.status, 'reason': resp.reason, 'headers': resp.getheaders()}) callback_resp, callback_content = http.request(task.result_callback_url, method='POST', body=json.dumps(result_data)) else: zmq_remotes = frozenset(task.url.split(',')) if not zmq_remotes in self.zmq_sockets: sock = self.zmq.socket(zmq.REQ) lock = gevent.coros.Semaphore() for remote in zmq_remotes: sock.connect(remote) self.zmq_sockets[zmq_remotes] = (sock, lock) else: sock, lock = self.zmq_sockets[zmq_remotes] try: lock.acquire() # Because send/recv have to be done together sock.send(task.url) resp = sock.recv() except zmq.ZMQError: raise finally: lock.release() self.scheduler.send('success:%s\n' % task.id)
def _backend_server(self, socket, address): for line in util.line_protocol(socket): action, payload = line.split(':', 1) if action == 'schedule': task = Task.unserialize(payload) task.schedule(self.dispatcher) self.scheduled[task.id] = task socket.send('scheduled:%s\n' % task.id) print "scheduled: %s" % task.id elif action == 'cancel': task_id = payload print "canceled: %s" % task_id self.scheduled.pop(task_id).cancel() elif action == 'reschedule': task_id, eta = payload.split(':', 1) eta = int(eta) print "rescheduled: %s for %s" % (task_id, eta) self.scheduled[task_id].reschedule(self.dispatcher, eta)