def _dispatcher(self):
http = httplib2.Http()
while True:
try:
task = Task.unserialize(self.queue.get())
timeout = gevent.Timeout(constants.WORKER_TIMEOUT)
timeout.start()
self.scheduler.send('start:%s\n' % task.id)
if task.url.startswith('http'):
headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"}
resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params)
else:
zmq_remotes = frozenset(task.url.split(','))
if not zmq_remotes in self.zmq_sockets:
sock = self.zmq.socket(zmq.REQ)
lock = gevent.coros.Semaphore()
for remote in zmq_remotes:
sock.connect(remote)
self.zmq_sockets[zmq_remotes] = (sock, lock)
else:
sock, lock = self.zmq_sockets[zmq_remotes]
try:
lock.acquire() # Because send/recv have to be done together
sock.send(task.url)
resp = sock.recv()
except zmq.ZMQError:
raise
finally:
lock.release()
self.scheduler.send('success:%s\n' % task.id)
except (gevent.Timeout, zmq.ZMQError, TaskFailure), e:
self.scheduler.send('failure:%s:%s\n' % (task.id, str(e)))
finally:
def _dispatcher(self):
http = httplib2.Http()
while True:
try:
task = Task.unserialize(self.queue.get())
timeout = gevent.Timeout(constants.WORKER_TIMEOUT)
timeout.start()
self.scheduler.send('start:%s\n' % task.id)
if task.url.startswith('http'):
headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"}
resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params)
if task.result_callback_url and content:
result_content_type = resp.get_header('Content-type')
result_data = content
if util.walks_like_json(result_content_type):
result_data = json.loads(content)
elif util.talks_like_form(result_content_type):
result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ])
else:
try:
result_data = json.loads(content)
except Exception, e:
try:
result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ])
except Exception, e:
pass
result_data.update({'status': resp.status, 'reason': resp.reason, 'headers': resp.getheaders()})
callback_resp, callback_content = http.request(task.result_callback_url, method='POST', body=json.dumps(result_data))
else:
zmq_remotes = frozenset(task.url.split(','))
if not zmq_remotes in self.zmq_sockets:
sock = self.zmq.socket(zmq.REQ)
lock = gevent.coros.Semaphore()
for remote in zmq_remotes:
sock.connect(remote)
self.zmq_sockets[zmq_remotes] = (sock, lock)
else:
sock, lock = self.zmq_sockets[zmq_remotes]
try:
lock.acquire() # Because send/recv have to be done together
sock.send(task.url)
resp = sock.recv()
except zmq.ZMQError:
raise
finally:
lock.release()
self.scheduler.send('success:%s\n' % task.id)
def _backend_server(self, socket, address):
for line in util.line_protocol(socket):
action, payload = line.split(':', 1)
if action == 'schedule':
task = Task.unserialize(payload)
task.schedule(self.dispatcher)
self.scheduled[task.id] = task
socket.send('scheduled:%s\n' % task.id)
print "scheduled: %s" % task.id
elif action == 'cancel':
task_id = payload
print "canceled: %s" % task_id
self.scheduled.pop(task_id).cancel()
elif action == 'reschedule':
task_id, eta = payload.split(':', 1)
eta = int(eta)
print "rescheduled: %s for %s" % (task_id, eta)
self.scheduled[task_id].reschedule(self.dispatcher, eta)
