def __init__(self, queue_name, content_type, body):
self.queue_name = queue_name
if util.walks_like_json(content_type):
data = json.loads(body)
self.url = data['url']
self.method = data.get('method', 'POST')
countdown = data.get('countdown')
self.eta = data.get('eta')
self.result_callback_url = data.get('result_callback_url')
self.params = json.dumps(data.get('params', {}))
elif util.talks_like_form(content_type):
data = urlparse.parse_qs(body)
self.url = data['task.url'][0]
self.method = data.get('task.method', ['POST'])[0]
countdown = data.get('task.countdown', [None])[0]
self.eta = data.get('task.eta', [None])[0]
self.result_callback_url = data.get('task.result_callback_url'[None])[0]
self.params = json.dumps(dict([(k,v[0]) for k,v in data.items() if not k.startswith('task.')]))
else:
raise NotImplementedError("content type not supported: %s" % content_type)
if countdown and not self.eta:
self.eta = int(time.time()+int(countdown))
self.id = str(uuid.uuid4()) # vs time.time() is about 100 req/sec slower
self.replica_hosts = []
self.replica_offset = 0
self._greenlet = None
self._serialize_cache = None
def _dispatcher(self):
http = httplib2.Http()
while True:
try:
task = Task.unserialize(self.queue.get())
timeout = gevent.Timeout(constants.WORKER_TIMEOUT)
timeout.start()
self.scheduler.send('start:%s\n' % task.id)
if task.url.startswith('http'):
headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"}
resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params)
if task.result_callback_url and content:
result_content_type = resp.get_header('Content-type')
result_data = content
if util.walks_like_json(result_content_type):
result_data = json.loads(content)
elif util.talks_like_form(result_content_type):
result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ])
else:
try:
result_data = json.loads(content)
except Exception, e:
try:
result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ])
except Exception, e:
pass
result_data.update({'status': resp.status, 'reason': resp.reason, 'headers': resp.getheaders()})
callback_resp, callback_content = http.request(task.result_callback_url, method='POST', body=json.dumps(result_data))
else:
zmq_remotes = frozenset(task.url.split(','))
if not zmq_remotes in self.zmq_sockets:
sock = self.zmq.socket(zmq.REQ)
lock = gevent.coros.Semaphore()
for remote in zmq_remotes:
sock.connect(remote)
self.zmq_sockets[zmq_remotes] = (sock, lock)
else:
sock, lock = self.zmq_sockets[zmq_remotes]
try:
lock.acquire() # Because send/recv have to be done together
sock.send(task.url)
resp = sock.recv()
except zmq.ZMQError:
raise
finally:
lock.release()
self.scheduler.send('success:%s\n' % task.id)
def __init__(self, content_type, body):
self.id = str(uuid.uuid4())
if util.walks_like_json(content_type):
data = json.loads(body)
self.url = data.get('url')
self.method = data.get('method', 'POST')
self.result_callback_url = data.get('result_callback_url')
self.params = json.dumps(data.get('params', {}))
self.adapter_names = data.get('adapter_names')
elif util.talks_like_form(content_type):
data = urlparse.parse_qs(body)
self.url = data.get('task.url',[])[0]
self.method = data.get('task.method', ['POST'])[0]
self.result_callback_url = data.get('task.result_callback_url'[None])[0]
self.params = json.dumps(dict([(k,v[0]) for k,v in data.items() if not k.startswith('task.')]))
self.adapter_names = data.get('adapter_names')
else:
raise NotImplementedError("content type not supported: %s" % content_type)
