def __init__(self, queue_name, content_type, body): self.queue_name = queue_name if util.walks_like_json(content_type): data = json.loads(body) self.url = data['url'] self.method = data.get('method', 'POST') countdown = data.get('countdown') self.eta = data.get('eta') self.result_callback_url = data.get('result_callback_url') self.params = json.dumps(data.get('params', {})) elif util.talks_like_form(content_type): data = urlparse.parse_qs(body) self.url = data['task.url'][0] self.method = data.get('task.method', ['POST'])[0] countdown = data.get('task.countdown', [None])[0] self.eta = data.get('task.eta', [None])[0] self.result_callback_url = data.get('task.result_callback_url'[None])[0] self.params = json.dumps(dict([(k,v[0]) for k,v in data.items() if not k.startswith('task.')])) else: raise NotImplementedError("content type not supported: %s" % content_type) if countdown and not self.eta: self.eta = int(time.time()+int(countdown)) self.id = str(uuid.uuid4()) # vs time.time() is about 100 req/sec slower self.replica_hosts = [] self.replica_offset = 0 self._greenlet = None self._serialize_cache = None
def _dispatcher(self): http = httplib2.Http() while True: try: task = Task.unserialize(self.queue.get()) timeout = gevent.Timeout(constants.WORKER_TIMEOUT) timeout.start() self.scheduler.send('start:%s\n' % task.id) if task.url.startswith('http'): headers = {"User-Agent": "Miyamoto/0.1", "X-Task": task.id, "X-Queue": task.queue_name, "Content-type": "application/json"} resp, content = http.request(task.url, method=task.method, headers=headers, body=task.params) if task.result_callback_url and content: result_content_type = resp.get_header('Content-type') result_data = content if util.walks_like_json(result_content_type): result_data = json.loads(content) elif util.talks_like_form(result_content_type): result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ]) else: try: result_data = json.loads(content) except Exception, e: try: result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(content).items() ]) except Exception, e: pass result_data.update({'status': resp.status, 'reason': resp.reason, 'headers': resp.getheaders()}) callback_resp, callback_content = http.request(task.result_callback_url, method='POST', body=json.dumps(result_data)) else: zmq_remotes = frozenset(task.url.split(',')) if not zmq_remotes in self.zmq_sockets: sock = self.zmq.socket(zmq.REQ) lock = gevent.coros.Semaphore() for remote in zmq_remotes: sock.connect(remote) self.zmq_sockets[zmq_remotes] = (sock, lock) else: sock, lock = self.zmq_sockets[zmq_remotes] try: lock.acquire() # Because send/recv have to be done together sock.send(task.url) resp = sock.recv() except zmq.ZMQError: raise finally: lock.release() self.scheduler.send('success:%s\n' % task.id)
def __init__(self, content_type, body): self.id = str(uuid.uuid4()) if util.walks_like_json(content_type): data = json.loads(body) self.url = data.get('url') self.method = data.get('method', 'POST') self.result_callback_url = data.get('result_callback_url') self.params = json.dumps(data.get('params', {})) self.adapter_names = data.get('adapter_names') elif util.talks_like_form(content_type): data = urlparse.parse_qs(body) self.url = data.get('task.url',[])[0] self.method = data.get('task.method', ['POST'])[0] self.result_callback_url = data.get('task.result_callback_url'[None])[0] self.params = json.dumps(dict([(k,v[0]) for k,v in data.items() if not k.startswith('task.')])) self.adapter_names = data.get('adapter_names') else: raise NotImplementedError("content type not supported: %s" % content_type)
def _call_callback(task, adapter_name, resp): try: # if caller specified a result_callback_url result_content_type = resp.headers.get('content-type') result_data = {} if resp.content is not None: if resp.json: result_data = resp.json elif util.talks_like_form(result_content_type): result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(resp.content).items() ]) else: try: result_data = dict([(k,v[0]) for k,v in urlparse.parse_qs(resp.content).items() ]) except Exception, e: pass if not result_data: result_data = {} result_data.update({'adapter_name': adapter_name,\ 'task_id': task.id, 'status': resp.status_code, 'reason': resp.reason,\ 'content-type': result_content_type}) callback_resp = requests.request('POST',\ task.result_callback_url, data=json.dumps(result_data))