def decode_request(self, message): obj = dict_to_bytes(super(Decoder, self).decode(message)) return self._request_model(url=to_native_str(obj[b'url']), method=obj[b'method'], headers=obj[b'headers'], cookies=obj[b'cookies'], meta=obj[b'meta'])
def decode(self, message): message = dict_to_bytes(super(Decoder, self).decode(message)) if message[b'type'] == b'links_extracted': request = self._request_from_object(message[b'r']) links = [self._request_from_object(link) for link in message[b'links']] return ('links_extracted', request, links) if message[b'type'] == b'page_crawled': response = self._response_from_object(message[b'r']) return ('page_crawled', response) if message[b'type'] == b'request_error': request = self._request_from_object(message[b'r']) return ('request_error', request, to_native_str(message[b'error'])) if message[b'type'] == b'update_score': return ('update_score', self._request_from_object(message[b'r']), message[b'score'], message[b'schedule']) if message[b'type'] == b'add_seeds': seeds = [] for seed in message[b'seeds']: request = self._request_from_object(seed) seeds.append(request) return ('add_seeds', seeds) if message[b'type'] == b'new_job_id': return ('new_job_id', int(message[b'job_id'])) if message[b'type'] == b'offset': return ('offset', int(message[b'partition_id']), int(message[b'offset'])) return TypeError('Unknown message type')
def decode(self, message): message = dict_to_bytes(super(Decoder, self).decode(message)) if message[b'type'] == b'links_extracted': request = self._request_from_object(message[b'r']) links = [ self._request_from_object(link) for link in message[b'links'] ] return ('links_extracted', request, links) if message[b'type'] == b'page_crawled': response = self._response_from_object(message[b'r']) return ('page_crawled', response) if message[b'type'] == b'request_error': request = self._request_from_object(message[b'r']) return ('request_error', request, to_native_str(message[b'error'])) if message[b'type'] == b'update_score': return ('update_score', self._request_from_object(message[b'r']), message[b'score'], message[b'schedule']) if message[b'type'] == b'add_seeds': seeds = [] for seed in message[b'seeds']: request = self._request_from_object(seed) seeds.append(request) return ('add_seeds', seeds) if message[b'type'] == b'new_job_id': return ('new_job_id', int(message[b'job_id'])) if message[b'type'] == b'offset': return ('offset', int(message[b'partition_id']), int(message[b'offset'])) return TypeError('Unknown message type')