Esempio n. 1
0
 def decode_request(self, message):
     obj = dict_to_bytes(super(Decoder, self).decode(message))
     return self._request_model(url=to_native_str(obj[b'url']),
                                method=obj[b'method'],
                                headers=obj[b'headers'],
                                cookies=obj[b'cookies'],
                                meta=obj[b'meta'])
Esempio n. 2
0
 def decode_request(self, message):
     obj = dict_to_bytes(super(Decoder, self).decode(message))
     return self._request_model(url=to_native_str(obj[b'url']),
                                method=obj[b'method'],
                                headers=obj[b'headers'],
                                cookies=obj[b'cookies'],
                                meta=obj[b'meta'])
Esempio n. 3
0
 def decode(self, message):
     message = dict_to_bytes(super(Decoder, self).decode(message))
     if message[b'type'] == b'links_extracted':
         request = self._request_from_object(message[b'r'])
         links = [self._request_from_object(link) for link in message[b'links']]
         return ('links_extracted', request, links)
     if message[b'type'] == b'page_crawled':
         response = self._response_from_object(message[b'r'])
         return ('page_crawled', response)
     if message[b'type'] == b'request_error':
         request = self._request_from_object(message[b'r'])
         return ('request_error', request, to_native_str(message[b'error']))
     if message[b'type'] == b'update_score':
         return ('update_score', self._request_from_object(message[b'r']), message[b'score'], message[b'schedule'])
     if message[b'type'] == b'add_seeds':
         seeds = []
         for seed in message[b'seeds']:
             request = self._request_from_object(seed)
             seeds.append(request)
         return ('add_seeds', seeds)
     if message[b'type'] == b'new_job_id':
         return ('new_job_id', int(message[b'job_id']))
     if message[b'type'] == b'offset':
         return ('offset', int(message[b'partition_id']), int(message[b'offset']))
     return TypeError('Unknown message type')
Esempio n. 4
0
 def decode(self, message):
     message = dict_to_bytes(super(Decoder, self).decode(message))
     if message[b'type'] == b'links_extracted':
         request = self._request_from_object(message[b'r'])
         links = [
             self._request_from_object(link) for link in message[b'links']
         ]
         return ('links_extracted', request, links)
     if message[b'type'] == b'page_crawled':
         response = self._response_from_object(message[b'r'])
         return ('page_crawled', response)
     if message[b'type'] == b'request_error':
         request = self._request_from_object(message[b'r'])
         return ('request_error', request, to_native_str(message[b'error']))
     if message[b'type'] == b'update_score':
         return ('update_score', self._request_from_object(message[b'r']),
                 message[b'score'], message[b'schedule'])
     if message[b'type'] == b'add_seeds':
         seeds = []
         for seed in message[b'seeds']:
             request = self._request_from_object(seed)
             seeds.append(request)
         return ('add_seeds', seeds)
     if message[b'type'] == b'new_job_id':
         return ('new_job_id', int(message[b'job_id']))
     if message[b'type'] == b'offset':
         return ('offset', int(message[b'partition_id']),
                 int(message[b'offset']))
     return TypeError('Unknown message type')