def process(self): if len(self.tasks) == 0: if self.queue_empty: self.queue_empty() log.msg('Queue now empty') return task = self.tasks.pop(0) url = task['url'] file_location = task['download_location'] file_id = task.get('id') defer = webclient.get(url, '', use_tor=self.use_tor) defer.addCallback(self._process_response, file_location, file_id) defer.addErrback(self._process_error, file_location, file_id)
def load_torrents(self, callback, errback, **kwargs): """ Creates an instance of PirateBayClient twisted client. sends of request and exeutes callback functions on response :param callback: This is a callback function that gets passed from the caller. The function passes a list of returned torrent objects. The list is empty if no torrents exist """ def _process_error(failure): log.err(str(failure)) log.err(failure.getTraceback()) errback() def _process_torrents(data): if data is None: log.err('No response from Pirate Bay. Trying again') errback() return document = html.fromstring(data) torrents = [ self._build_torrent(row) for row in self._get_torrent_rows(document)] callback(torrents, **kwargs) def _process_response(response): log.msg('Response Code: {}'.format(response.code)) d = readBody(response) d.addCallbacks(_process_torrents) d.addErrback(_process_error) return d defer = webclient.get(str(self.url), '', use_tor=self.use_tor) defer.addCallback(_process_response) defer.addErrback(_process_error)