def simple_request_to_dict(request, spider=None): """Convert Request object to a dict. If a spider is given, it will try to find out the name of the spider method used in the callback and store that as the callback. """ cb = request.callback if callable(cb): cb = _find_method(spider, cb) eb = request.errback if callable(eb): eb = _find_method(spider, eb) d = { 'url': to_unicode(request.url), # urls should be safe (safe_string_url) 'callback': cb, 'errback': eb, # 'method': request.method, # 'headers': dict(request.headers), # 'body': request.body, # 'cookies': request.cookies, 'meta': request.meta, # '_encoding': request._encoding, # 'priority': request.priority, # 'dont_filter': request.dont_filter, # 'flags': request.flags, # 'cb_kwargs': request.cb_kwargs, } if type(request) is not Request: d['_class'] = request.__module__ + '.' + request.__class__.__name__ return d
def push(self, request): url = request.url cb = request.callback if callable(cb): cb = _find_method(self.spider, cb) eb = request.errback if callable(eb): eb = _find_method(self.spider, eb) d = {'url': to_unicode(url), 'callback': cb, 'errback': eb} d = self.serializer.dumps(d) self.server.rpush(self.key, d)
def push(self, request): """Push a request""" url = request.url cb = request.callback if callable(cb): cb = _find_method(self.spider, cb) data = '%s--%s' % (cb, url) self.server.lpush(self.key, data)
def push(self, request): """Push a request""" url = request.url cb = request.callback if callable(cb): cb = _find_method(self.spider, cb) data = '%s--%s' % (cb, url) self.server.lpush(self.key, data)