def simple_request_to_dict(request, spider=None):
"""Convert Request object to a dict.
If a spider is given, it will try to find out the name of the spider method
used in the callback and store that as the callback.
"""
cb = request.callback
if callable(cb):
cb = _find_method(spider, cb)
eb = request.errback
if callable(eb):
eb = _find_method(spider, eb)
d = {
'url':
to_unicode(request.url), # urls should be safe (safe_string_url)
'callback': cb,
'errback': eb,
# 'method': request.method,
# 'headers': dict(request.headers),
# 'body': request.body,
# 'cookies': request.cookies,
'meta': request.meta,
# '_encoding': request._encoding,
# 'priority': request.priority,
# 'dont_filter': request.dont_filter,
# 'flags': request.flags,
# 'cb_kwargs': request.cb_kwargs,
}
if type(request) is not Request:
d['_class'] = request.__module__ + '.' + request.__class__.__name__
return d
def push(self, request):
url = request.url
cb = request.callback
if callable(cb):
cb = _find_method(self.spider, cb)
eb = request.errback
if callable(eb):
eb = _find_method(self.spider, eb)
d = {'url': to_unicode(url), 'callback': cb, 'errback': eb}
d = self.serializer.dumps(d)
self.server.rpush(self.key, d)
def push(self, request):
"""Push a request"""
url = request.url
cb = request.callback
if callable(cb):
cb = _find_method(self.spider, cb)
data = '%s--%s' % (cb, url)
self.server.lpush(self.key, data)
def push(self, request):
"""Push a request"""
url = request.url
cb = request.callback
if callable(cb):
cb = _find_method(self.spider, cb)
data = '%s--%s' % (cb, url)
self.server.lpush(self.key, data)