def enqueue(subspider, spider): url = settings['SUBSPIDER_DESTINATION'] spider_args = settings['SUBSPIDER_DEFAULTS'].copy() spider_args.update(subspider.get_members()) spider_args['spider'] = subspider.spider error = '' message = '' try: response = urlpost(url, data=urlencode(spider_args)) status = response.getcode() if status!= 200: error += 'response status %s for %s'%(status, url) else: json = loads(''.join(response.readlines())) message += '200 OK: "%s" for "%s"'%(json, url) except IOError, e: error += '`%s` at %s'%(e, url)
def enqueue(self, subspider, spider): data = subspider.get_members() data.update(self.SUBSPIDER_DEFAULTS) data['spider'] = subspider.spider return self.json(self.SUBSPIDER_DESTINATION, data=data, method='POST')