def __init__(self, settings, user_agent='Scrapy'):
DownloaderBaseMiddleware.__init__(self, settings)
UserAgentMiddleware.__init__(self)
user_agent_list = settings.get('USER_AGENT_LIST')
if not user_agent_list:
ua = settings.get('USER_AGENT', user_agent)
self.user_agent_list = [ua]
else:
self.user_agent_list = \
[i.strip() for i in user_agent_list.decode("utf-8").split('\n') if i.strip()]
self.default_agent = user_agent
self.choicer = self.choice()
self.user_agent = self.choicer.__next__() or user_agent
def __init__(self, user_agent=''):
UserAgentMiddleware.__init__(self)
self.user_agent = user_agent
def get_spider_and_mw(self, default_useragent):
crawler = get_crawler(Spider, {'USER_AGENT': default_useragent})
spider = crawler._create_spider('foo')
return spider, UserAgentMiddleware.from_crawler(crawler)
def get_spider_and_mw(self, default_useragent):
crawler = get_crawler(Spider, {'USER_AGENT': default_useragent})
spider = crawler._create_spider('foo')
return spider, UserAgentMiddleware.from_crawler(crawler)
