def pytest_funcarg__spider(request):
"""Use scrapy's overrides to start the spider w/ specific settings"""
settings.overrides['LOG_ENABLED'] = True
settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
spider = GeneralSpider()
# Create initial batch
now = spider.get_now_time()
spider.batch = model.Batch.objects.create(
kickoff_time=now, finish_time=now)
spider.batch.save()
# Generate two user agents to use in tests
factories.UserAgentFactory.build(ua_string="Firefox / 11.0")
factories.UserAgentFactory.build(ua_string="Chrome / 20.0")
# Set spider batch_user_agents
spider.batch_user_agents = []
for ua in list(model.UserAgent.objects.all()):
batch_user_agent = factories.BatchUserAgentFactory.create(
batch=spider.batch, ua_string=ua.ua_string)
spider.batch_user_agents.append(batch_user_agent)
# Delete created batch from database when test is done
request.addfinalizer(lambda: spider.batch.delete())
return spider
def pytest_funcarg__spider(request):
"""Use scrapy's overrides to start a spider w/ specific settings"""
# This is necessary because the spider errors when a source file is not
# provided.
settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
settings.overrides['LOG_ENABLED'] = True
# Initialize and return spider
spider = GeneralSpider()
now = spider.get_now_time()
spider.batch = model.Batch.objects.create(kickoff_time=now,
finish_time=now)
spider.batch.save()
# Delete created batch from database when test is done
request.addfinalizer(lambda: spider.batch.delete())
return spider
def pytest_funcarg__spider(request):
"""Use scrapy's overrides to start a spider w/ specific settings"""
# This is necessary because the spider errors when a source file is not
# provided.
settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
settings.overrides['LOG_ENABLED'] = True
# Initialize and return spider
spider = GeneralSpider()
now = spider.get_now_time()
spider.batch = model.Batch.objects.create(
kickoff_time=now, finish_time=now)
spider.batch.save()
# Delete created batch from database when test is done
request.addfinalizer(lambda: spider.batch.delete())
return spider
