Example #1
0
def pytest_funcarg__spider(request):
    """Use scrapy's overrides to start the spider w/ specific settings"""

    settings.overrides['LOG_ENABLED'] = True
    settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
    spider = GeneralSpider()

    # Create initial batch
    now = spider.get_now_time()
    spider.batch = model.Batch.objects.create(
        kickoff_time=now, finish_time=now)
    spider.batch.save()

    # Generate two user agents to use in tests
    factories.UserAgentFactory.build(ua_string="Firefox / 11.0")
    factories.UserAgentFactory.build(ua_string="Chrome / 20.0")

    # Set spider batch_user_agents
    spider.batch_user_agents = []
    for ua in list(model.UserAgent.objects.all()):
        batch_user_agent = factories.BatchUserAgentFactory.create(
            batch=spider.batch, ua_string=ua.ua_string)
        spider.batch_user_agents.append(batch_user_agent)

    # Delete created batch from database when test is done
    request.addfinalizer(lambda: spider.batch.delete())
    return spider
Example #2
0
def pytest_funcarg__spider(request):
    """Use scrapy's overrides to start a spider w/ specific settings"""
    # This is necessary because the spider errors when a source file is not
    # provided.
    settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
    settings.overrides['LOG_ENABLED'] = True

    # Initialize and return spider
    spider = GeneralSpider()
    now = spider.get_now_time()
    spider.batch = model.Batch.objects.create(kickoff_time=now,
                                              finish_time=now)
    spider.batch.save()

    # Delete created batch from database when test is done
    request.addfinalizer(lambda: spider.batch.delete())
    return spider
Example #3
0
def pytest_funcarg__spider(request):
    """Use scrapy's overrides to start a spider w/ specific settings"""
    # This is necessary because the spider errors when a source file is not
    # provided.
    settings.overrides['URLS'] = u"spade/tests/sitelists/urls.txt"
    settings.overrides['LOG_ENABLED'] = True

    # Initialize and return spider
    spider = GeneralSpider()
    now = spider.get_now_time()
    spider.batch = model.Batch.objects.create(
        kickoff_time=now, finish_time=now)
    spider.batch.save()

    # Delete created batch from database when test is done
    request.addfinalizer(lambda: spider.batch.delete())
    return spider