def test_run_spiders(self, popen):
tasks.run_spiders()
spiders = '/mnt/code/data_collection/gazette/spiders/*.py'
spider_calls = []
for module in sorted(glob.glob(spiders)):
name = module.split('/')[-1].split('.')[0]
if name not in ['__init__', 'base']:
new_call = call(['scrapy', 'crawl', name],
cwd='data_collection')
spider_calls.append(new_call)
self.assertEqual(popen.mock_calls, spider_calls)
def test_run_spiders(self, popen):
tasks.run_spiders()
spiders = "/mnt/code/data_collection/gazette/spiders/*.py"
spider_calls = []
for module in sorted(glob.glob(spiders)):
name = module.split("/")[-1].split(".")[0]
if name not in ["__init__", "base"]:
new_call = call(["scrapy", "crawl", name],
cwd="data_collection")
spider_calls.append(new_call)
self.assertEqual(popen.mock_calls, spider_calls)
def test_run_spiders(self, popen):
tasks.run_spiders()
current_folder = os.path.dirname(os.path.realpath(__file__))
spiders_glob = os.path.join(current_folder, "..", "data_collection",
"gazette", "spiders", "*.py")
spider_calls = []
for module in sorted(glob.glob(spiders_glob)):
name = module.split("/")[-1].split(".")[0]
if name not in ["__init__", "base"]:
new_call = call(["scrapy", "crawl", name],
cwd="data_collection")
spider_calls.append(new_call)
assert popen.mock_calls == spider_calls
def test_run_spiders_using_unsupported_timerange(self, popen):
with pytest.raises(ValueError):
tasks.run_spiders(["rs_porto_alegre"], timerange="past_month")
def test_run_spiders_using_week_timerange(self, popen):
tasks.run_spiders(["rs_porto_alegre"], timerange="past_week")
command = [
"scrapy", "crawl", "rs_porto_alegre", "-a", "start_date=2018-01-01"
]
popen.assert_called_with(command, cwd="data_collection")
def test_run_spiders_using_unsupported_timerange(self, popen):
with self.assertRaises(ValueError):
tasks.run_spiders(['rs_porto_alegre'], timerange='past_month')
def test_run_spiders_using_week_timerange(self, popen):
tasks.run_spiders(['rs_porto_alegre'], timerange='past_week')
command = [
'scrapy', 'crawl', 'rs_porto_alegre', '-a', 'start_date=2018-01-01'
]
popen.assert_called_with(command, cwd='data_collection')