def test_download_files_breaks_up_long_file_list(self, patched_call):
files_to_download = [
scraper.RemoteFile('2016/10/26/DNE.%d' % i, 0)
for i in range(100070)
]
files_downloaded = []
def verify_contents(args):
# Verify that the third-to-last argument to check_call is a filename
# that contains the right data (specifically, the filenames). This
# test needs to be kept in sync with the order of command-line
# arguments passed to the rsync call.
file_with_filenames = args[-3]
files = file(file_with_filenames).read().split('\0')
self.assertTrue(len(files) > 0)
self.assertTrue(len(files) <= 1000)
files_downloaded.extend(files)
return 0
patched_call.side_effect = verify_contents
scraper.download_files('/usr/bin/timeout', '/bin/true', 'localhost/',
files_to_download, '/tmp')
self.assertEqual(set(x.filename for x in files_to_download),
set(files_downloaded))
self.assertEqual(patched_call.call_count, 101)
def test_download_files_fails_and_dies(self):
with testfixtures.LogCapture() as log:
with self.assertRaises(scraper.RecoverableScraperException):
scraper.download_files(
'/usr/bin/timeout', '/bin/false', 'localhost/', [
scraper.RemoteFile('2016/10/26/DNE1', 0),
scraper.RemoteFile('2016/10/26/DNE2', 0)
], '/tmp')
self.assertIn('ERROR', [x.levelname for x in log.records])
def test_download_files(self, patched_check_call):
files_to_download = ['2016/10/26/DNE1', '2016/10/26/DNE2']
def verify_contents(args):
# Verify that the third-to-last argument to check_call is a filename
# that contains the right data (specifically, the filenames). This
# test needs to be kept in sync with the order of command-line
# arguments passed to the rsync call.
file_with_filenames = args[-3]
files_downloaded = file(file_with_filenames).read().split('\0')
self.assertEqual(files_to_download, files_downloaded)
patched_check_call.side_effect = verify_contents
scraper.download_files('/bin/true', 'localhost/', files_to_download,
'/tmp')
patched_check_call.assert_called_once()
def test_download_files(self, patched_call):
files_to_download = [
scraper.RemoteFile('2016/10/26/DNE1', 0),
scraper.RemoteFile('2016/10/26/DNE2', 0)
]
def verify_contents(args):
# Verify that the third-to-last argument to check_call is a filename
# that contains the right data (specifically, the filenames). This
# test needs to be kept in sync with the order of command-line
# arguments passed to the rsync call.
file_with_filenames = args[-3]
files_downloaded = file(file_with_filenames).read().split('\0')
self.assertEqual(files_downloaded,
[x.filename for x in files_to_download])
return 0
patched_call.side_effect = verify_contents
self.assertEqual(patched_call.call_count, 0)
scraper.download_files('/usr/bin/timeout', '/bin/true', 'localhost/',
files_to_download, '/tmp')
self.assertEqual(patched_call.call_count, 1)
def test_download_files_with_empty_does_nothing(self, _log):
# If the next line doesn't raise SystemExit then the test passes
scraper.download_files('/usr/bin/timeout', '/bin/false', 'localhost/',
[], '/tmp')
def test_download_files_fails_and_dies(self, log):
with self.assertRaises(scraper.RecoverableScraperException):
scraper.download_files('/bin/false', 'localhost/',
['2016/10/26/DNE1', '2016/10/26/DNE2'],
'/tmp')
self.assertIn('ERROR', [x.levelname for x in log.records])