def test_download_single_file(self, exists, isdir, asyncio_, main):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test')
downloader.download('test.xz')
asyncio_.get_event_loop.assert_called_with()
loop = asyncio_.get_event_loop.return_value
self.assertTrue(loop.run_until_complete.called)
main.assert_called_once_with(loop, ('test.xz',))
def test_download_multiple_files(self, exists, isdir, asyncio_, main):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test', bucket='bucket', region_name='south')
downloader.download(range(3))
asyncio_.get_event_loop.assert_called_with()
loop = asyncio_.get_event_loop.return_value
self.assertTrue(loop.run_until_complete.called)
main.assert_called_once_with(loop, (1, 2))
def test_download_timeout(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
with self.assertRaises(TimeoutError):
downloader = Downloader('test')
downloader.url = Mock(return_value="http://www.google.com:81/")
loop = asyncio.get_event_loop()
with ClientSession(loop=loop) as client:
yield from downloader.fetch_file(client, '2016-12-06-reibursements.xz')
def test_download_timeout(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
with self.assertRaises(TimeoutError):
downloader = Downloader('test',
bucket='serenata-de-amor-data',
region_name='a-east-1',
timeout=0.001)
downloader.url = Mock(return_value="http://www.google.com:81/")
loop = asyncio.get_event_loop()
with ClientSession(loop=loop) as client:
yield from downloader.fetch_file(
client, '2016-12-06-reibursements.xz')
def test_init(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test', timeout=1)
self.assertEqual(os.path.abspath('test'), downloader.target)
self.assertEqual(0, downloader.total)
self.assertEqual(1, downloader.timeout)
def test_init(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test', bucket='bucket', region_name='south')
self.assertEqual('bucket', downloader.bucket)
self.assertEqual('south', downloader.region)
self.assertEqual(os.path.abspath('test'), downloader.target)
self.assertEqual(0, downloader.total)
def __init__(self, local_directory=None):
if not local_directory:
local_directory = 'data'
self.local = LocalDatasets(local_directory)
self.remote = RemoteDatasets()
self.downloader = Downloader(local_directory,
bucket=self.remote.bucket,
**self.remote.credentials)
def __init__(self, local_directory=None, timeout=None):
if not local_directory:
local_directory = 'data'
self.local = LocalDatasets(local_directory)
self.downloader = Downloader(local_directory, timeout=timeout)
def test_download_no_file(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test', bucket='bucket', region_name='south')
self.assertIsNone(downloader.download(''))
self.assertIsNone(downloader.download([]))
def test_init_file_target(self, exists, isdir):
exists.return_value = True
isdir.return_value = False
with self.assertRaises(FileNotFoundError):
Downloader('test', bucket='bucket', region_name='south')
def test_init_no_region_no_bucket(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
with self.assertRaises(RuntimeError):
Downloader('test')
def test_init_no_timeout(self, exists, isdir):
downloader = Downloader('test', bucket='bucket', region_name='south')
self.assertEqual(None, downloader.timeout)
def test_init_no_timeout(self, exists, isdir):
downloader = Downloader('test')
self.assertEqual(None, downloader.timeout)
def test_url(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test', bucket='bucket', region_name='south')
expected = 'https://s3-south.amazonaws.com/bucket/test.xz'
self.assertEqual(expected, downloader.url('test.xz'))
def test_init_file_target(self, exists, isdir):
exists.return_value = True
isdir.return_value = False
with self.assertRaises(FileNotFoundError):
Downloader('test')
def test_download_no_file(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test')
self.assertIsNone(downloader.download(''))
self.assertIsNone(downloader.download([]))
def test_url(self, exists, isdir):
exists.return_value = True
isdir.return_value = True
downloader = Downloader('test')
expected = 'https://nyc3.digitaloceanspaces.com/serenata-de-amor-data/test.xz'
self.assertEqual(expected, downloader.url('test.xz'))
