def test_download_single_file(self, exists, isdir, asyncio_, main): exists.return_value = True isdir.return_value = True downloader = Downloader('test') downloader.download('test.xz') asyncio_.get_event_loop.assert_called_with() loop = asyncio_.get_event_loop.return_value self.assertTrue(loop.run_until_complete.called) main.assert_called_once_with(loop, ('test.xz',))
def test_download_multiple_files(self, exists, isdir, asyncio_, main): exists.return_value = True isdir.return_value = True downloader = Downloader('test', bucket='bucket', region_name='south') downloader.download(range(3)) asyncio_.get_event_loop.assert_called_with() loop = asyncio_.get_event_loop.return_value self.assertTrue(loop.run_until_complete.called) main.assert_called_once_with(loop, (1, 2))
def test_download_timeout(self, exists, isdir): exists.return_value = True isdir.return_value = True with self.assertRaises(TimeoutError): downloader = Downloader('test') downloader.url = Mock(return_value="http://www.google.com:81/") loop = asyncio.get_event_loop() with ClientSession(loop=loop) as client: yield from downloader.fetch_file(client, '2016-12-06-reibursements.xz')
def test_download_timeout(self, exists, isdir): exists.return_value = True isdir.return_value = True with self.assertRaises(TimeoutError): downloader = Downloader('test', bucket='serenata-de-amor-data', region_name='a-east-1', timeout=0.001) downloader.url = Mock(return_value="http://www.google.com:81/") loop = asyncio.get_event_loop() with ClientSession(loop=loop) as client: yield from downloader.fetch_file( client, '2016-12-06-reibursements.xz')
def test_init(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test', timeout=1) self.assertEqual(os.path.abspath('test'), downloader.target) self.assertEqual(0, downloader.total) self.assertEqual(1, downloader.timeout)
def test_init(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test', bucket='bucket', region_name='south') self.assertEqual('bucket', downloader.bucket) self.assertEqual('south', downloader.region) self.assertEqual(os.path.abspath('test'), downloader.target) self.assertEqual(0, downloader.total)
def __init__(self, local_directory=None): if not local_directory: local_directory = 'data' self.local = LocalDatasets(local_directory) self.remote = RemoteDatasets() self.downloader = Downloader(local_directory, bucket=self.remote.bucket, **self.remote.credentials)
def __init__(self, local_directory=None, timeout=None): if not local_directory: local_directory = 'data' self.local = LocalDatasets(local_directory) self.downloader = Downloader(local_directory, timeout=timeout)
def test_download_no_file(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test', bucket='bucket', region_name='south') self.assertIsNone(downloader.download('')) self.assertIsNone(downloader.download([]))
def test_init_file_target(self, exists, isdir): exists.return_value = True isdir.return_value = False with self.assertRaises(FileNotFoundError): Downloader('test', bucket='bucket', region_name='south')
def test_init_no_region_no_bucket(self, exists, isdir): exists.return_value = True isdir.return_value = True with self.assertRaises(RuntimeError): Downloader('test')
def test_init_no_timeout(self, exists, isdir): downloader = Downloader('test', bucket='bucket', region_name='south') self.assertEqual(None, downloader.timeout)
def test_init_no_timeout(self, exists, isdir): downloader = Downloader('test') self.assertEqual(None, downloader.timeout)
def test_url(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test', bucket='bucket', region_name='south') expected = 'https://s3-south.amazonaws.com/bucket/test.xz' self.assertEqual(expected, downloader.url('test.xz'))
def test_init_file_target(self, exists, isdir): exists.return_value = True isdir.return_value = False with self.assertRaises(FileNotFoundError): Downloader('test')
def test_download_no_file(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test') self.assertIsNone(downloader.download('')) self.assertIsNone(downloader.download([]))
def test_url(self, exists, isdir): exists.return_value = True isdir.return_value = True downloader = Downloader('test') expected = 'https://nyc3.digitaloceanspaces.com/serenata-de-amor-data/test.xz' self.assertEqual(expected, downloader.url('test.xz'))