def _download(self, emerge_execution):
url = emerge_execution.session.get_download_url_by_id(
self.outbound_source.file_id)
absolute_offset = self.outbound_source.offset + self.relative_offset
download_dest = DownloadDestBytes()
range_ = (absolute_offset, absolute_offset + self.length - 1)
emerge_execution.services.download_manager.download_file_from_url(
url, download_dest, range_=range_)
return download_dest.get_bytes_written()
def _read_object(self, key: str) -> bytes:
for i in range(self._read_object_attempts):
data_io = DownloadDestBytes()
try:
self.bucket.download_file_by_name(key, data_io)
# This is overly broad!
except B2Error as exception:
if isinstance(exception, FileNotPresent):
raise FileNotFoundError(
'Object {} not found.'.format(key)) from None
else:
if i + 1 < self._read_object_attempts:
sleep_time = (2**(i + 1)) + (random.randint(0, 1000) /
1000)
logger.warning(
'Download of object with key {} to B2 failed, will try again in {:.2f} seconds. Exception thrown was {}'
.format(key, sleep_time, str(exception)))
time.sleep(sleep_time)
continue
raise
else:
break
return data_io.get_bytes_written()
def setUp(self):
super(DownloadTests, self).setUp()
self.file_info = self.bucket.upload_bytes(six.b('hello world'),
'file1')
self.download_dest = DownloadDestBytes()
self.progress_listener = StubProgressListener()
class DownloadTests(object):
def setUp(self):
super(DownloadTests, self).setUp()
self.file_info = self.bucket.upload_bytes(six.b('hello world'),
'file1')
self.download_dest = DownloadDestBytes()
self.progress_listener = StubProgressListener()
def _verify(self, expected_result):
assert self.download_dest.get_bytes_written() == six.b(expected_result)
assert self.progress_listener.is_valid()
def test_download_by_id_no_progress(self):
self.bucket.download_file_by_id(self.file_info.id_, self.download_dest)
def test_download_by_name_no_progress(self):
self.bucket.download_file_by_name('file1', self.download_dest)
def test_download_by_name_progress(self):
self.bucket.download_file_by_name('file1', self.download_dest,
self.progress_listener)
self._verify('hello world')
def test_download_by_id_progress(self):
self.bucket.download_file_by_id(self.file_info.id_, self.download_dest,
self.progress_listener)
self._verify('hello world')
def test_download_by_id_progress_partial(self):
self.bucket.download_file_by_id(self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(3, 9))
self._verify('lo worl')
def test_download_by_id_progress_exact_range(self):
self.bucket.download_file_by_id(self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(0, 10))
self._verify('hello world')
def test_download_by_id_progress_range_one_off(self):
with self.assertRaises(
InvalidRange,
msg=
'A range of 0-11 was requested (size of 12), but cloud could only serve 11 of that',
):
self.bucket.download_file_by_id(
self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(0, 11),
)
def test_download_by_id_progress_partial_inplace_overwrite(self):
# LOCAL is
# 12345678901234567890
#
# and then:
#
# hello world
# |||||||
# |||||||
# vvvvvvv
#
# 123lo worl1234567890
with TempDir() as d:
path = os.path.join(d, 'file2')
download_dest = PreSeekedDownloadDest(seek_target=3,
local_file_path=path)
data = six.b('12345678901234567890')
write_file(path, data)
self.bucket.download_file_by_id(
self.file_info.id_,
download_dest,
self.progress_listener,
range_=(3, 9),
)
self._check_local_file_contents(path,
six.b('123lo worl1234567890'))
def test_download_by_id_progress_partial_shifted_overwrite(self):
# LOCAL is
# 12345678901234567890
#
# and then:
#
# hello world
# |||||||
# \\\\\\\
# \\\\\\\
# \\\\\\\
# \\\\\\\
# \\\\\\\
# |||||||
# vvvvvvv
#
# 1234567lo worl567890
with TempDir() as d:
path = os.path.join(d, 'file2')
download_dest = PreSeekedDownloadDest(seek_target=7,
local_file_path=path)
data = six.b('12345678901234567890')
write_file(path, data)
self.bucket.download_file_by_id(
self.file_info.id_,
download_dest,
self.progress_listener,
range_=(3, 9),
)
self._check_local_file_contents(path,
six.b('1234567lo worl567890'))
def _check_local_file_contents(self, path, expected_contents):
with open(path, 'rb') as f:
contents = f.read()
self.assertEqual(contents, expected_contents)
def _check_file_contents(self, file_name, expected_contents):
download = DownloadDestBytes()
self.bucket.download_file_by_name(file_name, download)
self.assertEqual(expected_contents, download.get_bytes_written())