def __init__(self, b2fuse, file_info, new_file=False):
super(B2FileDisk, self).__init__(b2fuse, file_info)
self.temp_filename = os.path.join(self.b2fuse.temp_folder,
self.file_info['fileName'])
folder = os.path.join(self.b2fuse.temp_folder,
os.path.dirname(self.file_info['fileName']))
if not os.path.exists(folder):
os.makedirs(folder)
self._dirty = False
if os.path.exists(self.temp_filename):
os.remove(self.temp_filename)
self.temp_file = open(self.temp_filename, "wr+b")
if new_file:
self._dirty = True
else:
download_dest = DownloadDestBytes()
self.b2fuse.bucket_api.download_file_by_id(
self.file_info['fileId'], download_dest)
self.temp_file.write(download_dest.get_bytes_written())
def test_download_by_id_progress(self):
file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1')
download = DownloadDestBytes()
progress_listener = StubProgressListener()
self.bucket.download_file_by_id(file_info.id_, download,
progress_listener)
self.assertEqual("11: 11 closed", progress_listener.get_history())
assert download.get_bytes_written() == six.b('hello world')
def download(self, inode):
download_dest = DownloadDestBytes()
file_id = self.file_info_store.get_file_id(inode)
try:
self.bucket_api.download_file_by_id(file_id, download_dest)
except B2ConnectionError:
raise ConnectionError
return BytesIO(download_dest.get_bytes_written())
def __init__(self, b2fuse, file_info, new_file=False):
super(B2SequentialFileMemory, self).__init__(b2fuse, file_info)
self._dirty = False
if new_file:
self.data = array.array('c')
self._dirty = True
else:
download_dest = DownloadDestBytes()
self.b2fuse.bucket_api.download_file_by_id(self.file_info['fileId'], download_dest)
self.data = array.array('c', download_dest.get_bytes_written())
def test_download_by_id_progress(self):
file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1')
download = DownloadDestBytes()
progress_listener = StubProgressListener()
self.bucket.download_file_by_id(file_info.id_, download, progress_listener, range_=(3, 9))
self.assertEqual("6: 6 closed", progress_listener.get_history())
assert download.bytes_io.getvalue() == six.b('lo wor'), download.bytes_io.getvalue()
def _read_object(self, key: str) -> bytes:
for i in range(self._read_object_attempts):
data_io = DownloadDestBytes()
try:
self.bucket.download_file_by_name(key, data_io)
except (B2Error, B2ConnectionError) as exception:
if isinstance(exception, FileNotPresent):
raise FileNotFoundError(
'Object {} not found.'.format(key)) from None
else:
if i + 1 < self._read_object_attempts:
sleep_time = (2**(i + 1)) + (random.randint(0, 1000) /
1000)
logger.warning(
'Download of object with key {} to B2 failed, will try again in {:.2f} seconds.'
.format(key, sleep_time))
time.sleep(sleep_time)
continue
raise
else:
break
return data_io.get_bytes_written()
def _check_file_contents(self, file_name, expected_contents):
download = DownloadDestBytes()
self.bucket.download_file_by_name(file_name, download)
self.assertEqual(expected_contents, download.bytes_io.getvalue())
def test_download_by_name_no_progress(self):
self.bucket.upload_bytes(six.b('hello world'), 'file1')
download = DownloadDestBytes()
self.bucket.download_file_by_name('file1', download)
def test_download_by_name_progress(self):
self.bucket.upload_bytes(six.b('hello world'), 'file1')
download = DownloadDestBytes()
progress_listener = StubProgressListener()
self.bucket.download_file_by_name('file1', download, progress_listener)
self.assertEqual("11: 11 closed", progress_listener.get_history())
def test_download_by_id_no_progress(self):
file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1')
download = DownloadDestBytes()
self.bucket.download_file_by_id(file_info.id_, download)
def setUp(self):
super(DownloadTests, self).setUp()
self.file_info = self.bucket.upload_bytes(six.b('hello world'),
'file1')
self.download_dest = DownloadDestBytes()
self.progress_listener = StubProgressListener()
class DownloadTests(object):
def setUp(self):
super(DownloadTests, self).setUp()
self.file_info = self.bucket.upload_bytes(six.b('hello world'),
'file1')
self.download_dest = DownloadDestBytes()
self.progress_listener = StubProgressListener()
def _verify(self, expected_result):
assert self.download_dest.get_bytes_written() == six.b(expected_result)
assert self.progress_listener.is_valid()
def test_download_by_id_no_progress(self):
self.bucket.download_file_by_id(self.file_info.id_, self.download_dest)
def test_download_by_name_no_progress(self):
self.bucket.download_file_by_name('file1', self.download_dest)
def test_download_by_name_progress(self):
self.bucket.download_file_by_name('file1', self.download_dest,
self.progress_listener)
self._verify('hello world')
def test_download_by_id_progress(self):
self.bucket.download_file_by_id(self.file_info.id_, self.download_dest,
self.progress_listener)
self._verify('hello world')
def test_download_by_id_progress_partial(self):
self.bucket.download_file_by_id(self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(3, 9))
self._verify('lo worl')
def test_download_by_id_progress_exact_range(self):
self.bucket.download_file_by_id(self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(0, 10))
self._verify('hello world')
def test_download_by_id_progress_range_one_off(self):
with self.assertRaises(
InvalidRange,
msg=
'A range of 0-11 was requested (size of 12), but cloud could only serve 11 of that',
):
self.bucket.download_file_by_id(
self.file_info.id_,
self.download_dest,
self.progress_listener,
range_=(0, 11),
)
def test_download_by_id_progress_partial_inplace_overwrite(self):
# LOCAL is
# 12345678901234567890
#
# and then:
#
# hello world
# |||||||
# |||||||
# vvvvvvv
#
# 123lo worl1234567890
with TempDir() as d:
path = os.path.join(d, 'file2')
download_dest = PreSeekedDownloadDest(seek_target=3,
local_file_path=path)
data = six.b('12345678901234567890')
write_file(path, data)
self.bucket.download_file_by_id(
self.file_info.id_,
download_dest,
self.progress_listener,
range_=(3, 9),
)
self._check_local_file_contents(path,
six.b('123lo worl1234567890'))
def test_download_by_id_progress_partial_shifted_overwrite(self):
# LOCAL is
# 12345678901234567890
#
# and then:
#
# hello world
# |||||||
# \\\\\\\
# \\\\\\\
# \\\\\\\
# \\\\\\\
# \\\\\\\
# |||||||
# vvvvvvv
#
# 1234567lo worl567890
with TempDir() as d:
path = os.path.join(d, 'file2')
download_dest = PreSeekedDownloadDest(seek_target=7,
local_file_path=path)
data = six.b('12345678901234567890')
write_file(path, data)
self.bucket.download_file_by_id(
self.file_info.id_,
download_dest,
self.progress_listener,
range_=(3, 9),
)
self._check_local_file_contents(path,
six.b('1234567lo worl567890'))
def _check_local_file_contents(self, path, expected_contents):
with open(path, 'rb') as f:
contents = f.read()
self.assertEqual(contents, expected_contents)
