def test_download_by_id_progress(self): file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1') download = DownloadDestBytes() progress_listener = StubProgressListener() self.bucket.download_file_by_id(file_info.id_, download, progress_listener, range_=(3, 9)) self.assertEqual("6: 6 closed", progress_listener.get_history()) assert download.get_bytes_written() == six.b( 'lo wor'), download.get_bytes_written()
def __init__(self, b2fuse, file_info, new_file=False): super(B2FileDisk, self).__init__(b2fuse, file_info) self.temp_filename = os.path.join(self.b2fuse.temp_folder, self.file_info['fileName']) folder = os.path.join(self.b2fuse.temp_folder, os.path.dirname(self.file_info['fileName'])) if not os.path.exists(folder): os.makedirs(folder) self._dirty = False if os.path.exists(self.temp_filename): os.remove(self.temp_filename) self.temp_file = open(self.temp_filename, "wr+b") if new_file: self._dirty = True else: download_dest = DownloadDestBytes() self.b2fuse.bucket_api.download_file_by_id( self.file_info['fileId'], download_dest) self.temp_file.write(download_dest.get_bytes_written())
def download(self, inode): download_dest = DownloadDestBytes() file_id = self.file_info_store.get_file_id(inode) try: self.bucket_api.download_file_by_id(file_id, download_dest) except B2ConnectionError: raise ConnectionError return BytesIO(download_dest.get_bytes_written())
def __init__(self, b2fuse, file_info, new_file=False): super(B2SequentialFileMemory, self).__init__(b2fuse, file_info) self._dirty = False if new_file: self.data = array.array('c') self._dirty = True else: download_dest = DownloadDestBytes() self.b2fuse.bucket_api.download_file_by_id(self.file_info['fileId'], download_dest) self.data = array.array('c', download_dest.get_bytes_written())
def _read_object(self, key: str) -> bytes: for i in range(self._read_object_attempts): data_io = DownloadDestBytes() try: self.bucket.download_file_by_name(key, data_io) except (B2Error, B2ConnectionError) as exception: if isinstance(exception, FileNotPresent): raise FileNotFoundError( 'Object {} not found.'.format(key)) from None else: if i + 1 < self._read_object_attempts: sleep_time = (2**(i + 1)) + (random.randint(0, 1000) / 1000) logger.warning( 'Download of object with key {} to B2 failed, will try again in {:.2f} seconds.' .format(key, sleep_time)) time.sleep(sleep_time) continue raise else: break return data_io.get_bytes_written()
def _check_file_contents(self, file_name, expected_contents): download = DownloadDestBytes() self.bucket.download_file_by_name(file_name, download) self.assertEqual(expected_contents, download.get_bytes_written())
class DownloadTests(object): def setUp(self): super(DownloadTests, self).setUp() self.file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1') self.download_dest = DownloadDestBytes() self.progress_listener = StubProgressListener() def _verify(self, expected_result): assert self.download_dest.get_bytes_written() == six.b(expected_result) assert self.progress_listener.is_valid() def test_download_by_id_no_progress(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest) def test_download_by_name_no_progress(self): self.bucket.download_file_by_name('file1', self.download_dest) def test_download_by_name_progress(self): self.bucket.download_file_by_name('file1', self.download_dest, self.progress_listener) self._verify('hello world') def test_download_by_id_progress(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest, self.progress_listener) self._verify('hello world') def test_download_by_id_progress_partial(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest, self.progress_listener, range_=(3, 9)) self._verify('lo worl') def test_download_by_id_progress_exact_range(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 10)) self._verify('hello world') def test_download_by_id_progress_range_one_off(self): with self.assertRaises( InvalidRange, msg= 'A range of 0-11 was requested (size of 12), but cloud could only serve 11 of that', ): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 11), ) def test_download_by_id_progress_partial_inplace_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # hello world # ||||||| # ||||||| # vvvvvvv # # 123lo worl1234567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = six.b('12345678901234567890') write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, six.b('123lo worl1234567890')) def test_download_by_id_progress_partial_shifted_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # hello world # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567lo worl567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = six.b('12345678901234567890') write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, six.b('1234567lo worl567890')) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: contents = f.read() self.assertEqual(contents, expected_contents)