def copy_from_iter(self, src_blob: "GSBlob") -> Generator[int, None, None]:
assert isinstance(src_blob, type(self))
if self.url != src_blob.url:
if not src_blob._gs_bucket.user_project:
# TODO: always use rewrite when it supports requester pays buckets
dst_gs_blob = self._gs_bucket.blob(self.key)
src_gs_blob = src_blob._gs_bucket.blob(src_blob.key)
token: Optional[str] = None
total_bytes_written = 0
while True:
try:
token, bytes_written, _ = dst_gs_blob.rewrite(
src_gs_blob, token)
except gcp_exceptions.NotFound as e:
raise blobstore.BlobNotFoundError() from e
chunk_size = bytes_written - total_bytes_written
total_bytes_written = bytes_written
yield chunk_size
if token is None:
break
else:
size = src_blob.size()
if size <= GSBlobStore.chunk_size:
self.put(src_blob.get())
yield size
else:
with self.part_writer() as writer:
for part in src_blob.iter_content():
writer.put_part(part)
yield len(part)
def __init__(self, path: str):
try:
self.size = os.path.getsize(path)
except FileNotFoundError:
raise blobstore.BlobNotFoundError(f"Could not find {path}")
self.chunk_size = LocalBlobStore.chunk_size
self._number_of_parts = ceil(self.size / self.chunk_size) if 0 < self.size else 1
self.handle = open(path, "rb")
def _get_native_blob(bucket: Union[str, GSNativeBucket],
key: str,
credentials: Optional[dict] = None,
billing_project: Optional[str] = None) -> GSNativeBlob:
bucket = _get_native_bucket(bucket, credentials, billing_project)
blob = bucket.get_blob(key)
if blob is None:
raise blobstore.BlobNotFoundError(
f"Could not find gs://{bucket.name}/{key}")
return blob
def wrapper(self, *args, **kwargs):
try:
for item in generator(self, *args, **kwargs):
yield item
except (HTTPError, ConnectionError) as ex:
raise blobstore.BlobNotFoundError(f"Could not find {self.url}") from ex
def wrapper(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except (HTTPError, ConnectionError) as ex:
raise blobstore.BlobNotFoundError(f"Could not find {self.url}") from ex
def wrapper(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except FileNotFoundError as ex:
raise blobstore.BlobNotFoundError(f"Could not find {self.url}") from ex
def download_iter(self, target: str) -> Generator[int, None, None]:
if not os.path.isfile(self._path):
raise blobstore.BlobNotFoundError(f"Could not find {self.url}")
if self._path != target:
shutil.copyfile(self._path, target)
yield self.size()