def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self.gcs.list_blobs(prefix=prefix):
if blob.name.endswith("/"):
continue
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
self.download_file(blob.name, local_dest_path)
def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key, _ in self._get_matching_s3_keys_generator(prefix):
if key.endswith("/"):
continue
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
self.download_file(key, local_dest_path)
def download_file(self, key, local_path):
util.mkdir_p(os.path.dirname(local_path))
try:
self.s3.download_file(self.bucket, key, local_path)
return local_path
except Exception as e:
raise CortexException(
'key "{}" in bucket "{}" could not be accessed; '.format(
key, self.bucket) +
"it may not exist, or you may not have sufficient permissions"
) from e
def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key, _ in self._get_matching_s3_keys_generator(
prefix, include_dir_objects=True):
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
if not local_dest_path.endswith("/"):
self.download_file(key, local_dest_path)
else:
util.mkdir_p(os.path.dirname(local_dest_path))
def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self._gcs_matching_blobs_generator(
prefix=prefix, include_dir_objects=True):
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
if not local_dest_path.endswith("/"):
self.download_file(blob.name, local_dest_path)
else:
util.mkdir_p(os.path.dirname(local_dest_path))
def download_file(self, key: str, local_path: str):
"""
Download file to the specified local path.
"""
blob = self.gcs.get_blob(blob_name=key)
if not isinstance(blob, storage.blob.Blob):
raise CortexException(
f'key "{key}" in bucket "{self.gcs.name}" not found')
util.mkdir_p(os.path.dirname(local_path))
try:
blob.download_to_filename(local_path)
except gexp.NotFound:
raise CortexException(
f'key "{key}" in bucket "{self.gcs.name}" not found')
def download_and_unzip(self, key, local_dir):
util.mkdir_p(local_dir)
local_zip = os.path.join(local_dir, "zip.zip")
self.download_file(key, local_zip)
util.extract_zip(local_zip, delete_zip_file=True)
