def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key in self._get_matching_s3_keys_generator(prefix):
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
self.download_file(key, local_dest_path)
def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self.gcs.list_blobs(prefix=prefix):
if blob.name.endswith("/"):
continue
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
self.download_file(blob.name, local_dest_path)
def deconstruct_gcs_path(gcs_path: str) -> Tuple[str, str]:
path = util.trim_prefix(gcs_path, "gs://")
bucket = path.split("/")[0]
key = os.path.join(*path.split("/")[1:])
return bucket, key
def deconstruct_s3_path(s3_path) -> Tuple[str, str]:
path = util.trim_prefix(s3_path, "s3://")
bucket = path.split("/")[0]
key = os.path.join(*path.split("/")[1:])
return (bucket, key)
