def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key in self._get_matching_s3_keys_generator(prefix):
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
self.download_file(key, local_dest_path)
def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self.gcs.list_blobs(prefix=prefix):
if blob.name.endswith("/"):
continue
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
self.download_file(blob.name, local_dest_path)
def _is_gcs_dir(self, dir_path: str) -> bool:
prefix = util.ensure_suffix(dir_path, "/")
return len(list(self.gcs.list_blobs(max_results=2, prefix=prefix))) > 1
def _is_s3_dir(self, dir_path):
prefix = util.ensure_suffix(dir_path, "/")
return self._is_s3_prefix(prefix)
