def _is_gcs_dir(self, dir_path: str) -> bool:
prefix = util.ensure_suffix(dir_path, "/")
matching_blobs = list(
self._gcs_matching_blobs_generator(max_results=2,
prefix=prefix,
include_dir_objects=True))
return len(matching_blobs) > 0
def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self.gcs.list_blobs(prefix=prefix):
if blob.name.endswith("/"):
continue
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
self.download_file(blob.name, local_dest_path)
def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key, _ in self._get_matching_s3_keys_generator(prefix):
if key.endswith("/"):
continue
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
self.download_file(key, local_dest_path)
def download_dir_contents(self, prefix, local_dir):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for key, _ in self._get_matching_s3_keys_generator(
prefix, include_dir_objects=True):
rel_path = util.trim_prefix(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
if not local_dest_path.endswith("/"):
self.download_file(key, local_dest_path)
else:
util.mkdir_p(os.path.dirname(local_dest_path))
def download_dir_contents(self, prefix: str, local_dir: str):
util.mkdir_p(local_dir)
prefix = util.ensure_suffix(prefix, "/")
for blob in self._gcs_matching_blobs_generator(
prefix=prefix, include_dir_objects=True):
relative_path = util.trim_prefix(blob.name, prefix)
local_dest_path = os.path.join(local_dir, relative_path)
if not local_dest_path.endswith("/"):
self.download_file(blob.name, local_dest_path)
else:
util.mkdir_p(os.path.dirname(local_dest_path))
def _is_s3_dir(self, dir_path):
prefix = util.ensure_suffix(dir_path, "/")
return self._is_s3_prefix(prefix)
def _is_gcs_dir(self, dir_path: str) -> bool:
prefix = util.ensure_suffix(dir_path, "/")
return len(list(self.gcs.list_blobs(max_results=2, prefix=prefix))) > 1