def _get_dir(self, prefix, local_dir):
prefix = util.add_suffix_unless_present(prefix, "/")
util.mkdir_p(local_dir)
for key in self._get_matching_s3_keys_generator(prefix):
rel_path = util.remove_prefix_if_present(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
self.download_file(key, local_dest_path)
def download_dir_from_s3(prefix, local_dir, bucket, client_config={}):
prefix = util.add_suffix_unless_present(prefix, "/")
util.mkdir_p(local_dir)
for key in get_matching_s3_keys_generator(bucket, prefix, client_config=client_config):
rel_path = util.remove_prefix_if_present(key, prefix)
local_dest_path = os.path.join(local_dir, rel_path)
download_file_from_s3(key, local_dest_path, bucket, client_config=client_config)
def deconstruct_s3_path(s3_path):
path = util.remove_prefix_if_present(s3_path, "s3://")
bucket = path.split("/")[0]
key = os.path.join(*path.split("/")[1:])
return (bucket, key)