def download_dir_contents(self, prefix: str, local_dir: str): util.mkdir_p(local_dir) prefix = util.ensure_suffix(prefix, "/") for blob in self.gcs.list_blobs(prefix=prefix): if blob.name.endswith("/"): continue relative_path = util.trim_prefix(blob.name, prefix) local_dest_path = os.path.join(local_dir, relative_path) self.download_file(blob.name, local_dest_path)
def download_dir_contents(self, prefix, local_dir): util.mkdir_p(local_dir) prefix = util.ensure_suffix(prefix, "/") for key, _ in self._get_matching_s3_keys_generator(prefix): if key.endswith("/"): continue rel_path = util.trim_prefix(key, prefix) local_dest_path = os.path.join(local_dir, rel_path) self.download_file(key, local_dest_path)
def download_file(self, key, local_path): util.mkdir_p(os.path.dirname(local_path)) try: self.s3.download_file(self.bucket, key, local_path) return local_path except Exception as e: raise CortexException( 'key "{}" in bucket "{}" could not be accessed; '.format( key, self.bucket) + "it may not exist, or you may not have sufficient permissions" ) from e
def download_dir_contents(self, prefix, local_dir): util.mkdir_p(local_dir) prefix = util.ensure_suffix(prefix, "/") for key, _ in self._get_matching_s3_keys_generator( prefix, include_dir_objects=True): rel_path = util.trim_prefix(key, prefix) local_dest_path = os.path.join(local_dir, rel_path) if not local_dest_path.endswith("/"): self.download_file(key, local_dest_path) else: util.mkdir_p(os.path.dirname(local_dest_path))
def download_dir_contents(self, prefix: str, local_dir: str): util.mkdir_p(local_dir) prefix = util.ensure_suffix(prefix, "/") for blob in self._gcs_matching_blobs_generator( prefix=prefix, include_dir_objects=True): relative_path = util.trim_prefix(blob.name, prefix) local_dest_path = os.path.join(local_dir, relative_path) if not local_dest_path.endswith("/"): self.download_file(blob.name, local_dest_path) else: util.mkdir_p(os.path.dirname(local_dest_path))
def download_file(self, key: str, local_path: str): """ Download file to the specified local path. """ blob = self.gcs.get_blob(blob_name=key) if not isinstance(blob, storage.blob.Blob): raise CortexException( f'key "{key}" in bucket "{self.gcs.name}" not found') util.mkdir_p(os.path.dirname(local_path)) try: blob.download_to_filename(local_path) except gexp.NotFound: raise CortexException( f'key "{key}" in bucket "{self.gcs.name}" not found')
def download_and_unzip(self, key, local_dir): util.mkdir_p(local_dir) local_zip = os.path.join(local_dir, "zip.zip") self.download_file(key, local_zip) util.extract_zip(local_zip, delete_zip_file=True)