def download_file(self, key, local_path, bucket_name=None, use_basename=True): """ Download a file from S3. Args: key: `str`. S3 key that will point to the file. local_path: `str`. the path to download to. bucket_name: `str`. Name of the bucket in which to store the file. use_basename: `bool`. whether or not to use the basename of the key. """ if not bucket_name: bucket_name, key = self.parse_s3_url(key) local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, key) check_dirname_exists(local_path) try: self.client.download_file(bucket_name, key, local_path) except ClientError as e: raise PolyaxonStoresException(e)
def download_file(self, blob, local_path, container_name=None, use_basename=True): """ Downloads a file from Google Cloud Storage. Args: blob: `str`. blob to download. local_path: `str`. the path to download to. container_name: `str`. the name of the container. use_basename: `bool`. whether or not to use the basename of the blob. """ if not container_name: container_name, _, blob = self.parse_wasbs_url(blob) local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, blob) check_dirname_exists(local_path) try: self.connection.get_blob_to_path(container_name, blob, local_path) except AzureHttpError as e: raise PolyaxonStoresException(e)
def download_file(self, blob, local_path, bucket_name=None, use_basename=True): """ Downloads a file from Google Cloud Storage. Args: blob: `str`. blob to download. local_path: `str`. the path to download to. bucket_name: `str`. the name of the bucket. use_basename: `bool`. whether or not to use the basename of the blob. """ if not bucket_name: bucket_name, blob = self.parse_gcs_url(blob) local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, blob) check_dirname_exists(local_path) blob = self.get_blob(blob=blob, bucket_name=bucket_name) blob.download_to_filename(local_path)
def download_dir(self, blob, local_path, container_name=None, use_basename=True): """ Download a directory from Google Cloud Storage. Args: blob: `str`. blob to download. local_path: `str`. the path to download to. container_name: `str`. the name of the container. use_basename: `bool`. whether or not to use the basename of the key. """ if not container_name: container_name, _, blob = self.parse_wasbs_url(blob) local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, blob) try: check_dirname_exists(local_path, is_dir=True) except PolyaxonStoresException: os.makedirs(local_path) results = self.list(container_name=container_name, key=blob, delimiter='/') # Create directories for prefix in sorted(results['prefixes']): direname = os.path.join(local_path, prefix) prefix = os.path.join(blob, prefix) # Download files under self.download_dir(blob=prefix, local_path=direname, container_name=container_name, use_basename=False) # Download files for file_key in results['blobs']: file_key = file_key[0] filename = os.path.join(local_path, file_key) file_key = os.path.join(blob, file_key) self.download_file(blob=file_key, local_path=filename, container_name=container_name, use_basename=False)
def download_dir(self, key, local_path, bucket_name=None, use_basename=True): """ Download a directory from S3. Args: key: `str`. S3 key that will point to the file. local_path: `str`. the path to download to. bucket_name: `str`. Name of the bucket in which to store the file. use_basename: `bool`. whether or not to use the basename of the key. """ if not bucket_name: bucket_name, key = self.parse_s3_url(key) local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, key) try: check_dirname_exists(local_path, is_dir=True) except PolyaxonStoresException: os.makedirs(local_path) results = self.list(bucket_name=bucket_name, prefix=key, delimiter='/') # Create directories for prefix in sorted(results['prefixes']): direname = os.path.join(local_path, prefix) prefix = os.path.join(key, prefix) # Download files under self.download_dir(key=prefix, local_path=direname, bucket_name=bucket_name, use_basename=False) # Download files for file_key in results['keys']: file_key = file_key[0] filename = os.path.join(local_path, file_key) file_key = os.path.join(key, file_key) self.download_file(key=file_key, local_path=filename, bucket_name=bucket_name, use_basename=False)