def download_file(self,
key,
local_path,
bucket_name=None,
use_basename=True):
"""
Download a file from S3.
Args:
key: `str`. S3 key that will point to the file.
local_path: `str`. the path to download to.
bucket_name: `str`. Name of the bucket in which to store the file.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, key)
check_dirname_exists(local_path)
try:
self.client.download_file(bucket_name, key, local_path)
except ClientError as e:
raise PolyaxonStoresException(e)
def download_file(self,
blob,
local_path,
container_name=None,
use_basename=True):
"""
Downloads a file from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the blob.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
check_dirname_exists(local_path)
try:
self.connection.get_blob_to_path(container_name, blob, local_path)
except AzureHttpError as e:
raise PolyaxonStoresException(e)
def download_file(self,
blob,
local_path,
bucket_name=None,
use_basename=True):
"""
Downloads a file from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
bucket_name: `str`. the name of the bucket.
use_basename: `bool`. whether or not to use the basename of the blob.
"""
if not bucket_name:
bucket_name, blob = self.parse_gcs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
check_dirname_exists(local_path)
blob = self.get_blob(blob=blob, bucket_name=bucket_name)
blob.download_to_filename(local_path)
def download_dir(self,
blob,
local_path,
container_name=None,
use_basename=True):
"""
Download a directory from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
try:
check_dirname_exists(local_path, is_dir=True)
except PolyaxonStoresException:
os.makedirs(local_path)
results = self.list(container_name=container_name,
key=blob,
delimiter='/')
# Create directories
for prefix in sorted(results['prefixes']):
direname = os.path.join(local_path, prefix)
prefix = os.path.join(blob, prefix)
# Download files under
self.download_dir(blob=prefix,
local_path=direname,
container_name=container_name,
use_basename=False)
# Download files
for file_key in results['blobs']:
file_key = file_key[0]
filename = os.path.join(local_path, file_key)
file_key = os.path.join(blob, file_key)
self.download_file(blob=file_key,
local_path=filename,
container_name=container_name,
use_basename=False)
def download_dir(self,
key,
local_path,
bucket_name=None,
use_basename=True):
"""
Download a directory from S3.
Args:
key: `str`. S3 key that will point to the file.
local_path: `str`. the path to download to.
bucket_name: `str`. Name of the bucket in which to store the file.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, key)
try:
check_dirname_exists(local_path, is_dir=True)
except PolyaxonStoresException:
os.makedirs(local_path)
results = self.list(bucket_name=bucket_name, prefix=key, delimiter='/')
# Create directories
for prefix in sorted(results['prefixes']):
direname = os.path.join(local_path, prefix)
prefix = os.path.join(key, prefix)
# Download files under
self.download_dir(key=prefix,
local_path=direname,
bucket_name=bucket_name,
use_basename=False)
# Download files
for file_key in results['keys']:
file_key = file_key[0]
filename = os.path.join(local_path, file_key)
file_key = os.path.join(key, file_key)
self.download_file(key=file_key,
local_path=filename,
bucket_name=bucket_name,
use_basename=False)
