def download_file(self,
blob,
local_path,
container_name=None,
use_basename=True):
"""
Downloads a file from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the blob.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
check_dirname_exists(local_path)
try:
self.connection.get_blob_to_path(container_name, blob, local_path)
except AzureHttpError as e:
raise PolyaxonStoresException(e)
def download_file(self,
key,
local_path,
bucket_name=None,
use_basename=True):
"""
Download a file from S3.
Args:
key: `str`. S3 key that will point to the file.
local_path: `str`. the path to download to.
bucket_name: `str`. Name of the bucket in which to store the file.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, key)
check_dirname_exists(local_path)
try:
self.client.download_file(bucket_name, key, local_path)
except ClientError as e:
raise PolyaxonStoresException(e)
def upload_dir(self,
dirname,
blob,
container_name=None,
use_basename=True):
"""
Uploads a local directory to to Google Cloud Storage.
Args:
dirname: `str`. name of the directory to upload.
blob: `str`. blob to upload to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the directory.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
if use_basename:
blob = append_basename(blob, dirname)
# Turn the path to absolute paths
dirname = os.path.abspath(dirname)
with get_files_in_current_directory(dirname) as files:
for f in files:
file_blob = os.path.join(blob, os.path.relpath(f, dirname))
self.upload_file(
filename=f,
blob=file_blob,
container_name=container_name,
use_basename=False,
)
def download_file(self,
blob,
local_path,
bucket_name=None,
use_basename=True):
"""
Downloads a file from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
bucket_name: `str`. the name of the bucket.
use_basename: `bool`. whether or not to use the basename of the blob.
"""
if not bucket_name:
bucket_name, blob = self.parse_gcs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
check_dirname_exists(local_path)
try:
blob = self.get_blob(blob=blob, bucket_name=bucket_name)
blob.download_to_filename(local_path)
except (NotFound, GoogleAPIError) as e:
raise PolyaxonStoresException(e)
def download_dir(self,
blob,
local_path,
container_name=None,
use_basename=True):
"""
Download a directory from Google Cloud Storage.
Args:
blob: `str`. blob to download.
local_path: `str`. the path to download to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, blob)
try:
check_dirname_exists(local_path, is_dir=True)
except PolyaxonStoresException:
os.makedirs(local_path)
results = self.list(container_name=container_name,
key=blob,
delimiter="/")
# Create directories
for prefix in sorted(results["prefixes"]):
direname = os.path.join(local_path, prefix)
prefix = os.path.join(blob, prefix)
# Download files under
self.download_dir(
blob=prefix,
local_path=direname,
container_name=container_name,
use_basename=False,
)
# Download files
for file_key in results["blobs"]:
file_key = file_key[0]
filename = os.path.join(local_path, file_key)
file_key = os.path.join(blob, file_key)
self.download_file(
blob=file_key,
local_path=filename,
container_name=container_name,
use_basename=False,
)
def download_dir(self,
key,
local_path,
bucket_name=None,
use_basename=True):
"""
Download a directory from S3.
Args:
key: `str`. S3 key that will point to the file.
local_path: `str`. the path to download to.
bucket_name: `str`. Name of the bucket in which to store the file.
use_basename: `bool`. whether or not to use the basename of the key.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, key)
try:
check_dirname_exists(local_path, is_dir=True)
except PolyaxonStoresException:
os.makedirs(local_path)
results = self.list(bucket_name=bucket_name, prefix=key, delimiter="/")
# Create directories
for prefix in sorted(results["prefixes"]):
direname = os.path.join(local_path, prefix)
prefix = os.path.join(key, prefix)
# Download files under
self.download_dir(
key=prefix,
local_path=direname,
bucket_name=bucket_name,
use_basename=False,
)
# Download files
for file_key in results["keys"]:
file_key = file_key[0]
filename = os.path.join(local_path, file_key)
file_key = os.path.join(key, file_key)
self.download_file(
key=file_key,
local_path=filename,
bucket_name=bucket_name,
use_basename=False,
)
def upload_dir(
self,
dirname,
key,
bucket_name=None,
overwrite=False,
encrypt=False,
acl=None,
use_basename=True,
):
"""
Uploads a local directory to S3.
Args:
dirname: `str`. name of the directory to upload.
key: `str`. S3 key that will point to the file.
bucket_name: `str`. Name of the bucket in which to store the file.
overwrite: `bool`. A flag to decide whether or not to overwrite the key
if it already exists. If replace is False and the key exists, an
error will be raised.
encrypt: `bool`. If True, the file will be encrypted on the server-side
by S3 and will be stored in an encrypted form while at rest in S3.
acl: `str`. ACL to use for uploading, e.g. "public-read".
use_basename: `bool`. whether or not to use the basename of the directory.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
if use_basename:
key = append_basename(key, dirname)
# Turn the path to absolute paths
dirname = os.path.abspath(dirname)
with get_files_in_current_directory(dirname) as files:
for f in files:
file_key = os.path.join(key, os.path.relpath(f, dirname))
self.upload_file(
filename=f,
key=file_key,
bucket_name=bucket_name,
overwrite=overwrite,
encrypt=encrypt,
acl=acl,
use_basename=False,
)
def upload_file(
self,
filename,
key,
bucket_name=None,
overwrite=False,
encrypt=False,
acl=None,
use_basename=True,
):
"""
Uploads a local file to S3.
Args:
filename: `str`. name of the file to upload.
key: `str`. S3 key that will point to the file.
bucket_name: `str`. Name of the bucket in which to store the file.
overwrite: `bool`. A flag to decide whether or not to overwrite the key
if it already exists. If replace is False and the key exists, an
error will be raised.
encrypt: `bool`. If True, the file will be encrypted on the server-side
by S3 and will be stored in an encrypted form while at rest in S3.
acl: `str`. ACL to use for uploading, e.g. "public-read".
use_basename: `bool`. whether or not to use the basename of the filename.
"""
if not bucket_name:
bucket_name, key = self.parse_s3_url(key)
if use_basename:
key = append_basename(key, filename)
if not overwrite and self.check_key(key, bucket_name):
raise PolyaxonStoresException(
"The key {} already exists.".format(key))
extra_args = {}
if encrypt:
extra_args["ServerSideEncryption"] = self.ENCRYPTION
if acl:
extra_args["ACL"] = acl
self.client.upload_file(filename,
bucket_name,
key,
ExtraArgs=extra_args)
def upload_file(self, filename, blob, bucket_name=None, use_basename=True):
"""
Uploads a local file to Google Cloud Storage.
Args:
filename: `str`. the file to upload.
blob: `str`. blob to upload to.
bucket_name: `str`. the name of the bucket.
use_basename: `bool`. whether or not to use the basename of the filename.
"""
if not bucket_name:
bucket_name, blob = self.parse_gcs_url(blob)
if use_basename:
blob = append_basename(blob, filename)
bucket = self.get_bucket(bucket_name)
bucket.blob(blob).upload_from_filename(filename)
def upload_file(self,
filename,
blob,
container_name=None,
use_basename=True):
"""
Uploads a local file to Google Cloud Storage.
Args:
filename: `str`. the file to upload.
blob: `str`. blob to upload to.
container_name: `str`. the name of the container.
use_basename: `bool`. whether or not to use the basename of the filename.
"""
if not container_name:
container_name, _, blob = self.parse_wasbs_url(blob)
if use_basename:
blob = append_basename(blob, filename)
self.connection.create_blob_from_path(container_name, blob, filename)
def test_append_basename(self):
assert append_basename("foo", "bar") == "foo/bar"
assert append_basename("foo", "moo/bar") == "foo/bar"
assert append_basename("/foo", "bar") == "/foo/bar"
assert append_basename("/foo/moo", "bar") == "/foo/moo/bar"
assert append_basename("/foo/moo", "boo/bar.txt") == "/foo/moo/bar.txt"