def create_bucket(self, bucket: str):
"""create bucket with `bucket` as name"""
try:
self.client.create_bucket(Bucket=bucket)
except self.client.meta.client.exceptions.BucketAlreadyExists as e:
logger.error("failed to create bucket {}".format(e))
raise RuntimeError
def __init__(self):
super().__init__()
try:
AWSStore.client = boto3.client("s3")
self.client = AWSStore.client
except aws_error as e:
logger.error("cannot login to aws")
logger.info("set credentials using environment variables")
logger.info(
"please see: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#environment-variables"
)
def download(self, bucket, object_name, file_name):
"""download file from s3 given an object name and bucket."""
try:
with open(file_name, "wb") as fh:
try:
self.client.download_file(bucket, object_name, file_name)
except aws_error as e:
logger.error("client error: {}".format(e))
sys.exit(1)
except IOError as e:
logger.error("io error: {}".format(e))
logger.info("object: {} downloaded to file: {}".format(
object_name, file_name))
def __init__(self):
super().__init__()
try:
GCStore.client = google_store.Client()
self.client = GCStore.client
except GCSCREDError:
logger.error((
"\ncredentials file is expected"
"\nuse: GOOGLE_APPLICATION_CREDENTIALS env var to set the path to the credential file"
"\nsee: https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable"
))
logger.error("credentials not given, exiting..")
sys.exit(1)
def upload(self, bucket: str, file_name):
"""aws upload file to aws."""
try:
self.client.upload_file(file_name, bucket, file_name)
except S3UploadFailedError:
try:
self.create_bucket(bucket)
except RuntimeError:
logger.error("runtime error raised, exiting")
sys.exit(1)
self.client.upload_file(file_name, bucket, file_name)
except aws_error as e:
logger.error("client error: {}".format(e))
sys.exit(1)
logger.info("file {} upload to s3".format(file_name))
def __init__(self):
connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
if connect_str is None:
logger.error(
"AZURE_STORAGE_CONNECTION_STRING env variable not set")
sys.exit(1)
try:
AZRStore.client = BlobServiceClient.from_connection_string(
connect_str)
except ValueError:
logger.error(
"AZURE_STORAGE_CONNECTION_STRING string malformed, check the env variable."
)
sys.exit(1)
self.client = AZRStore.client
def upload(self, bucket_name: str, file_name):
"""upload file to google cloud."""
create_b = False
try:
bucket = self.client.get_bucket(bucket_name)
except GCS_Forbidden:
logger.error(
"you dont have access, to the bucket, trying to create a new one.."
)
create_b = True
except GCS_NotFound as e:
logger.error("bucket name not found, trying to create a new one")
logger.error("exception: {}".format(e))
create_b = True
if create_b:
bucket = self.create_bucket(bucket_name)
blob = bucket.blob(file_name)
blob.upload_from_filename(file_name)
logger.info("file saved, URL:{}".format(blob.public_url))
return json.dumps(
dict(
bucket_name=blob.bucket.name,
object_name=blob.name,
public_url=blob.public_url,
))