def create_bucket(self, bucket: str): """create bucket with `bucket` as name""" try: self.client.create_bucket(Bucket=bucket) except self.client.meta.client.exceptions.BucketAlreadyExists as e: logger.error("failed to create bucket {}".format(e)) raise RuntimeError
def __init__(self): super().__init__() try: AWSStore.client = boto3.client("s3") self.client = AWSStore.client except aws_error as e: logger.error("cannot login to aws") logger.info("set credentials using environment variables") logger.info( "please see: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#environment-variables" )
def download(self, bucket, object_name, file_name): """download file from s3 given an object name and bucket.""" try: with open(file_name, "wb") as fh: try: self.client.download_file(bucket, object_name, file_name) except aws_error as e: logger.error("client error: {}".format(e)) sys.exit(1) except IOError as e: logger.error("io error: {}".format(e)) logger.info("object: {} downloaded to file: {}".format( object_name, file_name))
def __init__(self): super().__init__() try: GCStore.client = google_store.Client() self.client = GCStore.client except GCSCREDError: logger.error(( "\ncredentials file is expected" "\nuse: GOOGLE_APPLICATION_CREDENTIALS env var to set the path to the credential file" "\nsee: https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable" )) logger.error("credentials not given, exiting..") sys.exit(1)
def upload(self, bucket: str, file_name): """aws upload file to aws.""" try: self.client.upload_file(file_name, bucket, file_name) except S3UploadFailedError: try: self.create_bucket(bucket) except RuntimeError: logger.error("runtime error raised, exiting") sys.exit(1) self.client.upload_file(file_name, bucket, file_name) except aws_error as e: logger.error("client error: {}".format(e)) sys.exit(1) logger.info("file {} upload to s3".format(file_name))
def __init__(self): connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING") if connect_str is None: logger.error( "AZURE_STORAGE_CONNECTION_STRING env variable not set") sys.exit(1) try: AZRStore.client = BlobServiceClient.from_connection_string( connect_str) except ValueError: logger.error( "AZURE_STORAGE_CONNECTION_STRING string malformed, check the env variable." ) sys.exit(1) self.client = AZRStore.client
def upload(self, bucket_name: str, file_name): """upload file to google cloud.""" create_b = False try: bucket = self.client.get_bucket(bucket_name) except GCS_Forbidden: logger.error( "you dont have access, to the bucket, trying to create a new one.." ) create_b = True except GCS_NotFound as e: logger.error("bucket name not found, trying to create a new one") logger.error("exception: {}".format(e)) create_b = True if create_b: bucket = self.create_bucket(bucket_name) blob = bucket.blob(file_name) blob.upload_from_filename(file_name) logger.info("file saved, URL:{}".format(blob.public_url)) return json.dumps( dict( bucket_name=blob.bucket.name, object_name=blob.name, public_url=blob.public_url, ))