def unpickle_from_bucket(bucket, name): """Unpickle from s3 bucket and return result. Args: bucket(string): the name of the s3 bucket name(string): the filename Returns: the unpickled data object """ s3client = Session().client( "s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, ) print(f"downloading {name} from {bucket}") with TemporaryFile() as tmpf: s3client.download_fileobj(bucket, f"{name}.pickle", tmpf) tmpf.seek(0) tmpf.flush() unpickled = load(tmpf) return unpickled
def ReadFileFromBucket(bucket_name: str, file_name: str, type: str = 'gcs', auth_via_project=None, credentials_file: str = None) -> str: try: if 'gcs' in type or 'google' in type: if auth_via_project: storage_client = storage.Client() else: PWD = path.realpath(path.dirname(__file__)) credentials_file = path.join(PWD, credentials_file) storage_client = storage.Client.from_service_account_json( credentials_file) if storage_client.bucket(bucket_name).blob(file_name).exists(): file_size: int = storage_client.bucket(bucket_name).get_blob( file_name).size file_text: str = storage_client.bucket(bucket_name).blob( file_name).download_as_text() if file_size > 0: return file_text.rstrip() else: return None if 's3' in type or 'aws' in type: s3_client = Session().client("s3") fh = BytesIO() s3_client.download_fileobj(bucket_name, file_name, fh) fh.seek(0) return fh.getvalue() return "" except Exception as e: raise (e)