def unpickle_from_bucket(bucket, name):
"""Unpickle from s3 bucket and return result.
Args:
bucket(string): the name of the s3 bucket
name(string): the filename
Returns: the unpickled data object
"""
s3client = Session().client(
"s3",
endpoint_url="http://127.0.0.1:9000",
aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY,
)
print(f"downloading {name} from {bucket}")
with TemporaryFile() as tmpf:
s3client.download_fileobj(bucket, f"{name}.pickle", tmpf)
tmpf.seek(0)
tmpf.flush()
unpickled = load(tmpf)
return unpickled
def ReadFileFromBucket(bucket_name: str,
file_name: str,
type: str = 'gcs',
auth_via_project=None,
credentials_file: str = None) -> str:
try:
if 'gcs' in type or 'google' in type:
if auth_via_project:
storage_client = storage.Client()
else:
PWD = path.realpath(path.dirname(__file__))
credentials_file = path.join(PWD, credentials_file)
storage_client = storage.Client.from_service_account_json(
credentials_file)
if storage_client.bucket(bucket_name).blob(file_name).exists():
file_size: int = storage_client.bucket(bucket_name).get_blob(
file_name).size
file_text: str = storage_client.bucket(bucket_name).blob(
file_name).download_as_text()
if file_size > 0:
return file_text.rstrip()
else:
return None
if 's3' in type or 'aws' in type:
s3_client = Session().client("s3")
fh = BytesIO()
s3_client.download_fileobj(bucket_name, file_name, fh)
fh.seek(0)
return fh.getvalue()
return ""
except Exception as e:
raise (e)
