def main():
minio = Minio(STORAGE_ENDPOINT,
access_key=AWSAccessKeyId,
secret_key=AWSSecretKey)
content = BytesIO(b'Some Data to be stored')
key_id = 'YOUR-KMS-KEY'
context = {'Key1': 'Value1', 'Key2': 'Value2'}
# Create an SSE-KMS object with a Valid KMS key_id and context
sse_kms_obj = SSE_KMS(key_id, context)
# Put object with special headers from SSE_C object which encrypt object in S3 with provided key
minio.put_object(STORAGE_BUCKET,
'test_crypt.txt',
content,
content.getbuffer().nbytes,
sse=sse_kms_obj)
# Get decrypted object with same headers
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt.txt')
print(obj.read())
def main():
content = BytesIO(b'Hello again')
key = b'32byteslongsecretkeymustprovided'
encryption_key = base64.b64encode(key).decode()
encryption_key_md5 = base64.b64encode(hashlib.md5(key).digest()).decode()
minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey)
# Put object with special headers which encrypt object in S3 with provided key
minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes,
metadata={
'x-amz-server-side-encryption-customer-algorithm': 'AES256',
'x-amz-server-side-encryption-customer-key': encryption_key,
'x-amz-server-side-encryption-customer-key-MD5': encryption_key_md5
})
# Get decrypted object with same headers
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt1.txt', request_headers={
'x-amz-server-side-encryption-customer-algorithm': 'AES256',
'x-amz-server-side-encryption-customer-key': encryption_key,
'x-amz-server-side-encryption-customer-key-MD5': encryption_key_md5
})
print(obj.read())
def process_post(store: Minio, post: CleanPost, data_dir: str) -> CleanPost:
likes = post.likes
comments = post.comments
date = post.date
_id = post.id
rel = relevance(likes, comments)
print(f"~ [{_id}]: L: {likes}, C: {comments}, D: {date}, R: {rel}")
filepath = path.join(data_dir, post.id)
bucket = os.getenv("S3_BUCKET", "")
endpoint = os.getenv("S3_ENDPOINT", "")
folder_name = os.getenv("S3_DATA_DIR_NAME", "laciudadinvisible")
destination = folder_name + "/" + post.id + ".jpg"
full_filepath = filepath + ".jpg"
download_pic(full_filepath, post.image_uri)
post_processed_img = crop_image(full_filepath)
store.fput_object(bucket,
destination,
post_processed_img,
content_type="image/jpg",
metadata={"x-amz-acl": "public-read"})
link = f"https://{bucket}.{endpoint}/{destination}"
full_comments = post.comments_content
p = CleanPost(
id=post.id,
date=post.date,
likes=post.likes,
comments=post.comments,
hashtags=post.hashtags,
mentions=post.mentions,
relevance=rel,
image_uri=link,
description=post.description,
comments_content=full_comments,
)
return p
def main():
minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey)
content = BytesIO(b'Hello again')
#Create an SSE_S3 object
sse_s3_obj = SSE_S3()
# Put object with from SSE_S3 object which encrypt object in S3 with provided key
minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, sse=sse_s3_obj)
# Get decrypted object with same headers
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt.txt')
print(obj.read())
def main():
minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey)
content = BytesIO(b'Hello again')
#Create an SSE_S3 object
sse_s3_obj = SSE_S3()
# Put object with from SSE_S3 object which encrypt object in S3 with provided key
minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, sse=sse_s3_obj)
# Get decrypted object with same headers
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt.txt')
print(obj.read())
def main():
minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey)
content = BytesIO(b'Some Data to be stored')
key_id = 'YOUR-KMS-KEY'
context = {'Key1':'Value1', 'Key2':'Value2'}
# Create an SSE-KMS object with a Valid KMS key_id and context
sse_kms_obj = SSE_KMS(key_id, context)
# Put object with special headers from SSE_C object which encrypt object in S3 with provided key
minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, sse=sse_kms_obj)
# Get decrypted object with same headers
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt.txt')
print(obj.read())
def delete_objects(client: Minio, object_ids: List[str]) -> None:
"""
Delete objects stored in Minio
:param client: Minio client
:param object_ids: List of Splitgraph object IDs to delete
"""
# Expand the list of objects into actual files we store in Minio
all_object_ids = [o + suffix for o in object_ids for suffix in ("", ".schema", ".footer")]
list(client.remove_objects(S3_BUCKET, map(DeleteObject, all_object_ids)))
def main():
content = BytesIO(b'Hello again')
minio = Minio(STORAGE_ENDPOINT,
access_key=AWSAccessKeyId,
secret_key=AWSSecretKey)
# Create an SSE-C object with a 32 byte customer_key
key = b'32byteslongsecretkeymustprovided'
ssec = SseCustomerKey(key)
# Put object with SSE_C object passed as a param
minio.put_object(STORAGE_BUCKET,
'test_crypt.txt',
content,
content.getbuffer().nbytes,
sse=ssec)
# Copy encrypted object on Server-Side from Source to Destination
obj = minio.copy_object(STORAGE_BUCKET,
'test_crypt_copy.txt',
STORAGE_BUCKET + '/test_crypt.txt',
source_sse=ssec,
sse=ssec)
# Get decrypted object with SSE_C object passed in as param
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt_copy.txt', sse=ssec)
print(obj.read())
def list_objects(client: Minio) -> List[str]:
"""
List objects stored in Minio
:param client: Minio client
:return: List of Splitgraph object IDs
"""
return [
o.object_name
for o in client.list_objects(bucket_name=S3_BUCKET)
if not o.object_name.endswith(".footer") and not o.object_name.endswith(".schema")
]
def prompt_config() -> Config:
print("current config: ")
old_config = get_config()
show_config(old_config)
print(
f"{Fore.GREEN}you can leave any field empty to leave this field untouched then.{Fore.RESET}"
)
api_server = input(
"Please input your api server: ") or old_config.api_server
s3_endpoint = input(
"Please input your s3 endpoint (without http:// prefix, TLS isn't supported for now): "
) or old_config.s3_endpoint
s3_access_key = input(
"Please input your s3 access key: ") or old_config.s3_access_key
s3_secret_key = getpass(
"Please input your s3 secret access key: ") or old_config.s3_secret_key
logging.info("dialing to s3 storage using the current config...")
try:
client = Minio(s3_endpoint, s3_access_key, s3_secret_key, secure=False)
names = map(lambda b: b.name, client.list_buckets())
print(f"Which bucket you store your workloads?")
# leave empty string here for not changed. mapping it to a user-friend string
workload_bucket = select(
["", *names],
lambda b: b or "<not changed>") or old_config.workload_bucket
except Exception as e:
cont = prompt_ok(
f"{Fore.RED}cannot connect to s3 ({e}){Fore.RESET}, continue?",
False)
if not cont:
exit(1)
workload_bucket = input("What bucket you store your workloads? "
) or old_config.workload_bucket
return Config(api_server=api_server,
s3_endpoint=s3_endpoint,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
workload_bucket=workload_bucket)
def main():
content = BytesIO(b'Hello again')
minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey)
# Create an SSE-C object with a 32 byte customer_key
key = b'32byteslongsecretkeymustprovided'
sse_customer_key = SSE_C(key)
# Put object with SSE_C object passed as a param
minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, sse=sse_customer_key)
# Create a a copy_SSE-C object to copy an object from source to destination object on the Server-Side
copy_sse_customer_key = copy_SSE_C(key)
#Copy encrypted object on Server-Side from Source to Destination
obj = minio.copy_object(STORAGE_BUCKET, 'test_crypt_copy.txt', STORAGE_BUCKET+'/test_crypt.txt',
source_sse=copy_sse_customer_key, sse=sse_customer_key)
# Get decrypted object with SSE_C object passed in as param
obj = minio.get_object(STORAGE_BUCKET, 'test_crypt_copy.txt', sse=sse_customer_key)
print(obj.read())
def main():
content = BytesIO(b'Hello again')
key = b'32byteslongsecretkeymustprovided'
encryption_key = base64.b64encode(key).decode()
encryption_key_md5 = base64.b64encode(hashlib.md5(key).digest()).decode()
minio = Minio(STORAGE_ENDPOINT,
access_key=AWSAccessKeyId,
secret_key=AWSSecretKey)
# Put object with special headers which encrypt object in S3 with provided
# key
minio.put_object(STORAGE_BUCKET,
'test_crypt.txt',
content,
content.getbuffer().nbytes,
metadata={
'x-amz-server-side-encryption-customer-algorithm':
'AES256',
'x-amz-server-side-encryption-customer-key':
encryption_key,
'x-amz-server-side-encryption-customer-key-MD5':
encryption_key_md5
})
# Get decrypted object with same headers
obj = minio.get_object(
STORAGE_BUCKET,
'test_crypt1.txt',
request_headers={
'x-amz-server-side-encryption-customer-algorithm': 'AES256',
'x-amz-server-side-encryption-customer-key': encryption_key,
'x-amz-server-side-encryption-customer-key-MD5': encryption_key_md5
})
print(obj.read())
from minio.api import Minio
from minio.deleteobjects import DeleteObject
from splitgraph.config import CONFIG
S3_HOST = CONFIG["SG_S3_HOST"]
S3_PORT = CONFIG["SG_S3_PORT"]
S3_SECRET_KEY = CONFIG["SG_S3_PWD"]
S3_BUCKET = CONFIG["SG_S3_BUCKET"]
S3_ACCESS_KEY = CONFIG["SG_S3_KEY"]
S3_SECURE = CONFIG["SG_S3_SECURE"] == "true"
MINIO = Minio(
"%s:%s" % (S3_HOST, S3_PORT),
access_key=S3_ACCESS_KEY,
secret_key=S3_SECRET_KEY,
secure=S3_SECURE,
)
_EXP = timedelta(seconds=60)
def get_object_upload_urls(s3_host: str, object_ids: List[str]) -> List[List[str]]:
"""
Return a list of pre-signed URLs that each part of an object can be downloaded from.
:param s3_host: S3 host that the objects are stored on
:param object_ids: List of object IDs
:return: A list of lists [(object URL, object footer URL, object schema URL)]
"""
# Currently s3_host is ignored: this is to future-proof for choosing between multiple
def process_post(loader: Instaloader, post: Post, store: Minio,
data_dir: str) -> CleanPost:
likes = post.likes
comments = post.comments
date = post.date_local
_id = post.shortcode
rel = relevance(likes, comments)
print(f"\n~ [{_id}]: L: {likes}, C: {comments}, D: {date}, R: {rel}")
filepath = path.join(data_dir, post.shortcode)
loader.download_pic(filepath, post.url, post.date)
print("")
bucket = os.getenv("S3_BUCKET", "")
endpoint = os.getenv("S3_ENDPOINT", "")
folder_name = os.getenv("S3_DATA_DIR_NAME", "laciudadinvisible")
destination = folder_name + "/" + post.shortcode + ".jpg"
full_filepath = filepath + ".jpg"
store.fput_object(bucket,
destination,
full_filepath,
content_type="image/jpg",
metadata={"x-amz-acl": "public-read"})
link = f"https://{bucket}.{endpoint}/{destination}"
full_comments: List[str] = []
for comment in post.get_comments():
full_comments.append(comment.text)
p = CleanPost(
id=post.shortcode,
date=str(post.date_local),
likes=post.likes,
comments=post.comments,
hashtags=post.caption_hashtags,
mentions=post.caption_mentions,
relevance=rel,
image_uri=link,
description=post.caption,
comments_content=full_comments,
)
# coll_name = "Posts"
# try:
# db.create_collection(coll_name)
# except:
# print("error at execute create collection")
# posts = db.collection(coll_name)
# posts.insert(asdict(p))
# if path.exists(full_filepath):
# os.remove(full_filepath)
return p
