def _s3_query():
r = _elasticache_connect()
s3, bucket = _s3_connect()
f = random.randint(0, BATCH)
fname = 'dummy' + str(f)
key = 'S3:' + (os.environ['S3_BUCKET']) + ':' + fname
hex_dig = hashlib.sha256(key.encode('utf-8')).hexdigest()
if r.exists(hex_dig):
start = datetime.now()
data = pickle.loads(r.get(hex_dig))
stop = datetime.now()
diff = (stop-start).total_seconds()
payload = json.dumps({"Response": "CACHE HIT",
"Duration": str(diff),
"Measurement": "Seconds",
"Payload": fname,
"Payload Size": size(sys.getsizeof(data))},
indent=1)
else:
start = datetime.now()
obj = s3.Object(os.environ['S3_BUCKET'], fname)
data = obj.get()['Body'].read().decode('utf-8')
p_data = pickle.dumps(data)
r.set(hex_dig, p_data, ex=900)
stop = datetime.now()
diff = (stop-start).total_seconds()
payload = json.dumps({"Response": "CACHE MISS",
"Duration": str(diff),
"Measurement": "Seconds",
"Payload": fname,
"Payload Size": size(sys.getsizeof(data))},
indent=1)
return payload
def _s3_prepare():
s3, bucket = _s3_connect()
start = datetime.now()
for i in range(0, BATCH):
data = ''.join(random.choices(string.ascii_uppercase +
string.digits, k = 5120000))
bucket.put_object(Key="dummy" + str(i),
Body=data)
stop = datetime.now()
diff = (stop-start).total_seconds()
payload = json.dumps({"Response": "S3 data successfully created",
"Objects Created": str(BATCH),
"Duration": str(diff),
"Measurement": "Seconds"}, indent=1)
return payload
def _s3_clean():
counter = 0
start = datetime.now()
s3, bucket = _s3_connect()
keys = bucket.objects.all()
for obj in bucket.objects.all():
counter += 1
bucket.Object(obj.key).delete()
stop = datetime.now()
diff = (stop-start).total_seconds()
payload = json.dumps({"Response": "S3 data successfully deleted",
"Objects Deleted": str(counter),
"Duration": str(diff),
"Measurement": "Seconds"}, indent=1)
return payload