def process(event, function_name, user_name):
bucket_name = event.split('/')[0]
file_name = event.split('/')[1]
function_id = calculate_checksum(function_name)
mc = connect_minio()
obj = getObject(mc, file_name, bucket_name)
img_checksum = calculate_checksum(obj)
couch = connect_couchdb()
createDBsIfNotExist(couch)
hashedUsername = authenticateUser(user_name)
state, userdocId = getUpdateDocOfUser(couch, hashedUsername, function_id,
img_checksum)
if state is not None:
print("\n**Duplicate Data**")
print(state)
#create data if not present
addInputDataIfNotExist(couch, userdocId, function_id, img_checksum)
command = "wsk -i action invoke weatherhit"
execute(command)
print(function_name + " Invoked Successfully")
time.sleep(5)
obj = getObject(mc, "minio_log.json", "store")
with open(obj) as json_file:
data = json.load(json_file)
ref = data['reference']
addMinioRef(couch, userdocId, function_id, img_checksum, ref)
print("\n**Unique Data**")
return ref
def kafka_consumer(topic_name,function_name,user_name):
consumer = KafkaConsumer(
topic_name,
bootstrap_servers=['172.18.0.2:9092'],
auto_offset_reset='latest')
for message in consumer:
json_data = json.loads(message.value)
bucket_name = json_data['Key'].split('/')[0]
if (json_data['EventName'] == 's3:ObjectCreated:Put' and bucket_name == 'test1'):
#print('\nInput Bucket :', bucket_name)
#print('Input File name :', json_data['Key'].split('/')[1])
with open('kafka_log.json', 'w') as outfile:
json.dump(json_data, outfile)
minioClient = connect_minio()
try:
start = time.time()
minioClient.fput_object('store', 'kafka_log.json', 'kafka_log.json')
output_reference = process(json_data['Key'],function_name,user_name)
print('Output File Reference :', output_reference)
end = time.time()
print("Total Time Execution : "+str(end - start)+" sec(s)\n")
except ResponseError as err:
print(err)
def process(event, function_name):
bucket_name = event.split('/')[0]
file_name = event.split('/')[1]
function_id = calculate_checksum(function_name)
#connect couchdb
couch = connect_couchdb()
#connect minio
mc = connect_minio()
#fetch file from
obj = getObject(mc, file_name, bucket_name)
img_checksum = calculate_checksum(obj)
#create function if not present
addFunctionIfNotExist(couch, function_id)
# Check if same data is available in the couch db
state = verfiyDataAvailable(couch, function_id, img_checksum, "sanity")
if state is not None:
print("\n**Duplicate Data**")
return state
#create data if not present
addInputDataIfNotExist(couch, function_id, img_checksum)
#create command that makes an action
action_name = "random"
command = "wsk -i action invoke weather"
execute(command)
print(function_name + " Invoked Successfully")
time.sleep(5)
obj = getObject(mc, "minio_log.json", "store")
with open(obj) as json_file:
data = json.load(json_file)
ref = data['reference']
addMinioRef(couch, function_id, img_checksum, ref)
print("\n**Unique Data**")
return ref
def process(event):
bucket_name = event.split('/')[0]
file_name = event.split('/')[1]
function_id = "shfdguhrgfgekfvgsdv"
#connect couchdb
couch = connect_couchdb()
#connect minio
mc = connect_minio()
#fetch file from
obj = getObject(mc, file_name, bucket_name)
img_checksum = calculate_checksum(obj)
#if bucket_name == "input":
#create function if not present
addFunctionIfNotExist(couch, function_id)
# Check if same data is available in the couch db
state = verfiyDataAvailable(couch,function_id,img_checksum,"sanity")
if state is not None:
return state
#create data if not present
addInputDataIfNotExist(couch,function_id,img_checksum)
command = "wsk -i action invoke sprint"
execute(command)
time.sleep(5)
obj = getObject(mc, "minio_log.json", "store")
with open(obj) as json_file:
data = json.load(json_file)
ref = data['reference']
#print(ref)
addMinioRef(couch, function_id, img_checksum, ref)
return ref
Options:
-h --help Show this screen.
--version Show version.
--i Sanity Input Bucket
--o Sanity Output Bucket
--f Function Name
"""
from docopt import docopt
from connectMinio import connect_minio, createBucket
from kafkaConnect import kafka_consumer
from pyfiglet import Figlet
from connectCouchdb import connect_couchdb, addUserIfNotExist
if __name__ == '__main__':
arguments = docopt(__doc__, version='sanity 1.0')
f = Figlet(font='slant')
print(f.renderText('Sanity'))
input_bucket = arguments.get('<input>')
output_bucket = arguments.get('<output>')
function_name = arguments.get('<function/action>')
user_name = arguments.get('<username>')
print("\n***Welcome " + user_name + "***\n")
mc = connect_minio()
#createBucket(mc,input_bucket)
#createBucket(mc, output_bucket)
kafka_consumer("in-bucket-notifications", function_name, user_name)