def save_to_db(db_object, result_to_save):
try:
print("*****************SAVING TO DB******************************")
if db_object.text:
db_object.text = db_object.text + ' ' + result_to_save
else:
db_object.text = result_to_save
db_object.save()
print("*****************SAVED TO DB******************************")
except Exception as e:
print(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
ERR_LOGGER(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
def update_state(file):
payload = {
'topic_name': globals.RECEIVE_TOPIC,
'client_id': globals.CLIENT_ID,
'value': file
}
try:
requests.request("POST", globals.DASHBOARD_URL, data=payload)
except Exception as e:
print(f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
ERR_LOGGER(
f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
def save_to_db(db_object, labels, scores):
try:
print("*****************SAVING TO DB******************************")
print("in save")
print(db_object)
print(db_object.id)
db_object.labels = labels
db_object.scores = scores
db_object.save()
print("*****************SAVED TO DB******************************")
except Exception as e:
print(" ERROR IN SAVE TO DB")
ERR_LOGGER(str(e) + " ERROR IN SAVE TO DB")
def save_to_db(db_object, labels, scores):
try:
print("*****************SAVING TO DB******************************")
print("in save")
print(db_object)
print(db_object.id)
db_object.labels = labels
db_object.scores = scores
db_object.save()
print("*****************SAVED TO DB******************************")
except Exception as e:
print(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
ERR_LOGGER(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
def update_state(file_name):
payload = {
'parent_name': globals.PARENT_NAME,
'group_name': globals.GROUP_NAME,
'container_name': globals.RECEIVE_TOPIC,
'file_name': file_name,
'client_id': globals.CLIENT_ID
}
try:
requests.request("POST", globals.DASHBOARD_URL, data=payload)
except Exception as e:
print(f"{e} EXCEPTION IN UPDATE STATE API CALL......")
ERR_LOGGER(f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
def predict(file_name):
print('THIS IS THE ', file_name, os.path.isfile(file_name))
try:
objects = object_api(file_name)
# objects = ' '.join(objects)
os.remove(file_name)
return objects
except Exception as e:
print(str(e) + "Exception in predict")
ERR_LOGGER(str(e) + " Exception in predict")
return ""
# response = requests.request("POST", 'http://api:5000/upload/', files = files)
# print(response.text.encode('utf8'))
print(pyfiglet.figlet_format(str(globals.RECEIVE_TOPIC)))
print(pyfiglet.figlet_format("INDEXING CONTAINER"))
print("Connected to Kafka at " + globals.KAFKA_HOSTNAME + ":" +
globals.KAFKA_PORT)
print("Kafka Consumer topic for this Container is " +
globals.RECEIVE_TOPIC)
for message in init.consumer_obj:
message = message.value
db_key = str(message)
print(db_key, 'db_key')
try:
db_object = Cache.objects.get(pk=db_key)
except Exception as e:
print("EXCEPTION IN UPDATE STATE API CALL......")
ERR_LOGGER(
str(e) +
" EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
continue
file_name = db_object.file_name
# init.redis_obj.set(globals.RECEIVE_TOPIC, file_name)
print("#############################################")
print("########## PROCESSING FILE " + file_name)
print("#############################################")
if db_object.is_doc_type:
"""document"""
if db_object.contains_images:
images_array = []
for image in db_object.files:
pdf_image = str(uuid.uuid4()) + ".jpg"
with open(pdf_image, 'wb') as file_to_save:
file_to_save.write(image.file.read())
if __name__ == "__main__":
print("Connected to Kafka at " + globals.KAFKA_HOSTNAME + ":" +
globals.KAFKA_PORT)
print("Kafka Consumer topic for this Container is " +
globals.RECEIVE_TOPIC)
for message in init.consumer_obj:
message = message.value
db_key = str(message)
print(db_key, 'db_key')
try:
db_object = Cache.objects.get(pk=db_key)
except Exception as e:
print(f"{e} EXCEPTION IN GET PK... continue")
ERR_LOGGER(f"{e} EXCEPTION IN GET PK... continue")
continue
file_name = db_object.file_name
final_labels = db_object.labels
final_scores = db_object.scores
print("#############################################")
print("########## PROCESSING FILE " + file_name)
print("#############################################")
if db_object.is_doc_type:
"""document"""
if db_object.contains_images:
images_array = []
"""document"""
if db_object.contains_images:
images_array = []
for image in db_object.files:
pdf_image = str(uuid.uuid4()) + ".jpg"
with open(pdf_image, 'wb') as file_to_save:
file_to_save.write(image.file.read())
images_array.append(pdf_image)
result_list = list()
for image in images_array:
try:
image_results = predict(image)
except Exception as e:
print(f"{e} Exception in predict")
ERR_LOGGER(
f"{e} Exception in predict FILE ID {FILE_ID}")
continue
result_list.append(image_results)
to_save = ' '.join(result_list)
print("to_save", to_save)
save_to_db(db_object, to_save)
print(
".....................FINISHED PROCESSING FILE....................."
)
update_state(file_name)
else:
pass
else:
"""image"""