예제 #1
0
def save_to_db(db_object, result_to_save):
    try:
        print("*****************SAVING TO DB******************************")
        if db_object.text:
            db_object.text = db_object.text + ' ' + result_to_save
        else:
            db_object.text = result_to_save
        db_object.save()
        print("*****************SAVED TO DB******************************")
    except Exception as e:
        print(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
        ERR_LOGGER(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
예제 #2
0
def update_state(file):
    payload = {
        'topic_name': globals.RECEIVE_TOPIC,
        'client_id': globals.CLIENT_ID,
        'value': file
    }
    try:
        requests.request("POST", globals.DASHBOARD_URL, data=payload)
    except Exception as e:
        print(f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
        ERR_LOGGER(
            f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
예제 #3
0
def save_to_db(db_object, labels, scores):
    try:
        print("*****************SAVING TO DB******************************")
        print("in save")
        print(db_object)
        print(db_object.id)
        db_object.labels = labels
        db_object.scores = scores
        db_object.save()
        print("*****************SAVED TO DB******************************")
    except Exception as e:
        print(" ERROR IN SAVE TO DB")
        ERR_LOGGER(str(e) + " ERROR IN SAVE TO DB")
예제 #4
0
def save_to_db(db_object, labels, scores):
    try:
        print("*****************SAVING TO DB******************************")
        print("in save")
        print(db_object)
        print(db_object.id)
        db_object.labels = labels
        db_object.scores = scores
        db_object.save()
        print("*****************SAVED TO DB******************************")
    except Exception as e:
        print(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
        ERR_LOGGER(f"{e} ERROR IN SAVE TO DB FILE ID {FILE_ID}")
예제 #5
0
def update_state(file_name):
    payload = {
        'parent_name': globals.PARENT_NAME,
        'group_name': globals.GROUP_NAME,
        'container_name': globals.RECEIVE_TOPIC,
        'file_name': file_name,
        'client_id': globals.CLIENT_ID
    }
    try:
        requests.request("POST", globals.DASHBOARD_URL, data=payload)
    except Exception as e:
        print(f"{e} EXCEPTION IN UPDATE STATE API CALL......")
        ERR_LOGGER(f"{e} EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
예제 #6
0
def predict(file_name):
    print('THIS IS THE ', file_name, os.path.isfile(file_name))

    try:
        objects = object_api(file_name)
        # objects = ' '.join(objects)
        os.remove(file_name)
        return objects
    except Exception as e:
        print(str(e) + "Exception in predict")
        ERR_LOGGER(str(e) + " Exception in predict")
        return ""


# response = requests.request("POST", 'http://api:5000/upload/', files = files)

# print(response.text.encode('utf8'))
예제 #7
0
 print(pyfiglet.figlet_format(str(globals.RECEIVE_TOPIC)))
 print(pyfiglet.figlet_format("INDEXING CONTAINER"))
 print("Connected to Kafka at " + globals.KAFKA_HOSTNAME + ":" +
       globals.KAFKA_PORT)
 print("Kafka Consumer topic for this Container is " +
       globals.RECEIVE_TOPIC)
 for message in init.consumer_obj:
     message = message.value
     db_key = str(message)
     print(db_key, 'db_key')
     try:
         db_object = Cache.objects.get(pk=db_key)
     except Exception as e:
         print("EXCEPTION IN UPDATE STATE API CALL......")
         ERR_LOGGER(
             str(e) +
             " EXCEPTION IN UPDATE STATE API CALL......FILE ID {FILE_ID}")
         continue
     file_name = db_object.file_name
     # init.redis_obj.set(globals.RECEIVE_TOPIC, file_name)
     print("#############################################")
     print("########## PROCESSING FILE " + file_name)
     print("#############################################")
     if db_object.is_doc_type:
         """document"""
         if db_object.contains_images:
             images_array = []
             for image in db_object.files:
                 pdf_image = str(uuid.uuid4()) + ".jpg"
                 with open(pdf_image, 'wb') as file_to_save:
                     file_to_save.write(image.file.read())
예제 #8
0

if __name__ == "__main__":
    print("Connected to Kafka at " + globals.KAFKA_HOSTNAME + ":" +
          globals.KAFKA_PORT)
    print("Kafka Consumer topic for this Container is " +
          globals.RECEIVE_TOPIC)
    for message in init.consumer_obj:
        message = message.value
        db_key = str(message)
        print(db_key, 'db_key')
        try:
            db_object = Cache.objects.get(pk=db_key)
        except Exception as e:
            print(f"{e} EXCEPTION IN GET PK... continue")
            ERR_LOGGER(f"{e} EXCEPTION IN GET PK... continue")
            continue

        file_name = db_object.file_name

        final_labels = db_object.labels
        final_scores = db_object.scores

        print("#############################################")
        print("########## PROCESSING FILE " + file_name)
        print("#############################################")

        if db_object.is_doc_type:
            """document"""
            if db_object.contains_images:
                images_array = []
예제 #9
0
            """document"""
            if db_object.contains_images:
                images_array = []
                for image in db_object.files:
                    pdf_image = str(uuid.uuid4()) + ".jpg"
                    with open(pdf_image, 'wb') as file_to_save:
                        file_to_save.write(image.file.read())
                    images_array.append(pdf_image)

                result_list = list()
                for image in images_array:
                    try:
                        image_results = predict(image)
                    except Exception as e:
                        print(f"{e} Exception in predict")
                        ERR_LOGGER(
                            f"{e} Exception in predict FILE ID {FILE_ID}")
                        continue

                    result_list.append(image_results)
                to_save = ' '.join(result_list)
                print("to_save", to_save)
                save_to_db(db_object, to_save)
                print(
                    ".....................FINISHED PROCESSING FILE....................."
                )
                update_state(file_name)
            else:
                pass

        else:
            """image"""