def post(self): json_data = request.get_json(force=True) app_context.init() app_context.application_context = json_data log_info( "Resource Layout_Detector_WF Layout_Detector service started", app_context.application_context) task_id = str("LD-" + str(time.time()).replace('.', '')[0:13]) task_starttime = eval(str(time.time()).replace('.', '')[0:13]) #json_data = request.get_json(force = True) try: error_validator = ValidationResponse(DOWNLOAD_FOLDER) if error_validator.format_error(json_data) is True: response_gen = Response(json_data, DOWNLOAD_FOLDER) response = response_gen.workflow_response( task_id, task_starttime) log_info( "Resource Layout_Detector_WF Layout_Detector api response completed", app_context.application_context) return jsonify(response) except FormatError as e: log_error( "Resource Layout_Detector_WF Input json format is not correct or dict_key is missing", app_context.application_context, e) return Status.ERR_request_input_format.value
def vision_ocr_request_worker(): file_ops = FileOperation() DOWNLOAD_FOLDER = file_ops.create_file_download_dir(config.download_folder) producer_tok = Producer(config.bootstrap_server) log_info("vision_ocr_request_worker : starting thread ", LOG_WITHOUT_CONTEXT) while True: data = processQueue.get(block=True) ################# task_id = str("vision_ocr" + str(time.time()).replace('.', '')) ################### task_starttime = str(time.time()).replace('.', '') input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format(data) log_info("vision_ocr_request_worker processing -- received message "+str(jobid), data) try: response_gen = Response(data, DOWNLOAD_FOLDER) file_value_response = response_gen.workflow_response(task_id, task_starttime, False) if file_value_response != None: if "errorID" not in file_value_response.keys(): push_output(producer_tok, config.output_topic, file_value_response, jobid, task_id,data) log_info("vision_ocr_request_worker : response send to topic %s"%(config.output_topic), LOG_WITHOUT_CONTEXT) else: log_info("vision_ocr_request_worker : error send to error handler", data) log_info('vision_ocr_request_worker - request in internal queue {}'.format(Queue.qsize()), data) processQueue.task_done() except Exception as e: log_exception("vision_ocr_request_worker ", LOG_WITHOUT_CONTEXT, e) controlQueue.put(1)
def processRequest(data): file_ops = FileOperation() producer_tok = Producer(config.bootstrap_server) DOWNLOAD_FOLDER = file_ops.file_download(config.download_folder) task_id = str("ANNO-" + str(time.time()).replace('.', '')) task_starttime = str(time.time()).replace('.', '') input_params, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format( data) log_info("processing -- received message " + str(jobid), data) try: response_gen = Response(data, DOWNLOAD_FOLDER) result_response = response_gen.workflow_response( task_id, task_starttime) if result_response != None: if "errorID" not in result_response.keys(): push_output(producer_tok, config.output_topic, result_response, jobid, task_id, data) log_info( "processing completed successfully, published at %s" % (config.output_topic), data) else: log_info("processing failed, informed WFM", data) except Exception as e: log_exception("exception encountered ", LOG_WITHOUT_CONTEXT, e)
def word_detector_request_worker(): file_ops = FileOperation() DOWNLOAD_FOLDER = file_ops.create_file_download_dir(config.download_folder) producer_tok = Producer(config.bootstrap_server) log_info("word_detector_request_worker : starting thread ", LOG_WITHOUT_CONTEXT) while True: data = Queue.get(block=True) ################# task_id = str("word_detector" + str(time.time()).replace('.', '')) ################### task_starttime = str(time.time()).replace('.', '') input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format( data) log_info( "word_detector_request_worker processing -- received message " + str(jobid), data) try: response_gen = Response(data, DOWNLOAD_FOLDER) file_value_response = response_gen.workflow_response( task_id, task_starttime, False) if file_value_response != None: push_output(producer_tok, config.output_topic, file_value_response, jobid, task_id, data) log_info( "word_detector_request_worker : response send to topic %s" % (config.output_topic), LOG_WITHOUT_CONTEXT) else: erro_obj = { 'code': 400, 'jobID': jobid, 'message': "Word detector failed" } producer_tok.push_data_to_queue( config.KAFKA_ANUVAAD_ETL_WF_ERROR_TOPIC, erro_obj) log_info( "word_detector_request_worker : error send to error handler", data) log_info( 'word_detector_request_worker - request in internal queue {}'. format(Queue.qsize()), data) Queue.task_done() except Exception as e: log_exception("word_detector_request_worker ", LOG_WITHOUT_CONTEXT, e)
def post(self): log_info("Resource BlockMergerWF", "Block-merger service started", None) task_id = str("BM-" + str(time.time()).replace('.', '')) task_starttime = str(time.time()).replace('.', '') json_data = request.get_json(force=True) try: error_validator = ValidationResponse(DOWNLOAD_FOLDER) if error_validator.format_error(json_data) is True: response_gen = Response(json_data, DOWNLOAD_FOLDER) response = response_gen.workflow_response( task_id, task_starttime) log_info("Resource BlockMergerWF", "Block-merger api response completed", None) return jsonify(response) except FormatError as e: log_error( "Resource BlockMergerWF", "Input json format is not correct or dict_key is missing", None, e) return Status.ERR_request_input_format.value
def block_merger_request_worker(): file_ops = FileOperation() DOWNLOAD_FOLDER = file_ops.create_file_download_dir(config.download_folder) producer_tok = Producer(config.bootstrap_server) while True: data = blockMergerQueue.get(block=True) task_id = str("BM-" + str(time.time()).replace('.', '')) task_starttime = str(time.time()).replace('.', '') input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format( data) log_info( "block_merger_request_worker processing -- received message " + str(jobid), data) response_gen = Response(data, DOWNLOAD_FOLDER) file_value_response = response_gen.workflow_response( task_id, task_starttime, False) if file_value_response != None: if "errorID" not in file_value_response.keys(): push_output(producer_tok, config.output_topic, file_value_response, jobid, task_id) log_info( "process_block_merger_kf : response send to topic %s" % (config.output_topic), None) else: log_info( "process_block_merger_kf : error send to error handler", jobid) log_info( 'block_merger_request_worker - request in internal queue {}'. format(blockMergerQueue.qsize()), jobid) blockMergerQueue.task_done()