def process_fc_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER =file_ops.file_download(config.download_folder)
task_id = str("FC-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
producer_tok = Producer(config.bootstrap_server)
# instatiation of consumer for respective topic
try:
consumer = consumer_validator()
log_info("process_fc_kf", "trying to receive value from consumer ", None)
for msg in consumer:
log_info("process_fc_kf", "value received from consumer", None)
data = msg.value
task_id = str("FC-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format(data)
response_gen = Response(data, DOWNLOAD_FOLDER)
file_value_response = response_gen.workflow_response(task_id, task_starttime)
if "errorID" not in file_value_response.keys():
push_output(producer_tok, config.tok_output_topic, file_value_response, jobid, task_id)
log_info("process_fc_kf", "response send to topic %s"%(config.tok_output_topic), None)
else:
log_info("process_fc_kf", "error send to error handler", jobid)
except KafkaConsumerError as e:
response_custom = CustomResponse(Status.ERR_STATUS.value, None, None)
response_custom['message'] = str(e)
file_ops.error_handler(response_custom, "KAFKA_CONSUMER_ERROR", True)
log_exception("process_fc_kf", "Consumer didn't instantiate", None, e)
except KafkaProducerError as e:
response_custom = e.code
response_custom['message'] = e.message
file_ops.error_handler(response_custom, "KAFKA_PRODUCER_ERROR", True)
log_exception("process_fc_kf", "response send to topic %s"%(config.tok_output_topic), response_custom['jobID'], e)
def process_tokenization_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER =file_ops.file_download(config.download_folder)
# instatiation of consumer for respective topic
try:
consumer_class = Consumer(config.input_topic, config.bootstrap_server)
consumer = consumer_class.consumer_instantiate()
log_info("process_tokenization_kf : trying to receive value from consumer ", None)
for msg in consumer:
data = msg.value
log_info("process_tokenization_kf : received input json from input topic consumer ", data)
task_id = str("TOK-" + str(time.time()).replace('.', '')[0:13])
task_starttime = eval(str(time.time()).replace('.', '')[0:13])
input_files, workflow_id, jobid, tool_name, step_order, user_id = file_ops.json_input_format(data)
response_gen = Response(data, DOWNLOAD_FOLDER)
file_value_response = response_gen.workflow_response(task_id, task_starttime)
if "errorID" not in file_value_response.keys():
producer = Producer()
producer.push_data_to_queue(config.output_topic, file_value_response, data, task_id)
else:
log_error("process_tokenization_kf : error send to error handler", data, None)
except KafkaConsumerError as e:
response_custom = CustomResponse(Status.ERR_STATUS.value, None, None)
response_custom.status_code['message'] = str(e)
file_ops.error_handler(response_custom.status_code, "KAFKA_CONSUMER_ERROR", True)
log_exception("process_tokenization_kf : Consumer didn't instantiate", None, e)
except KafkaProducerError as e:
response_custom = e.code
response_custom['message'] = e.message
file_ops.error_handler(response_custom, "KAFKA_PRODUCER_ERROR", True)
log_exception("process_tokenization_kf : response send to topic %s"%(config.output_topic), data, e)
from flask_restful import Resource
from flask.json import jsonify
from flask import request
from resources.response_generation import Response
from errors.error_validator import ValidationResponse
from errors.errors_exception import FormatError
from anuvaad_auditor.loghandler import log_info
from anuvaad_auditor.loghandler import log_error
from utilities.utils import FileOperation
from utilities.model_response import Status
import config
import time
# sentence tokenisation
file_ops = FileOperation()
DOWNLOAD_FOLDER = file_ops.file_download(config.download_folder)
# rest request for tokenisation workflow service
class SenTokenisePostWF(Resource):
# reading json request and reurnung final response
def post(self):
task_id = str("TOK-" + str(time.time()).replace('.', '')[0:13])
task_starttime = eval(str(time.time()).replace('.', '')[0:13])
json_data = request.get_json(force=True)
log_info("Resource SenTokenisePostWF : data from api request received",
json_data)
try:
error_validator = ValidationResponse(DOWNLOAD_FOLDER)
if error_validator.format_error(json_data) is True: