def process_fc_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER =file_ops.file_download(config.download_folder)
task_id = str("FC-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
producer_tok = Producer(config.bootstrap_server)
# instatiation of consumer for respective topic
try:
consumer = consumer_validator()
log_info("process_fc_kf", "trying to receive value from consumer ", None)
for msg in consumer:
log_info("process_fc_kf", "value received from consumer", None)
data = msg.value
task_id = str("FC-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format(data)
response_gen = Response(data, DOWNLOAD_FOLDER)
file_value_response = response_gen.workflow_response(task_id, task_starttime)
if "errorID" not in file_value_response.keys():
push_output(producer_tok, config.tok_output_topic, file_value_response, jobid, task_id)
log_info("process_fc_kf", "response send to topic %s"%(config.tok_output_topic), None)
else:
log_info("process_fc_kf", "error send to error handler", jobid)
except KafkaConsumerError as e:
response_custom = CustomResponse(Status.ERR_STATUS.value, None, None)
response_custom['message'] = str(e)
file_ops.error_handler(response_custom, "KAFKA_CONSUMER_ERROR", True)
log_exception("process_fc_kf", "Consumer didn't instantiate", None, e)
except KafkaProducerError as e:
response_custom = e.code
response_custom['message'] = e.message
file_ops.error_handler(response_custom, "KAFKA_PRODUCER_ERROR", True)
log_exception("process_fc_kf", "response send to topic %s"%(config.tok_output_topic), response_custom['jobID'], e)
def process_tokenization_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER =file_ops.file_download(config.download_folder)
# instatiation of consumer for respective topic
try:
consumer_class = Consumer(config.input_topic, config.bootstrap_server)
consumer = consumer_class.consumer_instantiate()
log_info("process_tokenization_kf : trying to receive value from consumer ", None)
for msg in consumer:
data = msg.value
log_info("process_tokenization_kf : received input json from input topic consumer ", data)
task_id = str("TOK-" + str(time.time()).replace('.', '')[0:13])
task_starttime = eval(str(time.time()).replace('.', '')[0:13])
input_files, workflow_id, jobid, tool_name, step_order, user_id = file_ops.json_input_format(data)
response_gen = Response(data, DOWNLOAD_FOLDER)
file_value_response = response_gen.workflow_response(task_id, task_starttime)
if "errorID" not in file_value_response.keys():
producer = Producer()
producer.push_data_to_queue(config.output_topic, file_value_response, data, task_id)
else:
log_error("process_tokenization_kf : error send to error handler", data, None)
except KafkaConsumerError as e:
response_custom = CustomResponse(Status.ERR_STATUS.value, None, None)
response_custom.status_code['message'] = str(e)
file_ops.error_handler(response_custom.status_code, "KAFKA_CONSUMER_ERROR", True)
log_exception("process_tokenization_kf : Consumer didn't instantiate", None, e)
except KafkaProducerError as e:
response_custom = e.code
response_custom['message'] = e.message
file_ops.error_handler(response_custom, "KAFKA_PRODUCER_ERROR", True)
log_exception("process_tokenization_kf : response send to topic %s"%(config.output_topic), data, e)
def process_HTML_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER = file_ops.create_file_download_dir(config.download_folder)
task_id = str("HTML2JSON-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
# instatiation of consumer for respective topic
try:
consumer_class = Consumer(config.html_input_topic,
config.bootstrap_server)
consumer = consumer_class.consumer_instantiate() #Consumer
log.info("--- consumer running -----")
except:
response = Status.ERR_Consumer.value
producer_html2json = Producer(config.bootstrap_server)
producer = producer_html2json.producer_fn()
producer.send(config.html_output_topic, value=response)
producer.flush()
log.error(
"error in kafka opertation producer flushed value on topic %s" %
(config.html_input_topic))
try:
log.info("trying to receive value from consumer ")
for msg in consumer:
log.info("value received from consumer")
data = msg.value
task_id = str("HTML2JSON-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
checking_response = CheckingResponse(data, task_id, task_starttime,
DOWNLOAD_FOLDER)
file_value_response = checking_response.main_response_wf()
try:
producer_html2json = Producer(config.bootstrap_server)
producer = producer_html2json.producer_fn()
producer.send(config.html_output_topic,
value=file_value_response)
producer.flush()
log.info("producer flushed value on topic %s" %
(config.html_output_topic))
except:
log.info(
"error occured in file operation of workflow and it is pushed to error queue"
)
except Exception as e:
log.error(
"error occured while listening message from consumer or flushing data to another queue %s"
% e)
for msg in consumer:
log.info("value received from consumer")
data = msg.value
input_files, workflow_id, jobid, tool_name, step_order = file_ops.input_format(
data)
task_id = str("HTML2JSON-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
response_custom = CustomResponse(Status.ERR_Producer.value, jobid,
task_id)
file_ops.error_handler(response_custom.status_code, True)
log.info(
"error in kafka opertation producer flushed value on error topic"
)
from utilities.utils import FileOperation
from utilities.model_response import CustomResponse
from errors.errors_exception import WorkflowkeyError
from errors.errors_exception import FileErrors
from errors.errors_exception import FileEncodingError
from errors.errors_exception import ServiceError
from utilities.model_response import Status
from errors.error_validator import ValidationResponse
from services.service import Tokenisation
from anuvaad_auditor.loghandler import log_info
from anuvaad_auditor.loghandler import log_exception
import time
import copy
file_ops = FileOperation()
class Response(object):
def __init__(self, json_data, DOWNLOAD_FOLDER):
self.json_data = json_data
self.DOWNLOAD_FOLDER = DOWNLOAD_FOLDER
def workflow_response(self, task_id, task_starttime):
input_key, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format(
self.json_data)
log_info("workflow_response : started the response generation",
self.json_data)
error_validator = ValidationResponse(self.DOWNLOAD_FOLDER)
tokenisation = Tokenisation(self.DOWNLOAD_FOLDER, self.json_data)
try:
error_validator.wf_keyerror(jobid, workflow_id, tool_name,
from flask_restful import Resource
from flask.json import jsonify
from flask import request
from resources.response_generation import Response
from errors.error_validator import ValidationResponse
from errors.errors_exception import FormatError
from anuvaad_auditor.loghandler import log_info
from anuvaad_auditor.loghandler import log_error
from utilities.utils import FileOperation
from utilities.model_response import Status
import config
import time
# sentence tokenisation
file_ops = FileOperation()
DOWNLOAD_FOLDER = file_ops.file_download(config.download_folder)
# rest request for tokenisation workflow service
class SenTokenisePostWF(Resource):
# reading json request and reurnung final response
def post(self):
task_id = str("TOK-" + str(time.time()).replace('.', '')[0:13])
task_starttime = eval(str(time.time()).replace('.', '')[0:13])
json_data = request.get_json(force=True)
log_info("Resource SenTokenisePostWF : data from api request received",
json_data)
try:
error_validator = ValidationResponse(DOWNLOAD_FOLDER)
if error_validator.format_error(json_data) is True:
import os
import json
from flask_restful import Resource
from flask.json import jsonify
from flask import request
from utilities.utils import FileOperation
from utilities.model_response import CheckingResponse
import werkzeug
from werkzeug.utils import secure_filename
import config
import logging
import time
file_ops = FileOperation()
DOWNLOAD_FOLDER =file_ops.create_file_download_dir(config.download_folder)
log = logging.getLogger('file')
# rest request for html2json workflow service
class Html2JsonConversionWF(Resource):
# reading json request and reurnung final response
def post(self):
log.info("HTML to JSON conversion started")
task_id = str("Html2Json-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
json_data = request.get_json(force = True)
checking_response = CheckingResponse(json_data, task_id, task_starttime, DOWNLOAD_FOLDER)
file_value_response = checking_response.main_response_wf(rest_request=True)
log.info("Conversion completed")
return file_value_response