Exemplo n.º 1
0
def classifyLexical():
    assert request.method == 'POST'
    text_file = request.form['textfile']

    if not text_file:
        text = request.form['text']
    else:
        file_name = os.path.join(route_folder, TEXT_DATA, text_file)
        inputTextReader = InputTextReader()
        text = inputTextReader.readFile(file_name)

    # predict lexical
    load_config(LEXICAL_MODEL_FOLDER, request.form['lexical_folder'])
    input_text = InputText(text)
    punctuations_probs = lexical_classifier.predict(input_text)
    (window_size, punctuation_pos, pos_tagging) = lexical_classifier.get_lexical_parameter()
    question_mark = sbd.config.getboolean('features', 'use_question_mark')
    classes = get_lexical_classes(question_mark)

    all_probs = convert_probabilities(len(input_text.tokens), punctuation_pos, punctuations_probs, classes)

    jsonConverter = JsonConverter(punctuation_pos, window_size, None, None, pos_tagging)
    data = jsonConverter.convert_lexical(input_text.tokens, all_probs)

    if not text_file:
        text_file = "custom_input"
    file_name = os.path.join(route_folder, TEXT_DATA, text_file + ".result")
    resultWriter = ResultWriter(classes)
    resultWriter.writeToFile(file_name, input_text.tokens, all_probs)

    return json.dumps(data)
Exemplo n.º 2
0
def load_info_to_json(model_str):
    file_name = get_file_name_to_read(model_str)

    file_name = put_file_name_to_correct_path("./data/", file_name)

    f = open(file_name, 'r')

    file_lines = f.readlines()
    f.close()

    file_name = get_file_name_to_write(model_str)

    file_name = put_file_name_to_correct_path("../virus_app/fixtures/",
                                              file_name)

    json_converter = JsonConverter()

    json_data_mas = json_converter.get_json_data_mas(file_lines, model_str)

    file_exist = os.path.exists(file_name)

    with open(file_name, "a") as write_file:
        if (not file_exist):
            write_file.write("[")
        for json_data in json_data_mas:
            json.dump(json_data, write_file)
            write_file.write(",")
Exemplo n.º 3
0
    def __init__(self, host: str, port: int):
        self.server = WebsocketServer(host=host, port=port)
        self.server.set_fn_new_client(self.new_client)
        self.server.set_fn_message_received(self.message_received)
        self.database = Database(DatabaseConfig())
        self.converter = JsonConverter()

        self.commands = {
            'getVersion': self.on_get_version,
            'getBatches': self.on_get_batches,
            'getPathsAtd': self.on_get_paths_atd,
            'getPathsDta': self.on_get_paths_dta,
            'deleteItem': self.on_delete_item,
            'insertItem': self.on_insert_item,
            'updateItem': self.on_update_item,
            'restoreFromBackups': self.on_restore_from_backups,
            'exportData': self.on_export_data,
            'exit': self.on_exit
        }
Exemplo n.º 4
0
def classifyAudioLexical():
    assert request.method == 'POST'
    # get example folder
    example_folder = os.path.join(route_folder, AUDIO_EXAMPLE_FOLDER, request.form['example'])
    ctm_file, pitch_file, energy_file = get_audio_files(example_folder)

    # parse ctm_file, pitch_file and energy_file
    parser = AudioParser()
    talks = parser.parse(ctm_file)

    # predict audio
    load_config(AUDIO_MODEL_FOLDER, request.form['audio_folder'])
    audio_probs = audio_classifier.predict(InputAudio(talks))

    # predict lexical
    load_config(LEXICAL_MODEL_FOLDER, request.form['lexical_folder'])
    input_text = InputText(talks)
    lexical_probs = lexical_classifier.predict(input_text)

    # get config parameter
    (lexical_window_size, lexical_punctuation_pos, pos_tagging) = lexical_classifier.get_lexical_parameter()
    (audio_window_size, audio_punctuation_pos) = audio_classifier.get_audio_parameter()

    # write audio results
    audio_classes = ["NONE", "PERIOD"]
    all_audio_probs = convert_probabilities(len(input_text.tokens), audio_punctuation_pos, audio_probs, audio_classes)
    file_name = os.path.join(route_folder, AUDIO_EXAMPLE_FOLDER, request.form['example'] + ".result")
    resultWriter = ResultWriter(audio_classes)
    resultWriter.writeToFile(file_name, input_text.tokens, all_audio_probs)

    # fusion
    fusion = ThresholdFusion()
    fusion.init_parameters(lexical_punctuation_pos, lexical_window_size, audio_punctuation_pos, audio_window_size)
    fusion_probs = fusion.fuse(len(input_text.tokens), lexical_probs, audio_probs)

    # convert it into json
    jsonConverter = JsonConverter(lexical_punctuation_pos, lexical_window_size, audio_punctuation_pos, audio_window_size, pos_tagging)
    data = jsonConverter.convert_fusion(input_text.tokens, fusion_probs, lexical_probs, audio_probs)
    return json.dumps(data)
Exemplo n.º 5
0
class Server:
    server: WebsocketServer
    database: Database
    commands: Dict[str, Any]

    def __init__(self, host: str, port: int):
        self.server = WebsocketServer(host=host, port=port)
        self.server.set_fn_new_client(self.new_client)
        self.server.set_fn_message_received(self.message_received)
        self.database = Database(DatabaseConfig())
        self.converter = JsonConverter()

        self.commands = {
            'getVersion': self.on_get_version,
            'getBatches': self.on_get_batches,
            'getPathsAtd': self.on_get_paths_atd,
            'getPathsDta': self.on_get_paths_dta,
            'deleteItem': self.on_delete_item,
            'insertItem': self.on_insert_item,
            'updateItem': self.on_update_item,
            'restoreFromBackups': self.on_restore_from_backups,
            'exportData': self.on_export_data,
            'exit': self.on_exit
        }

    def run(self):
        self.server.run_forever()

    def new_client(self, client, server):
        print('New connection from: %s.' % client['id'])

    def client_left(self, client, server):
        print('Connection from %s closed.' % client['id'])

    def message_received(self, client, server: WebsocketServer, message: str):
        # print('Received from client: %s, message: %s' % (client['id'], message))
        message = message.encode('raw_unicode_escape').decode('utf-8')
        if not message.startswith('cmd'):
            return
        messages = message.split('|')
        command = messages[1]
        handler = self.commands.get(command)

        if not handler is None:
            if (command == 'deleteItem' or command == 'insertItem'
                    or command == 'restoreFromBackups'
                    or command == 'exportData'):
                handler(client, server, messages[2], messages[3])
            elif command == 'updateItem':
                handler(client, server, messages[2], messages[3], messages[4])
            else:
                handler(client, server)

    def on_get_version(self, client, server: WebsocketServer):
        version = self.database.get_version()
        server.send_message(client, 'res|version|%s' % version[0])

    def on_get_batches(self, client, server: WebsocketServer):
        batches = self.database.get_batches()
        json_data = self.converter.batches_to_json(batches)
        server.send_message(client, 'res|batches|%s' % json_data)

    def on_get_paths_atd(self, client, server: WebsocketServer):
        paths = self.database.get_paths_atd()
        json_data = self.converter.paths_atd_to_json(paths)
        server.send_message(client, 'res|pathsAtd|%s' % json_data)

    def on_get_paths_dta(self, client, server: WebsocketServer):
        paths = self.database.get_paths_dta()
        json_data = self.converter.paths_dta_to_json(paths)
        server.send_message(client, 'res|pathsDta|%s' % json_data)

    def on_delete_item(
        self,
        client,
        server: WebsocketServer,
        table: str,
        json_data: str,
    ):
        if table == 'batch':
            json_data = self.converter.batch_json_to_tuple(json_data)
            self.database.delete_item_batch(json_data)
        if table == 'atdPath':
            json_data = self.converter.path_atd_json_to_tuple(json_data)
            self.database.delete_item_path_atd(json_data)
        if table == 'dtaPath':
            json_data = self.converter.path_dta_json_to_tuple(json_data)
            self.database.delete_item_path_dta(json_data)

    def on_insert_item(
        self,
        client,
        server: WebsocketServer,
        table: str,
        json_data: str,
    ):
        if table == 'batch':
            json_data = self.converter.batch_json_to_tuple(json_data)
            self.database.insert_item_batch(json_data)
        if table == 'atdPath':
            json_data = self.converter.path_atd_json_to_tuple(json_data)
            self.database.insert_item_path_atd(json_data)
        if table == 'dtaPath':
            json_data = self.converter.path_dta_json_to_tuple(json_data)
            self.database.insert_item_path_dta(json_data)

    def on_update_item(self, client, server: WebsocketServer, table: str,
                       json_data: str, json_data_before: str):
        if table == 'batch':
            json_data = self.converter.batch_json_to_tuple(json_data)
            json_data_before = self.converter.batch_json_to_tuple(
                json_data_before)
            self.database.update_item_batch(json_data, json_data_before)
        if table == 'atdPath':
            json_data = self.converter.path_atd_json_to_tuple(json_data)
            json_data_before = self.converter.path_atd_json_to_tuple(
                json_data_before)
            self.database.update_item_path_atd(json_data, json_data_before)
        if table == 'dtaPath':
            json_data = self.converter.path_dta_json_to_tuple(json_data)
            json_data_before = self.converter.path_dta_json_to_tuple(
                json_data_before)
            self.database.update_item_path_dta(json_data, json_data_before)

    def on_restore_from_backups(self, client, server: WebsocketServer,
                                table: str, json_data: str):
        if table == 'batch':
            json_data = self.converter.batches_json_to_tuple(json_data)
            self.database.restore_batches(json_data)
        if table == 'atdPath':
            json_data = self.converter.paths_atd_json_to_tuple(json_data)
            self.database.restore_paths_atd(json_data)
        if table == 'dtaPath':
            json_data = self.converter.paths_dta_json_to_tuple(json_data)
            self.database.restore_paths_dta(json_data)

    def on_export_data(self, client, server: WebsocketServer, file_name: str,
                       json_data: str):
        excel_data = self.converter.json_to_excel(json_data)
        ext = file_name.split('.')[1]
        excel: Any
        if ext == 'xls':
            excel = excel_data.xls
        if ext == 'xlsx':
            excel = excel_data.xlsx
        try:
            open('output/%s' % file_name, 'wb').write(excel)
            server.send_message(client, 'res|info|io')
        except:
            print('Error occured while trying to write file.')
            server.send_message(client, 'res|error|io')

    def on_exit(self, client, server: WebsocketServer):
        server.server_close()