def classifyLexical():
assert request.method == 'POST'
text_file = request.form['textfile']
if not text_file:
text = request.form['text']
else:
file_name = os.path.join(route_folder, TEXT_DATA, text_file)
inputTextReader = InputTextReader()
text = inputTextReader.readFile(file_name)
# predict lexical
load_config(LEXICAL_MODEL_FOLDER, request.form['lexical_folder'])
input_text = InputText(text)
punctuations_probs = lexical_classifier.predict(input_text)
(window_size, punctuation_pos, pos_tagging) = lexical_classifier.get_lexical_parameter()
question_mark = sbd.config.getboolean('features', 'use_question_mark')
classes = get_lexical_classes(question_mark)
all_probs = convert_probabilities(len(input_text.tokens), punctuation_pos, punctuations_probs, classes)
jsonConverter = JsonConverter(punctuation_pos, window_size, None, None, pos_tagging)
data = jsonConverter.convert_lexical(input_text.tokens, all_probs)
if not text_file:
text_file = "custom_input"
file_name = os.path.join(route_folder, TEXT_DATA, text_file + ".result")
resultWriter = ResultWriter(classes)
resultWriter.writeToFile(file_name, input_text.tokens, all_probs)
return json.dumps(data)
def load_info_to_json(model_str):
file_name = get_file_name_to_read(model_str)
file_name = put_file_name_to_correct_path("./data/", file_name)
f = open(file_name, 'r')
file_lines = f.readlines()
f.close()
file_name = get_file_name_to_write(model_str)
file_name = put_file_name_to_correct_path("../virus_app/fixtures/",
file_name)
json_converter = JsonConverter()
json_data_mas = json_converter.get_json_data_mas(file_lines, model_str)
file_exist = os.path.exists(file_name)
with open(file_name, "a") as write_file:
if (not file_exist):
write_file.write("[")
for json_data in json_data_mas:
json.dump(json_data, write_file)
write_file.write(",")
def __init__(self, host: str, port: int):
self.server = WebsocketServer(host=host, port=port)
self.server.set_fn_new_client(self.new_client)
self.server.set_fn_message_received(self.message_received)
self.database = Database(DatabaseConfig())
self.converter = JsonConverter()
self.commands = {
'getVersion': self.on_get_version,
'getBatches': self.on_get_batches,
'getPathsAtd': self.on_get_paths_atd,
'getPathsDta': self.on_get_paths_dta,
'deleteItem': self.on_delete_item,
'insertItem': self.on_insert_item,
'updateItem': self.on_update_item,
'restoreFromBackups': self.on_restore_from_backups,
'exportData': self.on_export_data,
'exit': self.on_exit
}
def classifyAudioLexical():
assert request.method == 'POST'
# get example folder
example_folder = os.path.join(route_folder, AUDIO_EXAMPLE_FOLDER, request.form['example'])
ctm_file, pitch_file, energy_file = get_audio_files(example_folder)
# parse ctm_file, pitch_file and energy_file
parser = AudioParser()
talks = parser.parse(ctm_file)
# predict audio
load_config(AUDIO_MODEL_FOLDER, request.form['audio_folder'])
audio_probs = audio_classifier.predict(InputAudio(talks))
# predict lexical
load_config(LEXICAL_MODEL_FOLDER, request.form['lexical_folder'])
input_text = InputText(talks)
lexical_probs = lexical_classifier.predict(input_text)
# get config parameter
(lexical_window_size, lexical_punctuation_pos, pos_tagging) = lexical_classifier.get_lexical_parameter()
(audio_window_size, audio_punctuation_pos) = audio_classifier.get_audio_parameter()
# write audio results
audio_classes = ["NONE", "PERIOD"]
all_audio_probs = convert_probabilities(len(input_text.tokens), audio_punctuation_pos, audio_probs, audio_classes)
file_name = os.path.join(route_folder, AUDIO_EXAMPLE_FOLDER, request.form['example'] + ".result")
resultWriter = ResultWriter(audio_classes)
resultWriter.writeToFile(file_name, input_text.tokens, all_audio_probs)
# fusion
fusion = ThresholdFusion()
fusion.init_parameters(lexical_punctuation_pos, lexical_window_size, audio_punctuation_pos, audio_window_size)
fusion_probs = fusion.fuse(len(input_text.tokens), lexical_probs, audio_probs)
# convert it into json
jsonConverter = JsonConverter(lexical_punctuation_pos, lexical_window_size, audio_punctuation_pos, audio_window_size, pos_tagging)
data = jsonConverter.convert_fusion(input_text.tokens, fusion_probs, lexical_probs, audio_probs)
return json.dumps(data)
class Server:
server: WebsocketServer
database: Database
commands: Dict[str, Any]
def __init__(self, host: str, port: int):
self.server = WebsocketServer(host=host, port=port)
self.server.set_fn_new_client(self.new_client)
self.server.set_fn_message_received(self.message_received)
self.database = Database(DatabaseConfig())
self.converter = JsonConverter()
self.commands = {
'getVersion': self.on_get_version,
'getBatches': self.on_get_batches,
'getPathsAtd': self.on_get_paths_atd,
'getPathsDta': self.on_get_paths_dta,
'deleteItem': self.on_delete_item,
'insertItem': self.on_insert_item,
'updateItem': self.on_update_item,
'restoreFromBackups': self.on_restore_from_backups,
'exportData': self.on_export_data,
'exit': self.on_exit
}
def run(self):
self.server.run_forever()
def new_client(self, client, server):
print('New connection from: %s.' % client['id'])
def client_left(self, client, server):
print('Connection from %s closed.' % client['id'])
def message_received(self, client, server: WebsocketServer, message: str):
# print('Received from client: %s, message: %s' % (client['id'], message))
message = message.encode('raw_unicode_escape').decode('utf-8')
if not message.startswith('cmd'):
return
messages = message.split('|')
command = messages[1]
handler = self.commands.get(command)
if not handler is None:
if (command == 'deleteItem' or command == 'insertItem'
or command == 'restoreFromBackups'
or command == 'exportData'):
handler(client, server, messages[2], messages[3])
elif command == 'updateItem':
handler(client, server, messages[2], messages[3], messages[4])
else:
handler(client, server)
def on_get_version(self, client, server: WebsocketServer):
version = self.database.get_version()
server.send_message(client, 'res|version|%s' % version[0])
def on_get_batches(self, client, server: WebsocketServer):
batches = self.database.get_batches()
json_data = self.converter.batches_to_json(batches)
server.send_message(client, 'res|batches|%s' % json_data)
def on_get_paths_atd(self, client, server: WebsocketServer):
paths = self.database.get_paths_atd()
json_data = self.converter.paths_atd_to_json(paths)
server.send_message(client, 'res|pathsAtd|%s' % json_data)
def on_get_paths_dta(self, client, server: WebsocketServer):
paths = self.database.get_paths_dta()
json_data = self.converter.paths_dta_to_json(paths)
server.send_message(client, 'res|pathsDta|%s' % json_data)
def on_delete_item(
self,
client,
server: WebsocketServer,
table: str,
json_data: str,
):
if table == 'batch':
json_data = self.converter.batch_json_to_tuple(json_data)
self.database.delete_item_batch(json_data)
if table == 'atdPath':
json_data = self.converter.path_atd_json_to_tuple(json_data)
self.database.delete_item_path_atd(json_data)
if table == 'dtaPath':
json_data = self.converter.path_dta_json_to_tuple(json_data)
self.database.delete_item_path_dta(json_data)
def on_insert_item(
self,
client,
server: WebsocketServer,
table: str,
json_data: str,
):
if table == 'batch':
json_data = self.converter.batch_json_to_tuple(json_data)
self.database.insert_item_batch(json_data)
if table == 'atdPath':
json_data = self.converter.path_atd_json_to_tuple(json_data)
self.database.insert_item_path_atd(json_data)
if table == 'dtaPath':
json_data = self.converter.path_dta_json_to_tuple(json_data)
self.database.insert_item_path_dta(json_data)
def on_update_item(self, client, server: WebsocketServer, table: str,
json_data: str, json_data_before: str):
if table == 'batch':
json_data = self.converter.batch_json_to_tuple(json_data)
json_data_before = self.converter.batch_json_to_tuple(
json_data_before)
self.database.update_item_batch(json_data, json_data_before)
if table == 'atdPath':
json_data = self.converter.path_atd_json_to_tuple(json_data)
json_data_before = self.converter.path_atd_json_to_tuple(
json_data_before)
self.database.update_item_path_atd(json_data, json_data_before)
if table == 'dtaPath':
json_data = self.converter.path_dta_json_to_tuple(json_data)
json_data_before = self.converter.path_dta_json_to_tuple(
json_data_before)
self.database.update_item_path_dta(json_data, json_data_before)
def on_restore_from_backups(self, client, server: WebsocketServer,
table: str, json_data: str):
if table == 'batch':
json_data = self.converter.batches_json_to_tuple(json_data)
self.database.restore_batches(json_data)
if table == 'atdPath':
json_data = self.converter.paths_atd_json_to_tuple(json_data)
self.database.restore_paths_atd(json_data)
if table == 'dtaPath':
json_data = self.converter.paths_dta_json_to_tuple(json_data)
self.database.restore_paths_dta(json_data)
def on_export_data(self, client, server: WebsocketServer, file_name: str,
json_data: str):
excel_data = self.converter.json_to_excel(json_data)
ext = file_name.split('.')[1]
excel: Any
if ext == 'xls':
excel = excel_data.xls
if ext == 'xlsx':
excel = excel_data.xlsx
try:
open('output/%s' % file_name, 'wb').write(excel)
server.send_message(client, 'res|info|io')
except:
print('Error occured while trying to write file.')
server.send_message(client, 'res|error|io')
def on_exit(self, client, server: WebsocketServer):
server.server_close()