def update_audio_database():
if request.method == 'POST':
conn, persistence, _ = get_database_objects()
id = request.form['id']
column = request.form['column']
table = request.form['table']
update_all_tags = True if request.form.get('all-tags') == 'on' else False
validation = request.form.get('validation')
tag = request.form.get('tag')
if validation:
value = validation
value = text_split_sentence_validation(value)
else:
value = tag
table = get_table_name(table)
if update_all_tags:
persistence.update_by_col(table, column, value, where_col='tag', col_value=f"'{tag}'")
else:
persistence.update_by_col(table, column, value, where_col='id', col_value=id)
conn.close()
return redirect(url_for('audios_list'))
def save_config():
if request.method == 'POST':
conn, persistance, _ = get_database_objects()
# Getting html form values
text_id = request.form['text_id']
voice_id = request.form['voice_id']
score_single = request.form['score_single']
score_multiple = request.form['score_multiple']
score_suggestions = request.form['score_suggestion']
success = persistance.update_config(
channel_text_id=f"'{text_id}'",
channel_voice_id=f"'{voice_id}'",
valid_score_single_pred=f"'{score_single}'",
valid_score_multiple_pred=f"'{score_multiple}'",
valid_score_suggestions=f"'{score_suggestions}'"
)
if not success:
flash('Algo errado aconteceu...')
else:
flash('Configurações salvas!')
conn.close()
return redirect(request.url)
def get_search_audios():
search = request.args.get('search_text')
column = request.args.get('column')
table_type = request.args.get('table')
tables_names = {
'music': 'musica_ambiente',
'ambience': 'som_ambiente',
}
table = tables_names.get(table_type, 'efeito_sonoro')
conn, _, query = get_database_objects()
s3_url = 'https://audios-rpg-sound.s3-us-west-1.amazonaws.com'
columns = f"id, tag, validation, filename, replace(filename, '{s3_url}', '')"
audios_query = query.query_all(table=table, column=columns, where_col=column, value=search) if not False else ''
conn.close()
return render_template(
'table_body.html',
audio=audios_query,
table=table,
route_update_audio=update_audio,
route_delete_audio=delete_audio_route,
sound_type=table_type
)
def get_sentences_by_label(table, label):
conn, _, query = get_database_objects()
sentences = query.query_audio_by_tag(table, label)
conn.close()
return sentences
def audios_list():
conn, persistance, query = get_database_objects()
s3_url = 'https://audios-rpg-sound.s3-us-west-1.amazonaws.com'
columns = f"id, tag, validation, filename, replace(filename, '{s3_url}', '')"
music_audios = query.query_all(table='musica_ambiente', column=columns, use_where=False) if not False else ''
sound_audios = query.query_all(table='efeito_sonoro', column=columns, use_where=False) if not False else ''
ambience_audios = query.query_all(table='som_ambiente', column=columns, use_where=False) if not False else ''
audios_query = [music_audios, sound_audios, ambience_audios]
titles = ['_', 'Músicas', 'Efeitos sonoros', 'Som ambiente']
titles_ids = ['_', 'music', 'sound', 'ambience']
conn.close()
return render_template(
'audios.html',
title='LISTA DE ÁUDIOS',
audios=audios_query,
titles=titles,
titles_ids=titles_ids,
route_update_audio=update_audio,
route_delete_audio=delete_audio_route
)
def load_tables(table):
conn, _, query = get_database_objects()
validations = query.query_all(table=table,
column='tag, validation',
use_where=False)
conn.close()
return validations
def get_load_suggestions():
conn, persistence, query = get_database_objects()
transaction = request.args.get('transaction')
id = request.args.get('id')
if transaction == 'search':
score = request.args.get('score')
try:
suggestions = query.query_all(
table='label_suggestions',
column='id, validation, suggested_label, suggested_table, round(score, 2)',
where_col='score',
where_type='bigger_than',
value=score
)
except:
conn.close()
return False
conn.close()
return render_template('suggestions_tbody.html', suggestions=suggestions)
elif transaction == 'update':
table, label, validation = get_suggestion_by_id(query, id)
update_successfull = update_suggestion(persistence, table, label, validation)
if not update_successfull:
flash('Algo errado aconteceu ao atualizar a sugestão. Tente novamente em alguns minutos...')
else:
delete_suggestion(persistence, id)
elif transaction == 'delete':
# Insert in not labels table
table, label, validation = get_suggestion_by_id(query, id)
persistence.insert(
table=f'not_{table}',
validation=f"'{validation}'",
tag=f"'{label}'"
)
delete_suggestion(persistence, id)
else:
pass
suggestions = query.query_all(
table='label_suggestions',
column='id, validation, suggested_label, suggested_table, round(score, 2)',
use_where=False
)
conn.close()
return render_template('suggestions_tbody.html', suggestions=suggestions)
def get_train_data_not_labels(label, table):
conn, _, query = get_database_objects()
validations = query.query_all(table=f'not_{table}',
column='validation',
where_col='tag',
value=label)
conn.close()
return validations
def insert_status(status, message):
conn, persistence, _ = get_database_objects()
persistence.truncate_insert(
table='train_status',
status=f"'{status}'",
message=f"'{message}'",
created_at='now()'
)
conn.close()
def train_model():
conn, persistence, _ = get_database_objects()
persistence.update_by_col(
table='train_status',
column='status',
value='WAITING',
where_col=1,
col_value=1
)
conn.close()
return train_page()
def start_prediction(sentence, table):
# Getting S3 list from pickle file saved while training model
bucket_name = os.environ.get('MODELS_S3_BUCKET')
download_path = f'{table}_models_list.p'
s3_path = f'{table}/{download_path}'
get_file_from_s3(bucket_name, s3_path, download_path)
bucket = load_zipped_pickle(f'/tmp/{download_path}')
predictions_to_check = []
# For each model in S3 list, make prediction
for file in bucket.get('Contents', []):
key = file.get('Key')
file = key.replace(table + '/models/', '')
if not file:
continue
get_file_from_s3(bucket_name, key, file)
nlp_batch = load_zipped_pickle(f'/tmp/{file}')
for nlp in tuple(nlp_batch, ):
prediction = nlp_predict(nlp, sentence)
if not prediction:
continue
else:
predictions_to_check.append(prediction)
conn, _, query = get_database_objects()
_, _, score_single, score_multiple, score_suggestions = query.query_config(
)
conn.close()
if not predictions_to_check:
return None
# Check if score prediction is ok
if len(predictions_to_check) > 1:
label_predicted, score_predicted = check_multiple_prediction(
predictions_to_check, score_multiple)
else:
label_predicted, score_predicted = check_single_prediction(
predictions_to_check, score_single)
# Keeping suggestions to improve model
suggestions = make_suggestions(sentence, predictions_to_check,
label_predicted, score_suggestions)
return label_predicted, str(score_predicted), str(suggestions)
def delete_audio():
if request.method == 'POST':
id = request.form['id']
table = request.form['table']
file = request.form['file'].rsplit('/', 1)[-1]
delete_file_from_s3(file)
table = get_table_name(table)
conn, persistence, _ = get_database_objects()
persistence.delete_by_id(table, id)
conn.close()
return redirect(url_for('audios_list'))
def open_config():
conn, _, query = get_database_objects()
voice_id, text_id, score_single, score_multiple, score_suggestion = query.query_config()
conn.close()
return render_template(
'config.html',
title='CONFIGURAÇÕES',
submit_route=config,
voice_id=voice_id,
text_id=text_id,
score_single=score_single,
score_multiple=score_multiple,
score_suggestion=score_suggestion
)
def train_page():
conn, persistance, query = get_database_objects()
status, message, train_date = query.query_all(
table='train_status',
column="status, message, to_char(created_at - interval '3 hour', 'DD/MM/YYYY HH24:MI:SS')",
use_where=False
)[0]
persistance.update_timed_out_train_status()
conn.close()
return render_template(
'train.html',
title='TREINAR BOT',
status=status,
message=message,
train_date=train_date
)
def get_audio_by_label_function():
type = request.args.get('type')
label = request.args.get('label')
conn, _, query = get_database_objects()
try:
filename = query.query_all(
table=type,
column='filename',
where_col='tag',
value=label
)
import random
return random.choice(filename)[0]
except:
return ''
def upload_file():
conn, persistance, query = get_database_objects()
# Getting html form values
option = request.form['option']
validation = request.form['validacao']
tag = request.form['tag']
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
if file.filename == '':
flash('No file selected for uploading')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
else:
flash(f'Allowed file types are {ALLOWED_EXTENSIONS}')
return redirect(request.url)
result = persistance.insert(
table=option,
tag=f"'{tag}'",
validation=f"'{validation}'",
filename=f"'https://audios-rpg-sound.s3-us-west-1.amazonaws.com/{filename}'"
)
conn.close()
if result:
bucket = os.environ.get('S3_BUCKET')
send_file_to_s3(filename, bucket)
flash("Enviado com sucesso!")
else:
flash("Algo deu errado..")
return redirect(request.url)
def save_suggestions_function():
suggestions = request.args.get('suggestions')
audio_type = request.args.get('audio_type')
conn, persistence, _ = get_database_objects()
suggestions = ast.literal_eval(suggestions)
for suggestion in list(suggestions):
label, score, validation = suggestion
persistence.insert(
table='label_suggestions',
suggested_label=f"'{label}'",
suggested_table=f"'{audio_type}'",
score=score,
validation=f"'{validation}'"
)
conn.close()
return 'Suggestions saved!'
def delete_suggestions_function():
conn, persistence, _ = get_database_objects()
persistence.delete_all('label_suggestions')
conn.close()
return ''
def get_channels_id():
conn, _, query = get_database_objects()
voice_channel, text_channel = query.query_config()
conn.close()
return voice_channel, text_channel