state = CheckboxButtonGroup(labels=states,
active=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
year = CheckboxButtonGroup(labels=years,
active=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
# In[112]:
show(year)
# In[110]:
output_notebook()
# In[216]:
name.active = [0, 1]
# In[203]:
def speaker_filter(party_arr, prior_experience_arr, election_result_arr,
name_arr, state_arr, date_max, date_min):
query = {}
query['party'] = {'$in': party_arr}
query['incumbent'] = {'$in': prior_experience_arr}
query['election_result'] = {'$in': election_result_arr}
query['name'] = {'$in': name_arr}
query['state'] = {'$in': state_arr}
query['year'] = {'$lt': date_max, '$gt': date_min}
mongo_data = list(db.speakers.find(query, {'speaker_id': 1}))
ret_list = []
def modify_second_descriptive(doc):
# Captura de los argumentos pasados desde flask
args = doc.session_context.request.arguments
try:
periodo = int(args.get('periodo')[0])
tipo_var = args.get('tipo_var')[0].decode('ascii')
periodo_custom_start = args.get('periodo_custom_start')[0].decode(
'ascii')
periodo_custom_end = args.get('periodo_custom_end')[0].decode('ascii')
current_date = dt.strptime(
args.get('current_date')[0].decode('ascii'), '%d/%m/%Y')
except:
periodo = 0
tipo_var = ''
periodo_custom_start = ''
periodo_custom_end = ''
current_date = ''
if tipo_var == 'abs':
tipo_var = 'ABSOLUTAS'
elif tipo_var == 'rend':
tipo_var = 'RENDIMIENTOS'
print(
f'periodo: {periodo}, tipo_var: {tipo_var}, periodo_custom_start: {periodo_custom_start}, periodo_custom_end: {periodo_custom_end}, current_date: {current_date}'
)
# Creación/Carga en RAM del diccionario con las variables a modelizar
total_model_dict = load_or_create_model_vars(
model_vars_file='resources/total_model_dict.pkl',
mask_file='resources/model_variables_mask.xlsx',
sheets=[
'ID_INFLUENTE', 'ID_BIOS', 'ID_FANGOS', 'ID_HORNO', 'ID_EFLUENTE',
'ID_ELECTRICIDAD', 'YOKO', 'ANALITICA', 'METEO'
],
cols=['OUT', 'IN', 'MANIPULABLES', 'PROCESOS_IN'],
force_create=False)
# Inicialización del diccionario ordenado para almacenar los modelos creados
models = OrderedDict([])
try:
created_models = load_obj(name='resources/created_models.pkl')
except:
created_models = ['Calidad_Agua']
ruta_periodo = f'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv'
# Crear nuevo archivo custom si periodo=3
if periodo == 3:
create_custom_period(periodo_custom_start, periodo_custom_end)
ruta_periodo = F'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_CUSTOM.csv'
current_date = dt.strptime(periodo_custom_end, '%d/%m/%Y')
# Llamada al webservice de RapidMiner
json_perfil_document = call_webservice(
url=
'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Perfil_Out_JSON_v5?',
username='******',
password='******',
parameters={
'Ruta_periodo': ruta_periodo,
'Ruta_tipo_variable':
f'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_VARIABLES_{tipo_var}.csv',
'Normalizacion': 1
},
out_json=True)
# Extracción de los datos web
df_perfil = [json_normalize(data) for data in json_perfil_document]
# Asignación de los datos web a su variable correspondiente
prediction_df = df_perfil[3]
outlier_df = df_perfil[4]
# Creación de los gráficos y widgets permanentes en la interfaz
prediction_plot = create_prediction_plot(prediction_df, current_date)
outlier_plot = create_outlier_plot(outlier_df, prediction_df, tipo_var)
simulation_title = create_div_title(
'Creación, Simulación y Optimización de modelos')
model_title, add_model_button, model_select_menu = create_model_menu(
model_variables=list(total_model_dict.keys()))
create_model_spinner = Spinner(size=16)
recreate_button = Button(label='Recrear',
button_type='success',
height=35,
max_width=200,
min_width=200)
model_select_wb = widgetbox([
row([model_title, create_model_spinner.spinner],
sizing_mode='stretch_width',
max_width=400),
row([model_select_menu, recreate_button],
sizing_mode='stretch_width',
max_width=400), add_model_button
],
max_width=400,
sizing_mode='stretch_width')
created_models_title = create_div_title('Modelos creados')
created_models_checkbox = CheckboxButtonGroup(labels=list(models.keys()),
height=35)
created_models_checkbox.active = [0]
delete_model_button = Button(label='Eliminar',
button_type='danger',
height=35,
max_width=200)
created_models_wb = widgetbox(
[created_models_title, created_models_checkbox],
max_width=900,
sizing_mode='stretch_width')
# Callback para crear nuevamente el listado de variables de la mascara
def recreate_callback():
print('Recreando lista de variables para modelizar')
nonlocal total_model_dict
total_model_dict = load_or_create_model_vars(
model_vars_file='resources/total_model_dict.pkl',
mask_file='resources/model_variables_mask.xlsx',
sheets=[
'ID_INFLUENTE', 'ID_BIOS', 'ID_FANGOS', 'ID_HORNO',
'ID_EFLUENTE', 'ID_ELECTRICIDAD', 'YOKO', 'ANALITICA', 'METEO'
],
cols=['OUT', 'IN', 'MANIPULABLES', 'PROCESOS_IN'],
force_create=True)
model_select_menu.options = list(total_model_dict.keys())
model_select_menu.value = 'Calidad_Agua'
recreate_button.on_click(recreate_callback)
# Callbacks para los widgets de la interfaz
def prediction_callback(model='Calidad_Agua'):
create_model_spinner.show_spinner()
if initialize == False:
model_objective = model_select_menu.value
else:
model_objective = model
model_discretise = 5
# Verificar que el modelo no ha sido creado antes
if model_objective not in models:
# print(f'Objetivo: {model_objective}')
# print(f'Discretizacion: {model_discretise}')
# print(f'Ruta_periodo: /home/admin/Cartuja_Datos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv')
# print(f'IN_MODELO: {total_model_dict[model_objective]}')
# Llamar al servicio web EDAR_Cartuja_Prediccion con los nuevos parámetros
ruta_periodo = f'https://edar.vicomtech.org/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv'
# Crear nuevo archivo custom si periodo=3
if periodo == 3:
create_custom_period(periodo_custom_start, periodo_custom_end)
ruta_periodo = 'https://edar.vicomtech.org/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_CUSTOM.csv'
json_prediction_document = call_webservice(
url=
'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Prediccion_JSON_v5?',
username='******',
password='******',
parameters={
'Objetivo': model_objective,
'Discretizacion': model_discretise,
'Numero_Atributos': 4,
'Ruta_periodo': ruta_periodo,
'IN_MODELO': str(total_model_dict[model_objective])
},
out_json=True)
# Obtener datos
df_prediction = [
json_normalize(data) for data in json_prediction_document
]
decision_tree_df = df_prediction[0]
decision_tree_df = append_count(decision_tree_df)
confusion_df_raw = df_prediction[1].reindex(
columns=list(json_prediction_document[1][0].keys()))
confusion_df = create_df_confusion(confusion_df_raw)
weight_df = df_prediction[2]
pred_df = df_prediction[3]
ranges_df = df_prediction[4]
ranges_df.set_index('Name', inplace=True)
# ranges_df['Values']=ranges_df['Values'].replace(regex=r'\(.*\)',value='')
slider_df = create_df_sliders(weight_df, pred_df)
daily_pred_df = pred_df[[
'Fecha', model_objective, f'prediction({model_objective})'
]]
possible_targets = sorted(list(pred_df[model_objective].unique()))
# print(f'Targets: {possible_targets}')
var_influyentes = list(weight_df['Attribute'])
decision_tree_data = create_decision_tree_data(
decision_tree_df, model_objective)
# Crear nuevos gráficos
simul_or_optim_wb = SimulOptimWidget(
target=model_objective,
simul_df=slider_df,
possible_targets=possible_targets,
var_influyentes=var_influyentes,
periodo=periodo,
ranges=ranges_df,
periodo_custom_start=periodo_custom_start,
periodo_custom_end=periodo_custom_end)
daily_pred_plot = create_daily_pred_plot(daily_pred_df,
model_objective)
decision_tree_plot = create_decision_tree_plot()
decision_tree_graph = create_decision_tree_graph_renderer(
decision_tree_plot, decision_tree_data)
decision_tree_plot = append_labels_to_decision_tree(
decision_tree_plot, decision_tree_graph, decision_tree_data)
confusion_matrix = create_confusion_matrix(confusion_df)
weight_plot = create_attribute_weight_plot(weight_df,
model_objective)
corrects_plot = create_corrects_plot(confusion_df, model_objective)
model_title = create_div_title(f'Modelo - {model_objective}')
confusion_title = create_div_title(
f'Matriz de confusión - {model_objective}')
decision_tree_title = create_div_title(
f'Arbol de decisión - {model_objective}')
ranges_description = create_ranges_description(
possible_targets, model_objective)
new_plots = layout(
[[model_title], [simul_or_optim_wb.rb],
[
row([simul_or_optim_wb.wb, ranges_description],
min_width=1400,
sizing_mode='stretch_width')
], [daily_pred_plot],
[
column([confusion_title, confusion_matrix],
sizing_mode='stretch_width'), weight_plot,
corrects_plot
], [decision_tree_title], [decision_tree_plot]],
name=model_objective,
sizing_mode='stretch_width')
# Almacenar en RAM y algunos gráficos y en ROM algunos modelos creados
model_plots.children.append(new_plots)
if model_objective not in created_models:
created_models.append(model_objective)
save_obj(created_models, 'resources/created_models.pkl')
models.update({model_objective: new_plots})
models.move_to_end(model_objective, last=False)
created_models_checkbox.labels = list(models.keys())
created_models_checkbox.active = list(range(len(models.keys())))
create_model_spinner.hide_spinner()
add_model_button.on_click(prediction_callback)
# Callback para eliminar algunos modelos seleccionados
def remove_model_handler(new):
selected_labels = [
created_models_checkbox.labels[elements]
for elements in created_models_checkbox.active
]
try:
for element in selected_labels:
models.pop(element)
model_plots.children.remove(doc.get_model_by_name(element))
created_models.remove(element)
save_obj(created_models, 'resources/created_models.pkl')
except:
for element in selected_labels:
print(f"El modelo {element} no existe")
created_models_checkbox.labels = list(models.keys())
created_models_checkbox.active = list(range(len(models.keys())))
delete_model_button.on_click(remove_model_handler)
# Callback para mostrar u ocultar los gráficos de los modelos creados
def show_hide_plots(new):
selected_labels = [
created_models_checkbox.labels[elements] for elements in new
]
children = []
for element in selected_labels:
children.append(models[element])
model_plots.children = children
created_models_checkbox.on_click(show_hide_plots)
# Creación del layout inicial de la interfaz
model_plots = column([])
initialize = True
for model in created_models:
prediction_callback(model=model)
initialize = False
# Creación del layout estático de la interfaz
l = layout([[prediction_plot], [outlier_plot], [simulation_title],
[
model_select_wb,
column(created_models_wb,
delete_model_button,
sizing_mode='stretch_width')
], [model_plots]],
sizing_mode='stretch_both')
doc.add_root(l)
def modify_second_descriptive(doc):
models = OrderedDict([])
# Llamada al webservice de RapidMiner
json_perfil_document = call_webservice(
'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Perfil_Out_JSON?',
'rapidminer',
'rapidminer',
out_json=True)
# Extracción de los datos web
df_perfil = [json_normalize(data) for data in json_perfil_document]
# Asignación de los datos web a su variable correspondiente
prediction_df = df_perfil[3]
outlier_df = df_perfil[4]
# Creación de los gráficos y widgets permanentes en la interfaz
prediction_plot = create_prediction_plot(prediction_df)
outlier_plot = create_outlier_plot(outlier_df)
simulation_title = create_div_title('Simulación')
model_title, add_model_button, model_select_menu = create_model_menu()
model_select_wb = widgetbox(
[model_title, model_select_menu, add_model_button],
max_width=200,
sizing_mode='stretch_width')
created_models_title = create_div_title('Modelos creados')
created_models_checkbox = CheckboxButtonGroup(labels=list(models.keys()),
height=35)
created_models_checkbox.active = [0]
delete_model_button = Button(label='Eliminar',
button_type='danger',
height=45,
max_width=200)
created_models_wb = widgetbox(
[created_models_title, created_models_checkbox],
max_width=900,
sizing_mode='stretch_width')
# Callbacks para los widgets de la interfaz
def prediction_callback():
# Llamar al servicio web EDAR_Cartuja_Prediccion con los nuevos parámetros
model_objective = model_select_menu.value
model_discretise = 5
# Verificar que el modelo no ha sido creado antes
if model_objective not in models:
json_prediction_document = call_webservice(
'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Prediccion_JSON?',
'rapidminer',
'rapidminer', {
'Objetivo': model_objective,
'Discretizacion': model_discretise,
'Numero_Atributos': 4
},
out_json=True)
# Obtener datos
df_prediction = [
json_normalize(data) for data in json_prediction_document
]
decision_tree_df = df_prediction[0]
confusion_df_raw = df_prediction[1].reindex(
columns=list(json_prediction_document[1][0].keys()))
confusion_df = create_df_confusion(confusion_df_raw)
weight_df = df_prediction[2]
daily_pred_df = df_prediction[3][[
'timestamp', model_objective, f'prediction({model_objective})'
]]
decision_tree_data = create_decision_tree_data(
decision_tree_df, model_objective)
# Crear nuevos gráficos
daily_pred_plot = create_daily_pred_plot(daily_pred_df,
model_objective)
decision_tree_plot = create_decision_tree_plot()
decision_tree_graph = create_decision_tree_graph_renderer(
decision_tree_plot, decision_tree_data)
decision_tree_plot = append_labels_to_decision_tree(
decision_tree_plot, decision_tree_graph, decision_tree_data)
confusion_matrix = create_confusion_matrix(confusion_df)
weight_plot = create_attribute_weight_plot(weight_df,
model_objective)
corrects_plot = create_corrects_plot(confusion_df, model_objective)
confusion_title = create_div_title(
f'Matriz de confusión - {model_objective}')
decision_tree_title = create_div_title(
f'Arbol de decisión - {model_objective}')
new_plots = layout(
[[daily_pred_plot],
[
column([confusion_title, confusion_matrix],
sizing_mode='stretch_width'), weight_plot,
corrects_plot
], [decision_tree_title], [decision_tree_plot]],
name=model_objective,
sizing_mode='stretch_width')
model_plots.children.append(new_plots)
models.update({model_objective: new_plots})
models.move_to_end(model_objective, last=False)
created_models_checkbox.labels = list(models.keys())
created_models_checkbox.active = list(range(len(models.keys())))
add_model_button.on_click(prediction_callback)
def remove_options_handler(new):
selected_labels = [
created_models_checkbox.labels[elements]
for elements in created_models_checkbox.active
]
try:
for element in selected_labels:
models.pop(element)
model_plots.children.remove(doc.get_model_by_name(element))
except:
for element in selected_labels:
print(f"El modelo {element} no existe")
created_models_checkbox.labels = list(models.keys())
created_models_checkbox.active = list(range(len(models.keys())))
delete_model_button.on_click(remove_options_handler)
def show_hide_plots(new):
selected_labels = [
created_models_checkbox.labels[elements] for elements in new
]
# model_plots.children = []
children = []
for element in selected_labels:
children.append(models[element])
model_plots.children = children
created_models_checkbox.on_click(show_hide_plots)
# Creación del layout dinámico de la interfaz
model_plots = column([])
prediction_callback()
# Creación del layout estático de la interfaz
l = layout([[prediction_plot], [outlier_plot], [simulation_title],
[
model_select_wb,
column(created_models_wb,
delete_model_button,
sizing_mode='stretch_width')
], [model_plots]],
sizing_mode='stretch_both')
doc.add_root(l)
