state = CheckboxButtonGroup(labels=states, active=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) year = CheckboxButtonGroup(labels=years, active=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) # In[112]: show(year) # In[110]: output_notebook() # In[216]: name.active = [0, 1] # In[203]: def speaker_filter(party_arr, prior_experience_arr, election_result_arr, name_arr, state_arr, date_max, date_min): query = {} query['party'] = {'$in': party_arr} query['incumbent'] = {'$in': prior_experience_arr} query['election_result'] = {'$in': election_result_arr} query['name'] = {'$in': name_arr} query['state'] = {'$in': state_arr} query['year'] = {'$lt': date_max, '$gt': date_min} mongo_data = list(db.speakers.find(query, {'speaker_id': 1})) ret_list = []
def modify_second_descriptive(doc): # Captura de los argumentos pasados desde flask args = doc.session_context.request.arguments try: periodo = int(args.get('periodo')[0]) tipo_var = args.get('tipo_var')[0].decode('ascii') periodo_custom_start = args.get('periodo_custom_start')[0].decode( 'ascii') periodo_custom_end = args.get('periodo_custom_end')[0].decode('ascii') current_date = dt.strptime( args.get('current_date')[0].decode('ascii'), '%d/%m/%Y') except: periodo = 0 tipo_var = '' periodo_custom_start = '' periodo_custom_end = '' current_date = '' if tipo_var == 'abs': tipo_var = 'ABSOLUTAS' elif tipo_var == 'rend': tipo_var = 'RENDIMIENTOS' print( f'periodo: {periodo}, tipo_var: {tipo_var}, periodo_custom_start: {periodo_custom_start}, periodo_custom_end: {periodo_custom_end}, current_date: {current_date}' ) # Creación/Carga en RAM del diccionario con las variables a modelizar total_model_dict = load_or_create_model_vars( model_vars_file='resources/total_model_dict.pkl', mask_file='resources/model_variables_mask.xlsx', sheets=[ 'ID_INFLUENTE', 'ID_BIOS', 'ID_FANGOS', 'ID_HORNO', 'ID_EFLUENTE', 'ID_ELECTRICIDAD', 'YOKO', 'ANALITICA', 'METEO' ], cols=['OUT', 'IN', 'MANIPULABLES', 'PROCESOS_IN'], force_create=False) # Inicialización del diccionario ordenado para almacenar los modelos creados models = OrderedDict([]) try: created_models = load_obj(name='resources/created_models.pkl') except: created_models = ['Calidad_Agua'] ruta_periodo = f'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv' # Crear nuevo archivo custom si periodo=3 if periodo == 3: create_custom_period(periodo_custom_start, periodo_custom_end) ruta_periodo = F'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_CUSTOM.csv' current_date = dt.strptime(periodo_custom_end, '%d/%m/%Y') # Llamada al webservice de RapidMiner json_perfil_document = call_webservice( url= 'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Perfil_Out_JSON_v5?', username='******', password='******', parameters={ 'Ruta_periodo': ruta_periodo, 'Ruta_tipo_variable': f'{SERVER_HOST}/archivos/EDAR4.0_EDAR_Cartuja_VARIABLES_{tipo_var}.csv', 'Normalizacion': 1 }, out_json=True) # Extracción de los datos web df_perfil = [json_normalize(data) for data in json_perfil_document] # Asignación de los datos web a su variable correspondiente prediction_df = df_perfil[3] outlier_df = df_perfil[4] # Creación de los gráficos y widgets permanentes en la interfaz prediction_plot = create_prediction_plot(prediction_df, current_date) outlier_plot = create_outlier_plot(outlier_df, prediction_df, tipo_var) simulation_title = create_div_title( 'Creación, Simulación y Optimización de modelos') model_title, add_model_button, model_select_menu = create_model_menu( model_variables=list(total_model_dict.keys())) create_model_spinner = Spinner(size=16) recreate_button = Button(label='Recrear', button_type='success', height=35, max_width=200, min_width=200) model_select_wb = widgetbox([ row([model_title, create_model_spinner.spinner], sizing_mode='stretch_width', max_width=400), row([model_select_menu, recreate_button], sizing_mode='stretch_width', max_width=400), add_model_button ], max_width=400, sizing_mode='stretch_width') created_models_title = create_div_title('Modelos creados') created_models_checkbox = CheckboxButtonGroup(labels=list(models.keys()), height=35) created_models_checkbox.active = [0] delete_model_button = Button(label='Eliminar', button_type='danger', height=35, max_width=200) created_models_wb = widgetbox( [created_models_title, created_models_checkbox], max_width=900, sizing_mode='stretch_width') # Callback para crear nuevamente el listado de variables de la mascara def recreate_callback(): print('Recreando lista de variables para modelizar') nonlocal total_model_dict total_model_dict = load_or_create_model_vars( model_vars_file='resources/total_model_dict.pkl', mask_file='resources/model_variables_mask.xlsx', sheets=[ 'ID_INFLUENTE', 'ID_BIOS', 'ID_FANGOS', 'ID_HORNO', 'ID_EFLUENTE', 'ID_ELECTRICIDAD', 'YOKO', 'ANALITICA', 'METEO' ], cols=['OUT', 'IN', 'MANIPULABLES', 'PROCESOS_IN'], force_create=True) model_select_menu.options = list(total_model_dict.keys()) model_select_menu.value = 'Calidad_Agua' recreate_button.on_click(recreate_callback) # Callbacks para los widgets de la interfaz def prediction_callback(model='Calidad_Agua'): create_model_spinner.show_spinner() if initialize == False: model_objective = model_select_menu.value else: model_objective = model model_discretise = 5 # Verificar que el modelo no ha sido creado antes if model_objective not in models: # print(f'Objetivo: {model_objective}') # print(f'Discretizacion: {model_discretise}') # print(f'Ruta_periodo: /home/admin/Cartuja_Datos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv') # print(f'IN_MODELO: {total_model_dict[model_objective]}') # Llamar al servicio web EDAR_Cartuja_Prediccion con los nuevos parámetros ruta_periodo = f'https://edar.vicomtech.org/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_{periodo}.csv' # Crear nuevo archivo custom si periodo=3 if periodo == 3: create_custom_period(periodo_custom_start, periodo_custom_end) ruta_periodo = 'https://edar.vicomtech.org/archivos/EDAR4.0_EDAR_Cartuja_ID_PERIOD_CUSTOM.csv' json_prediction_document = call_webservice( url= 'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Prediccion_JSON_v5?', username='******', password='******', parameters={ 'Objetivo': model_objective, 'Discretizacion': model_discretise, 'Numero_Atributos': 4, 'Ruta_periodo': ruta_periodo, 'IN_MODELO': str(total_model_dict[model_objective]) }, out_json=True) # Obtener datos df_prediction = [ json_normalize(data) for data in json_prediction_document ] decision_tree_df = df_prediction[0] decision_tree_df = append_count(decision_tree_df) confusion_df_raw = df_prediction[1].reindex( columns=list(json_prediction_document[1][0].keys())) confusion_df = create_df_confusion(confusion_df_raw) weight_df = df_prediction[2] pred_df = df_prediction[3] ranges_df = df_prediction[4] ranges_df.set_index('Name', inplace=True) # ranges_df['Values']=ranges_df['Values'].replace(regex=r'\(.*\)',value='') slider_df = create_df_sliders(weight_df, pred_df) daily_pred_df = pred_df[[ 'Fecha', model_objective, f'prediction({model_objective})' ]] possible_targets = sorted(list(pred_df[model_objective].unique())) # print(f'Targets: {possible_targets}') var_influyentes = list(weight_df['Attribute']) decision_tree_data = create_decision_tree_data( decision_tree_df, model_objective) # Crear nuevos gráficos simul_or_optim_wb = SimulOptimWidget( target=model_objective, simul_df=slider_df, possible_targets=possible_targets, var_influyentes=var_influyentes, periodo=periodo, ranges=ranges_df, periodo_custom_start=periodo_custom_start, periodo_custom_end=periodo_custom_end) daily_pred_plot = create_daily_pred_plot(daily_pred_df, model_objective) decision_tree_plot = create_decision_tree_plot() decision_tree_graph = create_decision_tree_graph_renderer( decision_tree_plot, decision_tree_data) decision_tree_plot = append_labels_to_decision_tree( decision_tree_plot, decision_tree_graph, decision_tree_data) confusion_matrix = create_confusion_matrix(confusion_df) weight_plot = create_attribute_weight_plot(weight_df, model_objective) corrects_plot = create_corrects_plot(confusion_df, model_objective) model_title = create_div_title(f'Modelo - {model_objective}') confusion_title = create_div_title( f'Matriz de confusión - {model_objective}') decision_tree_title = create_div_title( f'Arbol de decisión - {model_objective}') ranges_description = create_ranges_description( possible_targets, model_objective) new_plots = layout( [[model_title], [simul_or_optim_wb.rb], [ row([simul_or_optim_wb.wb, ranges_description], min_width=1400, sizing_mode='stretch_width') ], [daily_pred_plot], [ column([confusion_title, confusion_matrix], sizing_mode='stretch_width'), weight_plot, corrects_plot ], [decision_tree_title], [decision_tree_plot]], name=model_objective, sizing_mode='stretch_width') # Almacenar en RAM y algunos gráficos y en ROM algunos modelos creados model_plots.children.append(new_plots) if model_objective not in created_models: created_models.append(model_objective) save_obj(created_models, 'resources/created_models.pkl') models.update({model_objective: new_plots}) models.move_to_end(model_objective, last=False) created_models_checkbox.labels = list(models.keys()) created_models_checkbox.active = list(range(len(models.keys()))) create_model_spinner.hide_spinner() add_model_button.on_click(prediction_callback) # Callback para eliminar algunos modelos seleccionados def remove_model_handler(new): selected_labels = [ created_models_checkbox.labels[elements] for elements in created_models_checkbox.active ] try: for element in selected_labels: models.pop(element) model_plots.children.remove(doc.get_model_by_name(element)) created_models.remove(element) save_obj(created_models, 'resources/created_models.pkl') except: for element in selected_labels: print(f"El modelo {element} no existe") created_models_checkbox.labels = list(models.keys()) created_models_checkbox.active = list(range(len(models.keys()))) delete_model_button.on_click(remove_model_handler) # Callback para mostrar u ocultar los gráficos de los modelos creados def show_hide_plots(new): selected_labels = [ created_models_checkbox.labels[elements] for elements in new ] children = [] for element in selected_labels: children.append(models[element]) model_plots.children = children created_models_checkbox.on_click(show_hide_plots) # Creación del layout inicial de la interfaz model_plots = column([]) initialize = True for model in created_models: prediction_callback(model=model) initialize = False # Creación del layout estático de la interfaz l = layout([[prediction_plot], [outlier_plot], [simulation_title], [ model_select_wb, column(created_models_wb, delete_model_button, sizing_mode='stretch_width') ], [model_plots]], sizing_mode='stretch_both') doc.add_root(l)
def modify_second_descriptive(doc): models = OrderedDict([]) # Llamada al webservice de RapidMiner json_perfil_document = call_webservice( 'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Perfil_Out_JSON?', 'rapidminer', 'rapidminer', out_json=True) # Extracción de los datos web df_perfil = [json_normalize(data) for data in json_perfil_document] # Asignación de los datos web a su variable correspondiente prediction_df = df_perfil[3] outlier_df = df_perfil[4] # Creación de los gráficos y widgets permanentes en la interfaz prediction_plot = create_prediction_plot(prediction_df) outlier_plot = create_outlier_plot(outlier_df) simulation_title = create_div_title('Simulación') model_title, add_model_button, model_select_menu = create_model_menu() model_select_wb = widgetbox( [model_title, model_select_menu, add_model_button], max_width=200, sizing_mode='stretch_width') created_models_title = create_div_title('Modelos creados') created_models_checkbox = CheckboxButtonGroup(labels=list(models.keys()), height=35) created_models_checkbox.active = [0] delete_model_button = Button(label='Eliminar', button_type='danger', height=45, max_width=200) created_models_wb = widgetbox( [created_models_title, created_models_checkbox], max_width=900, sizing_mode='stretch_width') # Callbacks para los widgets de la interfaz def prediction_callback(): # Llamar al servicio web EDAR_Cartuja_Prediccion con los nuevos parámetros model_objective = model_select_menu.value model_discretise = 5 # Verificar que el modelo no ha sido creado antes if model_objective not in models: json_prediction_document = call_webservice( 'http://rapidminer.vicomtech.org/api/rest/process/EDAR_Cartuja_Prediccion_JSON?', 'rapidminer', 'rapidminer', { 'Objetivo': model_objective, 'Discretizacion': model_discretise, 'Numero_Atributos': 4 }, out_json=True) # Obtener datos df_prediction = [ json_normalize(data) for data in json_prediction_document ] decision_tree_df = df_prediction[0] confusion_df_raw = df_prediction[1].reindex( columns=list(json_prediction_document[1][0].keys())) confusion_df = create_df_confusion(confusion_df_raw) weight_df = df_prediction[2] daily_pred_df = df_prediction[3][[ 'timestamp', model_objective, f'prediction({model_objective})' ]] decision_tree_data = create_decision_tree_data( decision_tree_df, model_objective) # Crear nuevos gráficos daily_pred_plot = create_daily_pred_plot(daily_pred_df, model_objective) decision_tree_plot = create_decision_tree_plot() decision_tree_graph = create_decision_tree_graph_renderer( decision_tree_plot, decision_tree_data) decision_tree_plot = append_labels_to_decision_tree( decision_tree_plot, decision_tree_graph, decision_tree_data) confusion_matrix = create_confusion_matrix(confusion_df) weight_plot = create_attribute_weight_plot(weight_df, model_objective) corrects_plot = create_corrects_plot(confusion_df, model_objective) confusion_title = create_div_title( f'Matriz de confusión - {model_objective}') decision_tree_title = create_div_title( f'Arbol de decisión - {model_objective}') new_plots = layout( [[daily_pred_plot], [ column([confusion_title, confusion_matrix], sizing_mode='stretch_width'), weight_plot, corrects_plot ], [decision_tree_title], [decision_tree_plot]], name=model_objective, sizing_mode='stretch_width') model_plots.children.append(new_plots) models.update({model_objective: new_plots}) models.move_to_end(model_objective, last=False) created_models_checkbox.labels = list(models.keys()) created_models_checkbox.active = list(range(len(models.keys()))) add_model_button.on_click(prediction_callback) def remove_options_handler(new): selected_labels = [ created_models_checkbox.labels[elements] for elements in created_models_checkbox.active ] try: for element in selected_labels: models.pop(element) model_plots.children.remove(doc.get_model_by_name(element)) except: for element in selected_labels: print(f"El modelo {element} no existe") created_models_checkbox.labels = list(models.keys()) created_models_checkbox.active = list(range(len(models.keys()))) delete_model_button.on_click(remove_options_handler) def show_hide_plots(new): selected_labels = [ created_models_checkbox.labels[elements] for elements in new ] # model_plots.children = [] children = [] for element in selected_labels: children.append(models[element]) model_plots.children = children created_models_checkbox.on_click(show_hide_plots) # Creación del layout dinámico de la interfaz model_plots = column([]) prediction_callback() # Creación del layout estático de la interfaz l = layout([[prediction_plot], [outlier_plot], [simulation_title], [ model_select_wb, column(created_models_wb, delete_model_button, sizing_mode='stretch_width') ], [model_plots]], sizing_mode='stretch_both') doc.add_root(l)