def runSimpleModelWithParams(model_params,data): #model_params BATCH_SIZE = model_params["batchSize"] np.random.seed(RANDOM_SEED) NUM_EPOCHS = model_params['num_epochs'] #data AP_train,TRP_train = data[0] AP_dev,TRP_dev = data[1] #Check direction if (model_params["DirectionForward"]): X_train,Y_train,X_dev,Y_dev = TRP_train,AP_train,TRP_dev,AP_dev else: X_train,Y_train,X_dev,Y_dev = AP_train,TRP_train,AP_dev,TRP_dev model_params["OutputNames"],model_params["InputNames"] = model_params["InputNames"],model_params["OutputNames"] model = customModels.SimpleNNMultiOutputModel(model_params) if model_params["scaling"]: X_train,X_dev,_ = prep.scaleData(X_train,X_dev) metrics_vect = {} for k in (model_params["OutputNames"]): if "ph" in k: metrics_vect[k] = [atan_mse] + ["mse"] + [modulo_2pi_error] + [phase_mse]+ [smape] else: metrics_vect[k] = ["mse"] + [smape] + [custom_loss] losses_vect = {} losses_weights = {} for k in (model_params["OutputNames"]): if "ph" in k: if model_params["phase_loss"] == "mse": losses_vect[k] = "mse" elif model_params["phase_loss"] == "phase_mse": losses_vect[k] = phase_mse elif model_params["phase_loss"] == "atan_mse": losses_vect[k] = atan_mse else: raise NotImplementedError("not valid phase_loss") #losses_weights[k] = 1 elif k in model_params["OutputNames"]: losses_vect[k] = "mse" #losses_vect[k] = custom_loss #losses_weights[k] = 1 else: losses_vect[k] = "mse" #losses_weights[k] = 1 if model_params["use_weights"]: wights = (Y_train.mean(axis = 0)).tolist() else: wights = [1]*len(model_params["OutputNames"]) for i,k in enumerate(model_params["OutputNames"]): f = model_params["weights_factor"] w = wights[i] losses_weights[k] = (1/w**2) #losses_vect["phi_1"] = atan_mse #losses_vect["phi_2"] = atan_mse #losses_vect["phi_3"] = phase_mse model.compile( optimizer = "adam", loss = losses_vect, loss_weights = losses_weights, metrics = metrics_vect ) #CALBACKS SETUP print("Local Training Params:") customUtils.print_dic_params(model_params,True,delimitor = "_",kvdelimitor = "_" ) print("="*50) callback_list = customCallbacks.addCallBacks(model_params) results = model.fit( X_train, dict(zip(model_params["OutputNames"],[row for row in Y_train.T])), epochs = NUM_EPOCHS , batch_size = model_params["batchSize"], validation_data = ( X_dev, dict(zip(model_params["OutputNames"],[row for row in Y_dev.T])) ), verbose =0, callbacks = callback_list, shuffle = True ) #run eval: model_location = os.path.join('models',model_params["model_name"] + '.hdf5') with open(os.path.join('model_params',model_params["model_name"] + '.json'), 'w') as fp: json.dump(model_params, fp, sort_keys=True) #eval_ensemble.run_ensemble_eval(models_locations = [model_location]) #pdb.set_trace() mse_total_train = run_eval_base(model_location,dataset = "train",email = model_params["email"]) mse_total_dev = run_eval_base(model_location,dataset = "dev",email = model_params["email"]) mse_total_test = run_eval_base(model_location,dataset = "test",email = model_params["email"]) return ( mse_total_train[0:len(model_params["OutputNames"])], mse_total_dev[0:len(model_params["OutputNames"])], mse_total_test[0:len(model_params["OutputNames"])] )
def runCustomAutpEncoderWithParams(model_params,data): #model_params np.random.seed(RANDOM_SEED) NUM_EPOCHS = model_params['num_epochs'] #data AP_train,TRP_train = data[0] AP_dev,TRP_dev = data[1] #Check direction if (model_params["DirectionForward"]): X_train,Y_train,X_dev,Y_dev = TRP_train,AP_train,TRP_dev,AP_dev else: X_train,Y_train,X_dev,Y_dev = AP_train,TRP_train,AP_dev,TRP_dev model_params["OutputNames"],model_params["InputNames"] = model_params["InputNames"],model_params["OutputNames"] if model_params["scaling"]: X_train,X_dev,_ = prep.scaleData(X_train,X_dev) model_params["InverseModelLocation"] = "models/Forward_BaseLine_MSE_run_0_time_2018_11_07_00_01_09.hdf5" # mse based #model_params["InverseModelLocation"] = "models/Fowrard_BaseLine_run_0_time_2018_11_05_19_05_19.hdf5" # atanmse based model_params["ForwardModelLocation"] = os.path.join('models',model_params["model_name"] + "forward" + '.hdf5') model_location = os.path.join('models',model_params["model_name"] + '.hdf5') weights_location = os.path.join('weights',model_params["model_name"] + '.h5') #model_forward = customModels.SimpleNNMultiOutputModelConcat(model_params) model_forward = customModels.DistNNMultiOutputModelCustom(model_params) #pdb.set_trace() model_reverse = run_eval.loadBestModel(model_params["InverseModelLocation"]) model_reverse.trainable = False #model_reverse.compile(optimizer = "adam", loss = ["mse","mse","mse",atan_mse,atan_mse,phase_mse]) inputs = model_forward.input outputs = model_forward.outputs + model_reverse(model_forward(model_forward.inputs)) combined_model = keras.models.Model(inputs = inputs , outputs = outputs ) #combined_model = keras.models.Model(inputs = model_forward.inputs , outputs = model_forward.outputs + model_reverse(model_forward.outputs) ) keras.utils.plot_model(combined_model,to_file=os.path.join('images',model_params["model_name"] + '.png'),show_shapes=True) #email_logger.send_email(text_mesage = "Test", title_text = "ModelImage",image_path = 'demo.png') #pdb.set_trace() model_forward.save(model_params["ForwardModelLocation"]) """ combined_model.compile( optimizer = keras.optimizers.adam(), loss = ["mse","mse","mse","mse","mse","mse",atan_mse,atan_mse,phase_mse], loss_weights = [10,10,10,1,1,1,10,10,10], #metrics = "mse" ) """ combined_model.compile( #optimizer = keras.optimizers.Adadelta(lr = 1.0), optimizer = keras.optimizers.RMSprop(lr = 0.001), #optimizer = keras.optimizers.Nadam(), #optimizer = keras.optimizers.Adam(lr=0.001), loss = [custom_loss,"mse","mse","mse","mse","mse","mse"], #custom loss - zero loss_weights = [1,0.1,0.1,0.1,0.3,0.3,0.3], metrics = {"out_TRP":[custom_metroc_TRP,custom_loss_TRP]} ) combined_model.save(model_location) #CALBACKS SETUP print("Local Training Params:") customUtils.print_dic_params(model_params,False ,delimitor = "_",kvdelimitor = "_" ) print("="*50) callback_list = customCallbacks.addCallBacks(model_params) input_train = [X_train] output_train = [Y_train,X_train[:,0],X_train[:,1],X_train[:,2],X_train[:,3],X_train[:,4],X_train[:,5]] input_dev = [X_dev] output_dev = [Y_dev,X_dev[:,0],X_dev[:,1],X_dev[:,2],X_dev[:,3],X_dev[:,4],X_dev[:,5]] """ input_train = [X_train] output_train = [Y_train[:,0],Y_train[:,1],Y_train[:,2],X_train[:,0],X_train[:,1],X_train[:,2],X_train[:,3],X_train[:,4],X_train[:,5]] input_dev = [X_dev] output_dev = [Y_dev[:,0],Y_dev[:,1],Y_dev[:,2],X_dev[:,0],X_dev[:,1],X_dev[:,2],X_dev[:,3],X_dev[:,4],X_dev[:,5]] """ """ input_train = [X_train] output_train = [X_train[:,0],X_train[:,1],X_train[:,2],X_train[:,3],X_train[:,4],X_train[:,5]] input_dev = [X_dev] output_dev = [X_dev[:,0],X_dev[:,1],X_dev[:,2],X_dev[:,3],X_dev[:,4],X_dev[:,5]] """ results = combined_model.fit( input_train,output_train, epochs = NUM_EPOCHS , batch_size = model_params["batchSize"], validation_data = (input_dev,output_dev), verbose =2, #callbacks = [customCallbacks.CustomSaveModel(model_params = model_params,model = combined_model)], callbacks = callback_list, shuffle = True ) print("*"*50) print("Finished Training - Saving Models") print("*"*50) with open(os.path.join('model_params',model_params["model_name"] + '.json'), 'w') as fp: json.dump(model_params, fp, sort_keys=True) mse_total_dev = run_eval_base(model_location,dataset = "dev",email = model_params["email"]) mse_total_train = run_eval_base(model_location,dataset = "train",email = model_params["email"]) mse_total_test = run_eval_base(model_location,dataset = "test",email = model_params["email"]) pdb.set_trace() return ( mse_total_train[0:len(model_params["OutputNames"])], mse_total_dev[0:len(model_params["OutputNames"])], mse_total_test[0:len(model_params["OutputNames"])] )
def run_ensemble_eval(models_locations=None): mse_total_list = [] smaple_total_list = [] rmse_total_list = [] for dataset in ["train", "dev", "test"]: Y_pred_list = [] #Load Data Y_true_dev = 0 for ix, model_location in enumerate(models_locations): best_model = run_eval.loadBestModel(model_location) path, model_name_with_ext = os.path.split(model_location) model_name, file_extension = os.path.splitext(model_name_with_ext) model_params_loc = os.path.join('model_params', model_name + ".json") with open(model_params_loc) as jf: model_params = json.load(jf) X_train, Y_train, X_dev, Y_dev = run_eval.loadData(model_params, dataset=dataset) Y_true_dev = np.transpose(Y_dev) if model_params["scaling"]: X_train, X_dev, scaler = prep.scaleData(X_train, X_dev) Y_pred_dev = best_model.predict(X_dev) output = np.zeros( (Y_pred_dev[0].shape[0], len(model_params["OutputNames"]))) for i in range(len(model_params["OutputNames"])): temp = Y_pred_dev[i] output[:, i] = temp[:, 0] Y_pred_dev = np.transpose(output) #print mse_models mse = ((Y_pred_dev - Y_true_dev)**2).mean(axis=-1) mape = 100 * (np.abs(Y_pred_dev - Y_true_dev) / (Y_true_dev + EPSILON)).mean(axis=-1) smape = 2 * 100 * (np.abs(Y_pred_dev - Y_true_dev) / np.abs(Y_true_dev + Y_pred_dev + EPSILON)).mean( axis=-1) rmse = (np.sqrt((Y_pred_dev - Y_true_dev)**2)).mean(axis=-1) non_valid_p_percent = 100 * np.where(Y_pred_dev[1, :] > Y_pred_dev[ 2, :])[0].shape[0] / Y_pred_dev[1, :].shape[0] #Print Errors print("Dataset: ", dataset) print("Model: ", ix, ":", model_location) print("non Valid P: ", non_valid_p_percent) print("MSE: ", mse) print("SMAPE: ", smape) #print("RMSE: ", rmse) Y_pred_list.append(Y_pred_dev) #pdb.set_trace() #?,3 if (len(models_locations) > 1): Y_pred_enseble = sum(Y_pred_list) / len(Y_pred_list) mse = ((Y_pred_enseble - Y_true_dev)**2).mean(axis=-1) smape = 2 * 100 * ( np.abs(Y_pred_enseble - Y_true_dev) / np.abs(Y_true_dev + Y_pred_enseble + EPSILON)).mean(axis=-1) rmse = (np.sqrt((Y_pred_enseble - Y_true_dev)**2)).mean(axis=-1) mse_total_list.append(mse) smaple_total_list.append(smape) rmse_total_list.append(rmse) print("Dataset: ", dataset) print("Model: Ensamble") print("non Valid P: ", non_valid_p_percent) print("MSE: ", mse) print("SMAPE: ", smape)
def runDistModelWithParams(model_params,data): #model_params np.random.seed(RANDOM_SEED) NUM_EPOCHS = model_params['num_epochs'] #data AP_train,TRP_train = data[0] AP_dev,TRP_dev = data[1] #Check direction if (model_params["DirectionForward"]): X_train,Y_train,X_dev,Y_dev = TRP_train,AP_train,TRP_dev,AP_dev else: X_train,Y_train,X_dev,Y_dev = AP_train,TRP_train,AP_dev,TRP_dev model_params["OutputNames"],model_params["InputNames"] = model_params["InputNames"],model_params["OutputNames"] if model_params["scaling"]: X_train,X_dev,_ = prep.scaleData(X_train,X_dev) model = customModels.DistNNMultiOutputModel(model_params) metrics_vect = {} for k in (model_params["OutputNames"]): if "ph" in k: metrics_vect[k] = [atan_mse] + ["mse"] + [modulo_2pi_error] + [mean_squared_error_mu] + [smape] elif k in ["T","R","P"]: metrics_vect[k] = [nll] + [log_likelihood_normal_cost] + [mean_squared_error_mu] +[acc_dist] + [cv_test] + ["mse"] + [smape] else: metrics_vect[k] = ["mse"] + [mean_squared_error_mu] + [smape] losses_vect = {} losses_weights = {} for k in (model_params["OutputNames"]): if "ph" in k: losses_vect[k] = "mse" #losses_weights[k] = 1 elif k in ["T","R","P"]: losses_vect[k] = log_likelihood_normal_cost #losses_vect[k] = custom_loss #losses_weights[k] = 1 else: losses_vect[k] = "mse" #losses_weights[k] = 1 if model_params["use_weights"]: wights = (Y_train.mean(axis = 0)).tolist() else: wights = [1]*len(model_params["OutputNames"]) for i,k in enumerate(model_params["OutputNames"]): f = model_params["weights_factor"] w = wights[i] losses_weights[k] = w #0.4/0.12/0.4 model.compile( optimizer = "adam", loss = losses_vect, loss_weights = losses_weights, metrics = metrics_vect ) #CALBACKS SETUP print("Local Training Params:") customUtils.print_dic_params(model_params,True,delimitor = "_",kvdelimitor = "_" ) print("="*50) callback_list = customCallbacks.addCallBacks(model_params) results = model.fit( X_train, dict(zip(model_params["OutputNames"],[np.stack((row,row), axis = 0).T for row in Y_train.T])), epochs = NUM_EPOCHS , batch_size = model_params["batchSize"], validation_data = ( X_dev, dict(zip(model_params["OutputNames"],[np.stack((row,row), axis = 0).T for row in Y_dev.T])), ), verbose =0, callbacks = callback_list, shuffle = True ) model_location = os.path.join('models',model_params["model_name"] + '.hdf5') with open(os.path.join('model_params',model_params["model_name"] + '.json'), 'w') as fp: json.dump(model_params, fp, sort_keys=True) mse_total_train = run_eval_base(model_location,dataset = "train",email = model_params["email"]) mse_total_dev = run_eval_base(model_location,dataset = "dev",email = model_params["email"]) mse_total_test = run_eval_base(model_location,dataset = "test",email = model_params["email"]) return ( mse_total_train[0:len(model_params["OutputNames"])], mse_total_dev[0:len(model_params["OutputNames"])], mse_total_test[0:len(model_params["OutputNames"])] )
def run_ensemble_eval_reverse_model(models_locations=None): #LOAD REvERSE MODEL reverse_model_loc = "models/Fowrard_BaseLine_run_0_time_2018_11_05_19_05_19.hdf5" best_reverse_model = run_eval.loadBestModel(reverse_model_loc) path, model_name_with_ext = os.path.split(reverse_model_loc) model_name, file_extension = os.path.splitext(model_name_with_ext) model_params_loc = os.path.join('model_params', model_name + ".json") with open(model_params_loc) as jf: reverse_model_params = json.load(jf) #for dataset in ["train","dev","test"]: for dataset in ["dev"]: Y_pred_list = [] #Load Data Y_true_dev = 0 for ix, model_location in enumerate(models_locations): best_model = run_eval.loadBestModel(model_location) path, model_name_with_ext = os.path.split(model_location) model_name, file_extension = os.path.splitext(model_name_with_ext) model_params_loc = os.path.join('model_params', model_name + ".json") with open(model_params_loc) as jf: model_params = json.load(jf) X_train, Y_train, X_dev, Y_dev = run_eval.loadData(model_params, dataset=dataset) if model_params["scaling"]: X_train, X_dev, scaler = prep.scaleData(X_train, X_dev) Y_pred_dev = best_model.predict(X_dev) output = np.zeros( (Y_pred_dev[0].shape[0], len(model_params["OutputNames"]))) for i in range(len(model_params["OutputNames"])): temp = Y_pred_dev[i] output[:, i] = temp[:, 0] Y_pred_dev = output ## Add reverse Stuff if reverse_model_params["scaling"]: _, Y_dev, scaler_reverse = prep.scaleData(Y_train, Y_dev) _, Y_pred_dev, scaler_reverse = prep.scaleData( Y_train, Y_pred_dev) X_from_true = best_reverse_model.predict(Y_dev) X_from_pred = best_reverse_model.predict(Y_pred_dev) #pdb.set_trace() output1 = np.zeros((X_from_true[0].shape[0], len(reverse_model_params["OutputNames"]))) output2 = np.zeros((X_from_true[0].shape[0], len(reverse_model_params["OutputNames"]))) for i in range(len(reverse_model_params["OutputNames"])): temp1 = X_from_true[i] temp2 = X_from_pred[i] output1[:, i] = temp1[:, 0] output2[:, i] = temp2[:, 0] X_from_true = output1 X_from_pred = output2 #pdb.set_trace() ##NEED TO ADD CONVERT TO NUMPY angle = np.arctan2( np.sin(X_from_true[:, 3:6] - X_from_pred[:, 3:6]), np.cos(X_from_true[:, 3:6] - X_from_pred[:, 3:6])) mae_angs = np.abs(angle).mean(axis=0) mse_amp = ((X_from_pred[:, 0:3] - X_from_true[:, 0:3])**2).mean(axis=0) errors = np.sqrt(mse_amp).tolist() + mae_angs.tolist() print("Dataset: ", dataset) print("Model: ", ix, ":", model_location) print("Errors: ", errors) if (len(models_locations) > 1): print("Dataset: ", dataset) print("Model: Ensamble")