def test_fit(self): model1 = nn.NeuralNetwork() features, performances, rankings, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, self.train_performances, order="asc", skip_value=None) rankings = rankings.astype("int32") performances = performances / np.max(performances) print("raw features", features) features = self.imputer.fit_transform(features) features = self.scaler.fit_transform(features) print("scaled features", features) print(performances) print(rankings) maximum = np.max(performances) performances_max_inv = maximum - performances max_inv = np.max(performances_max_inv) performances_max_inv = performances_max_inv / max_inv max_entry = performances.max() scaled_perf = performances / performances.max() lambda_value = 0.5 model1.fit( 5, rankings, features, performances, sample_weights=None, lambda_value=0.5, # epsilon_value=1.0, regression_loss="Squared", num_epochs=150, learning_rate=0.001, batch_size=32, early_stop_interval=5, patience=16, hidden_layer_sizes=[32], activation_function="sigmoid") for i, (index, row) in enumerate(self.train_performances.iterrows()): instance_values = self.train_inst.loc[index].values imputed_row = self.imputer.transform([instance_values]) scaled_row = self.scaler.transform(imputed_row).flatten() print("predicted", model1.predict_performances(scaled_row)) print("truth performance", row.values) print("\n") print("Predicted Ranking", model1.predict_ranking(scaled_row)) print("True Ranking", np.argsort(np.argsort(row.values)) + 1) print("\n")
def test_regression(self): model1 = nn_hinge.NeuralNetworkSquaredHinge() inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, self.train_performances, max_pairs_per_instance=15, seed=15) print(inst) print(perf) print(rank) # perf = perf / np.max(perf) rank = rank.astype("int32") # weights = np.ones(perf.shape[0]) # weights = np.amin(perf, axis=1) # print("maxes", weights) # weights = -np.log(weights) # print("weights", weights) model1.fit(5, rank, inst, perf, sample_weights=None, lambda_value=0.0, epsilon_value=1.0, regression_loss="Squared", num_epochs=250, learning_rate=0.001, batch_size=32, early_stop_interval=8, patience=8, hidden_layer_sizes=[32], activation_function="sigmoid", seed=15) for index, row in self.test_inst.iterrows(): print("True Performances", self.test_performances.loc[index].values) print("Predicted Performances Model 1", model1.predict_performances(row.values)) print("\n") print("True Ranking", self.test_ranking.loc[index].values) print("Predicted Ranking Model 1", model1.predict_ranking(row.values)) print("\n") print("network weights", model1.network.get_weights())
def test_regression(self): model1 = lh.LinearHingeModel() inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, self.train_performances, max_pairs_per_instance=15, seed=15) print(inst) print(perf) print(rank) max_entry = perf.max() scaled_perf = perf / perf.max() sample_weights = sample_weights / sample_weights.max() # print(sample_weights) # sample_weights = 1 - sample_weights model1.fit_np(5, rank, inst, perf, sample_weights=None, lambda_value=0.0, epsilon_value=1.0, regression_loss="Squared", maxiter=100, log_losses=False, reg_param=0.001) for index, row in self.test_inst.iterrows(): print("True Performances", self.test_performances.loc[index].values) print("Predicted Performances Model 1", model1.predict_performances(row.values)) print("\n") print("True Ranking", self.test_ranking.loc[index].values) print("Predicted Ranking Model 1", model1.predict_ranking(row.values)) print("\n") print("loss hist", model1.loss_history)
perf_max = 1 if scale_target_to_unit_interval: perf_max = np.max(perf) perf = perf / perf_max print("perf", perf) train_performances = pd.DataFrame(data=perf, index=train_performances.index, columns=train_performances.columns) print(order) inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( train_features, train_performances, max_pairs_per_instance=max_pairs_per_instance, seed=seed, order=order, skip_value=None) sample_weights = sample_weights / sample_weights.max() if not use_weighted_samples: sample_weights = np.ones(len(sample_weights)) print("sample weights", sample_weights) rank = rank.astype("int32") model = neural_net.NeuralNetwork() model.fit(len(scenario.algorithms), rank, inst,
def test_regression(self): model = nn.NeuralNetwork() # rankings = util.ordering_to_ranking_list(self.train_ranking.values) order = "asc" perf = self.train_performances.to_numpy() perf_bar = perf inst_bar = self.train_inst.to_numpy() maximum = np.max(perf) perf_max_inv = maximum - perf max_inv = np.max(perf_max_inv) perf_max_inv = perf_max_inv / max_inv order = "desc" train_performances_max_inv = pd.DataFrame( data=perf_max_inv, index=self.train_performances.index, columns=self.train_performances.columns) inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, train_performances_max_inv, max_pairs_per_instance=15, seed=15, order="desc") # inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( # self.train_inst, # self.train_performances, # max_pairs_per_instance=15, # seed=15) rank = rank.astype("int32") weights = weights / weights.max() model.fit(5, rank, inst, perf, lambda_value=0.0, regression_loss="Squared", num_epochs=500, learning_rate=0.001, hidden_layer_sizes=[10], activation_function="relu", early_stop_interval=100, batch_size=64, sample_weights=None, log_losses=True, es_val_ratio=0.3, patience=5) for index, row in self.test_inst.iterrows(): print("True Performances", self.test_performances.loc[index].values) print("Predicted Performances", maximum - model.predict_performances(row.values) * max_inv) print("\n") print("True Ranking", self.test_ranking.loc[index].values) print( "Predicted Ranking c", np.argsort( np.argsort(maximum - model.predict_performances(row.values) * max_inv)) + 1) print("\n") print("network weights", model.network.get_weights())
def test_regression(self): model1 = ll.LogLinearModel(use_exp_for_regression=False, use_reciprocal_for_regression=False) order = "asc" perf = self.train_performances.to_numpy() perf_bar = perf inst_bar = self.train_inst.to_numpy() maximum = np.max(perf) perf_max_inv = maximum - perf max_inv = np.max(perf_max_inv) perf_max_inv = perf_max_inv / max_inv order = "desc" train_performances_max_inv = pd.DataFrame( data=perf_max_inv, index=self.train_performances.index, columns=self.train_performances.columns) inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, train_performances_max_inv, max_pairs_per_instance=15, seed=15, order=order) # sample_weights = sample_weights / sample_weights.max() print(inst) print(perf) print(rank) lambda_value = 0.0 # print(sample_weights) model1.fit_np(5, rank, inst, perf, lambda_value=lambda_value, regression_loss="Squared", maxiter=100, log_losses=True, sample_weights=None, reg_param=-0.001) # model1.save_loss_history("loss_history1.csv") baselines = [LinearRegression() for i in range(0, 5)] for label in range(0, 5): baselines[label].fit(inst_bar, perf_bar[:, label]) for index, row in self.test_inst.iterrows(): print("True Performances", self.test_performances.loc[index].values) print("row values", row.values) # print( # "Predicted Performances Model LR", ",".join([ # str(baseline.predict(row.values.reshape(1, -1))) # for baseline in baselines # ])) print("Predicted Performances Model 1", maximum - (model1.predict_performances(row.values) * max_inv)) print("\n") print("True Ranking", self.test_ranking.loc[index].values) corras_ranking = pd.Series(data= maximum - (model1.predict_performances(row.values) * max_inv)).rank( method="min").fillna(-1).astype("int16").to_numpy() print("Model1 predicted ranking1", corras_ranking) print("\n") print("lr params", baselines[1].coef_) print("corras params", model1.weights)
def test_regression(self): model1 = ll.LogLinearModel(use_exp_for_regression=False) perf_bar = self.train_performances.to_numpy() inst_bar = self.train_inst.to_numpy() features, performances, rankings, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights( self.train_inst, self.train_performances, order="asc") features = features[:1001] performances = performances[:1001] rankings = rankings[:1001] # quad_feature_transform = PolynomialFeatures(2) # print("features", features.shape) # quad_features = quad_feature_transform.fit_transform(features[:999]) # print("quad features", quad_features.shape) # feature_vec = features[1000] # print("feature vec", feature_vec.shape) # quad_feature_vec = quad_feature_transform.transform( # feature_vec.reshape(1, -1)) # print("quad feature vec", quad_feature_vec.shape) performances = performances / np.max(performances) print("raw features", features) features = self.imputer.fit_transform(features) features = self.scaler.fit_transform(features) print("scaled features", features) print(performances) print(rankings) maximum = np.max(performances) performances_max_inv = maximum - performances max_inv = np.max(performances_max_inv) performances_max_inv = performances_max_inv / max_inv lambda_value = 1.0 model1.fit_np(2, rankings, features, performances_max_inv, lambda_value=lambda_value, regression_loss="Squared", maxiter=100, log_losses=True, sample_weights=None) # baselines = [ # LinearRegression(), # LinearRegression(), # LinearRegression(), # LinearRegression(), # LinearRegression() # ] # for label in range(0, 5): # print(perf_bar[:, label].shape) # print(inst_bar.shape) # baselines[label].fit(inst_bar, perf_bar[:, label]) taus = [] for i, (index, row) in enumerate(self.train_performances.iterrows()): instance_values = self.train_inst.loc[index].values imputed_row = self.imputer.transform([instance_values]) scaled_row = self.scaler.transform(imputed_row).flatten() print("predicted", maximum - model1.predict_performances(scaled_row) * max_inv) # print( # "Predicted Performances Model LR", ",".join([ # str(baseline.predict(scaled_row.reshape(1, -1))) # for baseline in baselines # ])) print("truth performance", row.values) print("\n") print("Predicted Ranking", model1.predict_ranking(scaled_row)) print("True Ranking", np.argsort(np.argsort(row.values)) + 1) print("\n") taus.append(kendalltau(np.argsort(np.argsort(row.values)) + 1, model1.predict_ranking(scaled_row))[0]) print("Average tau", np.mean(taus)) # sns.set_style("darkgrid") # df = model1.get_loss_history_frame() # print(df) # df = df.rename(columns={"NLL":"PL-NLL"}) # df["$\lambda$ PL-NLL"] = lambda_value * df["PL-NLL"] # df["$(1 - \lambda)$ MSE"] = (1 - lambda_value) * df["MSE"] # df["TOTAL_LOSS"] = df["$\lambda$ PL-NLL"] + df["$(1 - \lambda)$ MSE"] # df = df.melt(id_vars=["iteration"]) # plt.clf() # # plt.tight_layout() # # plt.annotate(text,(0,0), (0,-40), xycoords="axes fraction", textcoords="offset points", va="top") # print(df.head()) # lp = sns.lineplot(x="iteration", y="value", hue="variable", data=df) # plt.title(self.scenario.scenario) # plt.show() # if __name__ == "__main__": # unittest.main()