def test_fit(self):
model1 = nn.NeuralNetwork()
features, performances, rankings, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst,
self.train_performances,
order="asc",
skip_value=None)
rankings = rankings.astype("int32")
performances = performances / np.max(performances)
print("raw features", features)
features = self.imputer.fit_transform(features)
features = self.scaler.fit_transform(features)
print("scaled features", features)
print(performances)
print(rankings)
maximum = np.max(performances)
performances_max_inv = maximum - performances
max_inv = np.max(performances_max_inv)
performances_max_inv = performances_max_inv / max_inv
max_entry = performances.max()
scaled_perf = performances / performances.max()
lambda_value = 0.5
model1.fit(
5,
rankings,
features,
performances,
sample_weights=None,
lambda_value=0.5,
# epsilon_value=1.0,
regression_loss="Squared",
num_epochs=150,
learning_rate=0.001,
batch_size=32,
early_stop_interval=5,
patience=16,
hidden_layer_sizes=[32],
activation_function="sigmoid")
for i, (index, row) in enumerate(self.train_performances.iterrows()):
instance_values = self.train_inst.loc[index].values
imputed_row = self.imputer.transform([instance_values])
scaled_row = self.scaler.transform(imputed_row).flatten()
print("predicted", model1.predict_performances(scaled_row))
print("truth performance", row.values)
print("\n")
print("Predicted Ranking", model1.predict_ranking(scaled_row))
print("True Ranking", np.argsort(np.argsort(row.values)) + 1)
print("\n")
def test_regression(self):
model1 = nn_hinge.NeuralNetworkSquaredHinge()
inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst,
self.train_performances,
max_pairs_per_instance=15,
seed=15)
print(inst)
print(perf)
print(rank)
# perf = perf / np.max(perf)
rank = rank.astype("int32")
# weights = np.ones(perf.shape[0])
# weights = np.amin(perf, axis=1)
# print("maxes", weights)
# weights = -np.log(weights)
# print("weights", weights)
model1.fit(5,
rank,
inst,
perf,
sample_weights=None,
lambda_value=0.0,
epsilon_value=1.0,
regression_loss="Squared",
num_epochs=250,
learning_rate=0.001,
batch_size=32,
early_stop_interval=8,
patience=8,
hidden_layer_sizes=[32],
activation_function="sigmoid", seed=15)
for index, row in self.test_inst.iterrows():
print("True Performances",
self.test_performances.loc[index].values)
print("Predicted Performances Model 1",
model1.predict_performances(row.values))
print("\n")
print("True Ranking", self.test_ranking.loc[index].values)
print("Predicted Ranking Model 1",
model1.predict_ranking(row.values))
print("\n")
print("network weights", model1.network.get_weights())
def test_regression(self):
model1 = lh.LinearHingeModel()
inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst,
self.train_performances,
max_pairs_per_instance=15,
seed=15)
print(inst)
print(perf)
print(rank)
max_entry = perf.max()
scaled_perf = perf / perf.max()
sample_weights = sample_weights / sample_weights.max()
# print(sample_weights)
# sample_weights = 1 - sample_weights
model1.fit_np(5,
rank,
inst,
perf,
sample_weights=None,
lambda_value=0.0,
epsilon_value=1.0,
regression_loss="Squared",
maxiter=100,
log_losses=False,
reg_param=0.001)
for index, row in self.test_inst.iterrows():
print("True Performances",
self.test_performances.loc[index].values)
print("Predicted Performances Model 1",
model1.predict_performances(row.values))
print("\n")
print("True Ranking", self.test_ranking.loc[index].values)
print("Predicted Ranking Model 1",
model1.predict_ranking(row.values))
print("\n")
print("loss hist", model1.loss_history)
perf_max = 1
if scale_target_to_unit_interval:
perf_max = np.max(perf)
perf = perf / perf_max
print("perf", perf)
train_performances = pd.DataFrame(data=perf,
index=train_performances.index,
columns=train_performances.columns)
print(order)
inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
train_features,
train_performances,
max_pairs_per_instance=max_pairs_per_instance,
seed=seed,
order=order,
skip_value=None)
sample_weights = sample_weights / sample_weights.max()
if not use_weighted_samples:
sample_weights = np.ones(len(sample_weights))
print("sample weights", sample_weights)
rank = rank.astype("int32")
model = neural_net.NeuralNetwork()
model.fit(len(scenario.algorithms),
rank,
inst,
def test_regression(self):
model = nn.NeuralNetwork()
# rankings = util.ordering_to_ranking_list(self.train_ranking.values)
order = "asc"
perf = self.train_performances.to_numpy()
perf_bar = perf
inst_bar = self.train_inst.to_numpy()
maximum = np.max(perf)
perf_max_inv = maximum - perf
max_inv = np.max(perf_max_inv)
perf_max_inv = perf_max_inv / max_inv
order = "desc"
train_performances_max_inv = pd.DataFrame(
data=perf_max_inv,
index=self.train_performances.index,
columns=self.train_performances.columns)
inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst,
train_performances_max_inv,
max_pairs_per_instance=15,
seed=15,
order="desc")
# inst, perf, rank, weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
# self.train_inst,
# self.train_performances,
# max_pairs_per_instance=15,
# seed=15)
rank = rank.astype("int32")
weights = weights / weights.max()
model.fit(5,
rank,
inst,
perf,
lambda_value=0.0,
regression_loss="Squared",
num_epochs=500,
learning_rate=0.001,
hidden_layer_sizes=[10],
activation_function="relu",
early_stop_interval=100,
batch_size=64,
sample_weights=None,
log_losses=True,
es_val_ratio=0.3,
patience=5)
for index, row in self.test_inst.iterrows():
print("True Performances",
self.test_performances.loc[index].values)
print("Predicted Performances",
maximum - model.predict_performances(row.values) * max_inv)
print("\n")
print("True Ranking", self.test_ranking.loc[index].values)
print(
"Predicted Ranking c",
np.argsort(
np.argsort(maximum -
model.predict_performances(row.values) *
max_inv)) + 1)
print("\n")
print("network weights", model.network.get_weights())
def test_regression(self):
model1 = ll.LogLinearModel(use_exp_for_regression=False,
use_reciprocal_for_regression=False)
order = "asc"
perf = self.train_performances.to_numpy()
perf_bar = perf
inst_bar = self.train_inst.to_numpy()
maximum = np.max(perf)
perf_max_inv = maximum - perf
max_inv = np.max(perf_max_inv)
perf_max_inv = perf_max_inv / max_inv
order = "desc"
train_performances_max_inv = pd.DataFrame(
data=perf_max_inv,
index=self.train_performances.index,
columns=self.train_performances.columns)
inst, perf, rank, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst,
train_performances_max_inv,
max_pairs_per_instance=15,
seed=15,
order=order)
# sample_weights = sample_weights / sample_weights.max()
print(inst)
print(perf)
print(rank)
lambda_value = 0.0
# print(sample_weights)
model1.fit_np(5,
rank,
inst,
perf,
lambda_value=lambda_value,
regression_loss="Squared",
maxiter=100,
log_losses=True,
sample_weights=None,
reg_param=-0.001)
# model1.save_loss_history("loss_history1.csv")
baselines = [LinearRegression() for i in range(0, 5)]
for label in range(0, 5):
baselines[label].fit(inst_bar, perf_bar[:, label])
for index, row in self.test_inst.iterrows():
print("True Performances",
self.test_performances.loc[index].values)
print("row values", row.values)
# print(
# "Predicted Performances Model LR", ",".join([
# str(baseline.predict(row.values.reshape(1, -1)))
# for baseline in baselines
# ]))
print("Predicted Performances Model 1", maximum - (model1.predict_performances(row.values) * max_inv))
print("\n")
print("True Ranking", self.test_ranking.loc[index].values)
corras_ranking = pd.Series(data= maximum - (model1.predict_performances(row.values) * max_inv)).rank(
method="min").fillna(-1).astype("int16").to_numpy()
print("Model1 predicted ranking1", corras_ranking)
print("\n")
print("lr params", baselines[1].coef_)
print("corras params", model1.weights)
def test_regression(self):
model1 = ll.LogLinearModel(use_exp_for_regression=False)
perf_bar = self.train_performances.to_numpy()
inst_bar = self.train_inst.to_numpy()
features, performances, rankings, sample_weights = util.construct_numpy_representation_with_ordered_pairs_of_rankings_and_features_and_weights(
self.train_inst, self.train_performances, order="asc")
features = features[:1001]
performances = performances[:1001]
rankings = rankings[:1001]
# quad_feature_transform = PolynomialFeatures(2)
# print("features", features.shape)
# quad_features = quad_feature_transform.fit_transform(features[:999])
# print("quad features", quad_features.shape)
# feature_vec = features[1000]
# print("feature vec", feature_vec.shape)
# quad_feature_vec = quad_feature_transform.transform(
# feature_vec.reshape(1, -1))
# print("quad feature vec", quad_feature_vec.shape)
performances = performances / np.max(performances)
print("raw features", features)
features = self.imputer.fit_transform(features)
features = self.scaler.fit_transform(features)
print("scaled features", features)
print(performances)
print(rankings)
maximum = np.max(performances)
performances_max_inv = maximum - performances
max_inv = np.max(performances_max_inv)
performances_max_inv = performances_max_inv / max_inv
lambda_value = 1.0
model1.fit_np(2,
rankings,
features,
performances_max_inv,
lambda_value=lambda_value,
regression_loss="Squared",
maxiter=100,
log_losses=True, sample_weights=None)
# baselines = [
# LinearRegression(),
# LinearRegression(),
# LinearRegression(),
# LinearRegression(),
# LinearRegression()
# ]
# for label in range(0, 5):
# print(perf_bar[:, label].shape)
# print(inst_bar.shape)
# baselines[label].fit(inst_bar, perf_bar[:, label])
taus = []
for i, (index, row) in enumerate(self.train_performances.iterrows()):
instance_values = self.train_inst.loc[index].values
imputed_row = self.imputer.transform([instance_values])
scaled_row = self.scaler.transform(imputed_row).flatten()
print("predicted", maximum - model1.predict_performances(scaled_row) * max_inv)
# print(
# "Predicted Performances Model LR", ",".join([
# str(baseline.predict(scaled_row.reshape(1, -1)))
# for baseline in baselines
# ]))
print("truth performance", row.values)
print("\n")
print("Predicted Ranking", model1.predict_ranking(scaled_row))
print("True Ranking", np.argsort(np.argsort(row.values)) + 1)
print("\n")
taus.append(kendalltau(np.argsort(np.argsort(row.values)) + 1, model1.predict_ranking(scaled_row))[0])
print("Average tau", np.mean(taus))
# sns.set_style("darkgrid")
# df = model1.get_loss_history_frame()
# print(df)
# df = df.rename(columns={"NLL":"PL-NLL"})
# df["$\lambda$ PL-NLL"] = lambda_value * df["PL-NLL"]
# df["$(1 - \lambda)$ MSE"] = (1 - lambda_value) * df["MSE"]
# df["TOTAL_LOSS"] = df["$\lambda$ PL-NLL"] + df["$(1 - \lambda)$ MSE"]
# df = df.melt(id_vars=["iteration"])
# plt.clf()
# # plt.tight_layout()
# # plt.annotate(text,(0,0), (0,-40), xycoords="axes fraction", textcoords="offset points", va="top")
# print(df.head())
# lp = sns.lineplot(x="iteration", y="value", hue="variable", data=df)
# plt.title(self.scenario.scenario)
# plt.show()
# if __name__ == "__main__":
# unittest.main()
