def test_select_best_model_ranking_inf_skip():
X = load_wn18()
X['test'] = X['test'][::1000]
model_class = ComplEx
param_grid = in_dict = {
"batches_count": [10],
"seed": 0,
"epochs": [1],
"k": [150],
"eta": [10],
"loss": ["self_adversarial"],
"loss_params": {
},
"embedding_model_params": {
},
"regularizer": [None],
"regularizer_params": {
},
"optimizer": ["adagrad"],
"optimizer_params": {
"lr": [1000, 0.1]
},
'verbose':True
}
best_model, best_params, best_mrr_train, ranks_test, mrr_test = select_best_model_ranking(model_class,
X,
param_grid)
assert(best_params["optimizer_params"]["lr"] == 0.1)
def test_select_best_model_ranking():
X = load_wn18()
model_class = ComplEx
param_grid = in_dict = {
"batches_count": [500],
"seed": 0,
"epochs": [2000],
"k": [10, 150],
"eta": [10],
"loss": ["nll"],
"loss_params": {
},
"embedding_model_params": {
},
"regularizer": [None],
"regularizer_params": {
},
"optimizer": ["adagrad"],
"optimizer_params": {
"lr": [0.1, 0.01, 0.001]
}
}
best_model, best_params, best_mrr_train, ranks_test, mrr_test = select_best_model_ranking(model_class,
X,
param_grid)
print(type(best_model).__name__, best_params, best_mrr_train, mrr_test)
assert best_params['k'] == 150
def test_select_best_model_ranking_random():
X = load_wn18rr()
model_class = TransE
param_grid = {
"batches_count": [50],
"seed": 0,
"epochs": [1],
"k": [2, 50],
"eta": [1],
"loss": ["nll"],
"loss_params": {},
"embedding_model_params": {},
"regularizer": [None],
"regularizer_params": {},
"optimizer": ["adagrad"],
"optimizer_params": {
"lr": lambda: np.log(np.random.uniform(1.00001, 1.1))
}
}
best_model, best_params, best_mrr_train, ranks_test, test_results, experimental_history = select_best_model_ranking(
model_class,
X['train'],
X['valid'][::5],
X['test'][::10],
param_grid,
max_combinations=10)
assert best_params['k'] in (2, 50)
assert np.log(1.00001) <= best_params['optimizer_params']['lr'] <= np.log(
100)
assert len(experimental_history) == 10
assert set(i["model_params"]["k"] for i in experimental_history) == {2, 50}
assert np.all([
np.log(1.00001) <= i["model_params"]["optimizer_params"]["lr"] <=
np.log(100) for i in experimental_history
])
assert len(
set(
frozenset(_flatten_nested_keys(i["model_params"]).items())
for i in experimental_history)) == 10
assert set(
test_results.keys()) == {"mrr", "mr", "hits_1", "hits_3", "hits_10"}
assert all(r >= 0 for r in test_results.values())
assert all(not np.isnan(r) for r in test_results.values())
param_grid = {
"batches_count": [50],
"seed": 0,
"epochs": [300],
"k": [100, 200],
"eta": [5, 10, 15, 20],
"loss": ["pairwise", "nll", "multiclass_nll"],
"loss_params": {
"margin": [2]
},
"embedding_model_params": {},
"regularizer": ["LP", None],
"regularizer_params": {
"p": [1, 3],
"lambda": [1e-4, 1e-5]
},
"optimizer": ["adagrad", "adam", "sgd"],
"optimizer_params": {
"lr": lambda: np.random.uniform(0.0001, 0.01)
},
"verbose": False
}
select_best_model_ranking(model_class,
X_train,
X_valid,
X_test,
param_grid,
max_combinations=100,
use_filter=True,
verbose=True,
early_stopping=True)
def test_select_best_model_ranking_grid():
X = load_wn18rr()
model_class = TransE
param_grid = {
"batches_count": [50],
"seed": 0,
"epochs": [1],
"k": [2, 50],
"eta": [1],
"loss": ["nll"],
"loss_params": {},
"embedding_model_params": {},
"regularizer": [None],
"regularizer_params": {},
"optimizer": ["adagrad"],
"optimizer_params": {
"lr": [1000.0, 0.0001]
}
}
best_model, best_params, best_mrr_train, ranks_test, test_results, experimental_history = select_best_model_ranking(
model_class, X['train'], X['valid'][::5], X['test'][::10], param_grid)
assert best_params['k'] in (2, 50)
assert best_params['optimizer_params']['lr'] == 0.0001
assert len(experimental_history) == 4
assert set(i["model_params"]["k"] for i in experimental_history) == {2, 50}
assert set(i["model_params"]["optimizer_params"]["lr"]
for i in experimental_history) == {1000.0, 0.0001}
assert len(
set(
frozenset(_flatten_nested_keys(i["model_params"]).items())
for i in experimental_history)) == 4
assert set(
test_results.keys()) == {"mrr", "mr", "hits_1", "hits_3", "hits_10"}
print(test_results.values())
assert all(r >= 0 for r in test_results.values())
assert all(not np.isnan(r) for r in test_results.values())