def test_select_best_model_ranking_inf_skip(): X = load_wn18() X['test'] = X['test'][::1000] model_class = ComplEx param_grid = in_dict = { "batches_count": [10], "seed": 0, "epochs": [1], "k": [150], "eta": [10], "loss": ["self_adversarial"], "loss_params": { }, "embedding_model_params": { }, "regularizer": [None], "regularizer_params": { }, "optimizer": ["adagrad"], "optimizer_params": { "lr": [1000, 0.1] }, 'verbose':True } best_model, best_params, best_mrr_train, ranks_test, mrr_test = select_best_model_ranking(model_class, X, param_grid) assert(best_params["optimizer_params"]["lr"] == 0.1)
def test_select_best_model_ranking(): X = load_wn18() model_class = ComplEx param_grid = in_dict = { "batches_count": [500], "seed": 0, "epochs": [2000], "k": [10, 150], "eta": [10], "loss": ["nll"], "loss_params": { }, "embedding_model_params": { }, "regularizer": [None], "regularizer_params": { }, "optimizer": ["adagrad"], "optimizer_params": { "lr": [0.1, 0.01, 0.001] } } best_model, best_params, best_mrr_train, ranks_test, mrr_test = select_best_model_ranking(model_class, X, param_grid) print(type(best_model).__name__, best_params, best_mrr_train, mrr_test) assert best_params['k'] == 150
def test_select_best_model_ranking_random(): X = load_wn18rr() model_class = TransE param_grid = { "batches_count": [50], "seed": 0, "epochs": [1], "k": [2, 50], "eta": [1], "loss": ["nll"], "loss_params": {}, "embedding_model_params": {}, "regularizer": [None], "regularizer_params": {}, "optimizer": ["adagrad"], "optimizer_params": { "lr": lambda: np.log(np.random.uniform(1.00001, 1.1)) } } best_model, best_params, best_mrr_train, ranks_test, test_results, experimental_history = select_best_model_ranking( model_class, X['train'], X['valid'][::5], X['test'][::10], param_grid, max_combinations=10) assert best_params['k'] in (2, 50) assert np.log(1.00001) <= best_params['optimizer_params']['lr'] <= np.log( 100) assert len(experimental_history) == 10 assert set(i["model_params"]["k"] for i in experimental_history) == {2, 50} assert np.all([ np.log(1.00001) <= i["model_params"]["optimizer_params"]["lr"] <= np.log(100) for i in experimental_history ]) assert len( set( frozenset(_flatten_nested_keys(i["model_params"]).items()) for i in experimental_history)) == 10 assert set( test_results.keys()) == {"mrr", "mr", "hits_1", "hits_3", "hits_10"} assert all(r >= 0 for r in test_results.values()) assert all(not np.isnan(r) for r in test_results.values())
param_grid = { "batches_count": [50], "seed": 0, "epochs": [300], "k": [100, 200], "eta": [5, 10, 15, 20], "loss": ["pairwise", "nll", "multiclass_nll"], "loss_params": { "margin": [2] }, "embedding_model_params": {}, "regularizer": ["LP", None], "regularizer_params": { "p": [1, 3], "lambda": [1e-4, 1e-5] }, "optimizer": ["adagrad", "adam", "sgd"], "optimizer_params": { "lr": lambda: np.random.uniform(0.0001, 0.01) }, "verbose": False } select_best_model_ranking(model_class, X_train, X_valid, X_test, param_grid, max_combinations=100, use_filter=True, verbose=True, early_stopping=True)
def test_select_best_model_ranking_grid(): X = load_wn18rr() model_class = TransE param_grid = { "batches_count": [50], "seed": 0, "epochs": [1], "k": [2, 50], "eta": [1], "loss": ["nll"], "loss_params": {}, "embedding_model_params": {}, "regularizer": [None], "regularizer_params": {}, "optimizer": ["adagrad"], "optimizer_params": { "lr": [1000.0, 0.0001] } } best_model, best_params, best_mrr_train, ranks_test, test_results, experimental_history = select_best_model_ranking( model_class, X['train'], X['valid'][::5], X['test'][::10], param_grid) assert best_params['k'] in (2, 50) assert best_params['optimizer_params']['lr'] == 0.0001 assert len(experimental_history) == 4 assert set(i["model_params"]["k"] for i in experimental_history) == {2, 50} assert set(i["model_params"]["optimizer_params"]["lr"] for i in experimental_history) == {1000.0, 0.0001} assert len( set( frozenset(_flatten_nested_keys(i["model_params"]).items()) for i in experimental_history)) == 4 assert set( test_results.keys()) == {"mrr", "mr", "hits_1", "hits_3", "hits_10"} print(test_results.values()) assert all(r >= 0 for r in test_results.values()) assert all(not np.isnan(r) for r in test_results.values())