def check_vault(num_tests=100):
"""This function checks the complete regression vault that is distributed as part of
the package.
"""
fname = (os.path.dirname(grmpy.__file__) +
"/test/resources/old_regression_vault.grmpy.json")
tests = json.load(open(fname))
if num_tests > len(tests):
print("The specified number of evaluations is larger than the number"
" of entries in the regression_test vault.\n"
"Therefore the test runs the complete test battery.")
else:
tests = [tests[i] for i in np.random.choice(len(tests), num_tests)]
for test in tests:
stat, dict_, criteria = test
print_dict(dict_transformation(dict_))
init_dict = read("test.grmpy.yml")
df = simulate("test.grmpy.yml")
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
x0 = start_values(init_dict, df, "init")
criteria_ = calculate_criteria(init_dict, X1, X0, Z1, Z0, Y1, Y0, x0)
np.testing.assert_almost_equal(criteria_, criteria)
np.testing.assert_almost_equal(np.sum(df.sum()), stat)
cleanup("regression")
def test3():
"""The fourth test checks whether the simulation process works if there are only
treated or untreated Agents by setting the number of agents to one. Additionally the
test checks if the start values for the estimation process are set to the init-
ialization file values due to perfect separation.
"""
constr = {"AGENTS": 1, "DETERMINISTIC": False}
for _ in range(10):
generate_random_dict(constr)
dict_ = read("test.grmpy.yml")
df = simulate("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, dict_)
start = start_values(dict_, D, X1, X0, Z1, Z0, Y1, Y0, "init")
np.testing.assert_equal(dict_["AUX"]["init_values"][:(-6)],
start[:(-4)])
def test9():
"""This test checks if the start_values function returns the init file values if the
start option is set to init.
"""
for _ in range(10):
constr = dict()
constr["DETERMINISTIC"] = False
generate_random_dict(constr)
dict_ = read("test.grmpy.yml")
true = []
for key_ in ["TREATED", "UNTREATED", "CHOICE"]:
true += list(dict_[key_]["params"])
df = simulate("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, dict_)
x0 = start_values(dict_, D, X1, X0, Z1, Z0, Y1, Y0, "init")[:-4]
np.testing.assert_array_equal(true, x0)
def create_vault(num_tests=100, seed=123):
"""This function creates a new regression vault."""
np.random.seed(seed)
tests = []
for _ in range(num_tests):
dict_ = generate_random_dict()
init_dict = read("test.grmpy.yml")
df = simulate("test.grmpy.yml")
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
x0 = start_values(init_dict, df, "init")
criteria = calculate_criteria(init_dict, X1, X0, Z1, Z0, Y1, Y0, x0)
stat = np.sum(df.sum())
tests += [(stat, dict_, criteria)]
cleanup()
json.dump(tests, open("regression_vault.grmpy.json", "w"))
def test10():
"""This test checks if the refactor auxiliary function returns an unchanged init
file if the maximum number of iterations is set to zero.
"""
for _ in range(10):
constr = dict()
constr["DETERMINISTIC"], constr["AGENTS"] = False, 1000
constr["MAXITER"], constr["START"], constr[
"OPTIMIZER"] = 0, "init", "BFGS"
generate_random_dict(constr)
dict_ = read("test.grmpy.yml")
df = simulate("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, dict_)
start = start_values(dict_, D, X1, X0, Z1, Z0, Y1, Y0, "init")
start = backward_transformation(start)
rslt = fit("test.grmpy.yml")
np.testing.assert_equal(start, rslt["opt_rslt"]["params"].values)
def test2():
"""This test runs a random selection of five regression tests from the our old
regression test battery.
"""
fname = TEST_RESOURCES_DIR + "/old_regression_vault.grmpy.json"
tests = json.load(open(fname))
random_choice = np.random.choice(range(len(tests)), 5)
tests = [tests[i] for i in random_choice]
for test in tests:
stat, dict_, criteria = test
print_dict(dict_transformation(dict_))
df = simulate("test.grmpy.yml")
init_dict = read("test.grmpy.yml")
start = start_values(init_dict, df, "init")
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
criteria_ = calculate_criteria(init_dict, X1, X0, Z1, Z0, Y1, Y0,
start)
np.testing.assert_almost_equal(np.sum(df.sum()), stat)
np.testing.assert_array_almost_equal(criteria, criteria_)
def test14():
"""This test checks wether our gradient functions work properly."""
constr = {"AGENTS": 10000, "DETERMINISTIC": False}
for _ in range(10):
generate_random_dict(constr)
init_dict = read("test.grmpy.yml")
print(init_dict["AUX"])
df = simulate("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
num_treated = X1.shape[1]
num_untreated = X1.shape[1] + X0.shape[1]
x0 = start_values(init_dict, D, X1, X0, Z1, Z0, Y1, Y0, "init")
x0_back = backward_transformation(x0)
llh_gradient_approx = approx_fprime_cs(
x0_back,
log_likelihood,
args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None,
False),
)
llh_gradient = gradient_hessian(x0_back, X1, X0, Z1, Z0, Y1, Y0)
min_inter_approx = approx_fprime_cs(
x0,
minimizing_interface,
args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None,
False),
)
_, min_inter_gradient = log_likelihood(x0_back, X1, X0, Z1, Z0, Y1, Y0,
num_treated, num_untreated,
None, True)
np.testing.assert_array_almost_equal(min_inter_approx,
min_inter_gradient,
decimal=5)
np.testing.assert_array_almost_equal(llh_gradient_approx,
llh_gradient,
decimal=5)
cleanup()
def simulate(init_file):
"""This function simulates a user-specified version of the generalized Roy model."""
init_dict = read_simulation(init_file)
# We perform some basic consistency checks regarding the user's request.
check_sim_init_dict(init_dict)
# Distribute information
seed = init_dict["SIMULATION"]["seed"]
# Set random seed to ensure recomputabiltiy
np.random.seed(seed)
# Simulate unobservables of the model
U = simulate_unobservables(init_dict)
# Simulate observables of the model
X = simulate_covariates(init_dict)
# Simulate endogeneous variables of the model
df = simulate_outcomes(init_dict, X, U)
# Write output file
df = write_output(init_dict, df)
# Calculate Criteria function value
if not init_dict["DETERMINISTIC"]:
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
x0 = start_values(init_dict, D, X1, X0, Z1, Z0, Y1, Y0, "init")
init_dict["AUX"]["criteria_value"] = calculate_criteria(
x0, X1, X0, Z1, Z0, Y1, Y0
)
# Print Log file
print_info(init_dict, df)
return df
def test3():
"""The test checks if the criteria function value of the simulated and the
'estimated' sample is equal if both samples include an identical number of
individuals.
"""
for _ in range(5):
constr = dict()
constr["DETERMINISTIC"], constr["AGENTS"], constr[
"START"] = False, 1000, "init"
constr["OPTIMIZER"], constr["SAME_SIZE"] = "SCIPY-BFGS", True
generate_random_dict(constr)
df1 = simulate("test.grmpy.yml")
rslt = fit("test.grmpy.yml")
init_dict = read("test.grmpy.yml")
_, df2 = simulate_estimation(rslt)
start = start_values(init_dict, df1, "init")
criteria = []
for data in [df1, df2]:
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(data, init_dict)
criteria += [
calculate_criteria(init_dict, X1, X0, Z1, Z0, Y1, Y0, start)
]
np.testing.assert_allclose(criteria[1], criteria[0], rtol=0.1)
seeds = np.random.randint(0, 1000, size=NUM_TESTS)
directory = os.path.dirname(__file__)
file_dir = os.path.join(directory, "old_regression_vault.grmpy.json")
if True:
tests = []
for seed in seeds:
np.random.seed(seed)
constr = dict()
constr["DETERMINISTIC"], constr["CATEGORICAL"] = False, False
dict_ = generate_random_dict(constr)
df = simulate("test.grmpy.yml")
stat = np.sum(df.sum())
init_dict = read("test.grmpy.yml")
start = start_values(init_dict, df, "init")
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
criteria = calculate_criteria(init_dict, X1, X0, Z1, Z0, Y1, Y0, start)
tests += [(stat, dict_, criteria)]
json.dump(tests, open(file_dir, "w"))
if True:
tests = json.load(open(file_dir))
for test in tests:
stat, dict_, criteria = test
print_dict(dict_)
init_dict = read("test.grmpy.yml")
df = simulate("test.grmpy.yml")
start = start_values(init_dict, df, "init")
criteria_ = calculate_criteria(init_dict, df, start)
np.testing.assert_array_almost_equal(criteria, criteria_)
def test13():
"""This test checks if functions that affect the estimation output adjustment work as
intended.
"""
for _ in range(5):
generate_random_dict({"DETERMINISTIC": False})
df = simulate("test.grmpy.yml")
init_dict = read("test.grmpy.yml")
start = start_values(init_dict, dict, "init")
_, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
init_dict["AUX"]["criteria"] = calculate_criteria(
init_dict, X1, X0, Z1, Z0, Y1, Y0, start)
init_dict["AUX"]["starting_values"] = backward_transformation(start)
aux_dict1 = {"crit": {"1": 10}}
x0, se = [np.nan] * len(start), [np.nan] * len(start)
index = np.random.randint(0, len(x0) - 1)
x0[index], se[index] = np.nan, np.nan
p_values, t_values = calculate_p_values(se, x0, df.shape[0])
np.testing.assert_array_equal([p_values[index], t_values[index]],
[np.nan, np.nan])
x_processed, crit_processed, _ = process_output(
init_dict, aux_dict1, x0, "notfinite")
np.testing.assert_equal(
[x_processed, crit_processed],
[
init_dict["AUX"]["starting_values"],
init_dict["AUX"]["criteria"]
],
)
check1, flag1 = check_rslt_parameters(init_dict, X1, X0, Z1, Z0, Y1,
Y0, aux_dict1, start)
check2, flag2 = check_rslt_parameters(init_dict, X1, X0, Z1, Z0, Y1,
Y0, aux_dict1, x0)
np.testing.assert_equal([check1, flag1], [False, None])
np.testing.assert_equal([check2, flag2], [True, "notfinite"])
opt_rslt = {
"fun": 1.0,
"success": 1,
"status": 1,
"message": "msg",
"nfev": 10000,
}
rslt = adjust_output(opt_rslt,
init_dict,
start,
X1,
X0,
Z1,
Z0,
Y1,
Y0,
dict_=aux_dict1)
np.testing.assert_equal(rslt["crit"], opt_rslt["fun"])
np.testing.assert_equal(rslt["warning"][0], "---")
x_linalign = [0.0000000000000001] * len(x0)
num_treated = init_dict["AUX"]["num_covars_treated"]
num_untreated = num_treated + init_dict["AUX"]["num_covars_untreated"]
se, hess_inv, conf_interval, p_values, t_values, _ = calculate_se(
x_linalign, init_dict, X1, X0, Z1, Z0, Y1, Y0, num_treated,
num_untreated)
np.testing.assert_equal(se, [np.nan] * len(x0))
np.testing.assert_equal(hess_inv, np.full((len(x0), len(x0)), np.nan))
np.testing.assert_equal(conf_interval, [[np.nan, np.nan]] * len(x0))
np.testing.assert_equal(t_values, [np.nan] * len(x0))
np.testing.assert_equal(p_values, [np.nan] * len(x0))
cleanup()
def test13():
"""This test checks if functions that affect the estimation output adjustment work as
intended.
"""
for _ in range(5):
generate_random_dict({"DETERMINISTIC": False})
df = simulate("test.grmpy.yml")
init_dict = read("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
rslt_cont = create_rslt_df(init_dict)
start = start_values(init_dict, D, X1, X0, Z1, Z0, Y1, Y0, "init")
init_dict["AUX"]["criteria"] = calculate_criteria(
start, X1, X0, Z1, Z0, Y1, Y0)
init_dict["AUX"]["starting_values"] = backward_transformation(start)
aux_dict1 = {"crit": {"1": 10}}
x0, se = [np.nan] * len(start), [np.nan] * len(start)
index = np.random.randint(0, len(x0) - 1)
x0[index], se[index] = np.nan, np.nan
x_processed, crit_processed, _ = process_output(
init_dict, aux_dict1, x0, "notfinite")
np.testing.assert_equal(
[x_processed, crit_processed],
[
init_dict["AUX"]["starting_values"],
init_dict["AUX"]["criteria"]
],
)
check1, flag1 = check_rslt_parameters(start, X1, X0, Z1, Z0, Y1, Y0,
aux_dict1)
check2, flag2 = check_rslt_parameters(x0, X1, X0, Z1, Z0, Y1, Y0,
aux_dict1)
np.testing.assert_equal([check1, flag1], [False, None])
np.testing.assert_equal([check2, flag2], [True, "notfinite"])
opt_rslt = {
"x": start,
"fun": 1.0,
"success": 1,
"status": 1,
"message": "msg",
"nit": 10000,
}
rslt = adjust_output(
opt_rslt,
init_dict,
rslt_cont,
start,
"BFGS",
"init",
X1,
X0,
Z1,
Z0,
Y1,
Y0,
aux_dict1,
)
np.testing.assert_equal(rslt["opt_info"]["crit"], opt_rslt["fun"])
np.testing.assert_equal(rslt["opt_info"]["warning"][0], "---")
x_linalign = [0] * len(x0)
(
se,
hess_inv,
conf_interval_low,
conf_interval_up,
p_values,
t_values,
_,
) = calculate_se(x_linalign, 1, X1, X0, Z1, Z0, Y1, Y0)
np.testing.assert_equal(se, [np.nan] * len(x0))
np.testing.assert_equal(hess_inv, np.full((len(x0), len(x0)), np.nan))
np.testing.assert_equal(conf_interval_low, [np.nan] * len(x0))
np.testing.assert_equal(conf_interval_up, [np.nan] * len(x0))
np.testing.assert_equal(t_values, [np.nan] * len(x0))
np.testing.assert_equal(p_values, [np.nan] * len(x0))