def func(maxrt):
stop = datetime.datetime.now() + maxrt
while datetime.datetime.now() < stop:
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
) = random_init()
exog_educ_shares.to_pickle("test.soepy.educ.shares.pkl")
exog_child_age_shares.to_pickle("test.soepy.child.age.shares.pkl")
exog_child_info.to_pickle("test.soepy.child.pkl")
exog_partner_shares.to_pickle("test.soepy.partner.shares.pkl")
exog_exper_shares_pt.to_pickle("test.soepy.pt.exp.shares.pkl")
exog_exper_shares_ft.to_pickle("test.soepy.ft.exp.shares.pkl")
exog_partner_arrival_info.to_pickle("test.soepy.partner.arrival.pkl")
exog_partner_separation_info.to_pickle(
"test.soepy.partner.separation.pkl")
simulate(random_model_params_df, model_spec_init_dict)
def test_simulation_func_exp(input_vault, test_id):
"""This test runs a random selection of test regression tests from
our regression test battery.
"""
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
expected_df_sim_func,
expected_df_sim_sol,
) = input_vault[test_id]
exog_educ_shares.to_pickle("test.soepy.educ.shares.pkl")
exog_child_age_shares.to_pickle("test.soepy.child.age.shares.pkl")
exog_child_info.to_pickle("test.soepy.child.pkl")
exog_partner_shares.to_pickle("test.soepy.partner.shares.pkl")
exog_exper_shares_pt.to_pickle("test.soepy.pt.exp.shares.pkl")
exog_exper_shares_ft.to_pickle("test.soepy.ft.exp.shares.pkl")
exog_partner_arrival_info.to_pickle("test.soepy.partner.arrival.pkl")
exog_partner_separation_info.to_pickle("test.soepy.partner.separation.pkl")
#
calculated_df_false = simulate(random_model_params_df,
model_spec_init_dict,
is_expected=False)
for edu_type in ["low", "middle", "high"]:
random_model_params_df.loc[(
"exp_returns_p_bias", f"gamma_p_bias_{edu_type}"), "value"] = (
random_model_params_df.loc[("exp_returns_p",
f"gamma_p_{edu_type}"), "value"] /
random_model_params_df.loc[("exp_returns_f",
f"gamma_f_{edu_type}"), "value"])
calculated_df_true = simulate(random_model_params_df,
model_spec_init_dict,
is_expected=True)
pd.testing.assert_series_equal(
calculated_df_false.sum(axis=0),
calculated_df_true.sum(axis=0),
)
cleanup()
def test_simulation_func(input_vault, test_id):
"""This test runs a random selection of test regression tests from
our regression test battery.
"""
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
expected_df_sim_func,
expected_df_sim_sol,
) = input_vault[test_id]
exog_educ_shares.to_pickle("test.soepy.educ.shares.pkl")
exog_child_age_shares.to_pickle("test.soepy.child.age.shares.pkl")
exog_child_info.to_pickle("test.soepy.child.pkl")
exog_partner_shares.to_pickle("test.soepy.partner.shares.pkl")
exog_exper_shares_pt.to_pickle("test.soepy.pt.exp.shares.pkl")
exog_exper_shares_ft.to_pickle("test.soepy.ft.exp.shares.pkl")
exog_partner_arrival_info.to_pickle("test.soepy.partner.arrival.pkl")
exog_partner_separation_info.to_pickle("test.soepy.partner.separation.pkl")
calculated_df = simulate(random_model_params_df, model_spec_init_dict)
pd.testing.assert_series_equal(
expected_df_sim_func,
calculated_df.sum(axis=0),
)
cleanup()
def update_sim_objectes():
vault_file = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
vault = {}
with open(vault_file, "rb") as file:
tests_sim_func = pickle.load(file)
for i in range(0, 100):
print(i)
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
expected_df_sim_func,
expected_df_sim_sol,
) = tests_sim_func[i]
exog_educ_shares.to_pickle("test.soepy.educ.shares.pkl")
exog_child_age_shares.to_pickle("test.soepy.child.age.shares.pkl")
exog_child_info.to_pickle("test.soepy.child.pkl")
exog_partner_shares.to_pickle("test.soepy.partner.shares.pkl")
exog_exper_shares_pt.to_pickle("test.soepy.pt.exp.shares.pkl")
exog_exper_shares_ft.to_pickle("test.soepy.ft.exp.shares.pkl")
exog_partner_arrival_info.to_pickle("test.soepy.partner.arrival.pkl")
exog_partner_separation_info.to_pickle("test.soepy.partner.separation.pkl")
calculated_df_sim = simulate(random_model_params_df, model_spec_init_dict)
vault[i] = (
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
calculated_df_sim.sum(axis=0),
expected_df_sim_sol,
)
with open(vault_file, "wb") as file:
pickle.dump(vault, file)
cleanup(options="regression")
def test_unit_data_frame_shape():
"""This test ensures that the shape of the simulated data frame corresponds to the
to the random specifications of our initialization file.
"""
for _ in range(5):
constr = dict()
constr["AGENTS"] = np.random.randint(10, 100)
constr["PERIODS"] = np.random.randint(1, 6)
constr["EDUC_MAX"] = np.random.randint(10, min(10 + constr["PERIODS"], 12))
random_init(constr)
df = simulate("test.soepy.pkl", "test.soepy.yml")
np.testing.assert_array_equal(df.shape[0], constr["AGENTS"] * constr["PERIODS"])
def create_vault(num_test=1000, seed=123456):
"""This function creates our regression vault."""
np.random.seed(seed)
seeds = np.random.randint(0, 1000, size=num_test)
vault = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
tests = []
for counter, seed in enumerate(seeds):
np.random.seed(seed)
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
) = random_init()
df = simulate("test.soepy.pkl", "test.soepy.yml")
tests += [
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
df,
)
]
cleanup("regression")
with open(vault, "wb") as file:
pickle.dump(tests, file)
def check_vault(num_test):
"""This function runs another simulation for each init file in our regression vault.
"""
vault = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
with open(vault, "rb") as file:
tests = pickle.load(file)
for test in tests[:num_test]:
model_spec_init_dict, random_model_params_df, expected_df = test
calculated_df = simulate(random_model_params_df, model_spec_init_dict)
for col in expected_df.columns.tolist():
expected_df[col].equals(calculated_df[col])
cleanup("regression")
def test_unit_nan():
"""This test ensures that the data frame only includes individuals that have
completed education.
"""
constr = {
"AGENTS": 200,
"PERIODS": 7,
"EDUC_YEARS": [0, np.random.randint(1, 3),
np.random.randint(4, 6)],
}
random_init(constr)
df = simulate("test.soepy.pkl", "test.soepy.yml")
np.testing.assert_equal(
df[df["Education_Level"] == 1]["Period"].min(),
constr["EDUC_YEARS"][1],
)
np.testing.assert_equal(
df[df["Education_Level"] == 2]["Period"].min(),
constr["EDUC_YEARS"][2],
)
def test1(idx):
"""This test runs a random selection of five regression tests from
our regression test battery.
"""
vault = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
with open(vault, "rb") as file:
tests = pickle.load(file)
test = tests[idx]
model_spec_init_dict, random_model_params_df, expected_df = test
calculated_df = simulate(random_model_params_df, model_spec_init_dict)
for col in expected_df.columns.tolist():
np.testing.assert_array_almost_equal(
expected_df[col][expected_df[col].notna()],
calculated_df[col][calculated_df[col].notna()],
)
def create_vault(num_test=1000, seed=123456):
"""This function creates our regression vault."""
np.random.seed(seed)
seeds = np.random.randint(0, 1000, size=num_test)
vault = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
tests = []
for counter, seed in enumerate(seeds):
np.random.seed(seed)
model_spec_init_dict, random_model_params_df = random_init()
df = simulate("test.soepy.pkl", "test.soepy.yml")
tests += [(model_spec_init_dict, random_model_params_df, df)]
cleanup("regression")
with open(vault, "wb") as file:
pickle.dump(tests, file)
def check_vault(num_test):
"""This function runs another simulation for each init file in our regression vault."""
vault = TEST_RESOURCES_DIR / "regression_vault.soepy.pkl"
with open(vault, "rb") as file:
tests = pickle.load(file)
for test in tests[:num_test]:
(
model_spec_init_dict,
random_model_params_df,
exog_educ_shares,
exog_child_age_shares,
exog_partner_shares,
exog_exper_shares_pt,
exog_exper_shares_ft,
exog_child_info,
exog_partner_arrival_info,
exog_partner_separation_info,
expected_df,
) = test
exog_educ_shares.to_pickle("test.soepy.educ.shares.pkl")
exog_child_age_shares.to_pickle("test.soepy.child.age.shares.pkl")
exog_child_info.to_pickle("test.soepy.child.pkl")
exog_partner_shares.to_pickle("test.soepy.partner.shares.pkl")
exog_exper_shares_pt.to_pickle("test.soepy.pt.exp.shares.pkl")
exog_exper_shares_ft.to_pickle("test.soepy.ft.exp.shares.pkl")
exog_partner_arrival_info.to_pickle("test.soepy.partner.arrival.pkl")
exog_partner_separation_info.to_pickle("test.soepy.partner.separation.pkl")
calculated_df = simulate(random_model_params_df, model_spec_init_dict)
for col in expected_df.columns.tolist():
expected_df[col].equals(calculated_df[col])
cleanup("regression")
def test_unit_nan():
"""This test ensures that the data frame contain only nan values if individuals are
still a in education.
"""
constr = {"AGENTS": 200}
random_init(constr)
df = simulate("test.soepy.pkl", "test.soepy.yml")
for year in [11, 12]:
df2 = df[(df["Years_of_Education"] == year) & (df["Period"] < year - 10)]
df2 = df2[
[
col
for col in df2.columns.values
if col not in ["Identifier", "Period", "Years_of_Education"]
]
]
a = np.empty(df2.shape)
a[:] = np.nan
np.testing.assert_array_equal(df2.values, a)
def test_1():
"""This test makes sure the full package works for random initialization files."""
random_init()
simulate("test.soepy.pkl", "test.soepy.yml")
def func(maxrt):
stop = datetime.datetime.now() + maxrt
while datetime.datetime.now() < stop:
random_init()
simulate("test.soepy.pkl", "test.soepy.yml")