def test_state_space_restrictions_by_traversing_forward(
num_periods, num_types, edu_starts, edu_max):
"""Test for inadmissible states in the state space.
The test is motivated by the addition of another restriction in
https://github.com/OpenSourceEconomics/respy/pull/145. To ensure that similar errors
do not happen again, this test takes all states in one period and indexes each of
the four subsequent states. If a state in the next period got hit return one else 0.
Repeating the same procedure for all periods, we get a list of hopefully of zeros
and ones for each state which indicates whether the state was indexed or not.
"""
@njit
def traverse_forward(states, indexer, indicator):
for i in range(states.shape[0]):
# Unpack parent state and get index.
period, exp_a, exp_b, edu, choice_lagged, type_ = states[i]
# Working in Occupation A in period + 1
k = indexer[period + 1, exp_a + 1, exp_b, edu, 0, type_]
indicator[k] = 1
# Working in Occupation B in period +1
k = indexer[period + 1, exp_a, exp_b + 1, edu, 1, type_]
indicator[k] = 1
# Schooling in period + 1. Note that adding an additional year of schooling
# is only possible for those that have strictly less than the maximum level
# of additional education allowed. This condition is necessary as there are
# states which have reached maximum education. Incrementing education by one
# would target an inadmissible state.
if edu >= edu_max:
pass
else:
k = indexer[period + 1, exp_a, exp_b, edu + 1, 2, type_]
indicator[k] = 1
# Staying at home in period + 1
k = indexer[period + 1, exp_a, exp_b, edu, 3, type_]
indicator[k] = 1
return indicator
state_space = StateSpace(num_periods, num_types, edu_starts, edu_max)
indicator = np.zeros(state_space.num_states)
for period in range(num_periods - 1):
states = state_space.get_attribute_from_period("states", period)
indicator = traverse_forward(states, state_space.indexer, indicator)
# Restrict indicator to states of the second period as the first is never indexed.
indicator = indicator[state_space.states_per_period[0]:]
assert (indicator == 1).all()
def write_interpolation_grid(respy_obj):
""" Write out an interpolation grid that can be used across
implementations.
"""
# Distribute class attribute
num_periods, num_points_interp, edu_spec, num_types = dist_class_attributes(
respy_obj, "num_periods", "num_points_interp", "edu_spec", "num_types")
# Determine maximum number of states
state_space = StateSpace(num_periods, num_types, edu_spec["start"],
edu_spec["max"])
states_number_period = state_space.states_per_period
max_states_period = max(states_number_period)
# Initialize container
booleans = np.full((max_states_period, num_periods), True)
# Iterate over all periods
for period in range(num_periods):
# Construct auxiliary objects
num_states = states_number_period[period]
any_interpolation = (num_states - num_points_interp) > 0
# Check applicability
if not any_interpolation:
continue
# Draw points for interpolation
indicators = np.random.choice(range(num_states),
size=(num_states - num_points_interp),
replace=False)
# Replace indicators
for i in range(num_states):
if i in indicators:
booleans[i, period] = False
# Write out to file
np.savetxt(".interpolation.respy.test", booleans, fmt="%s")
# Some information that is useful elsewhere.
return max_states_period
def scripts_check(request, respy_obj):
""" Wrapper for the estimation.
"""
# Distribute model parameters
num_periods, edu_spec, num_types, optim_paras = dist_class_attributes(
respy_obj, "num_periods", "edu_spec", "num_types", "optim_paras"
)
# We need to run additional checks if an estimation is requested.
if request == "estimate":
# Create the grid of the admissible states.
state_space = StateSpace(
num_periods, num_types, edu_spec["start"], edu_spec["max"], optim_paras
)
# We also check the structure of the dataset.
data_array = process_dataset(respy_obj).to_numpy()
num_rows = data_array.shape[0]
for j in range(num_rows):
period = int(data_array[j, 1])
# Extract observable components of state space as well as agent decision.
exp_a, exp_b, edu, choice_lagged = data_array[j, 4:].astype(int)
# First of all, we need to ensure that all observed years of schooling are
# larger than the initial condition of the model.
try:
np.testing.assert_equal(edu >= 0, True)
except AssertionError:
raise UserError(ERR_MSG)
# Get state indicator to obtain the systematic component of the agents
# rewards. This might fail either because the state is simply infeasible at
# any period or just not defined for the particular period requested.
try:
k = state_space.indexer[period, exp_a, exp_b, edu, choice_lagged - 1]
np.testing.assert_equal(k >= 0, True)
except (IndexError, AssertionError):
raise UserError(ERR_MSG)
# We also take a special look at the optimizer options.
respy_obj.check_estimation()
def respy_interface(respy_obj, request, data=None):
"""Provide the interface to the PYTHON functionality."""
# Distribute class attributes
(
optim_paras,
num_periods,
edu_spec,
is_debug,
num_draws_prob,
seed_prob,
num_draws_emax,
seed_emax,
is_interpolated,
num_points_interp,
maxfun,
optimizer_used,
tau,
optimizer_options,
seed_sim,
num_agents_sim,
file_sim,
precond_spec,
num_types,
num_paras,
num_agents_est,
) = dist_class_attributes(
respy_obj,
"optim_paras",
"num_periods",
"edu_spec",
"is_debug",
"num_draws_prob",
"seed_prob",
"num_draws_emax",
"seed_emax",
"is_interpolated",
"num_points_interp",
"maxfun",
"optimizer_used",
"tau",
"optimizer_options",
"seed_sim",
"num_agents_sim",
"file_sim",
"precond_spec",
"num_types",
"num_paras",
"num_agents_est",
)
if request == "estimate":
periods_draws_prob = create_draws(
num_periods, num_draws_prob, seed_prob, is_debug
)
periods_draws_emax = create_draws(
num_periods, num_draws_emax, seed_emax, is_debug
)
# Construct starting values
x_optim_free_unscaled_start = get_optim_paras(
optim_paras, num_paras, "free", is_debug
)
x_optim_all_unscaled_start = get_optim_paras(
optim_paras, num_paras, "all", is_debug
)
# Construct the state space
state_space = StateSpace(
num_periods, num_types, edu_spec["start"], edu_spec["max"]
)
# Collect arguments that are required for the criterion function.
# These must be in the correct order already.
args = (
is_interpolated,
num_points_interp,
is_debug,
data,
tau,
periods_draws_emax,
periods_draws_prob,
state_space,
)
# Special case where just one evaluation at the starting values is
# requested is accounted for. Note, that the relevant value of the
# criterion function is always the one indicated by the class attribute
# and not the value returned by the optimization algorithm.
num_free = optim_paras["paras_fixed"].count(False)
# Take only bounds from unfixed parameters and insert default bounds.
mask_paras_fixed = np.array(optim_paras["paras_fixed"])
paras_bounds_free_unscaled = np.array(optim_paras["paras_bounds"])[
~mask_paras_fixed
]
paras_bounds_free_unscaled[:, 0] = np.where(
paras_bounds_free_unscaled[:, 0] == None, # noqa: E711
-HUGE_FLOAT,
paras_bounds_free_unscaled[:, 0],
)
paras_bounds_free_unscaled[:, 1] = np.where(
paras_bounds_free_unscaled[:, 1] == None, # noqa: E711
HUGE_FLOAT,
paras_bounds_free_unscaled[:, 1],
)
record_estimation_scaling(
x_optim_free_unscaled_start,
None,
None,
None,
optim_paras["paras_fixed"],
True,
)
precond_matrix = get_precondition_matrix(
precond_spec,
optim_paras,
x_optim_all_unscaled_start,
args,
maxfun,
num_paras,
num_types,
)
x_optim_free_scaled_start = apply_scaling(
x_optim_free_unscaled_start, precond_matrix, "do"
)
paras_bounds_free_scaled = np.full((num_free, 2), np.nan)
for i in range(2):
paras_bounds_free_scaled[:, i] = apply_scaling(
paras_bounds_free_unscaled[:, i], precond_matrix, "do"
)
record_estimation_scaling(
x_optim_free_unscaled_start,
x_optim_free_scaled_start,
paras_bounds_free_scaled,
precond_matrix,
optim_paras["paras_fixed"],
False,
)
opt_obj = OptimizationClass(
x_optim_all_unscaled_start,
optim_paras["paras_fixed"],
precond_matrix,
num_types,
)
opt_obj.maxfun = maxfun
if maxfun == 0:
record_estimation_scalability("Start")
opt_obj.crit_func(x_optim_free_scaled_start, *args)
record_estimation_scalability("Finish")
success = True
message = "Single evaluation of criterion function at starting values."
elif optimizer_used == "SCIPY-BFGS":
bfgs_maxiter = optimizer_options["SCIPY-BFGS"]["maxiter"]
bfgs_gtol = optimizer_options["SCIPY-BFGS"]["gtol"]
bfgs_eps = optimizer_options["SCIPY-BFGS"]["eps"]
try:
rslt = fmin_bfgs(
opt_obj.crit_func,
x_optim_free_scaled_start,
args=args,
gtol=bfgs_gtol,
epsilon=bfgs_eps,
maxiter=bfgs_maxiter,
full_output=True,
disp=False,
)
success = rslt[6] not in [1, 2]
message = "Optimization terminated successfully."
if rslt[6] == 1:
message = "Maximum number of iterations exceeded."
elif rslt[6] == 2:
message = "Gradient and/or function calls not changing."
except MaxfunError:
success = False
message = "Maximum number of iterations exceeded."
elif optimizer_used == "SCIPY-LBFGSB":
lbfgsb_maxiter = optimizer_options["SCIPY-LBFGSB"]["maxiter"]
lbfgsb_maxls = optimizer_options["SCIPY-LBFGSB"]["maxls"]
lbfgsb_factr = optimizer_options["SCIPY-LBFGSB"]["factr"]
lbfgsb_pgtol = optimizer_options["SCIPY-LBFGSB"]["pgtol"]
lbfgsb_eps = optimizer_options["SCIPY-LBFGSB"]["eps"]
lbfgsb_m = optimizer_options["SCIPY-LBFGSB"]["m"]
try:
rslt = fmin_l_bfgs_b(
opt_obj.crit_func,
x_optim_free_scaled_start,
args=args,
approx_grad=True,
bounds=paras_bounds_free_scaled,
m=lbfgsb_m,
factr=lbfgsb_factr,
pgtol=lbfgsb_pgtol,
epsilon=lbfgsb_eps,
iprint=-1,
maxfun=maxfun,
maxiter=lbfgsb_maxiter,
maxls=lbfgsb_maxls,
)
success = rslt[2]["warnflag"] in [0]
message = rslt[2]["task"]
except MaxfunError:
success = False
message = "Maximum number of iterations exceeded."
elif optimizer_used == "SCIPY-POWELL":
powell_maxiter = optimizer_options["SCIPY-POWELL"]["maxiter"]
powell_maxfun = optimizer_options["SCIPY-POWELL"]["maxfun"]
powell_xtol = optimizer_options["SCIPY-POWELL"]["xtol"]
powell_ftol = optimizer_options["SCIPY-POWELL"]["ftol"]
try:
rslt = fmin_powell(
opt_obj.crit_func,
x_optim_free_scaled_start,
args,
powell_xtol,
powell_ftol,
powell_maxiter,
powell_maxfun,
disp=0,
)
success = rslt[5] not in [1, 2]
message = "Optimization terminated successfully."
if rslt[5] == 1:
message = "Maximum number of function evaluations."
elif rslt[5] == 2:
message = "Maximum number of iterations."
except MaxfunError:
success = False
message = "Maximum number of iterations exceeded."
else:
raise NotImplementedError
record_estimation_final(success, message)
record_estimation_stop()
elif request == "simulate":
# Draw draws for the simulation.
periods_draws_sims = create_draws(
num_periods, num_agents_sim, seed_sim, is_debug
)
# Draw standard normal deviates for the solution and evaluation step.
periods_draws_emax = create_draws(
num_periods, num_draws_emax, seed_emax, is_debug
)
# Collect arguments for different implementations of the simulation.
state_space = pyth_solve(
is_interpolated,
num_points_interp,
num_periods,
is_debug,
periods_draws_emax,
edu_spec,
optim_paras,
file_sim,
num_types,
)
simulated_data = pyth_simulate(
state_space,
num_agents_sim,
periods_draws_sims,
seed_sim,
file_sim,
edu_spec,
optim_paras,
is_debug,
)
args = (state_space, simulated_data)
else:
raise NotImplementedError("This request is not implemented.")
return args
def pyth_solve(
is_interpolated,
num_points_interp,
num_periods,
is_debug,
periods_draws_emax,
edu_spec,
optim_paras,
file_sim,
num_types,
):
"""Solve the model.
This function is a wrapper around state space creation and determining the optimal
decision in each state by backward induction.
Parameters
----------
is_interpolated : bool
Indicator for whether the expected maximum utility should be interpolated.
num_points_interp : int
Number of points used for the interpolation.
num_periods : int
Number of periods.
is_debug : bool
Flag for debugging.
periods_draws_emax : np.ndarray
Array with shape (num_periods, num_draws, num_choices) containing draws for the
Monte Carlo simulation of expected maximum utility.
edu_spec : dict
Information on education.
optim_paras : dict
Parameters affected by optimization.
file_sim : ???
Undocumented parameter.
num_types : int
Number of types.
"""
record_solution_progress(1, file_sim)
# Create the state space
state_space = StateSpace(num_periods, num_types, edu_spec["start"],
edu_spec["max"], optim_paras)
record_solution_progress(-1, file_sim)
record_solution_progress(2, file_sim)
record_solution_progress(-1, file_sim)
# Backward iteration procedure. There is a PYTHON and FORTRAN
# implementation available. If agents are myopic, the backward induction
# procedure is not called upon.
record_solution_progress(3, file_sim)
state_space = pyth_backward_induction(
periods_draws_emax,
state_space,
is_debug,
is_interpolated,
num_points_interp,
optim_paras,
file_sim,
True,
)
if optim_paras["delta"]:
record_solution_progress(-1, file_sim)
return state_space
def test_4(self):
""" Testing the return values for the total values in case of myopic
individuals for one period.
Note
----
The original test was designed to use Fortran rewards and calculate the total
values and rewards ex post in Python and see whether they match. As both
versions diverged in their implementation, we will implement the test with the
Python version and check the equality of Fortran and Python outputs at all
stages.
"""
constr = {"edu_spec": {"max": 99}}
params_spec, options_spec = generate_random_model(myopic=True,
point_constr=constr)
# The equality below does not hold if schooling is an inadmissible state.
respy_obj = RespyCls(params_spec, options_spec)
respy_obj, _ = respy_obj.simulate()
(
num_periods,
num_types,
optim_paras,
edu_spec,
mapping_state_idx,
periods_emax,
states_all,
periods_rewards_systematic,
states_number_period,
) = dist_class_attributes(
respy_obj,
"num_periods",
"num_types",
"optim_paras",
"edu_spec",
"mapping_state_idx",
"periods_emax",
"states_all",
"periods_rewards_systematic",
"states_number_period",
)
# We have to create the state space and calculate the rewards in the Python
# version as we later need the wages which are not part of
# ``periods_rewards_systematic``.
state_space = StateSpace(num_periods, num_types, edu_spec["start"],
edu_spec["max"], optim_paras)
# Check that rewards match
_, _, pyth, _ = state_space._get_fortran_counterparts()
# Set NaNs to -99.
mask = np.isnan(periods_rewards_systematic)
periods_rewards_systematic[mask] = MISSING_FLOAT
assert_almost_equal(pyth, periods_rewards_systematic)
period = np.random.choice(num_periods)
draws = np.random.normal(size=4)
# Internalize periods_emax
state_space._create_attributes_from_fortran_counterparts(periods_emax)
# Unpack necessary attributes
rewards_period = state_space.get_attribute_from_period(
"rewards", period)
emaxs_period = state_space.get_attribute_from_period("emaxs",
period)[:, :4]
max_education_period = (state_space.get_attribute_from_period(
"states", period)[:, 3] >= edu_spec["max"])
total_values, rewards_ex_post = get_continuation_value_and_ex_post_rewards(
rewards_period[:, -2:],
rewards_period[:, :4],
emaxs_period,
draws.reshape(1, -1),
optim_paras["delta"],
max_education_period,
)
np.testing.assert_equal(total_values, rewards_ex_post)
def test_6(self):
""" Further tests for the interpolation routines.
"""
params_spec, options_spec = generate_random_model()
respy_obj = RespyCls(params_spec, options_spec)
respy_obj = simulate_observed(respy_obj)
# Extract class attributes
(
periods_rewards_systematic,
mapping_state_idx,
seed_prob,
periods_emax,
num_periods,
states_all,
num_points_interp,
edu_spec,
num_draws_emax,
is_myopic,
is_debug,
is_interpolated,
optim_paras,
optimizer_options,
file_sim,
num_types,
) = dist_class_attributes(
respy_obj,
"periods_rewards_systematic",
"mapping_state_idx",
"seed_prob",
"periods_emax",
"num_periods",
"states_all",
"num_points_interp",
"edu_spec",
"num_draws_emax",
"is_myopic",
"is_debug",
"is_interpolated",
"optim_paras",
"optimizer_options",
"file_sim",
"num_types",
)
shocks_cholesky = optim_paras["shocks_cholesky"]
shocks_cov = shocks_cholesky.dot(shocks_cholesky.T)
coeffs_common = optim_paras["coeffs_common"]
coeffs_a = optim_paras["coeffs_a"]
coeffs_b = optim_paras["coeffs_b"]
delta = optim_paras["delta"]
# Add some additional objects required for the interfaces to the functions.
period = np.random.choice(num_periods)
periods_draws_emax = create_draws(num_periods, num_draws_emax,
seed_prob, is_debug)
draws_emax_standard = periods_draws_emax[period, :, :]
draws_emax_risk = transform_disturbances(draws_emax_standard,
np.zeros(4), shocks_cholesky)
# Initialize Python version and solve.
state_space = StateSpace(num_periods, num_types, edu_spec["start"],
edu_spec["max"], optim_paras)
# Integrate periods_emax in state_space
state_space.emaxs = np.column_stack((
np.zeros((state_space.num_states, 4)),
periods_emax[~np.isnan(periods_emax)
& (periods_emax != MISSING_FLOAT)],
))
# Fill emaxs_a - emaxs_home in the requested period
states_period = state_space.get_attribute_from_period("states", period)
# Do not get the emaxs from the previous period if we are in the last one.
if period != state_space.num_periods - 1:
state_space.emaxs = get_emaxs_of_subsequent_period(
states_period, state_space.indexer, state_space.emaxs,
edu_spec["max"])
num_states = state_space.states_per_period[period]
shifts = np.random.randn(4)
# Slight modification of request which assures that the interpolation code is
# working.
num_points_interp = min(num_points_interp, num_states)
# Get the IS_SIMULATED indicator for the subset of points which are used for the
# predication model.
is_simulated = get_simulated_indicator(num_points_interp, num_states,
period, is_debug)
# Unpack necessary attributes
rewards_period = state_space.get_attribute_from_period(
"rewards", period)
emaxs_period = state_space.get_attribute_from_period("emaxs",
period)[:, :4]
max_education = (state_space.get_attribute_from_period(
"states", period)[:, 3] >= edu_spec["max"])
# Construct the exogenous variables for all points of the state space.
exogenous, max_emax = get_exogenous_variables(rewards_period,
emaxs_period, shifts,
optim_paras["delta"],
max_education)
# Align output between Python and Fortran version.
py = (exogenous, max_emax)
f90 = fort_debug.wrapper_get_exogenous_variables(
period,
num_periods,
num_states,
periods_rewards_systematic,
shifts,
mapping_state_idx,
periods_emax,
states_all,
edu_spec["start"],
edu_spec["max"],
delta,
coeffs_common,
coeffs_a,
coeffs_b,
num_types,
)
assert_almost_equal(py[0], f90[0])
assert_almost_equal(py[1], f90[1])
# Construct endogenous variable so that the prediction model can be fitted.
endogenous = get_endogenous_variable(
rewards_period,
emaxs_period,
max_emax,
is_simulated,
draws_emax_risk,
optim_paras["delta"],
max_education,
)
f90 = fort_debug.wrapper_get_endogenous_variable(
period,
num_periods,
num_states,
periods_rewards_systematic,
mapping_state_idx,
periods_emax,
states_all,
is_simulated,
num_draws_emax,
max_emax,
draws_emax_risk,
edu_spec["start"],
edu_spec["max"],
shocks_cov,
delta,
coeffs_common,
coeffs_a,
coeffs_b,
)
assert_almost_equal(endogenous, replace_missing_values(f90))
py = get_predictions(endogenous, exogenous, max_emax, is_simulated)
f90 = fort_debug.wrapper_get_predictions(
endogenous,
exogenous,
max_emax,
is_simulated,
num_points_interp,
num_states,
file_sim,
False,
)
# This assertion fails if a column is all zeros.
if not exogenous.any(axis=0).any():
assert_array_almost_equal(py, f90)
def test_4(self):
""" Testing the core functions of the solution step for the equality of results
between the PYTHON and FORTRAN implementations.
"""
params_spec, options_spec = generate_random_model()
respy_obj = RespyCls(params_spec, options_spec)
# Ensure that backward induction routines use the same grid for the
# interpolation.
write_interpolation_grid(respy_obj)
# Extract class attributes
(
num_periods,
edu_spec,
optim_paras,
num_draws_emax,
seed_emax,
is_debug,
is_interpolated,
num_points_interp,
optimizer_options,
file_sim,
num_types,
) = dist_class_attributes(
respy_obj,
"num_periods",
"edu_spec",
"optim_paras",
"num_draws_emax",
"seed_emax",
"is_debug",
"is_interpolated",
"num_points_interp",
"optimizer_options",
"file_sim",
"num_types",
)
shocks_cholesky = optim_paras["shocks_cholesky"]
coeffs_common = optim_paras["coeffs_common"]
coeffs_home = optim_paras["coeffs_home"]
coeffs_edu = optim_paras["coeffs_edu"]
coeffs_a = optim_paras["coeffs_a"]
coeffs_b = optim_paras["coeffs_b"]
delta = optim_paras["delta"]
type_spec_shifts = optim_paras["type_shifts"]
type_spec_shares = optim_paras["type_shares"]
min_idx = edu_spec["max"] + 1
# Check the state space creation.
state_space = StateSpace(num_periods, num_types, edu_spec["start"],
edu_spec["max"], optim_paras)
states_all, mapping_state_idx, _, _ = state_space._get_fortran_counterparts(
)
pyth = (
states_all,
state_space.states_per_period,
mapping_state_idx,
state_space.states_per_period.max(),
)
f2py = fort_debug.wrapper_create_state_space(num_periods, num_types,
edu_spec["start"],
edu_spec["max"], min_idx)
for i in range(4):
# Slice Fortran output to shape of Python output.
if isinstance(f2py[i], np.ndarray):
f2py_reduced = f2py[i][tuple(map(slice, pyth[i].shape))]
else:
f2py_reduced = f2py[i]
assert_allclose(pyth[i], f2py_reduced)
_, _, pyth, _ = state_space._get_fortran_counterparts()
f2py = fort_debug.wrapper_calculate_rewards_systematic(
num_periods,
state_space.states_per_period,
states_all,
state_space.states_per_period.max(),
coeffs_common,
coeffs_a,
coeffs_b,
coeffs_edu,
coeffs_home,
type_spec_shares,
type_spec_shifts,
)
assert_allclose(pyth, f2py)
# Carry some results from the systematic rewards calculation for future use and
# create the required set of disturbances.
periods_draws_emax = create_draws(num_periods, num_draws_emax,
seed_emax, is_debug)
# Save result for next test.
periods_rewards_systematic = pyth.copy()
# Fix for hardcoded myopic agents.
optim_paras["delta"] = 0.00000000000000001
# Check backward induction procedure.
state_space = pyth_backward_induction(
periods_draws_emax,
state_space,
is_debug,
is_interpolated,
num_points_interp,
optim_paras,
file_sim,
False,
)
_, _, _, pyth = state_space._get_fortran_counterparts()
f2py = fort_debug.wrapper_backward_induction(
num_periods,
False,
state_space.states_per_period.max(),
periods_draws_emax,
num_draws_emax,
state_space.states_per_period,
periods_rewards_systematic,
mapping_state_idx,
states_all,
is_debug,
is_interpolated,
num_points_interp,
edu_spec["start"],
edu_spec["max"],
shocks_cholesky,
delta,
coeffs_common,
coeffs_a,
coeffs_b,
file_sim,
False,
)
assert_allclose(pyth, f2py)
def test_2(self):
""" Compare results between FORTRAN and PYTHON of selected hand-crafted
functions. In test_97() we test FORTRAN implementations against PYTHON intrinsic
routines.
"""
for _ in range(33):
# Create grid of admissible state space values.
num_edu_start = np.random.choice(range(1, 3))
num_periods = np.random.randint(1, 15)
num_types = np.random.randint(1, 3)
edu_spec = {}
edu_spec["start"] = np.random.choice(range(1, 10),
size=num_edu_start,
replace=False).tolist()
edu_spec["max"] = max(edu_spec["start"]) + np.random.randint(1, 5)
min_idx = edu_spec["max"] + 1
# FORTRAN
base_args = (num_periods, num_types)
state_space = StateSpace(*base_args, edu_spec["start"],
edu_spec["max"])
py_a, py_c, _, _ = state_space._get_fortran_counterparts()
py_b = state_space.states_per_period
py_d = py_b.max()
fort_a, fort_b, fort_c, fort_d = fort_debug.wrapper_create_state_space(
*base_args, edu_spec["start"], edu_spec["max"], min_idx)
# Ensure equivalence
rslts = [[fort_a, py_a], [fort_b, py_b], [fort_c, py_c],
[fort_d, py_d]]
for obj in rslts:
# Slice Fortran output to shape of Python output.
if isinstance(obj[0], np.ndarray):
obj[0] = obj[0][tuple(map(slice, obj[1].shape))]
assert_allclose(obj[0], obj[1])
for _ in range(100):
# Draw random request for testing purposes
num_covars = np.random.randint(2, 10)
num_agents = np.random.randint(100, 1000)
tiny = np.random.normal(size=num_agents)
beta = np.random.normal(size=num_covars)
# Generate sample
exog = np.random.sample((num_agents, num_covars))
exog[:, 0] = 1
endog = np.dot(exog, beta) + tiny
# Run OLS
beta_result = ols(y=endog, x=exog)
# Check parameters
py = beta_result
f90 = fort_debug.wrapper_get_coefficients(endog, exog, num_covars,
num_agents)
assert_almost_equal(py, f90)
# Check prediction
py = exog.dot(beta_result)
f90 = fort_debug.wrapper_point_predictions(exog, f90, num_agents)
assert_almost_equal(py, f90)
def test_1(self):
""" Testing the equality of an evaluation of the criterion function for a random
request.
"""
# Run evaluation for multiple random requests.
is_deterministic = np.random.choice([True, False], p=[0.10, 0.9])
is_interpolated = bool(np.random.choice([True, False], p=[0.10, 0.9]))
is_myopic = np.random.choice([True, False], p=[0.10, 0.9])
max_draws = np.random.randint(11, 100)
num_agents = np.random.randint(10, max_draws)
bound_constr = {"max_draws": max_draws}
point_constr = {
"interpolation": {"flag": is_interpolated},
"program": {"procs": 1, "threads": 1, "version": "python"},
"estimation": {"maxfun": 0, "agents": num_agents},
"simulation": {"agents": num_agents},
"num_periods": np.random.randint(1, 5),
}
num_types = np.random.randint(2, 5)
if is_interpolated:
point_constr["num_periods"] = np.random.randint(3, 5)
params_spec, options_spec = generate_random_model(
bound_constr=bound_constr,
point_constr=point_constr,
deterministic=is_deterministic,
myopic=is_myopic,
num_types=num_types,
)
edu_spec = options_spec["edu_spec"]
num_periods = point_constr["num_periods"]
# The use of the interpolation routines is a another special case. Constructing
# a request that actually involves the use of the interpolation routine is a
# little involved as the number of interpolation points needs to be lower than
# the actual number of states. And to know the number of states each period, I
# need to construct the whole state space.
if is_interpolated:
state_space = StateSpace(
num_periods, num_types, edu_spec["start"], edu_spec["max"]
)
max_states_period = state_space.states_per_period.max()
options_spec["interpolation"]["points"] = np.random.randint(
10, max_states_period
)
# Write out random components and interpolation grid to align the three
# implementations.
write_draws(num_periods, max_draws)
respy_obj = RespyCls(params_spec, options_spec)
write_interpolation_grid(respy_obj)
type_shares = respy_obj.attr["optim_paras"]["type_shares"]
write_types(type_shares, num_agents)
write_edu_start(edu_spec, num_agents)
write_lagged_start(num_agents)
# Clean evaluations based on interpolation grid,
base_val, base_data = None, None
for version in ["python", "fortran"]:
respy_obj = RespyCls(params_spec, options_spec)
# Modify the version of the program for the different requests.
respy_obj.unlock()
respy_obj.set_attr("version", version)
respy_obj.lock()
# Solve the model
respy_obj = simulate_observed(respy_obj)
# This parts checks the equality of simulated dataset for the different
# versions of the code.
data_frame = pd.read_csv("data.respy.dat", delim_whitespace=True)
if base_data is None:
base_data = data_frame.copy()
assert_frame_equal(base_data, data_frame)
# This part checks the equality of an evaluation of the criterion function.
_, crit_val = respy_obj.fit()
if base_val is None:
base_val = crit_val
np.testing.assert_allclose(base_val, crit_val, rtol=1e-05, atol=1e-06)
# We know even more for the deterministic case.
if is_deterministic:
assert crit_val in [-1.0, 0.0]