def mdl_resid(x=None,
targets=None,
weights=w,
save_to=None,
load_from=None,
return_format=['distance'],
store_path=None,
verbose=False,
draw=False,
graphs=False,
rel_diff=False,
cs_moments=False,
moments_repeat=5,
Tsim=30,
moments_save_name=None):
from model import Model
from setup import DivorceCosts
from simulations import Agents
from crosssection import CrossSection
from calibration_params import calibration_params
if type(x) is dict:
kwords = x
if 'targets' in x:
targets = x.pop('targets')
else:
lb, ub, xdef, keys, translator = calibration_params()
if x is None:
x = xdef
kwords = translator(x)
if verbose: print(kwords)
# this is for the default model
#dc_k = DivorceCosts(unilateral_divorce=True,assets_kept = 1.0,u_lost_m=0.00,u_lost_f=0.00,eq_split=1.0)
#dc_nk = DivorceCosts(unilateral_divorce=True,assets_kept = 1.0,u_lost_m=0.00,u_lost_f=0.00,eq_split=1.0)
def join_path(name, path):
return os.path.join(path, name)
if load_from is not None:
if type(load_from) is not list:
load_from = [load_from]
if store_path is not None:
load_from = [join_path(n, store_path) for n in load_from]
if save_to is not None:
if type(save_to) is not list:
save_to = [save_to]
if store_path is not None:
save_to = [join_path(n, store_path) for n in save_to]
if load_from is None:
mdl = Model(verbose=verbose, **kwords)
mdl_list = [mdl]
else:
mdl_list = [dill.load(open(l, 'rb+')) for l in load_from]
mdl = mdl_list[0]
if save_to is not None:
if len(save_to) > 1:
print('warning: too much stuff is save_to')
dill.dump(mdl, open(save_to[0], 'wb+'))
np.random.seed(18)
agents = Agents(mdl_list, verbose=verbose, fix_seed=False, T=Tsim)
if not cs_moments:
moments_list = [agents.compute_moments()] + [
Agents(mdl_list, verbose=False, T=Tsim,
fix_seed=False).compute_moments()
for _ in range(moments_repeat - 1)
]
else:
moments_list = [
CrossSection(mdl_list,
verbose=False,
N_total=30000,
fix_seed=False).compute_moments()
for _ in range(moments_repeat)
]
mom = {
key: np.mean([m[key] for m in moments_list], axis=0)
for key in moments_list[0].keys()
}
#mom_join = Agents( mdl_list, N=10000, T=18, female=False, verbose=False).aux_moments()
#mom_men = agents_extra.compute_moments()
#mom.update(mom_join)
if moments_save_name: # is not None can be ommited
filer('{}.pkl'.format(moments_save_name), mom, True)
if targets is None:
from targets import target_values
tar = target_values()
elif type(targets) is str:
from targets import target_values
tar = target_values(mode=targets)
else:
tar = targets
resid_all, resid_sc, dist = distance_to_targets(mom,
tar,
weights=weights,
report=verbose)
#if verbose:
# print('data moments are {}'.format(dat))
# print('simulated moments are {}'.format(sim))
tt = mdl_list[0].get_total_time()
print('Distance {}, time {}'.format(dist, tt))
out_dict = {
'distance': dist,
'all residuals': resid_all,
'scaled residuals': resid_sc,
'models': mdl_list,
'agents': agents,
'moments': mom
}
out = [out_dict[key] for key in return_format]
del (out_dict)
if 'models' not in return_format:
for m in mdl_list:
del (m)
del mdl_list
if 'agents' not in return_format:
del (agents)
if len(out) == 1: out = out[0]
return out
'pmeet_35': 0.40800438661571153,
'preg_21': 0.07677173244887601,
'preg_28': 0.06061469851763078,
'preg_35': 0.01825557056243586,
'u_shift_mar': 1.7329041070973545,
'util_alp': 0.6182672481649074,
'util_kap': 0.8081836080864513,
'util_qbar': 0.5163163798943308,
'disutil_marry_sm_mal_coef': 14.446603934890161,
'disutil_shotgun_coef': 0.4904309002252879,
'taste_shock_mult': 4.116448914683272,
'high education': True,
'n_psi': npsi
}
tar = target_values('high education')
out, mdl, agents, res, mom = mdl_resid(
x=x,
targets=tar,
return_format=[
'distance', 'models', 'agents', 'scaled residuals', 'moments'
],
#load_from='mdl.pkl',
verbose=False,
draw=True,
cs_moments=False,
moments_repeat=2)
mdl[0].time_statistics()
print('Done. Residual in point x0 is {}'.format(out))
del ((out, mdl, agents, res, mom))
'pmeet_35': 0.3537668845748406,
'preg_21': 0.03301912411330818,
'preg_28': 0.04337421298029398,
'preg_35': 0.06499192886306442,
'u_shift_mar': 1.4329636375369272,
'util_alp': 0.4980641699631754,
'util_kap': 0.8967664551452008,
'util_qbar': 1.0457400305750946,
'disutil_marry_sm_mal_coef': 14.444200806643657,
'disutil_shotgun_coef': 2.117334990378265,
'abortion_costs_mult': 8.430713206729187,
'p_abortion_access': 1.0,
'u_lost_divorce_mult': 2.690742151483817
}
tar = target_values(targ_mode)
out, mdl, agents, res, mom = mdl_resid(
x=x,
targets=tar,
return_format=[
'distance', 'models', 'agents', 'scaled residuals', 'moments'
],
#load_from='mdl.pkl',
verbose=True,
draw=True,
cs_moments=False,
moments_save_name='college move social stigma',
moments_repeat=2)
mdl[0].time_statistics()
def run(resume=False, high_e=True):
xinit, targ_mode = get_point(high_e, read_wisdom=False)
tar = target_values(targ_mode)
if resume:
x_load = filer('wisdom_refined.pkl', 0, 0)
prob_meet_load = x_load['pmeet_exo']
prob_preg_load = x_load['ppreg_exo']
xinit = x_load
out, mdl, agents, res, mom = mdl_resid(x=xinit,
targets=tar,
return_format=[
'distance', 'models', 'agents',
'scaled residuals', 'moments'
],
verbose=False)
print('initial distance is {}'.format(out))
if resume:
prob_meet_init = prob_meet_load
prob_preg_init = prob_preg_load
else:
prob_meet_init = np.array(
mdl[0].setup.pars['pmeet_t'][:mdl[0].setup.pars['Tmeet']])
prob_preg_init = np.array([
mdl[0].setup.upp_precomputed_fem[t][3]
for t in range(mdl[0].setup.pars['Tmeet'])
])
nopt = 10
yfactor = 1.5
for iopt in range(nopt):
print('running esimation round {}'.format(iopt))
print('estimating probabilities:')
prob_meet_est = 0.0
prob_preg_est = 0.0
nrep = 4 if iopt > 0 else 1
np.random.seed(12)
for rep in range(nrep):
o = AgentsEst(mdl, T=30, verbose=False, fix_seed=False)
prob_meet_est += (1 / nrep) * o.pmeet_exo.copy()
prob_preg_est += (1 / nrep) * o.ppreg_exo.copy()
print('estimated pmeet = {}'.format(prob_meet_est))
print('estimated ppreg = {}'.format(prob_preg_est))
# this does binary search
w = 1.0
factor = 0.5
ne = prob_meet_est.size
nw = 10
print('reference value is {}'.format(out))
y_previous = out
for i in range(nw):
prob_meet_w = w * prob_meet_est + (1 - w) * prob_meet_init[:ne]
prob_preg_w = w * prob_preg_est + (1 - w) * prob_preg_init[:ne]
xsearch = xinit.copy()
xsearch.update({
'pmeet_exo': prob_meet_w,
'ppreg_exo': prob_preg_w
})
out_w = mdl_resid(x=xsearch,
targets=tar,
return_format=['distance'],
verbose=False)
print('with weight = {}, distance is {}'.format(w, out_w))
if out_w < yfactor * out:
print('found a potentially imporving weight for yfactor = {}'.
format(yfactor))
break
else:
w = factor * w
if i < nw - 1:
print('trying new weight = {}'.format(w))
else:
print('no luck...')
xfix = {
k: xinit[k]
for k in [
'pmeet_21', 'pmeet_30', 'pmeet_40', 'preg_21', 'preg_28',
'preg_35'
]
}
lb, ub, _, keys, translator = calibration_params(xfix=xfix)
def tr(x):
xx = translator(x)
xx.update({'pmeet_exo': prob_meet_w, 'ppreg_exo': prob_preg_w})
return xx
x0 = [xinit[key] for key in keys]
x0, lb, ub = np.array(x0), np.array(lb), np.array(ub)
print('starting from {}'.format(tr(x0)))
tar = target_values('high education')
def q(pt):
#print('computing at point {}'.format(translator(pt)))
try:
ans = mdl_resid(tr(pt), return_format=['scaled residuals'])
except BaseException as a:
print('During optimization function evaluation failed at {}'.
format(pt))
print(a)
ans = np.array([1e6])
finally:
gc.collect()
return ans
res = dfols.solve(q,
x0,
rhobeg=0.02,
rhoend=1e-5,
maxfun=60,
bounds=(lb, ub),
scaling_within_bounds=True,
objfun_has_noise=False,
npt=len(x0) + 5,
user_params={
'restarts.use_restarts': True,
'restarts.rhoend_scale': 0.5,
'restarts.increase_npt': True
})
print(res)
print('Result is {}'.format(tr(res.x)))
filer('wisdom_refined.pkl', tr(res.x), True)
print('wrote to the file!')
xinit = tr(res.x)
out, mdl, agents, res, mom = mdl_resid(x=xinit,
return_format=[
'distance', 'models',
'agents',
'scaled residuals',
'moments'
])
if out > y_previous:
print('no reduction in function value obtained')
yfactor = 0.5 * yfactor + 0.5
y_previous = out
def run(adj_name, fix, educ_name, resume=False, noplot=False):
# at first this takes point x saved in estimates.py
# by calling get_point(high_e). x is a dict with parameter names and values
# then it applies adjustment fix (a dict) to the point x
# resulting file name is educ_name + adj_name
# adj_name -- name of adjustment
# fix -- dictionary containing parameters of setup.py to change
# educ_name -- name of education group ("col" or "hs")
# high_e -- input to get_point function, True for college, False for HS
high_e = (educ_name == 'col' or educ_name == 'college')
low_e = (educ_name == 'hs' or educ_name == 'high school')
assert (high_e or low_e), 'wrong specifier for education'
x, targ_mode = get_point(high_e, read_wisdom=False)
tar = target_values(targ_mode)
print('\n\n\n\n\n\n')
print('doing {} {}'.format(educ_name, adj_name))
x_new = x.copy()
fix = fix.copy()
if 'multiply' in fix:
mult = fix.pop('multiply')
for m in mult:
x_new[m] *= mult[m]
x_new.update(fix)
print(x_new)
name = '{} {}'.format(educ_name, adj_name)
fname = '{}.pkl'.format(name)
try:
if resume:
try:
mom = filer(fname, 0, 0, repeat=False)
skip = True
except:
skip = False
else:
skip = False
if not skip:
print("computing {}".format(fname))
out, mom = mdl_resid(x=x_new,
targets=tar,
return_format=['distance', 'moments'],
verbose=False,
draw=False,
cs_moments=False,
save_to='mdl for {}'.format(fname),
moments_save_name=name,
moments_repeat=5,
Tsim=42)
print("file {} saved".format(fname))
else:
print("file {} already exists".format(fname))
print('done, doing fit plots')
try:
if adj_name == 'baseline':
fp = FitPlots(targ_mode=targ_mode,
compare=None,
base='{} baseline.pkl'.format(educ_name),
compare_name='Data',
base_name='baseline',
moments_aux=None,
noplot=noplot)
else:
fp = FitPlots(targ_mode=targ_mode,
compare='{} baseline.pkl'.format(educ_name),
base=fname,
compare_name='baseline',
base_name=adj_name,
moments_aux=None,
noplot=noplot)
except:
print('something wrong with fit plots...')
except KeyboardInterrupt:
raise (KeyboardInterrupt)
except BaseException as a:
print("file {} {}.pkl could not be produced. Exception:".format(
name, adj_name))
print(a)
print(' ')
return mom, fp
os.environ['QT_QPA_PLATFORM'] = 'offscreen'
from residuals import mdl_resid
print('Hi!')
import os
os.environ['MKL_CBWR'] = 'AUTO'
from estimates import get_point
if __name__ == '__main__':
x_base, tm = get_point(True, read_wisdom=False) # high education
tar = target_values(tm)
x_comp, _ = get_point(False, read_wisdom=False) # low education
pmeets = ['pmeet_21', 'pmeet_30', 'pmeet_40']
ppregs = ['preg_21', 'preg_28', 'preg_35']
abortions = ['abortion_costs', 'p_abortion_access']
prefs = [
'disutil_marry_sm_mal', 'disutil_shotgun', 'u_lost_divorce',
'util_qbar'
]
# form xlist
xlist = [x_base]
he = ['high education'],
replace_fields = [('trend only', he), ('pmeets', pmeets),
('ppregs', ppregs), ('remar', ['disutil_marry_sm_mal']),
def main(read_wisdom=False,erase=False):
high_e = True
if erase:
try:
os.remove('az_dist_fem.pkl')
print('removed')
except:
pass
try:
os.remove('az_dist_mal.pkl')
print('removed')
except:
pass
x, targ_mode = get_point(high_e,read_wisdom=read_wisdom)
tar = target_values(targ_mode)
this_name = 'college baseline' if high_e else 'high school baseline'
out, mdl, agents, res, mom = mdl_resid(x=x,targets=tar,
return_format=['distance','models','agents','scaled residuals','moments'],
#load_from='mdl.pkl',
verbose=True,draw=True,cs_moments=False,
moments_save_name = this_name,
moments_repeat=1)
#agents.sim_graphs()
mdl[0].time_statistics()
print('Done. Residual in point x0 is {}'.format(out))
from fit_plot import FitPlots
fp = FitPlots(targ_mode=targ_mode,
compare=None,
base=this_name+'.pkl',
compare_name='data',
base_name='college baseline',
moments_aux=None) #,moments_aux=moments_aux)
'''
from fit_plot import FitPlots
fp = FitPlots(targ_mode=targ_mode,
compare='col baseline.pkl',
base='col double social stigma.pkl',
compare_name='baseline',
base_name='double stigma',
moments_aux=None) #,moments_aux=moments_aux)
'''
mdl[0].mar_graphs(t = 4)
return locals()