def get_point(high_e,read_wisdom=False):
if high_e:
x = {'sigma_psi': 0.02351795998155682, 'sigma_psi_init': 0.06319749482803798, 'pmeet_21': 0.36231065032307264, 'pmeet_30': 1.2185173549427066, 'pmeet_40': 1.0, 'preg_21': 0.006683178914886262, 'preg_28': 0.030446109898106186, 'preg_35': 0.026257274180916695, 'u_shift_mar': 0.24749413577501758, 'util_alp': 0.5739290298121097, 'util_kap': 8.484242183675391, 'util_qbar': 5.561954628246849, 'disutil_marry_sm_mal': 4.408415907989258, 'disutil_shotgun': 0.4715857724582857, 'abortion_costs': 1.8092065614536414, 'p_abortion_access': 0.9512267376733684, 'u_lost_divorce': 0.8892578980806901, 'mu_psi_init': -0.15876965206098093,
'high education': True}
if read_wisdom:
try:
print('read wisdom from file!')
o = filer('wisdom.pkl',0,0,repeat=False)[0]
x = calibration_params()[-1](o[1])
print(x)
print('saved distance is {}'.format(o[0]))
x.update({'high education': True})
except:
print('failed to read from wisdom file')
targ_mode = 'high education'
else:
x = {'sigma_psi': 0.04119975516565719, 'sigma_psi_init': 0.07184509981781, 'pmeet_21': 0.7300641341551373, 'pmeet_30': 0.38552526708748397, 'pmeet_40': 1.4132304041226518, 'preg_21': 0.1029100967053943, 'preg_28': 0.11241132276639117, 'preg_35': 0.11203564468462099, 'u_shift_mar': 0.338428482678413, 'util_alp': 0.5195282434982275, 'util_kap': 7.152398760885778, 'util_qbar': 0.0, 'disutil_marry_sm_mal': 3.18966037249299, 'disutil_shotgun': 0.3647670950676456, 'abortion_costs': 0.2962878054482049, 'p_abortion_access': 0.6662167114665236, 'u_lost_divorce': 0.5275074834332285, 'mu_psi_init': -0.24342175587968384,
'high education': False}
targ_mode = 'low education'
if read_wisdom:
try:
print('read wisdom from file!')
o = filer('wisdom.pkl',0,0,repeat=False)[0]
x = calibration_params()[-1](o[1])
print(x)
print('saved distance is {}'.format(o[0]))
x.update({'high education': False})
except:
print('failed to read from wisdom file')
return x, targ_mode
def tiktak(*,
N,
N_st,
xfix=None,
skip_global=False,
skip_local=False,
resume_global=False,
resume_local=False):
xl, xu, x0, keys, translator = calibration_params(xfix=xfix)
#Initial cheks
assert len(xl) == len(xu)
assert N >= N_st
############################################
#1 INITIALIZATION
###########################################
if not skip_global:
#First Create a Sobol Sequence
init = sobol_seq.i4_sobol_generate(
len(xl), N) # generate many draws from uniform
#init=init[:,0]
#Get point on the grid
x_init = xl * (1 - init) + xu * init
x_init = x_init.T
x_init = x_init.squeeze()
#Get fitness of initial points
pts = [('compute', translator(x_init[:, j])) for j in range(N)]
fx_init = compute_for_values(pts, resume=resume_global)
fx_init = (np.array(fx_init)).squeeze()
# !! not the optimizer returns squared value of mdl_resid
#Sort in ascending order of fitness
order = np.argsort(fx_init)
fx_init = fx_init[order]
x_init = x_init[:, order]
filer('sobol_results.pkl', (fx_init, x_init), True)
print('saved the results succesfully')
else:
(fx_init, x_init) = filer('sobol_results.pkl', None, False)
print('loaded the results from the file')
#Take only the first N_st realization
fx_init = fx_init[0:N_st]
x_init = x_init[:, 0:N_st]
if skip_local:
print('local minimizers are skipped')
return x_init[:, 0]
#Create a file with sobol sequence points
filer('sobol.pkl', x_init, True)
if not resume_local:
#List containing parameters and save them in file
param = list([(fx_init[0], x_init[:, 0])])
filer('wisdom.pkl', param, True)
i_start = 0
else:
param = filer('wisdom.pkl', None, False)
i_start = len(param)
vals = [('minimize', (i, N_st, xfix)) for i in range(i_start, N_st)]
compute_for_values(vals, timeout=7200.0)
param = filer('wisdom.pkl', None, write=False)
############################################
#3 TOPPING RULE
###########################################
#print(999,ite)
#Final Refinement
return param[0]
def fun(x):
assert type(x) is tuple, 'x must be a tuple!'
action = x[0]
args = x[1]
assert type(action) is str, 'x[0] should be string for action'
assert len(x) <= 2, 'too many things in x! x is (action,agrs)'
if action == 'test':
return mdl_resid()
elif action == 'compute':
return mdl_resid(args)
elif action == 'minimize':
import dfols
import pybobyqa
i, N_st, xfix = args
xl, xu, x0, keys, translator = calibration_params(xfix=xfix)
#Sort lists
def sortFirst(val):
return val[0]
#Get the starting point for local minimization
#Open File with best solution so far
param = filer('wisdom.pkl', 0, False)
param.sort(key=sortFirst)
print('f best so far is {} and x is {}'.format(param[0][0],
param[0][1]))
xm = param[0][1]
#Get right sobol sequence point
xt = filer('sobol.pkl', None, False)
#Determine the initial position
dump = min(max(0.1, ((i + 1) / N_st)**(0.5)), 0.995)
xc = dump * xm + (1 - dump) * xt[:, i]
xc = xc.squeeze()
print('The initial position is {}'.format(xc))
#Standard Way
def q(pt):
try:
ans = mdl_resid(translator(pt),
return_format=['scaled residuals'])[0]
except:
print('During optimization function evaluation failed at {}'.
format(pt))
ans = np.array([1e6])
finally:
gc.collect()
return ans
res = dfols.solve(q,
xc,
rhobeg=0.01,
rhoend=1e-4,
maxfun=100,
bounds=(xl, xu),
scaling_within_bounds=True,
objfun_has_noise=False,
print_progress=True)
#res=pybobyqa.solve(q, xc, rhobeg = 0.001, rhoend=1e-6, maxfun=80, bounds=(xl,xu),
# scaling_within_bounds=True,objfun_has_noise=False,print_progress=True)
print(res)
if res.flag == -1:
raise Exception('solver returned something creepy...')
fbest = mdl_resid(translator(
res.x))[0] # in prnciple, this can be inconsistent with
# squared sum of residuals
print('fbest is {} and res.f is {}'.format(fbest, res.f))
print('Final value is {}'.format(fbest))
param_new = filer('wisdom.pkl', None, False)
param_write = param_new + [(fbest, res.x)]
#Save Updated File
param_write.sort(key=sortFirst)
filer('wisdom.pkl', param_write, True)
return fbest
else:
raise Exception('unsupported action or format')
def mdl_resid(x=None,
targets=None,
weights=w,
save_to=None,
load_from=None,
return_format=['distance'],
store_path=None,
verbose=False,
draw=False,
graphs=False,
rel_diff=False,
cs_moments=False,
moments_repeat=5,
Tsim=30,
moments_save_name=None):
from model import Model
from setup import DivorceCosts
from simulations import Agents
from crosssection import CrossSection
from calibration_params import calibration_params
if type(x) is dict:
kwords = x
if 'targets' in x:
targets = x.pop('targets')
else:
lb, ub, xdef, keys, translator = calibration_params()
if x is None:
x = xdef
kwords = translator(x)
if verbose: print(kwords)
# this is for the default model
#dc_k = DivorceCosts(unilateral_divorce=True,assets_kept = 1.0,u_lost_m=0.00,u_lost_f=0.00,eq_split=1.0)
#dc_nk = DivorceCosts(unilateral_divorce=True,assets_kept = 1.0,u_lost_m=0.00,u_lost_f=0.00,eq_split=1.0)
def join_path(name, path):
return os.path.join(path, name)
if load_from is not None:
if type(load_from) is not list:
load_from = [load_from]
if store_path is not None:
load_from = [join_path(n, store_path) for n in load_from]
if save_to is not None:
if type(save_to) is not list:
save_to = [save_to]
if store_path is not None:
save_to = [join_path(n, store_path) for n in save_to]
if load_from is None:
mdl = Model(verbose=verbose, **kwords)
mdl_list = [mdl]
else:
mdl_list = [dill.load(open(l, 'rb+')) for l in load_from]
mdl = mdl_list[0]
if save_to is not None:
if len(save_to) > 1:
print('warning: too much stuff is save_to')
dill.dump(mdl, open(save_to[0], 'wb+'))
np.random.seed(18)
agents = Agents(mdl_list, verbose=verbose, fix_seed=False, T=Tsim)
if not cs_moments:
moments_list = [agents.compute_moments()] + [
Agents(mdl_list, verbose=False, T=Tsim,
fix_seed=False).compute_moments()
for _ in range(moments_repeat - 1)
]
else:
moments_list = [
CrossSection(mdl_list,
verbose=False,
N_total=30000,
fix_seed=False).compute_moments()
for _ in range(moments_repeat)
]
mom = {
key: np.mean([m[key] for m in moments_list], axis=0)
for key in moments_list[0].keys()
}
#mom_join = Agents( mdl_list, N=10000, T=18, female=False, verbose=False).aux_moments()
#mom_men = agents_extra.compute_moments()
#mom.update(mom_join)
if moments_save_name: # is not None can be ommited
filer('{}.pkl'.format(moments_save_name), mom, True)
if targets is None:
from targets import target_values
tar = target_values()
elif type(targets) is str:
from targets import target_values
tar = target_values(mode=targets)
else:
tar = targets
resid_all, resid_sc, dist = distance_to_targets(mom,
tar,
weights=weights,
report=verbose)
#if verbose:
# print('data moments are {}'.format(dat))
# print('simulated moments are {}'.format(sim))
tt = mdl_list[0].get_total_time()
print('Distance {}, time {}'.format(dist, tt))
out_dict = {
'distance': dist,
'all residuals': resid_all,
'scaled residuals': resid_sc,
'models': mdl_list,
'agents': agents,
'moments': mom
}
out = [out_dict[key] for key in return_format]
del (out_dict)
if 'models' not in return_format:
for m in mdl_list:
del (m)
del mdl_list
if 'agents' not in return_format:
del (agents)
if len(out) == 1: out = out[0]
return out
fix_values = False
if fix_values:
xfix = {'sigma_psi': 0.2476722655689144,
'sigma_psi_mult': 5.125801752198881,
'u_shift_mar': 1.7329041070973545,
'util_alp': 0.6182672481649074,
'util_kap': 0.8081836080864513,
'taste_shock_mult': 4.116448914683272
}
else:
xfix = None
lb, ub, xdef, keys, translator = calibration_params(xfix=xfix)
print('calibration adjusts {}'.format(keys))
print('')
print('')
print('running tic tac...')
print('')
print('')
#Tik Tak Optimization
param=tiktak(xfix=xfix,N=20000,N_st=250,skip_local=False,skip_global=True,
resume_global=False,resume_local=False)
def run(resume=False, high_e=True):
xinit, targ_mode = get_point(high_e, read_wisdom=False)
tar = target_values(targ_mode)
if resume:
x_load = filer('wisdom_refined.pkl', 0, 0)
prob_meet_load = x_load['pmeet_exo']
prob_preg_load = x_load['ppreg_exo']
xinit = x_load
out, mdl, agents, res, mom = mdl_resid(x=xinit,
targets=tar,
return_format=[
'distance', 'models', 'agents',
'scaled residuals', 'moments'
],
verbose=False)
print('initial distance is {}'.format(out))
if resume:
prob_meet_init = prob_meet_load
prob_preg_init = prob_preg_load
else:
prob_meet_init = np.array(
mdl[0].setup.pars['pmeet_t'][:mdl[0].setup.pars['Tmeet']])
prob_preg_init = np.array([
mdl[0].setup.upp_precomputed_fem[t][3]
for t in range(mdl[0].setup.pars['Tmeet'])
])
nopt = 10
yfactor = 1.5
for iopt in range(nopt):
print('running esimation round {}'.format(iopt))
print('estimating probabilities:')
prob_meet_est = 0.0
prob_preg_est = 0.0
nrep = 4 if iopt > 0 else 1
np.random.seed(12)
for rep in range(nrep):
o = AgentsEst(mdl, T=30, verbose=False, fix_seed=False)
prob_meet_est += (1 / nrep) * o.pmeet_exo.copy()
prob_preg_est += (1 / nrep) * o.ppreg_exo.copy()
print('estimated pmeet = {}'.format(prob_meet_est))
print('estimated ppreg = {}'.format(prob_preg_est))
# this does binary search
w = 1.0
factor = 0.5
ne = prob_meet_est.size
nw = 10
print('reference value is {}'.format(out))
y_previous = out
for i in range(nw):
prob_meet_w = w * prob_meet_est + (1 - w) * prob_meet_init[:ne]
prob_preg_w = w * prob_preg_est + (1 - w) * prob_preg_init[:ne]
xsearch = xinit.copy()
xsearch.update({
'pmeet_exo': prob_meet_w,
'ppreg_exo': prob_preg_w
})
out_w = mdl_resid(x=xsearch,
targets=tar,
return_format=['distance'],
verbose=False)
print('with weight = {}, distance is {}'.format(w, out_w))
if out_w < yfactor * out:
print('found a potentially imporving weight for yfactor = {}'.
format(yfactor))
break
else:
w = factor * w
if i < nw - 1:
print('trying new weight = {}'.format(w))
else:
print('no luck...')
xfix = {
k: xinit[k]
for k in [
'pmeet_21', 'pmeet_30', 'pmeet_40', 'preg_21', 'preg_28',
'preg_35'
]
}
lb, ub, _, keys, translator = calibration_params(xfix=xfix)
def tr(x):
xx = translator(x)
xx.update({'pmeet_exo': prob_meet_w, 'ppreg_exo': prob_preg_w})
return xx
x0 = [xinit[key] for key in keys]
x0, lb, ub = np.array(x0), np.array(lb), np.array(ub)
print('starting from {}'.format(tr(x0)))
tar = target_values('high education')
def q(pt):
#print('computing at point {}'.format(translator(pt)))
try:
ans = mdl_resid(tr(pt), return_format=['scaled residuals'])
except BaseException as a:
print('During optimization function evaluation failed at {}'.
format(pt))
print(a)
ans = np.array([1e6])
finally:
gc.collect()
return ans
res = dfols.solve(q,
x0,
rhobeg=0.02,
rhoend=1e-5,
maxfun=60,
bounds=(lb, ub),
scaling_within_bounds=True,
objfun_has_noise=False,
npt=len(x0) + 5,
user_params={
'restarts.use_restarts': True,
'restarts.rhoend_scale': 0.5,
'restarts.increase_npt': True
})
print(res)
print('Result is {}'.format(tr(res.x)))
filer('wisdom_refined.pkl', tr(res.x), True)
print('wrote to the file!')
xinit = tr(res.x)
out, mdl, agents, res, mom = mdl_resid(x=xinit,
return_format=[
'distance', 'models',
'agents',
'scaled residuals',
'moments'
])
if out > y_previous:
print('no reduction in function value obtained')
yfactor = 0.5 * yfactor + 0.5
y_previous = out
from numpy.random import random_sample as rs
from data_moments import dat_moments
from tiktak import tiktak
print('Hi!')
from residuals import mdl_resid
from calibration_params import calibration_params
if __name__ == '__main__':
#Build data moments and pickle them
dat_moments(period=1, sampling_number=4, transform=2) # refresh
#Initialize the file with parameters
lb, ub, x0, keys, translator = calibration_params(
) # bounds are set in a separate file
##### FIRST LET'S TRY TO RUN THE FUNCTION IN FEW POINTS
print('Testing the workers...')
from p_client import compute_for_values
pts = [lb + rs(lb.shape) * (ub - lb) for _ in range(1)]
pts = [('compute', translator(x)) for x in pts]
outs = compute_for_values(pts, timeout=72000.0)
print('Everything worked, output is {}'.format(outs))
print('')
print('')
print('running tic tac...')
print('')
print('')
import gc
import dfols
from residuals import mdl_resid
from targets import target_values
print('Hi!')
import os
os.environ['MKL_CBWR'] = 'AUTO'
if __name__ == '__main__':
from calibration_params import calibration_params
lb, ub, x0, keys, translator = calibration_params()
weight = 0.9
import numpy as np
x0, lb, ub = np.array(x0), np.array(lb), np.array(ub)
x_rand = lb + np.random.random_sample(len(lb)) * (ub - lb)
x_init = x0 * weight + (1 - weight) * x_rand
print('starting from {}'.format(x_init))
tar = target_values('high education')
def q(pt):
#print('computing at point {}'.format(translator(pt)))
Created on Mon Mar 9 09:13:02 2020
@author: egorkozlov
"""
from model import Model
import numpy as np
import os
os.environ['MKL_CBWR'] = 'AUTO'
from numba import config
config.NUMBA_NUM_THREADS = 2
x0 = np.array([0.2, 0.0710307, 1.11501, 0.543047, 0.050264, 0.005, -0.09])
from calibration_params import calibration_params
pars = calibration_params()[-1](x0)
print(pars)
pars.pop('alost')
mdl = Model(**pars, verbose=False, solve_till=0, display_v=False)
mdl.time_statistics()
decisions = mdl.decisions[-2]['Female, single']['Decision']
assert np.allclose(mdl.V[0]['Couple, M']['V'].mean(), -342.69626387623344)
assert np.allclose(mdl.V[0]['Female, single']['V'].mean(), -242.52742451161123)
def mdl_resid(x=None,
save_to=None,
load_from=None,
return_format=['distance'],
solve_transition=True,
store_path=None,
verbose=False,
calibration_report=False,
draw=False,
graphs=False,
rel_diff=True,
welf=False):
from model import Model
from setup import DivorceCosts
from simulations import Agents, AgentsPooled
from moments import moment
if type(x) is dict:
params = x
else:
from calibration_params import calibration_params
lb, ub, x0, keys, translator = calibration_params()
if verbose: print('Calibration adjusts {}'.format(keys))
if x is None: x = x0
params = translator(
x
) # this converts x to setup parameters according to what is in calibration_params
try:
ulost = params.pop(
'ulost'
) # ulost does not belong to setup parameters so this pop removes it
except:
ulost = 0.0
try:
alost = params.pop('alost')
except:
alost = 0.0
# this is for the default model
dc = DivorceCosts(unilateral_divorce=False,
assets_kept=1.0,
u_lost_m=ulost,
u_lost_f=ulost,
eq_split=1.0)
sc = DivorceCosts(unilateral_divorce=True,
assets_kept=1.0,
u_lost_m=0.00,
u_lost_f=0.00,
eq_split=0.0)
iter_name = 'default' if not verbose else 'default-timed'
def join_path(name, path):
return os.path.join(path, name)
if load_from is not None:
if type(load_from) is not list:
load_from = [load_from]
if store_path is not None:
load_from = [join_path(n, store_path) for n in load_from]
if save_to is not None:
if type(save_to) is not list:
save_to = [save_to]
if store_path is not None:
save_to = [join_path(n, store_path) for n in save_to]
if load_from is None:
if not solve_transition:
mdl = Model(iterator_name=iter_name,
divorce_costs=dc,
separation_costs=sc,
draw=draw,
**params)
mdl_list = [mdl]
else:
# specify the changes here manually
dc_before = DivorceCosts(unilateral_divorce=False,
assets_kept=1.0,
u_lost_m=ulost,
u_lost_f=ulost,
eq_split=1.0)
dc_after = DivorceCosts(unilateral_divorce=True,
assets_kept=1.0,
u_lost_m=ulost,
u_lost_f=ulost,
eq_split=1.0)
mdl_before = Model(iterator_name=iter_name,
divorce_costs=dc_before,
separation_costs=sc,
draw=draw,
**params)
mdl_after = Model(iterator_name=iter_name,
divorce_costs=dc_after,
separation_costs=sc,
draw=draw,
**params)
mdl = mdl_before # !!! check if this makes a difference
# I think that it is not used for anything other than getting
# setup for plotting
mdl_list = [mdl_before, mdl_after]
else:
mdl_list = [dill.load(open(l, 'rb+')) for l in load_from]
mdl = mdl_list[0]
if solve_transition:
if len(mdl_list) < 2:
print(
'Warning: you supplied only one model, so no transition is computed'
)
if save_to is not None:
if not solve_transition:
if len(save_to) > 1:
print('warning: too much stuff is save_to')
dill.dump(mdl, open(save_to[0], 'wb+'))
else:
if len(save_to) > 1:
[
dill.dump(m_i, open(n_i, 'wb+'))
for (m_i, n_i) in zip(mdl_list, save_to)
]
else:
print('Warning: file names have change to write two models, \
please provide the list of names if you do not want this'
)
dill.dump(mdl_before, open(save_to[0] + '_before', 'wb+'))
dill.dump(mdl_after, open(save_to[0] + '_after', 'wb+'))
##############################################################
# Build Markov transition processes for models from the data
#############################################################
#Import Data
with open('age_uni.pkl', 'rb') as file:
age_uni = pickle.load(file)
def get_transition(age_dist, welf=False):
#Transformation of age at uni from actual age to model periods
change = -np.ones(
1000, np.int32
) #the bigger is the size of this array, the more precise the final distribution
summa = 0.0
summa1 = 0.0
for i in age_dist:
summa += age_dist[i]
change[int(summa1 * len(change[:]) / sum(age_dist.values(
))):int(summa * len(change[:]) /
sum(age_dist.values()))] = (i - 18) / mdl.setup.pars['py']
summa1 += age_dist[i]
change = np.sort(change, axis=0)
#Now we compute the actual conditional probabilities
transition_matricest = list()
#First period treated differently
pr = np.sum(change <= 0) / (np.sum(change <= np.inf))
transition_matricest = transition_matricest + [
np.array([[1 - pr, pr], [0, 1]])
]
for t in range(mdl.setup.pars['T'] - 1):
if not welf:
pr = np.sum(change == t + 1) / (np.sum(change <= np.inf))
transition_matricest = transition_matricest + [
np.array([[1 - pr, pr], [0, 1]])
]
else:
transition_matricest = transition_matricest + [
np.array([[1, 0], [1, 0]])
]
return transition_matricest
transition_matricesf = get_transition(age_uni['female'], welf)
transition_matricesm = get_transition(age_uni['male'], welf)
#Get Number of simulated agent, malea and female
N = 45000
Nf = int(N * age_uni['share_female'])
Nm = N - Nf
agents_fem = Agents(mdl_list,
age_uni['female'],
female=True,
pswitchlist=transition_matricesf,
verbose=False,
N=Nf,
draw=draw)
agents_mal = Agents(mdl_list,
age_uni['male'],
female=False,
pswitchlist=transition_matricesm,
verbose=False,
N=Nm,
draw=draw)
agents_pooled = AgentsPooled([agents_fem, agents_mal])
#Compute moments
moments = moment(mdl_list, agents_pooled, agents_mal, draw=draw)
############################################################
#Build data moments and compare them with simulated ones
###########################################################
#Get Data Moments
with open('moments.pkl', 'rb') as file:
packed_data = pickle.load(file)
#Unpack Moments (see data_moments.py to check if changes)
#(hazm,hazs,hazd,mar,coh,fls_ratio,W)
hazm_d = packed_data['hazm']
hazs_d = packed_data['hazs']
hazd_d = packed_data['hazd']
mar_d = packed_data['emar']
coh_d = packed_data['ecoh']
fls_d = packed_data['fls_ratio']
wage_d = np.ones(1) * packed_data['wage_ratio']
div_d = np.ones(1) * packed_data['div_ratio']
beta_unid_d = np.ones(1) * packed_data['beta_unid']
mean_fls_d = np.ones(1) * packed_data['mean_fls']
W = packed_data['W']
dat = np.concatenate((hazm_d, hazs_d, hazd_d, mar_d, coh_d, fls_d, wage_d,
div_d, beta_unid_d, mean_fls_d),
axis=0)
#Get Simulated Data
Tret = mdl.setup.pars['Tret']
hazm_s = moments['hazard mar'][0:len(hazm_d)]
hazs_s = moments['hazard sep'][0:len(hazs_d)]
hazd_s = moments['hazard div'][0:len(hazd_d)]
mar_s = moments['share mar'][0:len(mar_d)]
coh_s = moments['share coh'][0:len(coh_d)]
beta_unid_s = np.ones(1) * moments['beta unid']
mean_fls_s = np.ones(1) * moments['mean_fls']
fls_s = moments['fls_ratio']
wage_s = np.ones(1) * moments['wage_ratio']
div_s = np.ones(1) * moments['div_ratio']
sim = np.concatenate((hazm_s, hazs_s, hazd_s, mar_s, coh_s, fls_s, wage_s,
div_s, beta_unid_s, mean_fls_s),
axis=0)
if len(dat) != len(sim):
sim = np.full_like(dat, 1.0e6)
res_all = (dat - sim)
if verbose:
print('data moments are {}'.format(dat))
print('simulated moments are {}'.format(sim))
resid_all = np.array(
[x if (not np.isnan(x) and not np.isinf(x)) else 1e6 for x in res_all])
resid_sc = resid_all * np.sqrt(np.diag(W)) # all residuals scaled
dist = np.dot(np.dot(resid_all, W), resid_all)
print('Distance is {}'.format(dist))
out_dict = {
'distance': dist,
'all residuals': resid_all,
'scaled residuals': resid_sc,
'models': mdl_list,
'agents': agents_pooled
}
out = [out_dict[key] for key in return_format]
del (out_dict)
# memory management
if ('models' not in return_format) | (draw == False):
for m in mdl_list:
del (m)
del mdl_list
if ('agents' not in return_format) | (draw == False):
del (agents_pooled, agents_fem, agents_mal)
return out
def fun(x):
assert type(x) is tuple, 'x must be a tuple!'
action = x[0]
args = x[1]
assert type(action) is str, 'x[0] should be string for action'
assert len(x) <= 2, 'too many things in x! x is (action,agrs)'
if action == 'test':
return mdl_resid()
elif action == 'compute':
return mdl_resid(args)
elif action == 'moments':
agents = mdl_resid(args, return_format=['agents'])
mom = agents.compute_moments()
return mom
elif action == 'minimize':
import dfols
i, N_st, xfix = args
xl, xu, x0, keys, translator = calibration_params(xfix=xfix)
#Sort lists
def sortFirst(val):
return val[0]
#Get the starting point for local minimization
#Open File with best solution so far
param = filer('wisdom.pkl', 0, False)
param.sort(key=sortFirst)
print('f best so far is {} and x is {}'.format(param[0][0],
param[0][1]))
xm = param[0][1]
#Get right sobol sequence point
xt = filer('sobol.pkl', None, False)
#Determine the initial position
dump = min(max(0.1, ((i + 1) / N_st)**(0.5)), 0.995)
xc = dump * xm + (1 - dump) * xt[:, i]
xc = xc.squeeze()
def q(pt):
try:
ans = mdl_resid(translator(pt),
moments_repeat=3,
return_format=['scaled residuals'])
except BaseException as a:
print('During optimization function evaluation failed at {}'.
format(pt))
print(a)
ans = np.array([1e6])
finally:
gc.collect()
return ans
res = dfols.solve(
q,
xc,
rhobeg=0.15,
rhoend=1e-6,
maxfun=npt,
bounds=(xl, xu),
#npt=len(xc)+5,
scaling_within_bounds=True,
#user_params={'tr_radius.gamma_dec':0.75,'tr_radius.gamma_inc':1.5,
# 'tr_radius.alpha1':0.5,'tr_radius.alpha2':0.75,
# 'regression.momentum_extra_steps':True,
#'restarts.use_restarts':True},
objfun_has_noise=True)
print(res)
if res.flag == -1:
raise Exception('solver returned something creepy...')
fbest = mdl_resid(translator(
res.x)) # in prnciple, this can be inconsistent with
# squared sum of residuals
print('fbest is {} and res.f is {}'.format(fbest, res.f))
print('Final value is {}'.format(fbest))
param_new = filer('wisdom.pkl', None, False)
param_write = param_new + [(fbest, res.x)]
#Save Updated File
param_write.sort(key=sortFirst)
filer('wisdom.pkl', param_write, True)
return fbest
else:
raise Exception('unsupported action or format')