def solve_policy(model, tol=1e-6, grid={}, distribution={}, integration_orders=None, maxit=500, maxit_howard=500, verbose=False, hook=None, initial_dr=None, pert_order=1):
'''
Solve for the value function and associated decision rule by iterating over
the value function.
Parameters
-----------
model :
``dtcscc`` model. Must contain a ``felicity`` function.
Returns
--------
dr : Markov decision rule
The solved decision rule/policy function
drv : decision rule
The solved value function
'''
assert (model.model_type == 'dtcscc')
def vprint(t):
if verbose:
print(t)
# transition(s, x, e, p, out), felicity(s, x, p, out)
transition = model.functions['transition']
felicity = model.functions['felicity']
controls_lb = model.functions['controls_lb']
controls_ub = model.functions['controls_ub']
parms = model.calibration['parameters']
discount = model.calibration['beta']
x0 = model.calibration['controls']
s0 = model.calibration['states']
r0 = felicity(s0, x0, parms)
approx = model.get_grid(**grid)
# a = approx.a
# b = approx.b
# orders = approx.orders
distrib = model.get_distribution(**distribution)
sigma = distrib.sigma
# Possibly use the approximation orders?
if integration_orders is None:
integration_orders = [3] * sigma.shape[0]
[nodes, weights] = gauss_hermite_nodes(integration_orders, sigma)
interp_type = approx.interpolation
drv = create_interpolator(approx, interp_type)
if initial_dr is None:
if pert_order == 1:
dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
else:
dr = initial_dr
grid = drv.grid
N = grid.shape[0] # Number of states
n_x = grid.shape[1] # Number of controls
controls = dr(grid)
controls_0 = np.zeros((N, n_x))
controls_0[:, :] = model.calibration['controls'][None, :]
values_0 = np.zeros((N, 1))
values_0[:, :] = r0/(1-discount)
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format('N', ' Error','Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
t1 = time.time()
# FIRST: value function iterations, 10 iterations to start
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
err_x = 100
err_x_0 = 100
if verbose:
print(stars)
print('Starting value function iteration')
print(stars)
while err_v > tol and it < 10:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
for n in range(N):
s = grid[n, :]
x = controls[n, :]
lb = controls_lb(s, parms)
ub = controls_ub(s, parms)
bnds = [e for e in zip(lb, ub)]
def valfun(xx):
return -choice_value(transition, felicity, s, xx, drv, nodes,
weights, parms, discount)[0]
res = minimize(valfun, x, bounds=bnds, tol=1e-4)
controls[n, :] = res.x
values[n, 0] = -valfun(res.x)
# compute error, update value and dr
err_x = abs(controls - controls_0).max()
err_v = abs(values - values_0).max()
t_end = time.time()
elapsed = t_end - t_start
values_0 = values
controls_0 = controls
gain_x = err_x / err_x_0
gain_v = err_v / err_v_0
err_x_0 = err_x
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
# SECOND: Howard improvement step, 10-20 iterations
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
if verbose:
print(stars)
print('Starting Howard improvement step')
print(stars)
while err_v > tol and it < maxit_howard:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
# controls = controls_0.copy() # No need to keep updating
for n in range(N):
s = grid[n, :]
x = controls[n, :]
values[n, 0] = choice_value(transition, felicity, s, x, drv, nodes,
weights, parms, discount)[0]
# compute error, update value function
err_v = abs(values - values_0).max()
values_0 = values
t_end = time.time()
elapsed = t_end - t_start
gain_v = err_v / err_v_0
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
# THIRD: Back to value function iteration until convergence.
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
err_x = 100
err_x_0 = 100
if verbose:
print(stars)
print('Starting value function iteration')
print(stars)
while err_v > tol and it < maxit:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
for n in range(N):
s = grid[n, :]
x = controls[n, :]
lb = controls_lb(s, parms)
ub = controls_ub(s, parms)
bnds = [e for e in zip(lb, ub)]
def valfun(xx):
return -choice_value(transition, felicity, s, xx, drv, nodes,
weights, parms, discount)[0]
res = minimize(valfun, x, bounds=bnds, tol=1e-4)
controls[n, :] = res.x
values[n, 0] = -valfun(res.x)
# compute error, update value and dr
err_x = abs(controls - controls_0).max()
err_v = abs(values - values_0).max()
t_end = time.time()
elapsed = t_end - t_start
values_0 = values
controls_0 = controls
gain_x = err_x / err_x_0
gain_v = err_v / err_v_0
err_x_0 = err_x
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
if it == maxit:
warnings.warn(UserWarning("Maximum number of iterations reached"))
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# final value function and decision rule
drv.set_values(values_0)
dr = create_interpolator(approx, interp_type)
dr.set_values(controls_0)
return dr, drv
def evaluate_policy(model, dr, tol=1e-8, grid={}, distribution={}, maxit=2000, verbose=False, hook=None,
integration_orders=None):
'''
Compute value function corresponding to policy ``dr``
Parameters:
-----------
model:
"dtcscc" model. Must contain a 'value' function.
dr:
decision rule to evaluate
Returns:
--------
decision rule:
value function (a function of the space similar to a decision rule
object)
'''
assert (model.model_type == 'dtcscc')
vfun = model.functions["value"]
gfun = model.functions['transition']
parms = model.calibration['parameters']
n_vals = len(model.symbols['values'])
t1 = time.time()
err = 1.0
it = 0
approx = model.get_grid(**grid)
interp_type = approx.interpolation
distrib = model.get_distribution(**distribution)
sigma = distrib.sigma
drv = create_interpolator(approx, approx.interpolation)
grid = drv.grid
N = drv.grid.shape[0]
controls = dr(grid)
guess_0 = model.calibration['values']
guess_0 = guess_0[None, :].repeat(N, axis=0)
if not integration_orders:
integration_orders = [3] * sigma.shape[0]
[nodes, weights] = gauss_hermite_nodes(integration_orders, sigma)
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format('N', ' Error', 'Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
err_0 = 1.0
while err > tol and it < maxit:
if hook:
hook()
t_start = time.time()
it += 1
# update spline coefficients with current values
drv.set_values(guess_0)
# update the geuss of value functions
guess = update_value(gfun, vfun, grid, controls, dr, drv,
nodes, weights, parms, n_vals)
# compute error
err = abs(guess - guess_0).max()
err_SA = err / err_0
err_0 = err
# update guess
guess_0[:] = guess.copy()
# print update to user, if verbose
t_end = time.time()
elapsed = t_end - t_start
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err, err_SA, elapsed))
if it == maxit:
warnings.warn(UserWarning("Maximum number of iterations reached"))
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
return drv
def parameterized_expectations_direct(model,
verbose=False,
dr0=None,
pert_order=1,
grid={},
distribution={},
maxit=100,
tol=1e-8):
'''
Finds a global solution for ``model`` using parameterized expectations
function. Requires the model to be written with controls as a direct
function of the model objects.
The algorithm iterates on the expectations function in the arbitrage
equation. It follows the discussion in section 9.9 of Miranda and
Fackler (2002).
Parameters
----------
model : NumericModel
"dtcscc" model to be solved
verbose : boolean
if True, display iterations
dr0 : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
grid: grid options
distribution: distribution options
maxit: maximum number of iterations
tol: tolerance criterium for successive approximations
Returns
-------
decision rule :
approximated solution
'''
t1 = time.time()
g = model.functions['transition']
d = model.functions['direct_response']
h = model.functions['expectation']
parms = model.calibration['parameters']
if dr0 is None:
if pert_order == 1:
dr0 = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
approx = model.get_endo_grid(**grid)
grid = approx.grid
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
expect = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
N = grid.shape[0]
z = np.zeros((N, len(model.symbols['expectations'])))
x_0 = dr0(grid)
x_0 = x_0.real # just in case ...
h_0 = h(grid, x_0, parms)
it = 0
err = 10
err_0 = 10
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'
headline = headline.format('N', ' Error', 'Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'
while err > tol and it <= maxit:
it += 1
t_start = time.time()
# dr.set_values(x_0)
expect.set_values(h_0)
z[...] = 0
for i in range(weights.shape[0]):
e = nodes[i, :]
S = g(grid, x_0, e, parms)
# evaluate expectation over the future state
z += weights[i] * expect(S)
# TODO: check that control is admissible
new_x = d(grid, z, parms)
new_h = h(grid, new_x, parms)
# update error
err = (abs(new_h - h_0).max())
# Update guess for decision rule and expectations function
x_0 = new_x
h_0 = new_h
# print error information if `verbose`
err_SA = err / err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed))
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final fime and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# Interpolation for the decision rule
dr.set_values(x_0)
return dr
def parameterized_expectations(model,
verbose=False,
dr0=None,
pert_order=1,
with_complementarities=True,
grid={},
distribution={},
maxit=100,
tol=1e-8,
inner_maxit=10,
direct=False):
'''
Find global solution for ``model`` via parameterized expectations.
Controls must be expressed as a direct function of equilibrium objects.
Algorithm iterates over the expectations function in the arbitrage equation.
Parameters:
----------
model : NumericModel
``dtcscc`` model to be solved
verbose : boolean
if True, display iterations
dr0 : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
grid : grid options
distribution : distribution options
maxit : maximum number of iterations
tol : tolerance criterium for successive approximations
inner_maxit : maximum number of iteration for inner solver
direct : if True, solve with direct method. If false, solve indirectly
Returns
-------
decision rule :
approximated solution
'''
t1 = time.time()
g = model.functions['transition']
h = model.functions['expectation']
d = model.functions['direct_response']
f = model.functions['arbitrage_exp'] # f(s, x, z, p, out)
parms = model.calibration['parameters']
if dr0 is None:
if pert_order == 1:
dr0 = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
approx = model.get_endo_grid(**grid)
grid = approx.grid
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
expect = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
N = grid.shape[0]
z = np.zeros((N, len(model.symbols['expectations'])))
x_0 = dr0(grid)
x_0 = x_0.real # just in case ...
h_0 = h(grid, x_0, parms)
it = 0
err = 10
err_0 = 10
verbit = True if verbose == 'full' else False
if with_complementarities is True:
lbfun = model.functions['controls_lb']
ubfun = model.functions['controls_ub']
lb = lbfun(grid, parms)
ub = ubfun(grid, parms)
else:
lb = None
ub = None
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'
headline = headline.format('N', ' Error', 'Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'
while err > tol and it <= maxit:
it += 1
t_start = time.time()
# dr.set_values(x_0)
expect.set_values(h_0)
# evaluate expectation over the future state
z[...] = 0
for i in range(weights.shape[0]):
e = nodes[i, :]
S = g(grid, x_0, e, parms)
z += weights[i] * expect(S)
if direct is True:
# Use control as direct function of arbitrage equation
new_x = d(grid, z, parms)
if with_complementarities is True:
new_x = np.minimum(new_x, ub)
new_x = np.maximum(new_x, lb)
else:
# Find control by solving arbitrage equation
def fun(x):
return f(grid, x, z, parms)
sdfun = SerialDifferentiableFunction(fun)
if with_complementarities is True:
[new_x, nit] = ncpsolve(sdfun,
lb,
ub,
x_0,
verbose=verbit,
maxit=inner_maxit)
else:
[new_x, nit] = serial_newton(sdfun, x_0, verbose=verbit)
new_h = h(grid, new_x, parms)
# update error
err = (abs(new_h - h_0).max())
# Update guess for decision rule and expectations function
x_0 = new_x
h_0 = new_h
# print error infomation if `verbose`
err_SA = err / err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed))
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final fime and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# Interpolation for the decision rule
dr.set_values(x_0)
return dr
def parameterized_expectations_direct(model, verbose=False, initial_dr=None,
pert_order=1, grid={}, distribution={},
maxit=100, tol=1e-8):
'''
Finds a global solution for ``model`` using parameterized expectations
function. Requires the model to be written with controls as a direct
function of the model objects.
The algorithm iterates on the expectations function in the arbitrage
equation. It follows the discussion in section 9.9 of Miranda and
Fackler (2002).
Parameters
----------
model : NumericModel
"dtcscc" model to be solved
verbose : boolean
if True, display iterations
initial_dr : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
grid: grid options
distribution: distribution options
maxit: maximum number of iterations
tol: tolerance criterium for successive approximations
Returns
-------
decision rule :
approximated solution
'''
t1 = time.time()
g = model.functions['transition']
d = model.functions['direct_response']
h = model.functions['expectation']
parms = model.calibration['parameters']
if initial_dr is None:
if pert_order == 1:
initial_dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
approx = model.get_grid(**grid)
grid = approx.grid
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
expect = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
N = grid.shape[0]
z = np.zeros((N, len(model.symbols['expectations'])))
x_0 = initial_dr(grid)
x_0 = x_0.real # just in case ...
h_0 = h(grid, x_0, parms)
it = 0
err = 10
err_0 = 10
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'
headline = headline.format('N', ' Error', 'Gain', 'Time')
stars = '-'*len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'
while err > tol and it <= maxit:
it += 1
t_start = time.time()
# dr.set_values(x_0)
expect.set_values(h_0)
z[...] = 0
for i in range(weights.shape[0]):
e = nodes[i, :]
S = g(grid, x_0, e, parms)
# evaluate expectation over the future state
z += weights[i]*expect(S)
# TODO: check that control is admissible
new_x = d(grid, z, parms)
new_h = h(grid, new_x, parms)
# update error
err = (abs(new_h - h_0).max())
# Update guess for decision rule and expectations function
x_0 = new_x
h_0 = new_h
# print error information if `verbose`
err_SA = err/err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed))
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final fime and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# Interpolation for the decision rule
dr.set_values(x_0)
return dr
def parameterized_expectations(model, verbose=False, initial_dr=None,
pert_order=1, with_complementarities=True,
grid={}, distribution={},
maxit=100, tol=1e-8, inner_maxit=10,
direct=False):
'''
Find global solution for ``model`` via parameterized expectations.
Controls must be expressed as a direct function of equilibrium objects.
Algorithm iterates over the expectations function in the arbitrage equation.
Parameters:
----------
model : NumericModel
``dtcscc`` model to be solved
verbose : boolean
if True, display iterations
initial_dr : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
grid : grid options
distribution : distribution options
maxit : maximum number of iterations
tol : tolerance criterium for successive approximations
inner_maxit : maximum number of iteration for inner solver
direct : if True, solve with direct method. If false, solve indirectly
Returns
-------
decision rule :
approximated solution
'''
t1 = time.time()
g = model.functions['transition']
h = model.functions['expectation']
d = model.functions['direct_response']
f = model.functions['arbitrage_exp'] # f(s, x, z, p, out)
parms = model.calibration['parameters']
if initial_dr is None:
if pert_order == 1:
initial_dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
approx = model.get_grid(**grid)
grid = approx.grid
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
expect = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
N = grid.shape[0]
z = np.zeros((N, len(model.symbols['expectations'])))
x_0 = initial_dr(grid)
x_0 = x_0.real # just in case ...
h_0 = h(grid, x_0, parms)
it = 0
err = 10
err_0 = 10
verbit = True if verbose == 'full' else False
if with_complementarities is True:
lbfun = model.functions['controls_lb']
ubfun = model.functions['controls_ub']
lb = lbfun(grid, parms)
ub = ubfun(grid, parms)
else:
lb = None
ub = None
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'
headline = headline.format('N', ' Error', 'Gain', 'Time')
stars = '-'*len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'
while err > tol and it <= maxit:
it += 1
t_start = time.time()
# dr.set_values(x_0)
expect.set_values(h_0)
# evaluate expectation over the future state
z[...] = 0
for i in range(weights.shape[0]):
e = nodes[i, :]
S = g(grid, x_0, e, parms)
z += weights[i]*expect(S)
if direct is True:
# Use control as direct function of arbitrage equation
new_x = d(grid, z, parms)
if with_complementarities is True:
new_x = np.minimum(new_x, ub)
new_x = np.maximum(new_x, lb)
else:
# Find control by solving arbitrage equation
def fun(x): return f(grid, x, z, parms)
sdfun = SerialDifferentiableFunction(fun)
if with_complementarities is True:
[new_x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit,
maxit=inner_maxit)
else:
[new_x, nit] = serial_newton(sdfun, x_0, verbose=verbit)
new_h = h(grid, new_x, parms)
# update error
err = (abs(new_h - h_0).max())
# Update guess for decision rule and expectations function
x_0 = new_x
h_0 = new_h
# print error infomation if `verbose`
err_SA = err/err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed))
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final fime and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# Interpolation for the decision rule
dr.set_values(x_0)
return dr
def time_iteration(model, verbose=False, initial_dr=None, pert_order=1,
with_complementarities=True, grid={}, distribution={},
maxit=500, tol=1e-8, inner_maxit=10, hook=None):
'''
Finds a global solution for ``model`` using backward time-iteration.
This algorithm iterates on the residuals of the arbitrage equations
Parameters
----------
model : NumericModel
"dtcscc" model to be solved
verbose : boolean
if True, display iterations
initial_dr : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
with_complementarities : boolean (True)
if False, complementarity conditions are ignored
grid: grid options
distribution: distribution options
maxit: maximum number of iterations
inner_maxit: maximum number of iteration for inner solver
tol: tolerance criterium for successive approximations
Returns
-------
decision rule :
approximated solution
'''
def vprint(t):
if verbose:
print(t)
f = model.functions['arbitrage']
g = model.functions['transition']
parms = model.calibration['parameters']
approx = model.get_grid(**grid)
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
if initial_dr is None:
if pert_order == 1:
initial_dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
vprint('Starting time iteration')
# TODO: transpose
grid = dr.grid
x_0 = initial_dr(grid)
x_0 = x_0.real # just in case...
# define objective function (residuals of arbitrage equations)
def fun(x):
return step_residual(grid, x, dr, f, g, parms, nodes, weights)
##
t1 = time.time()
err = 1
err_0 = 1
it = 0
verbit = True if verbose == 'full' else False
if with_complementarities:
lbfun = model.functions['controls_lb']
ubfun = model.functions['controls_ub']
lb = lbfun(grid, parms)
ub = ubfun(grid, parms)
else:
lb = None
ub = None
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} | {4:3} |'
headline = headline.format('N', ' Error', 'Gain', 'Time', 'nit')
stars = '-'*len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} | {4:3} |'
while err > tol and it < maxit:
# update counters
t_start = time.time()
it += 1
# update interpolation coefficients (NOTE: filters through `fun`)
dr.set_values(x_0)
# Derivative of objective function
sdfun = SerialDifferentiableFunction(fun)
# Apply solver with current decision rule for controls
if with_complementarities:
[x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit,
maxit=inner_maxit)
else:
[x, nit] = serial_newton(sdfun, x_0, verbose=verbit)
# update error and print if `verbose`
err = abs(x-x_0).max()
err_SA = err/err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed, nit))
# Update control vector
x_0[:] = x # x0 = x0 + (x-x0)
# call user supplied hook, if any
if hook:
hook(dr, it, err)
# warn and bail if we get inf
if False in np.isfinite(x_0):
print('iteration {} failed : non finite value')
return [x_0, x]
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final time and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
return dr
def time_iteration_direct(model, verbose=False, initial_dr=None,
pert_order=1, with_complementarities=True, grid={}, distribution={}, maxit=500,
tol=1e-8):
'''
Finds a global solution for ``model`` using backward time-iteration.
Requires the model to be written with controls as a direct function of the
model objects.
This algorithm iterates on the (directly expressed) decision rule, which is
a re-expression of the arbitrage equation.
Parameters
----------
model : NumericModel
"dtcscc" model to be solved
verbose : boolean
if True, display iterations
initial_dr : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
grid: grid options
distribution: distribution options
maxit: maximum number of iterations
tol: tolerance criterium for successive approximations
Returns
-------
decision rule :
approximated solution
'''
def vprint(t):
if verbose:
print(t)
g = model.functions['transition']
d = model.functions['direct_response']
h = model.functions['expectation']
parms = model.calibration['parameters']
approx = model.get_grid(**grid)
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
if initial_dr is None:
if pert_order == 1:
initial_dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
vprint('Starting direct response time iteration')
grid = approx.grid
N = grid.shape[0]
z = np.zeros((N, len(model.symbols['expectations'])))
x_0 = initial_dr(grid)
x_0 = x_0.real # just in case...
if with_complementarities:
lbfun = model.functions['controls_lb']
ubfun = model.functions['controls_ub']
lb = lbfun(grid, parms)
ub = ubfun(grid, parms)
else:
lb = None
ub = None
##
t1 = time.time()
err = 10
err_0 = 10
it = 0
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'
headline = headline.format('N', ' Error', 'Gain', 'Time')
stars = '-'*len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'
while err > tol and it <= maxit:
# update counters
t_start = time.time()
it += 1
# update interpolation coefficients
dr.set_values(x_0)
# Compute expectations function
z[...] = 0
for i in range(weights.shape[0]):
e = nodes[i, :]
S = g(grid, x_0, e, parms)
# evaluate future controls
X = dr(S)
z += weights[i]*h(S, X, parms)
# Update control
if with_complementarities:
new_x = d(grid, z, parms)
new_x = np.minimum(new_x, ub)
new_x = np.maximum(new_x, lb)
else:
new_x = d(grid, z, parms)
# update error
err = (abs(new_x - x_0).max())
# Update control vector
x_0[:] = new_x
# print error information if `verbose`
err_SA = err/err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed))
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final time and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
return dr
def evaluate_policy(model,
dr,
tol=1e-8,
grid={},
distribution={},
maxit=2000,
verbose=False,
hook=None,
integration_orders=None):
'''
Compute value function corresponding to policy ``dr``
Parameters:
-----------
model:
"dtcscc" model. Must contain a 'value' function.
dr:
decision rule to evaluate
Returns:
--------
decision rule:
value function (a function of the space similar to a decision rule
object)
'''
assert (model.model_type == 'dtcscc')
vfun = model.functions["value"]
gfun = model.functions['transition']
parms = model.calibration['parameters']
n_vals = len(model.symbols['values'])
t1 = time.time()
err = 1.0
it = 0
approx = model.get_grid(**grid)
interp_type = approx.interpolation
distrib = model.get_distribution(**distribution)
sigma = distrib.sigma
drv = create_interpolator(approx, approx.interpolation)
grid = drv.grid
N = drv.grid.shape[0]
controls = dr(grid)
guess_0 = model.calibration['values']
guess_0 = guess_0[None, :].repeat(N, axis=0)
if not integration_orders:
integration_orders = [3] * sigma.shape[0]
[nodes, weights] = gauss_hermite_nodes(integration_orders, sigma)
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format(
'N', ' Error', 'Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
err_0 = 1.0
while err > tol and it < maxit:
if hook:
hook()
t_start = time.time()
it += 1
# update spline coefficients with current values
drv.set_values(guess_0)
# update the geuss of value functions
guess = update_value(gfun, vfun, grid, controls, dr, drv, nodes,
weights, parms, n_vals)
# compute error
err = abs(guess - guess_0).max()
err_SA = err / err_0
err_0 = err
# update guess
guess_0[:] = guess.copy()
# print update to user, if verbose
t_end = time.time()
elapsed = t_end - t_start
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err, err_SA, elapsed))
if it == maxit:
warnings.warn(UserWarning("Maximum number of iterations reached"))
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
return drv
def solve_policy(model,
tol=1e-6,
grid={},
distribution={},
integration_orders=None,
maxit=500,
maxit_howard=500,
verbose=False,
hook=None,
initial_dr=None,
pert_order=1):
'''
Solve for the value function and associated decision rule by iterating over
the value function.
Parameters
-----------
model :
``dtcscc`` model. Must contain a ``felicity`` function.
Returns
--------
dr : Markov decision rule
The solved decision rule/policy function
drv : decision rule
The solved value function
'''
assert (model.model_type == 'dtcscc')
def vprint(t):
if verbose:
print(t)
# transition(s, x, e, p, out), felicity(s, x, p, out)
transition = model.functions['transition']
felicity = model.functions['felicity']
controls_lb = model.functions['controls_lb']
controls_ub = model.functions['controls_ub']
parms = model.calibration['parameters']
discount = model.calibration['beta']
x0 = model.calibration['controls']
s0 = model.calibration['states']
r0 = felicity(s0, x0, parms)
approx = model.get_grid(**grid)
# a = approx.a
# b = approx.b
# orders = approx.orders
distrib = model.get_distribution(**distribution)
sigma = distrib.sigma
# Possibly use the approximation orders?
if integration_orders is None:
integration_orders = [3] * sigma.shape[0]
[nodes, weights] = gauss_hermite_nodes(integration_orders, sigma)
interp_type = approx.interpolation
drv = create_interpolator(approx, interp_type)
if initial_dr is None:
if pert_order == 1:
dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
else:
dr = initial_dr
grid = drv.grid
N = grid.shape[0] # Number of states
n_x = grid.shape[1] # Number of controls
controls = dr(grid)
controls_0 = np.zeros((N, n_x))
controls_0[:, :] = model.calibration['controls'][None, :]
values_0 = np.zeros((N, 1))
values_0[:, :] = r0 / (1 - discount)
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format(
'N', ' Error', 'Gain', 'Time')
stars = '-' * len(headline)
print(stars)
print(headline)
print(stars)
t1 = time.time()
# FIRST: value function iterations, 10 iterations to start
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
err_x = 100
err_x_0 = 100
if verbose:
print(stars)
print('Starting value function iteration')
print(stars)
while err_v > tol and it < 10:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
for n in range(N):
s = grid[n, :]
x = controls[n, :]
lb = controls_lb(s, parms)
ub = controls_ub(s, parms)
bnds = [e for e in zip(lb, ub)]
def valfun(xx):
return -choice_value(transition, felicity, s, xx, drv, nodes,
weights, parms, discount)[0]
res = minimize(valfun, x, bounds=bnds, tol=1e-4)
controls[n, :] = res.x
values[n, 0] = -valfun(res.x)
# compute error, update value and dr
err_x = abs(controls - controls_0).max()
err_v = abs(values - values_0).max()
t_end = time.time()
elapsed = t_end - t_start
values_0 = values
controls_0 = controls
gain_x = err_x / err_x_0
gain_v = err_v / err_v_0
err_x_0 = err_x
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
# SECOND: Howard improvement step, 10-20 iterations
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
if verbose:
print(stars)
print('Starting Howard improvement step')
print(stars)
while err_v > tol and it < maxit_howard:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
# controls = controls_0.copy() # No need to keep updating
for n in range(N):
s = grid[n, :]
x = controls[n, :]
values[n, 0] = choice_value(transition, felicity, s, x, drv, nodes,
weights, parms, discount)[0]
# compute error, update value function
err_v = abs(values - values_0).max()
values_0 = values
t_end = time.time()
elapsed = t_end - t_start
gain_v = err_v / err_v_0
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
# THIRD: Back to value function iteration until convergence.
it = 0
err_v = 100
err_v_0 = 0.0
gain_v = 0.0
err_x = 100
err_x_0 = 100
if verbose:
print(stars)
print('Starting value function iteration')
print(stars)
while err_v > tol and it < maxit:
t_start = time.time()
it += 1
# update interpolation object with current values
drv.set_values(values_0)
values = values_0.copy()
controls = controls_0.copy()
for n in range(N):
s = grid[n, :]
x = controls[n, :]
lb = controls_lb(s, parms)
ub = controls_ub(s, parms)
bnds = [e for e in zip(lb, ub)]
def valfun(xx):
return -choice_value(transition, felicity, s, xx, drv, nodes,
weights, parms, discount)[0]
res = minimize(valfun, x, bounds=bnds, tol=1e-4)
controls[n, :] = res.x
values[n, 0] = -valfun(res.x)
# compute error, update value and dr
err_x = abs(controls - controls_0).max()
err_v = abs(values - values_0).max()
t_end = time.time()
elapsed = t_end - t_start
values_0 = values
controls_0 = controls
gain_x = err_x / err_x_0
gain_v = err_v / err_v_0
err_x_0 = err_x
err_v_0 = err_v
# print update to user, if verbose
if verbose:
print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format(
it, err_v, gain_v, elapsed))
if it == maxit:
warnings.warn(UserWarning("Maximum number of iterations reached"))
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
# final value function and decision rule
drv.set_values(values_0)
dr = create_interpolator(approx, interp_type)
dr.set_values(controls_0)
return dr, drv
def time_iteration(model, verbose=False, initial_dr=None, pert_order=1,
with_complementarities=True, grid={}, distribution={},
maxit=500, tol=1e-8, inner_maxit=10, hook=None):
'''
Finds a global solution for ``model`` using backward time-iteration.
This algorithm iterates on the residuals of the arbitrage equations
Parameters
----------
model : NumericModel
"dtcscc" model to be solved
verbose : boolean
if True, display iterations
initial_dr : decision rule
initial guess for the decision rule
pert_order : {1}
if no initial guess is supplied, the perturbation solution at order
``pert_order`` is used as initial guess
with_complementarities : boolean (True)
if False, complementarity conditions are ignored
grid: grid options
distribution: distribution options
maxit: maximum number of iterations
inner_maxit: maximum number of iteration for inner solver
tol: tolerance criterium for successive approximations
Returns
-------
decision rule :
approximated solution
'''
def vprint(t):
if verbose:
print(t)
f = model.functions['arbitrage']
g = model.functions['transition']
parms = model.calibration['parameters']
approx = model.get_grid(**grid)
interp_type = approx.interpolation
dr = create_interpolator(approx, interp_type)
distrib = model.get_distribution(**distribution)
nodes, weights = distrib.discretize()
if initial_dr is None:
if pert_order == 1:
initial_dr = approximate_controls(model)
if pert_order > 1:
raise Exception("Perturbation order > 1 not supported (yet).")
vprint('Starting time iteration')
# TODO: transpose
grid = dr.grid
x_0 = initial_dr(grid)
x_0 = x_0.real # just in case...
# define objective function (residuals of arbitrage equations)
def fun(x):
return step_residual(grid, x, dr, f, g, parms, nodes, weights)
##
t1 = time.time()
err = 1
err_0 = 1
it = 0
verbit = True if verbose == 'full' else False
if with_complementarities:
lbfun = model.functions['controls_lb']
ubfun = model.functions['controls_ub']
lb = lbfun(grid, parms)
ub = ubfun(grid, parms)
else:
lb = None
ub = None
if verbose:
headline = '|{0:^4} | {1:10} | {2:8} | {3:8} | {4:3} |'
headline = headline.format('N', ' Error', 'Gain', 'Time', 'nit')
stars = '-'*len(headline)
print(stars)
print(headline)
print(stars)
# format string for within loop
fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} | {4:3} |'
while err > tol and it < maxit:
# update counters
t_start = time.time()
it += 1
# update interpolation coefficients (NOTE: filters through `fun`)
dr.set_values(x_0)
# Derivative of objective function
sdfun = SerialDifferentiableFunction(fun)
# Apply solver with current decision rule for controls
if with_complementarities:
[x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit,
maxit=inner_maxit)
else:
[x, nit] = serial_newton(sdfun, x_0, verbose=verbit)
# update error and print if `verbose`
err = abs(x-x_0).max()
err_SA = err/err_0
err_0 = err
t_finish = time.time()
elapsed = t_finish - t_start
if verbose:
print(fmt_str.format(it, err, err_SA, elapsed, nit))
# Update control vector
x_0[:] = x # x0 = x0 + (x-x0)
# call user supplied hook, if any
if hook:
hook(dr, it, err)
# warn and bail if we get inf
if False in np.isfinite(x_0):
print('iteration {} failed : non finite value')
return [x_0, x]
if it == maxit:
import warnings
warnings.warn(UserWarning("Maximum number of iterations reached"))
# compute final time and do final printout if `verbose`
t2 = time.time()
if verbose:
print(stars)
print('Elapsed: {} seconds.'.format(t2 - t1))
print(stars)
return dr
