def solve_policy(model, tol=1e-6, grid={}, distribution={}, integration_orders=None, maxit=500, maxit_howard=500, verbose=False, hook=None, initial_dr=None, pert_order=1): ''' Solve for the value function and associated decision rule by iterating over the value function. Parameters ----------- model : ``dtcscc`` model. Must contain a ``felicity`` function. Returns -------- dr : Markov decision rule The solved decision rule/policy function drv : decision rule The solved value function ''' assert (model.model_type == 'dtcscc') def vprint(t): if verbose: print(t) # transition(s, x, e, p, out), felicity(s, x, p, out) transition = model.functions['transition'] felicity = model.functions['felicity'] controls_lb = model.functions['controls_lb'] controls_ub = model.functions['controls_ub'] parms = model.calibration['parameters'] discount = model.calibration['beta'] x0 = model.calibration['controls'] s0 = model.calibration['states'] r0 = felicity(s0, x0, parms) approx = model.get_grid(**grid) # a = approx.a # b = approx.b # orders = approx.orders distrib = model.get_distribution(**distribution) sigma = distrib.sigma # Possibly use the approximation orders? if integration_orders is None: integration_orders = [3] * sigma.shape[0] [nodes, weights] = gauss_hermite_nodes(integration_orders, sigma) interp_type = approx.interpolation drv = create_interpolator(approx, interp_type) if initial_dr is None: if pert_order == 1: dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") else: dr = initial_dr grid = drv.grid N = grid.shape[0] # Number of states n_x = grid.shape[1] # Number of controls controls = dr(grid) controls_0 = np.zeros((N, n_x)) controls_0[:, :] = model.calibration['controls'][None, :] values_0 = np.zeros((N, 1)) values_0[:, :] = r0/(1-discount) if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format('N', ' Error','Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) t1 = time.time() # FIRST: value function iterations, 10 iterations to start it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 err_x = 100 err_x_0 = 100 if verbose: print(stars) print('Starting value function iteration') print(stars) while err_v > tol and it < 10: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() for n in range(N): s = grid[n, :] x = controls[n, :] lb = controls_lb(s, parms) ub = controls_ub(s, parms) bnds = [e for e in zip(lb, ub)] def valfun(xx): return -choice_value(transition, felicity, s, xx, drv, nodes, weights, parms, discount)[0] res = minimize(valfun, x, bounds=bnds, tol=1e-4) controls[n, :] = res.x values[n, 0] = -valfun(res.x) # compute error, update value and dr err_x = abs(controls - controls_0).max() err_v = abs(values - values_0).max() t_end = time.time() elapsed = t_end - t_start values_0 = values controls_0 = controls gain_x = err_x / err_x_0 gain_v = err_v / err_v_0 err_x_0 = err_x err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) # SECOND: Howard improvement step, 10-20 iterations it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 if verbose: print(stars) print('Starting Howard improvement step') print(stars) while err_v > tol and it < maxit_howard: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() # controls = controls_0.copy() # No need to keep updating for n in range(N): s = grid[n, :] x = controls[n, :] values[n, 0] = choice_value(transition, felicity, s, x, drv, nodes, weights, parms, discount)[0] # compute error, update value function err_v = abs(values - values_0).max() values_0 = values t_end = time.time() elapsed = t_end - t_start gain_v = err_v / err_v_0 err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) # THIRD: Back to value function iteration until convergence. it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 err_x = 100 err_x_0 = 100 if verbose: print(stars) print('Starting value function iteration') print(stars) while err_v > tol and it < maxit: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() for n in range(N): s = grid[n, :] x = controls[n, :] lb = controls_lb(s, parms) ub = controls_ub(s, parms) bnds = [e for e in zip(lb, ub)] def valfun(xx): return -choice_value(transition, felicity, s, xx, drv, nodes, weights, parms, discount)[0] res = minimize(valfun, x, bounds=bnds, tol=1e-4) controls[n, :] = res.x values[n, 0] = -valfun(res.x) # compute error, update value and dr err_x = abs(controls - controls_0).max() err_v = abs(values - values_0).max() t_end = time.time() elapsed = t_end - t_start values_0 = values controls_0 = controls gain_x = err_x / err_x_0 gain_v = err_v / err_v_0 err_x_0 = err_x err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) if it == maxit: warnings.warn(UserWarning("Maximum number of iterations reached")) t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # final value function and decision rule drv.set_values(values_0) dr = create_interpolator(approx, interp_type) dr.set_values(controls_0) return dr, drv
def evaluate_policy(model, dr, tol=1e-8, grid={}, distribution={}, maxit=2000, verbose=False, hook=None, integration_orders=None): ''' Compute value function corresponding to policy ``dr`` Parameters: ----------- model: "dtcscc" model. Must contain a 'value' function. dr: decision rule to evaluate Returns: -------- decision rule: value function (a function of the space similar to a decision rule object) ''' assert (model.model_type == 'dtcscc') vfun = model.functions["value"] gfun = model.functions['transition'] parms = model.calibration['parameters'] n_vals = len(model.symbols['values']) t1 = time.time() err = 1.0 it = 0 approx = model.get_grid(**grid) interp_type = approx.interpolation distrib = model.get_distribution(**distribution) sigma = distrib.sigma drv = create_interpolator(approx, approx.interpolation) grid = drv.grid N = drv.grid.shape[0] controls = dr(grid) guess_0 = model.calibration['values'] guess_0 = guess_0[None, :].repeat(N, axis=0) if not integration_orders: integration_orders = [3] * sigma.shape[0] [nodes, weights] = gauss_hermite_nodes(integration_orders, sigma) if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format('N', ' Error', 'Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) err_0 = 1.0 while err > tol and it < maxit: if hook: hook() t_start = time.time() it += 1 # update spline coefficients with current values drv.set_values(guess_0) # update the geuss of value functions guess = update_value(gfun, vfun, grid, controls, dr, drv, nodes, weights, parms, n_vals) # compute error err = abs(guess - guess_0).max() err_SA = err / err_0 err_0 = err # update guess guess_0[:] = guess.copy() # print update to user, if verbose t_end = time.time() elapsed = t_end - t_start if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err, err_SA, elapsed)) if it == maxit: warnings.warn(UserWarning("Maximum number of iterations reached")) t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) return drv
def parameterized_expectations_direct(model, verbose=False, dr0=None, pert_order=1, grid={}, distribution={}, maxit=100, tol=1e-8): ''' Finds a global solution for ``model`` using parameterized expectations function. Requires the model to be written with controls as a direct function of the model objects. The algorithm iterates on the expectations function in the arbitrage equation. It follows the discussion in section 9.9 of Miranda and Fackler (2002). Parameters ---------- model : NumericModel "dtcscc" model to be solved verbose : boolean if True, display iterations dr0 : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess grid: grid options distribution: distribution options maxit: maximum number of iterations tol: tolerance criterium for successive approximations Returns ------- decision rule : approximated solution ''' t1 = time.time() g = model.functions['transition'] d = model.functions['direct_response'] h = model.functions['expectation'] parms = model.calibration['parameters'] if dr0 is None: if pert_order == 1: dr0 = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") approx = model.get_endo_grid(**grid) grid = approx.grid interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) expect = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() N = grid.shape[0] z = np.zeros((N, len(model.symbols['expectations']))) x_0 = dr0(grid) x_0 = x_0.real # just in case ... h_0 = h(grid, x_0, parms) it = 0 err = 10 err_0 = 10 if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |' headline = headline.format('N', ' Error', 'Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |' while err > tol and it <= maxit: it += 1 t_start = time.time() # dr.set_values(x_0) expect.set_values(h_0) z[...] = 0 for i in range(weights.shape[0]): e = nodes[i, :] S = g(grid, x_0, e, parms) # evaluate expectation over the future state z += weights[i] * expect(S) # TODO: check that control is admissible new_x = d(grid, z, parms) new_h = h(grid, new_x, parms) # update error err = (abs(new_h - h_0).max()) # Update guess for decision rule and expectations function x_0 = new_x h_0 = new_h # print error information if `verbose` err_SA = err / err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed)) if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final fime and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # Interpolation for the decision rule dr.set_values(x_0) return dr
def parameterized_expectations(model, verbose=False, dr0=None, pert_order=1, with_complementarities=True, grid={}, distribution={}, maxit=100, tol=1e-8, inner_maxit=10, direct=False): ''' Find global solution for ``model`` via parameterized expectations. Controls must be expressed as a direct function of equilibrium objects. Algorithm iterates over the expectations function in the arbitrage equation. Parameters: ---------- model : NumericModel ``dtcscc`` model to be solved verbose : boolean if True, display iterations dr0 : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess grid : grid options distribution : distribution options maxit : maximum number of iterations tol : tolerance criterium for successive approximations inner_maxit : maximum number of iteration for inner solver direct : if True, solve with direct method. If false, solve indirectly Returns ------- decision rule : approximated solution ''' t1 = time.time() g = model.functions['transition'] h = model.functions['expectation'] d = model.functions['direct_response'] f = model.functions['arbitrage_exp'] # f(s, x, z, p, out) parms = model.calibration['parameters'] if dr0 is None: if pert_order == 1: dr0 = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") approx = model.get_endo_grid(**grid) grid = approx.grid interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) expect = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() N = grid.shape[0] z = np.zeros((N, len(model.symbols['expectations']))) x_0 = dr0(grid) x_0 = x_0.real # just in case ... h_0 = h(grid, x_0, parms) it = 0 err = 10 err_0 = 10 verbit = True if verbose == 'full' else False if with_complementarities is True: lbfun = model.functions['controls_lb'] ubfun = model.functions['controls_ub'] lb = lbfun(grid, parms) ub = ubfun(grid, parms) else: lb = None ub = None if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |' headline = headline.format('N', ' Error', 'Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |' while err > tol and it <= maxit: it += 1 t_start = time.time() # dr.set_values(x_0) expect.set_values(h_0) # evaluate expectation over the future state z[...] = 0 for i in range(weights.shape[0]): e = nodes[i, :] S = g(grid, x_0, e, parms) z += weights[i] * expect(S) if direct is True: # Use control as direct function of arbitrage equation new_x = d(grid, z, parms) if with_complementarities is True: new_x = np.minimum(new_x, ub) new_x = np.maximum(new_x, lb) else: # Find control by solving arbitrage equation def fun(x): return f(grid, x, z, parms) sdfun = SerialDifferentiableFunction(fun) if with_complementarities is True: [new_x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit, maxit=inner_maxit) else: [new_x, nit] = serial_newton(sdfun, x_0, verbose=verbit) new_h = h(grid, new_x, parms) # update error err = (abs(new_h - h_0).max()) # Update guess for decision rule and expectations function x_0 = new_x h_0 = new_h # print error infomation if `verbose` err_SA = err / err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed)) if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final fime and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # Interpolation for the decision rule dr.set_values(x_0) return dr
def parameterized_expectations_direct(model, verbose=False, initial_dr=None, pert_order=1, grid={}, distribution={}, maxit=100, tol=1e-8): ''' Finds a global solution for ``model`` using parameterized expectations function. Requires the model to be written with controls as a direct function of the model objects. The algorithm iterates on the expectations function in the arbitrage equation. It follows the discussion in section 9.9 of Miranda and Fackler (2002). Parameters ---------- model : NumericModel "dtcscc" model to be solved verbose : boolean if True, display iterations initial_dr : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess grid: grid options distribution: distribution options maxit: maximum number of iterations tol: tolerance criterium for successive approximations Returns ------- decision rule : approximated solution ''' t1 = time.time() g = model.functions['transition'] d = model.functions['direct_response'] h = model.functions['expectation'] parms = model.calibration['parameters'] if initial_dr is None: if pert_order == 1: initial_dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") approx = model.get_grid(**grid) grid = approx.grid interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) expect = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() N = grid.shape[0] z = np.zeros((N, len(model.symbols['expectations']))) x_0 = initial_dr(grid) x_0 = x_0.real # just in case ... h_0 = h(grid, x_0, parms) it = 0 err = 10 err_0 = 10 if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |' headline = headline.format('N', ' Error', 'Gain', 'Time') stars = '-'*len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |' while err > tol and it <= maxit: it += 1 t_start = time.time() # dr.set_values(x_0) expect.set_values(h_0) z[...] = 0 for i in range(weights.shape[0]): e = nodes[i, :] S = g(grid, x_0, e, parms) # evaluate expectation over the future state z += weights[i]*expect(S) # TODO: check that control is admissible new_x = d(grid, z, parms) new_h = h(grid, new_x, parms) # update error err = (abs(new_h - h_0).max()) # Update guess for decision rule and expectations function x_0 = new_x h_0 = new_h # print error information if `verbose` err_SA = err/err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed)) if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final fime and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # Interpolation for the decision rule dr.set_values(x_0) return dr
def parameterized_expectations(model, verbose=False, initial_dr=None, pert_order=1, with_complementarities=True, grid={}, distribution={}, maxit=100, tol=1e-8, inner_maxit=10, direct=False): ''' Find global solution for ``model`` via parameterized expectations. Controls must be expressed as a direct function of equilibrium objects. Algorithm iterates over the expectations function in the arbitrage equation. Parameters: ---------- model : NumericModel ``dtcscc`` model to be solved verbose : boolean if True, display iterations initial_dr : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess grid : grid options distribution : distribution options maxit : maximum number of iterations tol : tolerance criterium for successive approximations inner_maxit : maximum number of iteration for inner solver direct : if True, solve with direct method. If false, solve indirectly Returns ------- decision rule : approximated solution ''' t1 = time.time() g = model.functions['transition'] h = model.functions['expectation'] d = model.functions['direct_response'] f = model.functions['arbitrage_exp'] # f(s, x, z, p, out) parms = model.calibration['parameters'] if initial_dr is None: if pert_order == 1: initial_dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") approx = model.get_grid(**grid) grid = approx.grid interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) expect = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() N = grid.shape[0] z = np.zeros((N, len(model.symbols['expectations']))) x_0 = initial_dr(grid) x_0 = x_0.real # just in case ... h_0 = h(grid, x_0, parms) it = 0 err = 10 err_0 = 10 verbit = True if verbose == 'full' else False if with_complementarities is True: lbfun = model.functions['controls_lb'] ubfun = model.functions['controls_ub'] lb = lbfun(grid, parms) ub = ubfun(grid, parms) else: lb = None ub = None if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |' headline = headline.format('N', ' Error', 'Gain', 'Time') stars = '-'*len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |' while err > tol and it <= maxit: it += 1 t_start = time.time() # dr.set_values(x_0) expect.set_values(h_0) # evaluate expectation over the future state z[...] = 0 for i in range(weights.shape[0]): e = nodes[i, :] S = g(grid, x_0, e, parms) z += weights[i]*expect(S) if direct is True: # Use control as direct function of arbitrage equation new_x = d(grid, z, parms) if with_complementarities is True: new_x = np.minimum(new_x, ub) new_x = np.maximum(new_x, lb) else: # Find control by solving arbitrage equation def fun(x): return f(grid, x, z, parms) sdfun = SerialDifferentiableFunction(fun) if with_complementarities is True: [new_x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit, maxit=inner_maxit) else: [new_x, nit] = serial_newton(sdfun, x_0, verbose=verbit) new_h = h(grid, new_x, parms) # update error err = (abs(new_h - h_0).max()) # Update guess for decision rule and expectations function x_0 = new_x h_0 = new_h # print error infomation if `verbose` err_SA = err/err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed)) if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final fime and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # Interpolation for the decision rule dr.set_values(x_0) return dr
def time_iteration(model, verbose=False, initial_dr=None, pert_order=1, with_complementarities=True, grid={}, distribution={}, maxit=500, tol=1e-8, inner_maxit=10, hook=None): ''' Finds a global solution for ``model`` using backward time-iteration. This algorithm iterates on the residuals of the arbitrage equations Parameters ---------- model : NumericModel "dtcscc" model to be solved verbose : boolean if True, display iterations initial_dr : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess with_complementarities : boolean (True) if False, complementarity conditions are ignored grid: grid options distribution: distribution options maxit: maximum number of iterations inner_maxit: maximum number of iteration for inner solver tol: tolerance criterium for successive approximations Returns ------- decision rule : approximated solution ''' def vprint(t): if verbose: print(t) f = model.functions['arbitrage'] g = model.functions['transition'] parms = model.calibration['parameters'] approx = model.get_grid(**grid) interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() if initial_dr is None: if pert_order == 1: initial_dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") vprint('Starting time iteration') # TODO: transpose grid = dr.grid x_0 = initial_dr(grid) x_0 = x_0.real # just in case... # define objective function (residuals of arbitrage equations) def fun(x): return step_residual(grid, x, dr, f, g, parms, nodes, weights) ## t1 = time.time() err = 1 err_0 = 1 it = 0 verbit = True if verbose == 'full' else False if with_complementarities: lbfun = model.functions['controls_lb'] ubfun = model.functions['controls_ub'] lb = lbfun(grid, parms) ub = ubfun(grid, parms) else: lb = None ub = None if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} | {4:3} |' headline = headline.format('N', ' Error', 'Gain', 'Time', 'nit') stars = '-'*len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} | {4:3} |' while err > tol and it < maxit: # update counters t_start = time.time() it += 1 # update interpolation coefficients (NOTE: filters through `fun`) dr.set_values(x_0) # Derivative of objective function sdfun = SerialDifferentiableFunction(fun) # Apply solver with current decision rule for controls if with_complementarities: [x, nit] = ncpsolve(sdfun, lb, ub, x_0, verbose=verbit, maxit=inner_maxit) else: [x, nit] = serial_newton(sdfun, x_0, verbose=verbit) # update error and print if `verbose` err = abs(x-x_0).max() err_SA = err/err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed, nit)) # Update control vector x_0[:] = x # x0 = x0 + (x-x0) # call user supplied hook, if any if hook: hook(dr, it, err) # warn and bail if we get inf if False in np.isfinite(x_0): print('iteration {} failed : non finite value') return [x_0, x] if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final time and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) return dr
def time_iteration_direct(model, verbose=False, initial_dr=None, pert_order=1, with_complementarities=True, grid={}, distribution={}, maxit=500, tol=1e-8): ''' Finds a global solution for ``model`` using backward time-iteration. Requires the model to be written with controls as a direct function of the model objects. This algorithm iterates on the (directly expressed) decision rule, which is a re-expression of the arbitrage equation. Parameters ---------- model : NumericModel "dtcscc" model to be solved verbose : boolean if True, display iterations initial_dr : decision rule initial guess for the decision rule pert_order : {1} if no initial guess is supplied, the perturbation solution at order ``pert_order`` is used as initial guess grid: grid options distribution: distribution options maxit: maximum number of iterations tol: tolerance criterium for successive approximations Returns ------- decision rule : approximated solution ''' def vprint(t): if verbose: print(t) g = model.functions['transition'] d = model.functions['direct_response'] h = model.functions['expectation'] parms = model.calibration['parameters'] approx = model.get_grid(**grid) interp_type = approx.interpolation dr = create_interpolator(approx, interp_type) distrib = model.get_distribution(**distribution) nodes, weights = distrib.discretize() if initial_dr is None: if pert_order == 1: initial_dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") vprint('Starting direct response time iteration') grid = approx.grid N = grid.shape[0] z = np.zeros((N, len(model.symbols['expectations']))) x_0 = initial_dr(grid) x_0 = x_0.real # just in case... if with_complementarities: lbfun = model.functions['controls_lb'] ubfun = model.functions['controls_ub'] lb = lbfun(grid, parms) ub = ubfun(grid, parms) else: lb = None ub = None ## t1 = time.time() err = 10 err_0 = 10 it = 0 if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |' headline = headline.format('N', ' Error', 'Gain', 'Time') stars = '-'*len(headline) print(stars) print(headline) print(stars) # format string for within loop fmt_str = '|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |' while err > tol and it <= maxit: # update counters t_start = time.time() it += 1 # update interpolation coefficients dr.set_values(x_0) # Compute expectations function z[...] = 0 for i in range(weights.shape[0]): e = nodes[i, :] S = g(grid, x_0, e, parms) # evaluate future controls X = dr(S) z += weights[i]*h(S, X, parms) # Update control if with_complementarities: new_x = d(grid, z, parms) new_x = np.minimum(new_x, ub) new_x = np.maximum(new_x, lb) else: new_x = d(grid, z, parms) # update error err = (abs(new_x - x_0).max()) # Update control vector x_0[:] = new_x # print error information if `verbose` err_SA = err/err_0 err_0 = err t_finish = time.time() elapsed = t_finish - t_start if verbose: print(fmt_str.format(it, err, err_SA, elapsed)) if it == maxit: import warnings warnings.warn(UserWarning("Maximum number of iterations reached")) # compute final time and do final printout if `verbose` t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) return dr
def evaluate_policy(model, dr, tol=1e-8, grid={}, distribution={}, maxit=2000, verbose=False, hook=None, integration_orders=None): ''' Compute value function corresponding to policy ``dr`` Parameters: ----------- model: "dtcscc" model. Must contain a 'value' function. dr: decision rule to evaluate Returns: -------- decision rule: value function (a function of the space similar to a decision rule object) ''' assert (model.model_type == 'dtcscc') vfun = model.functions["value"] gfun = model.functions['transition'] parms = model.calibration['parameters'] n_vals = len(model.symbols['values']) t1 = time.time() err = 1.0 it = 0 approx = model.get_grid(**grid) interp_type = approx.interpolation distrib = model.get_distribution(**distribution) sigma = distrib.sigma drv = create_interpolator(approx, approx.interpolation) grid = drv.grid N = drv.grid.shape[0] controls = dr(grid) guess_0 = model.calibration['values'] guess_0 = guess_0[None, :].repeat(N, axis=0) if not integration_orders: integration_orders = [3] * sigma.shape[0] [nodes, weights] = gauss_hermite_nodes(integration_orders, sigma) if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format( 'N', ' Error', 'Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) err_0 = 1.0 while err > tol and it < maxit: if hook: hook() t_start = time.time() it += 1 # update spline coefficients with current values drv.set_values(guess_0) # update the geuss of value functions guess = update_value(gfun, vfun, grid, controls, dr, drv, nodes, weights, parms, n_vals) # compute error err = abs(guess - guess_0).max() err_SA = err / err_0 err_0 = err # update guess guess_0[:] = guess.copy() # print update to user, if verbose t_end = time.time() elapsed = t_end - t_start if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err, err_SA, elapsed)) if it == maxit: warnings.warn(UserWarning("Maximum number of iterations reached")) t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) return drv
def solve_policy(model, tol=1e-6, grid={}, distribution={}, integration_orders=None, maxit=500, maxit_howard=500, verbose=False, hook=None, initial_dr=None, pert_order=1): ''' Solve for the value function and associated decision rule by iterating over the value function. Parameters ----------- model : ``dtcscc`` model. Must contain a ``felicity`` function. Returns -------- dr : Markov decision rule The solved decision rule/policy function drv : decision rule The solved value function ''' assert (model.model_type == 'dtcscc') def vprint(t): if verbose: print(t) # transition(s, x, e, p, out), felicity(s, x, p, out) transition = model.functions['transition'] felicity = model.functions['felicity'] controls_lb = model.functions['controls_lb'] controls_ub = model.functions['controls_ub'] parms = model.calibration['parameters'] discount = model.calibration['beta'] x0 = model.calibration['controls'] s0 = model.calibration['states'] r0 = felicity(s0, x0, parms) approx = model.get_grid(**grid) # a = approx.a # b = approx.b # orders = approx.orders distrib = model.get_distribution(**distribution) sigma = distrib.sigma # Possibly use the approximation orders? if integration_orders is None: integration_orders = [3] * sigma.shape[0] [nodes, weights] = gauss_hermite_nodes(integration_orders, sigma) interp_type = approx.interpolation drv = create_interpolator(approx, interp_type) if initial_dr is None: if pert_order == 1: dr = approximate_controls(model) if pert_order > 1: raise Exception("Perturbation order > 1 not supported (yet).") else: dr = initial_dr grid = drv.grid N = grid.shape[0] # Number of states n_x = grid.shape[1] # Number of controls controls = dr(grid) controls_0 = np.zeros((N, n_x)) controls_0[:, :] = model.calibration['controls'][None, :] values_0 = np.zeros((N, 1)) values_0[:, :] = r0 / (1 - discount) if verbose: headline = '|{0:^4} | {1:10} | {2:8} | {3:8} |'.format( 'N', ' Error', 'Gain', 'Time') stars = '-' * len(headline) print(stars) print(headline) print(stars) t1 = time.time() # FIRST: value function iterations, 10 iterations to start it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 err_x = 100 err_x_0 = 100 if verbose: print(stars) print('Starting value function iteration') print(stars) while err_v > tol and it < 10: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() for n in range(N): s = grid[n, :] x = controls[n, :] lb = controls_lb(s, parms) ub = controls_ub(s, parms) bnds = [e for e in zip(lb, ub)] def valfun(xx): return -choice_value(transition, felicity, s, xx, drv, nodes, weights, parms, discount)[0] res = minimize(valfun, x, bounds=bnds, tol=1e-4) controls[n, :] = res.x values[n, 0] = -valfun(res.x) # compute error, update value and dr err_x = abs(controls - controls_0).max() err_v = abs(values - values_0).max() t_end = time.time() elapsed = t_end - t_start values_0 = values controls_0 = controls gain_x = err_x / err_x_0 gain_v = err_v / err_v_0 err_x_0 = err_x err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) # SECOND: Howard improvement step, 10-20 iterations it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 if verbose: print(stars) print('Starting Howard improvement step') print(stars) while err_v > tol and it < maxit_howard: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() # controls = controls_0.copy() # No need to keep updating for n in range(N): s = grid[n, :] x = controls[n, :] values[n, 0] = choice_value(transition, felicity, s, x, drv, nodes, weights, parms, discount)[0] # compute error, update value function err_v = abs(values - values_0).max() values_0 = values t_end = time.time() elapsed = t_end - t_start gain_v = err_v / err_v_0 err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) # THIRD: Back to value function iteration until convergence. it = 0 err_v = 100 err_v_0 = 0.0 gain_v = 0.0 err_x = 100 err_x_0 = 100 if verbose: print(stars) print('Starting value function iteration') print(stars) while err_v > tol and it < maxit: t_start = time.time() it += 1 # update interpolation object with current values drv.set_values(values_0) values = values_0.copy() controls = controls_0.copy() for n in range(N): s = grid[n, :] x = controls[n, :] lb = controls_lb(s, parms) ub = controls_ub(s, parms) bnds = [e for e in zip(lb, ub)] def valfun(xx): return -choice_value(transition, felicity, s, xx, drv, nodes, weights, parms, discount)[0] res = minimize(valfun, x, bounds=bnds, tol=1e-4) controls[n, :] = res.x values[n, 0] = -valfun(res.x) # compute error, update value and dr err_x = abs(controls - controls_0).max() err_v = abs(values - values_0).max() t_end = time.time() elapsed = t_end - t_start values_0 = values controls_0 = controls gain_x = err_x / err_x_0 gain_v = err_v / err_v_0 err_x_0 = err_x err_v_0 = err_v # print update to user, if verbose if verbose: print('|{0:4} | {1:10.3e} | {2:8.3f} | {3:8.3f} |'.format( it, err_v, gain_v, elapsed)) if it == maxit: warnings.warn(UserWarning("Maximum number of iterations reached")) t2 = time.time() if verbose: print(stars) print('Elapsed: {} seconds.'.format(t2 - t1)) print(stars) # final value function and decision rule drv.set_values(values_0) dr = create_interpolator(approx, interp_type) dr.set_values(controls_0) return dr, drv