X = np.linspace(xmin, xmax, m) # vector of actions # Reward Function f = np.full((m, n), -np.inf) for k in range(m): f[k, S >= X[k]] = (X[k]**(1 - gamma)) / (1 - gamma) - cost * X[k] # State Transition Function g = np.zeros_like(f) for k in range(m): snext = alpha * (S - X[k]) - 0.5 * beta * (S - X[k])**2 g[k] = getindex(snext, S) # Model Structure model = DDPmodel(f, g, delta) model.solve() ## Analysis # Plot Optimal Policy demo.figure('Optimal Harvest Policy', 'Stock', 'Harvest') plt.plot(S, X[model.policy]) # Plot Value Function demo.figure('Optimal Value Function', 'Stock', 'Value') plt.plot(S, model.value) # Simulate Model nyrs = 20 t = np.arange(nyrs + 1)
# Reward Function f = np.full((m, n), -np.inf) for k in range(m): f[k, S >= X[k]] = (X[k] ** (1 - gamma)) / (1 - gamma) - cost * X[k] # State Transition Function g = np.zeros_like(f) for k in range(m): snext = alpha * (S - X[k]) - 0.5 * beta * (S - X[k]) ** 2 g[k] = getindex(snext, S) # Model Structure model = DDPmodel(f, g, delta) model.solve() ## Analysis # Plot Optimal Policy demo.figure('Optimal Harvest Policy', 'Stock', 'Harvest') plt.plot(S,X[model.policy]) # Plot Value Function demo.figure('Optimal Value Function', 'Stock', 'Value') plt.plot(S,model.value)
# Reward Function f = np.zeros((m, n)) f[0] = v # gets leisure f[1, 0] = u # gets benefit # State Transition Probability Matrix P = np.zeros((m, n, n)) P[0, :, 0] = 1 # remains unemployed P[1, 0, 0] = 1 - pfind # finds no job P[1, 0, 1] = pfind # finds job P[1, 1, 0] = pfire # gets fired P[1, 1, 1] = 1 - pfire # keeps job # Model Structure model = DDPmodel(f, P, delta) ## Solution # Solve Bellman Equation wage = np.arange(55, 66) xtable = np.zeros((wage.size, 2), dtype=int) for i, w in enumerate(wage): model.reward[1, 1] = w # vary wage xtable[i] = model.solve().policy # solve via policy iteration ## Analysis # Display Optimal Policy print(' Optimal Job Search Strategy') print(' (1=innactive, 2=active) ')
m = 500 # number of actions X = np.linspace(xmin, xmax, m) # vector of actions # Reward Function f = np.empty((m, n)) for k in range(m): f[k] = ((S * (1 - X[k]))**(1 - alpha)) / (1 - alpha) # State Transition Function g = np.empty_like(f) for k in range(m): snext = gamma * X[k] * S + (X[k] * S)**beta g[k] = getindex(snext, S) # Model Structure model = DDPmodel(f, g, delta).solve() ## Analysis # Plot Optimal Policy demo.figure('Optimal Investment', 'Wealth', 'Investment') plt.plot(S, X[model.policy] * S) # Plot Optimal Policy demo.figure('Optimal Consumption', 'Wealth', 'Consumption') plt.plot(S, S - X[model.policy] * S) # Plot Value Function demo.figure('Optimal Value Function', 'Wealth', 'Value') plt.plot(S, model.value)
# Reward Function f = np.full((m, n), -np.inf) for c, s in enumerate(S): for r, x in enumerate(X): if x <= s: f[r, c] = price * x - (x ** 2) / (1 + s) # State Transition Function g = np.empty_like(f) for r, x in enumerate(X): snext = S - x g[r] = getindex(snext, S) # Model Structure model = DDPmodel(f, g, delta) model.solve() # Analysis # Simulate Model sinit = S.max() nyrs = 15 t = np.arange(nyrs + 1) spath, xpath = model.simulate(sinit, nyrs) # Plot Optimal Policy demo.figure('Optimal Extraction Policy', 'Stock', 'Extraction') plt.plot(S, X[model.policy]) # Plot Value Function
# survives predation, finds food snext = S[i] - 1 + e[k] j = getindex(snext, S) P[k, i, j] += p[k] * q[k] # survives predation, finds no food snext = S[i] - 1 j = getindex(snext, S) P[k, i, j] += p[k] * (1 - q[k]) # Terminal Value Function vterm = np.ones(n) # terminal value: survive vterm[0] = 0 # terminal value: death # Model Structure model = DDPmodel(f, P, 1, T, vterm=vterm) model.solve() ## Analysis lims = [-0.5, emax + 0.5], [0, 1] # Plot Survial Probabilities, Period 0 demo.figure('Survival Probability (Period 0)', 'Stock of Energy', 'Probability', *lims) plt.bar(S, model.value[0], 1) # Plot Survial Probabilities, Period 5 demo.figure('Survival Probability (Period 5)', 'Stock of Energy', 'Probability', *lims) plt.bar(S, model.value[5], 1) plt.show()
# State Transition Probability Matrix P = np.zeros((2, n1, n2, n1, n2)) for i in range(n1): for j in range(n2): if i < 9: P[0, i, j, i+1, j] = 1 # Raise lactation number by 1, if keep else: P[0, i, j, 0] = 0.2, 0.6, 0.2 P[1, i, j, 0] = 0.2, 0.6, 0.2 # Optional replacement P.shape = 2, n, n # Model Structure model = DDPmodel(f, P, delta).solve(print=True) ## Analysis # Display Optimal Policy xtemp = model.policy.reshape((n1, n2)) header = '{:^8s} {:^8s} {:^8s} {:^8s}'.format('Age','Lo', 'Med', 'Hi') print('Optimal Policy') print(header) print(*('{:^8d} {:^8s} {:^8s} {:^8s}\n'.format(s, *X[x]) for s, x in zip(s1, xtemp))) # Plot Value Function demo.figure('Optimal Replacement Value', 'Age', 'Optimal Value (thousands)')
# State Transition Probability Matrix P = np.zeros((2, n1, n2, n1, n2)) for i in range(n1): for j in range(n2): if i < 9: P[0, i, j, i + 1, j] = 1 # Raise lactation number by 1, if keep else: P[0, i, j, 0] = 0.2, 0.6, 0.2 P[1, i, j, 0] = 0.2, 0.6, 0.2 # Optional replacement P.shape = 2, n, n # Model Structure model = DDPmodel(f, P, delta).solve(print=True) ## Analysis # Display Optimal Policy xtemp = model.policy.reshape((n1, n2)) header = '{:^8s} {:^8s} {:^8s} {:^8s}'.format('Age', 'Lo', 'Med', 'Hi') print('Optimal Policy') print(header) print(*('{:^8d} {:^8s} {:^8s} {:^8s}\n'.format(s, *X[x]) for s, x in zip(s1, xtemp))) # Plot Value Function demo.figure('Optimal Replacement Value', 'Age', 'Optimal Value (thousands)') plt.plot(s1, model.value.reshape((n1, n2)) / 1000)
f = np.full((m, n), -np.inf) for k in range(m): f[k, k:] = alpha1 * X[k]**beta1 + alpha2 * (S[k:] - X[k])**beta2 # State Transition Probability Matrix P = np.zeros((m, n, n)) for k in range(m): for i in range(n): for j in range(r.size): snext = min(S[i] - X[k] + r[j], maxcap) inext = getindex(snext, S) P[k, i, inext] = P[k, i, inext] + p[j] # Model Structure model = DDPmodel(f, P, delta) model.solve() ## Analysis # Plot Optimal Policy demo.figure('Optimal Irrigation Policy', 'Water Level', 'Irrigation', [-1, 31], [0, 6]) plt.plot(S, X[model.policy], '*') # Plot Value Function demo.figure('Optimal Value Function', 'Water Level', 'Value') plt.plot(S, model.value) # Simulate Model sinit = np.zeros(10000)
m = len(X) # number of actions # Reward Function f = np.empty((m, n)) y = -0.2 * S**2 + 2 * S + 8 # yield per lactation f[0] = price * y f[1] = f[0] - cost f[0, -1] = -np.inf # force replace at lactation 10 # State Transition Function g = np.ones_like(f) g[0] = np.minimum(np.arange(n) + 1, n - 1) # Raise lactation number by 1, if keep # Model Structure model = DDPmodel(f, g, delta) model.solve() # Plot Optimal Policy demo.figure('Optimal Replacement', 'Age', 'Optimal Decision', [0, n + 1], [-0.5, 1.5]) plt.plot(S, model.policy, '*', markersize=15) plt.yticks((0, 1), X) # Plot Value Function demo.figure('Optimal Value in Cow Replacement', 'Age', 'Value (thousands)') plt.plot(S, model.value / 1000) plt.show()
X = ['Keep', 'Replace'] # keep or replace m = len(X) # number of actions # Reward Function f = np.empty((m, n)) y = -0.2 * S ** 2 + 2 * S + 8 # yield per lactation f[0] = price * y f[1] = f[0] - cost f[0, -1] = -np.inf # force replace at lactation 10 # State Transition Function g = np.ones_like(f) g[0] = np.minimum(np.arange(n) + 1, n - 1) # Raise lactation number by 1, if keep # Model Structure model = DDPmodel(f, g, delta) model.solve() # Plot Optimal Policy demo.figure('Optimal Replacement', 'Age', 'Optimal Decision', [0, n + 1], [-0.5, 1.5]) plt.plot(S, model.policy, '*', markersize=15) plt.yticks((0, 1), X) # Plot Value Function demo.figure('Optimal Value in Cow Replacement', 'Age', 'Value (thousands)') plt.plot(S, model.value / 1000) plt.show()
# Reward Function f = np.empty((m ,n)) for k in range(m): f[k] = ((S * (1 - X[k])) ** (1-alpha)) / (1 - alpha) # State Transition Function g = np.empty_like(f) for k in range(m): snext = gamma * X[k] * S + (X[k] * S) ** beta g[k] = getindex(snext, S) # Model Structure model = DDPmodel(f, g, delta).solve() ## Analysis # Plot Optimal Policy demo.figure('Optimal Investment', 'Wealth', 'Investment') plt.plot(S, X[model.policy] * S) # Plot Optimal Policy demo.figure('Optimal Consumption', 'Wealth', 'Consumption') plt.plot(S, S - X[model.policy] * S) # Plot Value Function demo.figure('Optimal Value Function', 'Wealth', 'Value') plt.plot(S, model.value)
X = ['hold', 'exercise'] # vector of actions m = len(X) # number of actions # Reward Function f = np.zeros((m,n)) f[1] = strike - price # State Transition Probability Matrix P = np.zeros((m, n, n)) for i in range(n): P[0, i, min(i + 1, n - 1)] = q P[0, i, max(i - 1, 0)] = 1 - q # Model Structure model = DDPmodel(f, P, delta, horizon=N) model.solve() ## Analysis # Plot Optimal Exercise Boundary i, j = np.where(np.diff(model.policy[:-1], 1)) temp = (i * tau)[::-1] demo.figure('Put Option Optimal Exercise Boundary', 'Time to Maturity', 'Asset Price') plt.plot(temp, price[j]) # Plot Option Premium vs. Asset Price demo.figure('Put Option Value', 'Asset Price', 'Premium', [0, 2 * strike]) plt.plot([0, strike],[strike, 0], 'k--', lw=2) plt.plot(price, model.value[0], lw=3)