X = np.linspace(xmin, xmax, m) # vector of actions
# Reward Function
f = np.full((m, n), -np.inf)
for k in range(m):
f[k, S >= X[k]] = (X[k]**(1 - gamma)) / (1 - gamma) - cost * X[k]
# State Transition Function
g = np.zeros_like(f)
for k in range(m):
snext = alpha * (S - X[k]) - 0.5 * beta * (S - X[k])**2
g[k] = getindex(snext, S)
# Model Structure
model = DDPmodel(f, g, delta)
model.solve()
## Analysis
# Plot Optimal Policy
demo.figure('Optimal Harvest Policy', 'Stock', 'Harvest')
plt.plot(S, X[model.policy])
# Plot Value Function
demo.figure('Optimal Value Function', 'Stock', 'Value')
plt.plot(S, model.value)
# Simulate Model
nyrs = 20
t = np.arange(nyrs + 1)
# Reward Function
f = np.full((m, n), -np.inf)
for k in range(m):
f[k, S >= X[k]] = (X[k] ** (1 - gamma)) / (1 - gamma) - cost * X[k]
# State Transition Function
g = np.zeros_like(f)
for k in range(m):
snext = alpha * (S - X[k]) - 0.5 * beta * (S - X[k]) ** 2
g[k] = getindex(snext, S)
# Model Structure
model = DDPmodel(f, g, delta)
model.solve()
## Analysis
# Plot Optimal Policy
demo.figure('Optimal Harvest Policy', 'Stock', 'Harvest')
plt.plot(S,X[model.policy])
# Plot Value Function
demo.figure('Optimal Value Function', 'Stock', 'Value')
plt.plot(S,model.value)
# Reward Function
f = np.zeros((m, n))
f[0] = v # gets leisure
f[1, 0] = u # gets benefit
# State Transition Probability Matrix
P = np.zeros((m, n, n))
P[0, :, 0] = 1 # remains unemployed
P[1, 0, 0] = 1 - pfind # finds no job
P[1, 0, 1] = pfind # finds job
P[1, 1, 0] = pfire # gets fired
P[1, 1, 1] = 1 - pfire # keeps job
# Model Structure
model = DDPmodel(f, P, delta)
## Solution
# Solve Bellman Equation
wage = np.arange(55, 66)
xtable = np.zeros((wage.size, 2), dtype=int)
for i, w in enumerate(wage):
model.reward[1, 1] = w # vary wage
xtable[i] = model.solve().policy # solve via policy iteration
## Analysis
# Display Optimal Policy
print(' Optimal Job Search Strategy')
print(' (1=innactive, 2=active) ')
# Reward Function
f = np.zeros((m, n))
f[0] = v # gets leisure
f[1, 0] = u # gets benefit
# State Transition Probability Matrix
P = np.zeros((m, n, n))
P[0, :, 0] = 1 # remains unemployed
P[1, 0, 0] = 1 - pfind # finds no job
P[1, 0, 1] = pfind # finds job
P[1, 1, 0] = pfire # gets fired
P[1, 1, 1] = 1 - pfire # keeps job
# Model Structure
model = DDPmodel(f, P, delta)
## Solution
# Solve Bellman Equation
wage = np.arange(55, 66)
xtable = np.zeros((wage.size, 2), dtype=int)
for i, w in enumerate(wage):
model.reward[1, 1] = w # vary wage
xtable[i] = model.solve().policy # solve via policy iteration
## Analysis
# Display Optimal Policy
print(' Optimal Job Search Strategy')
print(' (1=innactive, 2=active) ')
m = 500 # number of actions
X = np.linspace(xmin, xmax, m) # vector of actions
# Reward Function
f = np.empty((m, n))
for k in range(m):
f[k] = ((S * (1 - X[k]))**(1 - alpha)) / (1 - alpha)
# State Transition Function
g = np.empty_like(f)
for k in range(m):
snext = gamma * X[k] * S + (X[k] * S)**beta
g[k] = getindex(snext, S)
# Model Structure
model = DDPmodel(f, g, delta).solve()
## Analysis
# Plot Optimal Policy
demo.figure('Optimal Investment', 'Wealth', 'Investment')
plt.plot(S, X[model.policy] * S)
# Plot Optimal Policy
demo.figure('Optimal Consumption', 'Wealth', 'Consumption')
plt.plot(S, S - X[model.policy] * S)
# Plot Value Function
demo.figure('Optimal Value Function', 'Wealth', 'Value')
plt.plot(S, model.value)
# Reward Function
f = np.full((m, n), -np.inf)
for c, s in enumerate(S):
for r, x in enumerate(X):
if x <= s:
f[r, c] = price * x - (x ** 2) / (1 + s)
# State Transition Function
g = np.empty_like(f)
for r, x in enumerate(X):
snext = S - x
g[r] = getindex(snext, S)
# Model Structure
model = DDPmodel(f, g, delta)
model.solve()
# Analysis
# Simulate Model
sinit = S.max()
nyrs = 15
t = np.arange(nyrs + 1)
spath, xpath = model.simulate(sinit, nyrs)
# Plot Optimal Policy
demo.figure('Optimal Extraction Policy', 'Stock', 'Extraction')
plt.plot(S, X[model.policy])
# Plot Value Function
# survives predation, finds food
snext = S[i] - 1 + e[k]
j = getindex(snext, S)
P[k, i, j] += p[k] * q[k]
# survives predation, finds no food
snext = S[i] - 1
j = getindex(snext, S)
P[k, i, j] += p[k] * (1 - q[k])
# Terminal Value Function
vterm = np.ones(n) # terminal value: survive
vterm[0] = 0 # terminal value: death
# Model Structure
model = DDPmodel(f, P, 1, T, vterm=vterm)
model.solve()
## Analysis
lims = [-0.5, emax + 0.5], [0, 1]
# Plot Survial Probabilities, Period 0
demo.figure('Survival Probability (Period 0)', 'Stock of Energy', 'Probability', *lims)
plt.bar(S, model.value[0], 1)
# Plot Survial Probabilities, Period 5
demo.figure('Survival Probability (Period 5)', 'Stock of Energy', 'Probability', *lims)
plt.bar(S, model.value[5], 1)
plt.show()
# State Transition Probability Matrix
P = np.zeros((2, n1, n2, n1, n2))
for i in range(n1):
for j in range(n2):
if i < 9:
P[0, i, j, i+1, j] = 1 # Raise lactation number by 1, if keep
else:
P[0, i, j, 0] = 0.2, 0.6, 0.2
P[1, i, j, 0] = 0.2, 0.6, 0.2 # Optional replacement
P.shape = 2, n, n
# Model Structure
model = DDPmodel(f, P, delta).solve(print=True)
## Analysis
# Display Optimal Policy
xtemp = model.policy.reshape((n1, n2))
header = '{:^8s} {:^8s} {:^8s} {:^8s}'.format('Age','Lo', 'Med', 'Hi')
print('Optimal Policy')
print(header)
print(*('{:^8d} {:^8s} {:^8s} {:^8s}\n'.format(s, *X[x]) for s, x in zip(s1, xtemp)))
# Plot Value Function
demo.figure('Optimal Replacement Value', 'Age', 'Optimal Value (thousands)')
# State Transition Probability Matrix
P = np.zeros((2, n1, n2, n1, n2))
for i in range(n1):
for j in range(n2):
if i < 9:
P[0, i, j, i + 1, j] = 1 # Raise lactation number by 1, if keep
else:
P[0, i, j, 0] = 0.2, 0.6, 0.2
P[1, i, j, 0] = 0.2, 0.6, 0.2 # Optional replacement
P.shape = 2, n, n
# Model Structure
model = DDPmodel(f, P, delta).solve(print=True)
## Analysis
# Display Optimal Policy
xtemp = model.policy.reshape((n1, n2))
header = '{:^8s} {:^8s} {:^8s} {:^8s}'.format('Age', 'Lo', 'Med', 'Hi')
print('Optimal Policy')
print(header)
print(*('{:^8d} {:^8s} {:^8s} {:^8s}\n'.format(s, *X[x])
for s, x in zip(s1, xtemp)))
# Plot Value Function
demo.figure('Optimal Replacement Value', 'Age', 'Optimal Value (thousands)')
plt.plot(s1, model.value.reshape((n1, n2)) / 1000)
f = np.full((m, n), -np.inf)
for k in range(m):
f[k, k:] = alpha1 * X[k]**beta1 + alpha2 * (S[k:] - X[k])**beta2
# State Transition Probability Matrix
P = np.zeros((m, n, n))
for k in range(m):
for i in range(n):
for j in range(r.size):
snext = min(S[i] - X[k] + r[j], maxcap)
inext = getindex(snext, S)
P[k, i, inext] = P[k, i, inext] + p[j]
# Model Structure
model = DDPmodel(f, P, delta)
model.solve()
## Analysis
# Plot Optimal Policy
demo.figure('Optimal Irrigation Policy', 'Water Level', 'Irrigation', [-1, 31],
[0, 6])
plt.plot(S, X[model.policy], '*')
# Plot Value Function
demo.figure('Optimal Value Function', 'Water Level', 'Value')
plt.plot(S, model.value)
# Simulate Model
sinit = np.zeros(10000)
m = len(X) # number of actions
# Reward Function
f = np.empty((m, n))
y = -0.2 * S**2 + 2 * S + 8 # yield per lactation
f[0] = price * y
f[1] = f[0] - cost
f[0, -1] = -np.inf # force replace at lactation 10
# State Transition Function
g = np.ones_like(f)
g[0] = np.minimum(np.arange(n) + 1,
n - 1) # Raise lactation number by 1, if keep
# Model Structure
model = DDPmodel(f, g, delta)
model.solve()
# Plot Optimal Policy
demo.figure('Optimal Replacement', 'Age', 'Optimal Decision', [0, n + 1],
[-0.5, 1.5])
plt.plot(S, model.policy, '*', markersize=15)
plt.yticks((0, 1), X)
# Plot Value Function
demo.figure('Optimal Value in Cow Replacement', 'Age', 'Value (thousands)')
plt.plot(S, model.value / 1000)
plt.show()
X = ['Keep', 'Replace'] # keep or replace
m = len(X) # number of actions
# Reward Function
f = np.empty((m, n))
y = -0.2 * S ** 2 + 2 * S + 8 # yield per lactation
f[0] = price * y
f[1] = f[0] - cost
f[0, -1] = -np.inf # force replace at lactation 10
# State Transition Function
g = np.ones_like(f)
g[0] = np.minimum(np.arange(n) + 1, n - 1) # Raise lactation number by 1, if keep
# Model Structure
model = DDPmodel(f, g, delta)
model.solve()
# Plot Optimal Policy
demo.figure('Optimal Replacement', 'Age', 'Optimal Decision', [0, n + 1], [-0.5, 1.5])
plt.plot(S, model.policy, '*', markersize=15)
plt.yticks((0, 1), X)
# Plot Value Function
demo.figure('Optimal Value in Cow Replacement', 'Age', 'Value (thousands)')
plt.plot(S, model.value / 1000)
plt.show()
# Reward Function
f = np.empty((m ,n))
for k in range(m):
f[k] = ((S * (1 - X[k])) ** (1-alpha)) / (1 - alpha)
# State Transition Function
g = np.empty_like(f)
for k in range(m):
snext = gamma * X[k] * S + (X[k] * S) ** beta
g[k] = getindex(snext, S)
# Model Structure
model = DDPmodel(f, g, delta).solve()
## Analysis
# Plot Optimal Policy
demo.figure('Optimal Investment', 'Wealth', 'Investment')
plt.plot(S, X[model.policy] * S)
# Plot Optimal Policy
demo.figure('Optimal Consumption', 'Wealth', 'Consumption')
plt.plot(S, S - X[model.policy] * S)
# Plot Value Function
demo.figure('Optimal Value Function', 'Wealth', 'Value')
plt.plot(S, model.value)
X = ['hold', 'exercise'] # vector of actions
m = len(X) # number of actions
# Reward Function
f = np.zeros((m,n))
f[1] = strike - price
# State Transition Probability Matrix
P = np.zeros((m, n, n))
for i in range(n):
P[0, i, min(i + 1, n - 1)] = q
P[0, i, max(i - 1, 0)] = 1 - q
# Model Structure
model = DDPmodel(f, P, delta, horizon=N)
model.solve()
## Analysis
# Plot Optimal Exercise Boundary
i, j = np.where(np.diff(model.policy[:-1], 1))
temp = (i * tau)[::-1]
demo.figure('Put Option Optimal Exercise Boundary', 'Time to Maturity', 'Asset Price')
plt.plot(temp, price[j])
# Plot Option Premium vs. Asset Price
demo.figure('Put Option Value', 'Asset Price', 'Premium', [0, 2 * strike])
plt.plot([0, strike],[strike, 0], 'k--', lw=2)
plt.plot(price, model.value[0], lw=3)