def _estimate(self, t, w_plus, z, value):
"""Estimate holding costs.
Args:
t: time of estimate
wplus: holdings
tau: time to estimate (default=t)
"""
try:
w_plus = w_plus[w_plus.index != self.cash_key]
w_plus = w_plus.values
except AttributeError:
w_plus = w_plus[:-1] # TODO fix when cvxpy pandas ready
try:
self.expression = cvx.multiply(
values_in_time(self.borrow_costs, t), cvx.neg(w_plus))
except TypeError:
self.expression = cvx.multiply(
values_in_time(self.borrow_costs, t).values, cvx.neg(w_plus))
try:
self.expression -= cvx.multiply(values_in_time(self.dividends, t),
w_plus)
except TypeError:
self.expression -= cvx.multiply(
values_in_time(self.dividends, t).values, w_plus)
return cvx.sum(self.expression), []
def __init__(self, m, n, r, eps=1e-4):
cvxopt.glpk.options["msg_lev"] = "GLP_MSG_OFF"
self.m = m
self.n = n
self.r = r
self.eps = eps
self.last_move = None
self.a = cp.Parameter(m) # Adjustments
self.C = cp.Parameter((m, m)) # C: Gradient inner products, G^T G
self.Ca = cp.Parameter(m) # d_bal^TG
self.rhs = cp.Parameter(m) # RHS of constraints for balancing
self.alpha = cp.Variable(m) # Variable to optimize
obj_bal = cp.Maximize(self.alpha @ self.Ca) # objective for balance
constraints_bal = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Simplex
self.C @ self.alpha >= self.rhs]
self.prob_bal = cp.Problem(obj_bal, constraints_bal) # LP balance
obj_dom = cp.Maximize(cp.sum(self.alpha @ self.C)) # obj for descent
constraints_res = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Restrict
self.alpha @ self.Ca >= -cp.neg(cp.max(self.Ca)),
self.C @ self.alpha >= 0]
constraints_rel = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Relaxed
self.C @ self.alpha >= 0]
self.prob_dom = cp.Problem(obj_dom, constraints_res) # LP dominance
self.prob_rel = cp.Problem(obj_dom, constraints_rel) # LP dominance
self.gamma = 0 # Stores the latest Optimum value of the LP problem
self.mu_rl = 0 # Stores the latest non-uniformity
def soiling_seperation_algorithm(observed, iterations=5, weights=None,
index_set=None, tau=0.85):
if weights is None:
weights = np.ones_like(observed)
if index_set is None:
index_set = ~np.isnan(observed)
zero_set = np.zeros(len(observed) - 1, dtype=np.bool)
eps = .01
n = len(observed)
s1 = cvx.Variable(n)
s2 = cvx.Variable(n)
s3 = cvx.Variable(n)
w = cvx.Parameter(n - 2, nonneg=True)
w.value = np.ones(len(observed) - 2)
for i in range(iterations):
# cvx.norm(cvx.multiply(s3, weights), p=2) \
cost = .1 * cvx.sum(tau * cvx.pos(s3) +(1 - tau) * cvx.neg(s3)) \
+ 10 * cvx.norm(cvx.diff(s2, k=2), p=2) \
+ .2 * cvx.norm(cvx.multiply(w, cvx.diff(s1, k=2)), p=1)
objective = cvx.Minimize(cost)
constraints = [
observed[index_set] == s1[index_set] + s2[index_set] + s3[index_set],
s2[365:] - s2[:-365] == 0,
cvx.sum(s2[:365]) == 0
# s1 <= 1
]
if np.sum(zero_set) > 0:
constraints.append(cvx.diff(s1, k=1)[zero_set] == 0)
problem = cvx.Problem(objective, constraints)
problem.solve(solver='MOSEK')
w.value = 1 / (eps + 1e2* np.abs(cvx.diff(s1, k=2).value)) # Reweight the L1 penalty
zero_set = np.abs(cvx.diff(s1, k=1).value) <= 5e-5 # Make nearly flat regions exactly flat (sparse 1st diff)
return s1.value, s2.value, s3.value
def est_freq_read(L):
"""Estimate expected genotype frequencies for an individual
using maximum likelihood. This is a convex optimization
problem i.e. a mixture distribution with fixed components
Args
----
L : np.array
p x 3 matrix of likelihoods
Returns
-------
pi_hat : np.array
3 x 1 vector of estimated mixture proportions
"""
# variable for mixture proportions
pi = cvx.Variable(3)
# objective function
objective = cvx.Minimize(cvx.neg(cvx.sum_entries(cvx.log(L * pi))))
# constraints of mixture proportions
constraints = [0 <= pi, pi <= 1, cvx.sum_entries(pi) == 1]
# intailize the optimization problem
prob = cvx.Problem(objective, constraints)
# sovle the problem
result = prob.solve()
# the optimal value for the mixture proportions
pi_hat = pi.value
return (pi_hat)
def risk_adj_return(x, s, Fs, Rs, Ds, mus, n, T, L, gamma=1):
'''
Compute objective function to portfolio problem
'''
obj = 0
CASH = np.eye(1, n, n - 1).T
for ii in range(T):
if ii == 0:
obj += s.T.dot(cvx.neg(x[ii*n:(ii+1)*n]).value) + gamma*(cvx.sum_squares(Fs[ii].T.dot(x[ii*n:(ii+1)*n])).value +\
Rs[ii]*cvx.sum_squares(x[ii*n:(ii+1)*n]).value) - np.dot(mus[ii].T,x[ii*n:(ii+1)*n])
+Ds[0] * cvx.sum_squares(x[:n]).value - (Ds[0] * CASH).T.dot(x[:n])
else:
obj += s.T.dot(cvx.neg(x[ii*n:(ii+1)*n]).value) + gamma*(cvx.sum_squares(Fs[ii].T.dot(x[ii*n:(ii+1)*n])).value +\
Rs[ii]*cvx.sum_squares(x[ii*n:(ii+1)*n]).value) - np.dot(mus[ii].T,x[ii*n:(ii+1)*n])
return float(obj + 0.5 * cvx.quad_form(x, L).value)
def get_constr_error(constr):
if isinstance(constr, cp.constraints.Equality):
error = cp.abs(constr.args[0] - constr.args[1])
elif isinstance(constr, cp.constraints.Inequality):
error = cp.pos(constr.args[0] - constr.args[1])
elif isinstance(constr, cp.constraints.PSD):
mat = constr.args[0] - constr.args[1]
error = cp.neg(cp.lambda_min(mat + mat.T) / 2)
return cp.sum(error)
def objective(ii):
full_obj = cvx.Minimize(
s.T * cvx.neg(Xs[ii]) + gamma *
(cvx.sum_squares(f[ii]) + Rs[ii] * cvx.sum_squares(Xs[ii])) -
mus[ii].T * Xs[ii] + .5 * LHAT_max_eigval *
cvx.sum_squares(Xs[ii] - cvx.reshape(x_prev[ii * n:(ii + 1) * n],
(n, ))) +
h_k[ii * n:(ii + 1) * n].T * Xs[ii])
return full_obj
def get_constr_error(constr):
if isinstance(constr, cvx.constraints.EqConstraint):
error = cvx.abs(constr.args[0] - constr.args[1])
elif isinstance(constr, cvx.constraints.LeqConstraint):
error = cvx.pos(constr.args[0] - constr.args[1])
elif isinstance(constr, cvx.constraints.PSDConstraint):
mat = constr.args[0] - constr.args[1]
error = cvx.neg(cvx.lambda_min(mat + mat.T)/2)
return cvx.sum_entries(error)
def quantile_reg(Y, X, quantile=0.5, lamb=0.01):
y, x = np.array(Y), np.array(X)
T, K = X.shape
b = cvx.Variable(K)
u = y - x * b
J = (quantile * cvx.sum_entries(cvx.pos(u)) +
(1 - quantile) * cvx.sum_entries(cvx.neg(u))) / T
L2 = cvx.norm(b)**2
prob = cvx.Problem(cvx.Minimize(J + lamb * L2))
prob.solve()
return np.asarray(b.value).reshape(-1)
def _genModelConstraints(self, x, prepared_constraints, prepared_option):
CVXConstraints = []
for iType, iConstraint in prepared_constraints.items():
if iType == "Box":
CVXConstraints.extend([
x <= iConstraint["ub"].flatten(),
x >= iConstraint["lb"].flatten()
])
elif iType == "LinearIn":
CVXConstraints.append(
iConstraint["A"] @ x <= iConstraint["b"].flatten())
elif iType == "LinearEq":
CVXConstraints.append(
iConstraint["Aeq"] @ x == iConstraint["beq"].flatten())
elif iType == "Quadratic":
for jSubConstraint in iConstraint:
if "X" in jSubConstraint:
jSigma = np.dot(
np.dot(jSubConstraint["X"], jSubConstraint["F"]),
jSubConstraint["X"].T) + np.diag(
jSubConstraint["Delta"].flatten())
jSigma = (jSigma + jSigma.T) / 2
elif "Sigma" in jSubConstraint:
jSigma = jSubConstraint["Sigma"]
CVXConstraints.append(
cvx.quad_form(x, jSigma) +
jSubConstraint["Mu"].T @ x <= jSubConstraint["q"])
elif iType == "L1":
for jSubConstraint in iConstraint:
CVXConstraints.append(
cvx.norm(x - jSubConstraint["c"].flatten(), p=1) <=
jSubConstraint["l"])
elif iType == "Pos":
for jSubConstraint in iConstraint:
CVXConstraints.append(
cvx.pos(x - jSubConstraint["c_pos"].flatten()) <=
jSubConstraint["l_pos"])
elif iType == "Neg":
for jSubConstraint in iConstraint:
CVXConstraints.append(
cvx.neg(x - jSubConstraint["c_neg"].flatten()) <=
jSubConstraint["l_neg"])
elif iType == "NonZeroNum":
raise __QS_Error__("不支持的约束条件: '非零数目约束'!")
return CVXConstraints
def finance_example_regular_solve(problem_data):
'''
Solves multi-period portfolio optimization problem by forming one large problem,
and invoking CVXPY's solve() method.
'''
L = problem_data['L']
n = problem_data['n']
m = problem_data['m']
T = problem_data['p']
gamma = problem_data['gamma']
tolerance = problem_data['tol']
mus = problem_data['mus']
Fs = problem_data['Fs']
Rs = problem_data['Rs']
Ds = problem_data['Ds']
s = problem_data['s']
SOLVER = problem_data['SOLVER']
print('Solving problem via CVXPY, no MM.')
CASH = np.eye(1, n, n - 1).T.reshape(n, ) #cash vector
x = cvx.Variable((n, T))
f = cvx.Variable((m, T))
objs_normal = 0 #Ds[0] * cvx.sum_squares(x[:,0]) - (np.dot(Ds[0],CASH).T * x[:,0])
constraints_normal = [x[:, 0] == CASH, x[:, T - 1] == CASH]
for ii in range(T):
constraints_normal += [
sum(x[:, ii]) == 1, f[:, ii] == Fs[ii].T * x[:, ii]
]
objs_normal += s.T * cvx.neg(x[:, ii]) + gamma * (cvx.sum_squares(
f[:, ii]) + Rs[ii] * cvx.sum_squares(x[:, ii])) - (mus[ii].T *
x[:, ii])
#Solve problem with CVXPY
t_start_reg = time.time()
(x_opt, optval) = solve_complete(x, objs_normal, constraints_normal, L, Ds,
SOLVER)
t_reg = time.time() - t_start_reg
return x_opt, optval, t_reg
def _solveMeanVarianceModel(self, nvar, prepared_objective,
prepared_constraints, prepared_option):
x = cvx.Variable(nvar)
Obj = 0
if "f" in prepared_objective: Obj += prepared_objective["f"].T @ x
if "X" in prepared_objective:
Sigma = np.dot(
np.dot(prepared_objective["X"], prepared_objective["F"]),
prepared_objective["X"].T) + np.diag(
prepared_objective["Delta"].flatten())
Sigma = (Sigma + Sigma.T) / 2
Obj += cvx.quad_form(x, Sigma)
elif "Sigma" in prepared_objective:
Obj += cvx.quad_form(x, prepared_objective["Sigma"])
if "Mu" in prepared_objective: Obj += prepared_objective["Mu"].T @ x
if "lambda1" in prepared_objective:
Obj += prepared_objective["lambda1"] * cvx.norm(
x - prepared_objective["c"].flatten(), p=1)
if "lambda2" in prepared_objective:
Obj += prepared_objective["lambda2"] * cvx.pos(
x - prepared_objective["c_pos"].flatten())
if "lambda3" in prepared_objective:
Obj += prepared_objective["lambda3"] * cvx.neg(
x - prepared_objective["c_neg"].flatten())
CVXConstraints = self._genModelConstraints(x, prepared_constraints,
prepared_option)
self._Model = cvx.Problem(cvx.Minimize(Obj), CVXConstraints)
self._x = x
self._Model.solve(**prepared_option)
Status = (1 if self._Model.status not in ("infeasible",
"unbounded") else 0)
return (self._x.value, {
"Status": Status,
"Msg": self._Model.status,
"solver_name": self._Model.solver_stats.solver_name,
"solve_time": self._Model.solver_stats.solve_time,
"setup_time": self._Model.solver_stats.setup_time,
"num_iters": self._Model.solver_stats.num_iters
})
problem_data['p'] = T
problem_data['gamma'] = gamma
problem_data['tol'] = tol
problem_data['max_iter'] = max_iter
problem_data['mus'] = mus
problem_data['Fs'] = Fs
problem_data['Rs'] = Rs
problem_data['Ds'] = Ds
problem_data['s'] = s
problem_data['SOLVER'] = 'OSQP'
print('Using %s as solver.' % problem_data['SOLVER'])
'''Solve static problem to get initial points for MM.'''
print('Solving static problem.')
t_start_static = time.time()
x_static = cvx.Variable((n, 1))
obj_static = s.T * cvx.neg(x_static) + gamma * (
cvx.sum_squares(Fs[0].T * x_static) +
Rs[0] * cvx.sum_squares(x_static)) - mus[0].T * x_static
constraints_static = [sum(x_static) == 1]
cvx.Problem(cvx.Minimize(obj_static), constraints_static).solve()
t_static = time.time() - t_start_static
print('Computation time was %s seconds for the static problem.' %
str(t_static))
print('Solve MPO problem using problem.solve().')
(x_opt, obj_opt, t_reg) = finance_example_regular_solve(problem_data)
print('Time it took for regular solve = %s' % t_reg)
'''instantiate x^0 with static problem solution.'''
x_prev = cvx.Parameter((n * T, 1))
x_prev.value = np.zeros((n * T, 1))
for ii in range(T):
def backtest(returns,
Z_returns,
benchmark,
means,
covs,
lev_lim,
bottom_sec_limit,
upper_sec_limit,
shorting_cost,
tcost,
MAXRISK,
kappa=None,
bid_ask=None):
_, num_assets = returns.shape
T, K = Z_returns.shape
Zs_time = np.zeros((T - 1, K))
value_strat, value_benchmark = 1, 1
vals_strat, vals_benchmark = [value_strat], [value_benchmark]
benchmark_returns = benchmark.loc[Z_returns.index].copy().values.flatten()
"""
On the fly computing for stratified model policy
"""
W = [np.zeros(18)]
W[0][8] = 1 #vti
for date in range(1, T):
# if date % 50 == 0: print(date)
dt = Z_returns.iloc[date].name.strftime("%Y-%m-%d")
node = tuple(Z_returns.iloc[date])
Zs_time[date - 1, :] = [*node]
if date == 1:
w_prev = W[0].copy()
else:
w_prev = W[-1].flatten().copy()
w = cp.Variable(num_assets)
#adding cash asset into returns and covariances
SIGMA = covs[node]
MU = means[node]
roll = 15
#get last 5 days tcs, lagged by one! so this doesnt include today's date
tau = np.maximum(bid_ask.loc[:dt].iloc[-(roll + 1):-1].mean().values,
0) / 2
obj = -w @ (MU + 1) + shorting_cost * (kappa @ cp.neg(w)) + tcost * (
cp.abs(w - w_prev)) @ tau
cons = [
cp.quad_form(w, SIGMA) <= MAXRISK * 100 * 100,
sum(w) == 1,
cp.norm1(w) <= lev_lim,
bottom_sec_limit * np.ones(num_assets) <= w,
w <= upper_sec_limit * np.ones(num_assets),
]
prob_sm = cp.Problem(cp.Minimize(obj), cons)
prob_sm.solve(verbose=False)
returns_date = 1 + returns[date, :]
#get TODAY's tc
tau_sim = bid_ask.loc[dt].values.flatten() / 2
value_strat *= returns_date @ w.value - (kappa @ cp.neg(w)).value - (
cp.abs(w - w_prev) @ tau_sim).value
vals_strat += [value_strat]
value_benchmark *= 1 + benchmark_returns[date]
vals_benchmark += [value_benchmark]
w_prev = w.value.copy() * returns_date
W += [w_prev.reshape(-1, 1)]
vals = pd.DataFrame(data=np.vstack([vals_strat, vals_benchmark]).T,
columns=["policy", "benchmark"],
index=Z_returns.index.rename("Date"))
Zs_time = pd.DataFrame(data=Zs_time,
index=Z_returns.index[1:],
columns=Z_returns.columns)
#calculate sharpe
rr = vals.pct_change()
sharpes = np.sqrt(250) * rr.mean() / rr.std()
returns = 250 * rr.mean()
return vals, Zs_time, W, sharpes, returns
def routing_algorithm(world, robots, mode="random", maximumSeconds=120):
"""
Route the world using various algorithms.
Parameters
----------
world: World object
world object as defined in world.py
robots: list
list of Robot objects that will be altered for the animation
mode: str
a string that represents the mode we are going to use
"""
assignments = [[] for _ in range(len(robots))]
if mode == "random":
"""Assign each task to a random robot multiple times to find a distribution."""
tasks = {}
for i, hospital in enumerate(world.hospitals):
if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[
hospital]['demand2'] != 0:
pointer = world.graph.nodes[hospital]
tasks[hospital] = np.array(
[pointer['demand1'], pointer['demand2']], dtype=float)
# Assign robots to tasks until there are none left
while len(tasks) > 0:
# print("Remaining tasks: ", tasks)
for i, robot in enumerate(robots):
if len(tasks) == 0:
break
random_goal = random.choice(list(tasks.keys()))
tasks[random_goal] -= np.array(
[robot.capacity, robot.capacity])
assignments[i].append(
[random_goal,
np.array([robot.capacity, robot.capacity])])
if np.all(tasks[random_goal] <= 0.):
del tasks[random_goal]
elif mode == "hungarian":
""" We want a cost matrix of size number_of_robots x no_of_tasks"""
number_of_robots = len(robots)
tasks = []
for hospital in world.hospitals:
if world.graph.nodes[hospital]['demand1'] != 0:
tasks.append(
(hospital, 0, world.graph.nodes[hospital]['demand1']))
if world.graph.nodes[hospital]['demand2'] != 0:
tasks.append(
(hospital, 1, world.graph.nodes[hospital]['demand2']))
cost_matrix = np.zeros((number_of_robots, len(tasks)))
# So now we have a list of tasks. Each task is a tuple: (hospital, demand type, demand)
# We also have a cost matrix of the correct size, now we fill it in with the time cost
for i, robot in enumerate(robots):
for j, task in enumerate(tasks):
#print("robot: {}, task: {}".format(i,j))
path_length = nx.shortest_path_length(world.graph,
robot.start_node,
task[0],
weight='length')
# Calculate time taken to go to that hospital
time_taken = path_length / robot.speed
cost_matrix[i][j] = time_taken
robot_ind, task_ind = linear_sum_assignment(cost_matrix)
for i, index in enumerate(robot_ind):
#print("robot_ind: ", i)
task = tasks[task_ind[i]]
assignments[i].append([
task[0],
np.array([(1 - task[1]) * robots[index].capacity,
task[1] * robots[index].capacity])
])
elif mode == "linear_separate_tasks":
# Takes all the tasks as separate tasks, i.e. does not join them together and the robot only delivers one type
# of task.
# Obtain all the demand and priority for goods 1 and 2
demand = []
tasks = []
types = []
priority = []
for i, hospital in enumerate(world.hospitals):
if world.graph.nodes[hospital]['demand1'] != 0:
demand.append(world.graph.nodes[hospital]['demand1'])
tasks.append(hospital)
types.append(0)
priority.append(world.graph.nodes[hospital]['priority'])
if world.graph.nodes[hospital]['demand2'] != 0:
demand.append(world.graph.nodes[hospital]['demand2'])
tasks.append(hospital)
types.append(1)
priority.append(world.graph.nodes[hospital]['priority'])
# Obtain the time cost matrix and the robots' capacity.
T = np.zeros((len(robots), len(demand)))
capacity = np.zeros(len(robots))
for i, robot in enumerate(robots):
capacity[i] = robot.capacity
for j, task in enumerate(tasks):
path_length = nx.shortest_path_length(world.graph,
robot.start_node,
task,
weight='length')
time_taken = path_length / robot.speed
T[i][j] = time_taken
# Change the demand and priority arrays to numpy ones
demand = np.array(demand)
priority = np.array(priority) * 10000
# Define the optimization problem
x = cp.Variable((len(robots), len(demand)), boolean=True)
cost = cp.sum(cp.multiply(T, x)) + cp.sum(
cp.neg(cp.multiply(cp.matmul(capacity, x) - demand, priority)))
objective = cp.Minimize(cost)
inequality = [cp.sum(x, axis=1) <= 1]
problem = cp.Problem(objective, inequality)
problem.solve()
# Assign the results to the robots and evaluate the costs
nonzero = x.value.nonzero()
for index in range(len(nonzero[0])):
i = nonzero[1][index]
j = nonzero[0][index]
robot = robots[j]
task = tasks[i]
assignments[j].append([
task,
np.array([(1 - types[i]) * robot.capacity,
types[i] * robot.capacity])
])
elif mode == "linear_joined_tasks":
# As above but allows the robots to deliver two goods at the same time
# Obtain all the demand and priority for goods 1 and 2
demand = []
tasks = []
priority = []
for i, hospital in enumerate(world.hospitals):
if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[
hospital]['demand2'] != 0:
pointer = world.graph.nodes[hospital]
demand.append([pointer['demand1'], pointer['demand2']])
tasks.append(hospital)
priority.append([pointer['priority'], pointer['priority']])
# Obtain the time cost matrix and the robots' capacity
T = np.zeros((len(robots), len(demand)))
capacity = np.zeros((len(robots), 2))
for i, robot in enumerate(robots):
capacity[i, 0] = robot.capacity
capacity[i, 1] = robot.capacity
for j, task in enumerate(tasks):
path_length = nx.shortest_path_length(world.graph,
robot.start_node,
task,
weight='length')
time_taken = path_length / robot.speed
T[i][j] = time_taken
# Change the demand and priority arrays to numpy ones
demand = np.array(demand)
priority = np.array(priority) * 10000
# Define the optimization problem
x = cp.Variable((len(robots), len(demand)), boolean=True)
cost = cp.sum(cp.multiply(T, x)) + cp.sum(
cp.neg(
cp.multiply(
cp.matmul(cp.transpose(x), capacity) - demand, priority)))
objective = cp.Minimize(cost)
inequality = [cp.sum(x, axis=1) <= 1]
problem = cp.Problem(objective, inequality)
problem.solve(verbose=True, tm_lim=60000)
# Assign the results to the robots and evaluate the costs
nonzero = x.value.nonzero()
# print(nonzero)
for index in range(len(nonzero[0])):
i = nonzero[1][index]
j = nonzero[0][index]
task = tasks[i]
assignments[j].append(
[task, np.array([capacity[j][0], capacity[j][1]])])
elif mode == "tsm":
# Method based on the m-travelling salesmen problem
# Obtain all the demand and priority for goods 1 and 2
demand = []
tasks = []
priority = []
for i, hospital in enumerate(world.hospitals):
if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[
hospital]['demand2'] != 0:
pointer = world.graph.nodes[hospital]
demand.append([pointer['demand1'], pointer['demand2']])
tasks.append(hospital)
priority.append([pointer['priority'], pointer['priority']])
# Obtain the time cost matrix and the robots' capacity
x = []
v = []
z = []
capacities = []
distances = []
constraints = []
costs = []
for i, robot in enumerate(robots):
# Create necessary variables
x.append(
cp.Variable((len(demand) + 1, len(demand) + 1), boolean=True))
v.append(cp.Variable(len(demand) + 1, boolean=True))
z.append(cp.Variable((len(demand), 1)))
capacities.append(cp.Variable((len(demand), 2)))
# Add constraints
constraints.append(cp.sum(
x[i], axis=0) == v[i]) # Note if city is moved from
constraints.append(cp.sum(x[i], axis=0) == cp.sum(
x[i], axis=1)) # Note if city is visited
constraints.append(z[i] - cp.transpose(z[i]) +
len(demand) * x[i][1:, 1:] <= len(demand) - 1)
constraints.append(cp.sum(x[i], axis=1) <= cp.sum(
x[i][0, :])) # We start from 0th node
constraints.append(
cp.sum(capacities[i], axis=0) <=
robot.capacity) # Make sure we are not over capacity
constraints.append(
cp.sum(capacities[i], axis=1) <=
1000 * v[i][1:]) # Check how many units are delivered
constraints.append(capacities[i] >= 0)
# Fill in the distance matrix
distances.append(np.zeros((len(demand) + 1, len(demand) + 1)))
for j, task in enumerate([robot.start_node] + tasks):
for k, other_task in enumerate([robot.start_node] + tasks):
if other_task != task:
path_length = nx.shortest_path_length(world.graph,
task,
other_task,
weight='length')
distances[i][j][k] = path_length / robot.speed
else:
distances[i][j][k] = 100000
costs.append(cp.sum(cp.multiply(x[i], distances[i])))
demand = np.array(demand)
priority = np.array(priority) * 10000
costs = cp.max(cp.hstack(costs)) + cp.sum(
cp.multiply(cp.pos(demand - sum(capacities)), priority))
objective = cp.Minimize(costs)
problem = cp.Problem(objective, constraints)
problem.solve(verbose=True,
solver=cp.CBC,
numberThreads=8,
logLevel=1,
maximumSeconds=maximumSeconds,
allowablePercentageGap=10)
# Assign the results to the robots and evaluate the costs
for i, x_i in enumerate(x):
print(f"variable {i + 1}: {x_i.value}")
print(f"variable {i + 1}: {np.sum(x_i.value * distances[i])}")
print(f"variable {i + 1}: {capacities[i].value}")
for i, robot in enumerate(robots):
node = np.argmax(x[i].value[0, :])
while node != 0:
if np.any(capacities[i].value[node - 1] != 0):
task = tasks[node - 1]
assignments[i].append(
[task, capacities[i].value[node - 1]])
node = np.argmax(x[i].value[node, :])
elif mode == "home":
# Home made method allowing to specify the depth of the solution to be considered N
# Obtain all the demand and priority for goods 1 and 2
demand = []
tasks = []
priority = []
N = 2
for i, hospital in enumerate(world.hospitals):
if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[
hospital]['demand2'] != 0:
pointer = world.graph.nodes[hospital]
demand.append([pointer['demand1'], pointer['demand2']])
tasks.append(hospital)
priority.append([pointer['priority'], pointer['priority']])
# Obtain the time cost matrix and the robots' capacity
x = [[] for _ in range(len(robots))]
capacities = []
constraints = []
costs = []
for i, robot in enumerate(robots):
# Obtain the time matrix for the robot
time_matrix = np.zeros((len(demand), len(demand)))
original = np.zeros(len(demand))
for j, task in enumerate(tasks):
path_length = nx.shortest_path_length(world.graph,
robot.start_node,
task,
weight='length')
original[j] = path_length / robot.speed
for k, other_task in enumerate(tasks):
if other_task != task:
path_length = nx.shortest_path_length(world.graph,
task,
other_task,
weight='length')
time_matrix[j][k] = path_length / robot.speed
else:
time_matrix[j][k] = 0
# Add the initial state
current_costs = []
x[i].append(cp.Variable((len(demand), 1), boolean=True))
constraints.append(cp.sum(x[i][0]) <= 1)
current_costs.append(cp.sum(cp.multiply(cp.vec(x[i][0]),
original)))
capacities.append(cp.Variable((len(demand), 2), integer=True))
# Loop over the other future states
for n in range(1, N):
x[i].append(cp.Variable((len(demand), 1), boolean=True))
current_costs.append(
cp.sum(
cp.multiply(
cp.pos(x[i][n - 1] + cp.transpose(x[i][n]) - 1),
time_matrix)))
constraints.append(cp.sum(x[i][n]) <= cp.sum(x[i][n - 1]))
# Add the capacities constraints
constraints.append(
cp.sum(capacities[i], axis=0) <=
robot.capacity) # Make sure we are not over capacity
constraints.append(
cp.sum(capacities[i], axis=1) <= 1000 * cp.vec(sum(x[i])))
constraints.append(capacities[i] >= 0)
# Add the current costs to the total costs
costs.append(sum(current_costs))
demand = np.array(demand)
priority = np.array(priority) * 10000
costs = cp.max(cp.hstack(costs)) + cp.sum(
cp.multiply(cp.pos(demand - sum(capacities)), priority))
objective = cp.Minimize(costs)
problem = cp.Problem(objective, constraints)
problem.solve(verbose=True,
solver=cp.CBC,
logLevel=1,
numberThreads=4,
maximumSeconds=120,
allowablePercentageGap=5)
# Assign the results to the robots and evaluate the costs
for robot_i, robot in enumerate(x):
print(f"ROBOT {robot_i}\n-------")
print(f"delivered capacities: \n{capacities[robot_i].value}")
print(f"step 0: {0}")
for i, x_i in enumerate(robot):
print(f"step {i + 1}: {x_i.value.T}")
for i, robot in enumerate(robots):
visited_tasks = set()
robot._current_node = robot.start_node
for j, x_i in enumerate(x[i]):
index = np.argmax(x_i.value)
task = tasks[index]
if np.any(capacities[i].value[index] != 0
) and task not in visited_tasks:
visited_tasks.add(task)
assignments[i].append([task, capacities[i].value[index]])
robot._current_node = task
else:
raise NotImplementedError
return assignments
def reg(self, X): return 1e10*cp.sum(cp.neg(X))
def __str__(self): return "nonnegative reg"
def reg(self, X):
return 1e10 * cp.sum_entries(cp.neg(X))
def train_network(X_f, X_i, X_s, Y_i, beta, M=int(5000), fuse_xxstar=False, abs_act=False, sdf=True, norm=2):
n_f, d = X_f.shape
n_i, d = X_i.shape
X_f = np.append(X_f,np.ones((n_f,1)),axis=1)
X_i = np.append(X_i,np.ones((n_i,1)),axis=1)
X_s = np.append(X_s,np.ones((n_i,1)),axis=1)
d += 1
Y_i = np.atleast_1d(Y_i.squeeze())
dual_norm = np.inf if norm==1 else int(1/(1-1/float(norm)))
## Finite approximation of all possible sign patterns
t0 = time.time()
if n_f + 2*n_i < 15:
D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar)
else:
D_f, D_i, D_s = random_signed_patterns(X_f, X_i, X_s, M, fuse_xxstar=fuse_xxstar)
m1 = D_f.shape[1]
print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.')
# Optimal CVX
Uopt1=cp.Variable((d,m1), value=np.random.randn(d,m1))
Uopt2=cp.Variable((d,m1), value=np.random.randn(d,m1))
constraints = []
loss = 0
# Feasible points
if n_f > 0:
ux_f_1 = cp.multiply(D_f,(X_f @ Uopt1))
ux_f_2 = cp.multiply(D_f,(X_f @ Uopt2))
y_f = cp.sum(ux_f_1 - ux_f_2,axis=1) if abs_act \
else cp.sum(cp.multiply((D_f+1)/2,(X_f @ (Uopt1 - Uopt2))),axis=1)
constraints += [
ux_f_1>=0,
ux_f_2>=0
]
Y_f_lb = np.max(Y_i[:, None] - np.linalg.norm(X_f[None, :, :] - X_i[:, None, :], axis=-1,
ord=dual_norm), axis=0)
if sdf:
loss_f = cp.sum(cp.abs(y_f - Y_f_lb))
else:
loss_f = cp.sum(cp.pos(y_f))
loss = loss + loss_f
lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb))
else:
loss_f = cp.Variable(value=0.)
lipsh_f = cp.Variable(value=0.)
# Infeasible points
if n_i > 0:
ux_i_1 = cp.multiply(D_i,(X_i @ Uopt1))
ux_s_1 = cp.multiply(D_s,(X_s @ Uopt1))
ux_i_2 = cp.multiply(D_i,(X_i @ Uopt2))
ux_s_2 = cp.multiply(D_s,(X_s @ Uopt2))
if abs_act:
y_i = cp.sum(ux_i_1 - ux_i_2,axis=1)
y_s = cp.sum(ux_s_1 - ux_s_2,axis=1)
if sdf:
if norm == 2:
constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1]
elif norm == 1:
constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1]
else:
y_i = cp.sum(cp.multiply((D_i+1)/2,(X_i @ (Uopt1 - Uopt2))),axis=1)
y_s = cp.sum(cp.multiply((D_s+1)/2,(X_s @ (Uopt1 - Uopt2))),axis=1)
if sdf:
if norm == 2:
constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1]
elif norm == 1:
constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1]
constraints += [
ux_i_1>=0,
ux_s_1>=0,
ux_i_2>=0,
ux_s_2>=0,
]
loss_i = cp.sum(cp.abs(Y_i - y_i))
loss_s = cp.sum(cp.abs(y_s))
loss = loss + loss_i + loss_s
lipsh_i = cp.sum(cp.neg(Y_i - y_i))
else:
loss_i = cp.Variable(value=0.)
loss_s = cp.Variable(value=0.)
lipsh_i = cp.Variable(value=0.)
lipsh_s = cp.Variable(value=0.)
# Regularization
if norm == 2:
reg = cp.mixed_norm(Uopt1[:-1].T,2,1) + cp.mixed_norm(Uopt2[:-1].T,2,1)
elif norm == 1:
reg = cp.Variable(nonneg=True)
for i in range(d-1):
for s in [-1,1]:
constraints += [cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <= reg]
# Solution
prob=cp.Problem(cp.Minimize(100*(loss + beta * reg)),constraints)
t0 = time.time()
options = dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8})
prob.solve(solver=cp.MOSEK, verbose=True, **options)
# prob.solve(solver=cp.SCS)
print(f'Status: {prob.status}, \n '
f'Value: {prob.value :.2E}, \n '
f'loss_f: {loss_f.value :.2E}, \n '
f'loss_i: {loss_i.value :.2E}, \n '
f'loss_s: {loss_s.value :.2E}, \n '
f'Reg: {reg.value : .2E}, \n '
f'lipsh_f: {lipsh_f.value :.2E}, \n '
f'lipsh_i: {lipsh_i.value :.2E}, \n '
f'Time: {time.time()-t0 :.2f}s')
if prob.status.lower() == 'infeasible':
st()
return None
u1, u2 = torch.tensor(Uopt1.value), torch.tensor(Uopt2.value)
st()
return u1, u2
#No (additional) constraints
constraints_i = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1]
prob_ai = cvx.Problem(obj_a, constraints_i)
risk_ai = prob_ai.solve()
#Long-only
constraints_ii = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1 , x >= 0]
prob_aii = cvx.Problem(obj_a, constraints_ii)
risk_aii = prob_aii.solve()
#Limit on total short position
#one = np.matrix(np.ones((n, 1)))
constraints_iii = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1]
for i in range (n):
constraints_iii += [1 * cvx.neg(x[i]) <= 0.5]
prob_aiii = cvx.Problem(obj_a, constraints_iii)
risk_aiii = prob_aiii.solve()
print('Uniform portfolio risk: {}'.format(risk_uniform))
print('No (additional) constraints risk: {}'.format(risk_ai))
print('Long-only risk: {}'.format(risk_aii))
print('Limit on total short position risk: {}'.format(risk_aiii))
# Part (b) Plot optimal risk-return trade-off curves
mus = np.logspace(0, 5, n)
mean_long = np.zeros(n)
std_long = np.zeros(n)
mean_totalshort = np.zeros(n)
std_totalshort = np.zeros(n)
constraints_long = [cvx.sum_entries(x) == 1, x>= 0]
def reg(self, X):
return 1e10 * cp.sum(cp.neg(X))
required_return = expected_return(x_unif)
x = cp.Variable((n, 1))
sum_to_one_constraint = np.ones((n, 1)).T @ x == 1
required_return_constraint = pbar.T @ x == required_return
objective = cp.Minimize(cp.quad_form(x, 2 * S))
min_risk_prob = cp.Problem(objective,
[sum_to_one_constraint, required_return_constraint])
min_risk_prob.solve()
print("Status = " + str(min_risk_prob.status))
print_portfolio(x.value, "min risk")
long_only_constraint = x >= 0
long_only_prob = cp.Problem(
objective,
[sum_to_one_constraint, required_return_constraint, long_only_constraint])
long_only_prob.solve()
print("Status = " + str(long_only_prob.status))
print_portfolio(x.value, "long only")
limit_short_constraint = np.ones((n, 1)).T @ cp.neg(x) <= 0.5
limit_short_prob = cp.Problem(objective, [
sum_to_one_constraint, required_return_constraint, limit_short_constraint
])
limit_short_prob.solve()
print("Status = " + str(limit_short_prob.status))
print_portfolio(x.value, "limit short")
print("min risk with expected return equals uniform portfolio")
x = cp.Variable(n)
constraints = \
[
cp.sum(x) == 1,
pMean.T @ x == pMean.T @ xU
]
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints)
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")
#%%
# What is the risk of a long-only portfolio ЁЭСетк░0?
print("x >= 0")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [x >= 0])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")
#%%
# What is the risk of a portfolio with a limit on total short position: 1ЁЭСЗ(ЁЭСетИТ)тЙд0.5 , where (ЁЭСетИТ)ЁЭСЦ=max{тИТЁЭСеЁЭСЦ,0} ?
print("limit on short")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)),
constraints + [cp.sum(cp.neg(x)) <= 0.5])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")
def optimization(
self, model="Classic", rm="MV", obj="Sharpe", rf=0, l=2, hist=True
):
r"""
This method that calculates the optimum portfolio according to the
optimization model selected by the user. The general problem that
solves is:
.. math::
\begin{align}
&\underset{x}{\text{optimize}} & & F(w)\\
&\text{s. t.} & & Aw \geq B\\
& & & \phi_{i}(w) \leq c_{i}\\
\end{align}
Where:
:math:`F(w)` is the objective function.
:math:`Aw \geq B` is a set of linear constraints.
:math:`\phi_{i}(w) \leq c_{i}` are constraints on maximum values of
several risk measures.
Parameters
----------
model : str can be 'Classic', 'BL' or 'FM'
The model used for optimize the portfolio.
The default is 'Classic'. Posible values are:
- 'Classic': use estimates of expected return vector and covariance matrix that depends on historical data.
- 'BL': use estimates of expected return vector and covariance matrix based on the Black Litterman model.
- 'FM': use estimates of expected return vector and covariance matrix based on a Risk Factor model specified by the user.
rm : str, optional
The risk measure used to optimze the portfolio.
The default is 'MV'. Posible values are:
- 'MV': Standard Deviation.
- 'MAD': Mean Absolute Deviation.
- 'MSV': Semi Standard Deviation.
- 'FLPM': First Lower Partial Moment (Omega Ratio).
- 'SLPM': Second Lower Partial Moment (Sortino Ratio).
- 'CVaR': Conditional Value at Risk.
- 'WR': Worst Realization (Minimax)
- 'MDD': Maximum Drawdown of uncompounded returns (Calmar Ratio).
- 'ADD': Average Drawdown of uncompounded returns.
- 'CDaR': Conditional Drawdown at Risk of uncompounded returns.
obj : str can be {'MinRisk', 'Utility', 'Sharpe' or 'MaxRet'.
Objective function of the optimization model.
The default is 'Sharpe'. Posible values are:
- 'MinRisk': Minimize the selected risk measure.
- 'Utility': Maximize the Utility function :math:`mu w - l \phi_{i}(w)`.
- 'Sharpe': Maximize the risk adjusted return ratio based on the selected risk measure.
- 'MaxRet': Maximize the expected return of the portfolio.
rf : float, optional
Risk free rate, must be in the same period of assets returns.
The default is 0.
l : scalar, optional
Risk aversion factor of the 'Utility' objective function.
The default is 2.
hist : bool, optional
Indicate if uses historical or factor estimation of returns to
calculate risk measures that depends on scenarios (All except
'MV' risk measure). The default is True.
Returns
-------
w : DataFrame
The weights of optimum portfolio.
"""
# General model Variables
mu = None
sigma = None
returns = None
if model == "Classic":
mu = np.matrix(self.mu)
sigma = np.matrix(self.cov)
returns = np.matrix(self.returns)
nav = np.matrix(self.nav)
elif model == "FM":
mu = np.matrix(self.mu_fm)
if hist == False:
sigma = np.matrix(self.cov_fm)
returns = np.matrix(self.returns_fm)
nav = np.matrix(self.nav_fm)
elif hist == True:
sigma = np.matrix(self.cov)
returns = np.matrix(self.returns)
nav = np.matrix(self.nav)
elif model == "BL":
mu = np.matrix(self.mu_bl)
if hist == False:
sigma = np.matrix(self.cov_bl)
elif hist == True:
sigma = np.matrix(self.cov)
returns = np.matrix(self.returns)
nav = np.matrix(self.nav)
elif model == "BL_FM":
mu = np.matrix(self.mu_bl_fm)
if hist == False:
sigma = np.matrix(self.cov_bl_fm)
returns = np.matrix(self.returns_fm)
nav = np.matrix(self.nav_fm)
elif hist == True:
sigma = np.matrix(self.cov)
returns = np.matrix(self.returns)
nav = np.matrix(self.nav)
# General Model Variables
returns = np.matrix(returns)
w = cv.Variable((mu.shape[1], 1))
k = cv.Variable((1, 1))
rf0 = cv.Parameter(nonneg=True)
rf0.value = rf
n = cv.Parameter(nonneg=True)
n.value = returns.shape[0]
ret = mu * w
# MV Model Variables
risk1 = cv.quad_form(w, sigma)
returns_1 = af.cov_returns(sigma) * 1000
n1 = cv.Parameter(nonneg=True)
n1.value = returns_1.shape[0]
risk1_1 = cv.norm(returns_1 * w, "fro") / cv.sqrt(n1 - 1)
# MAD Model Variables
madmodel = False
Y = cv.Variable((returns.shape[0], 1))
u = np.matrix(np.ones((returns.shape[0], 1)) * mu)
a = returns - u
risk2 = cv.sum(Y) / n
# madconstraints=[a*w >= -Y, a*w <= Y, Y >= 0]
madconstraints = [a * w <= Y, Y >= 0]
# Semi Variance Model Variables
risk3 = cv.norm(Y, "fro") / cv.sqrt(n - 1)
# CVaR Model Variables
alpha1 = self.alpha
VaR = cv.Variable(1)
alpha = cv.Parameter(nonneg=True)
alpha.value = alpha1
X = returns * w
Z = cv.Variable((returns.shape[0], 1))
risk4 = VaR + 1 / (alpha * n) * cv.sum(Z)
cvarconstraints = [Z >= 0, Z >= -X - VaR]
# Worst Realization (Minimax) Model Variables
M = cv.Variable(1)
risk5 = M
wrconstraints = [-X <= M]
# Lower Partial Moment Variables
lpmmodel = False
lpm = cv.Variable((returns.shape[0], 1))
lpmconstraints = [lpm >= 0]
if obj == "Sharpe":
lpmconstraints += [lpm >= rf0 * k - X]
else:
lpmconstraints += [lpm >= rf0 - X]
# First Lower Partial Moment (Omega) Model Variables
risk6 = cv.sum(lpm) / n
# Second Lower Partial Moment (Sortino) Model Variables
risk7 = cv.norm(lpm, "fro") / cv.sqrt(n - 1)
# Drawdown Model Variables
drawdown = False
if obj == "Sharpe":
X1 = k + nav * w
else:
X1 = 1 + nav * w
U = cv.Variable((nav.shape[0] + 1, 1))
ddconstraints = [U[1:] >= X1, U[1:] >= U[:-1]]
if obj == "Sharpe":
ddconstraints += [U[1:] >= k, U[0] == k]
else:
ddconstraints += [U[1:] >= 1, U[0] == 1]
# Maximum Drawdown Model Variables
MDD = cv.Variable(1)
risk8 = MDD
mddconstraints = [MDD >= U[1:] - X1]
# Average Drawdown Model Variables
risk9 = 1 / n * cv.sum(U[1:] - X1)
# Conditional Drawdown Model Variables
CDaR = cv.Variable(1)
Zd = cv.Variable((nav.shape[0], 1))
risk10 = CDaR + 1 / (alpha * n) * cv.sum(Zd)
cdarconstraints = [Zd >= U[1:] - X1 - CDaR, Zd >= 0]
# Tracking Error Model Variables
c = self.benchweights
if self.kindbench == True:
bench = np.matrix(returns) * c
else:
bench = self.benchindex
if obj == "Sharpe":
TE = cv.norm(returns * w - bench * k, "fro") / cv.sqrt(n - 1)
else:
TE = cv.norm(returns * w - bench, "fro") / cv.sqrt(n - 1)
# Problem aditional linear constraints
if obj == "Sharpe":
constraints = [
cv.sum(w) == self.upperlng * k,
k >= 0,
mu * w - rf0 * k == 1,
]
if self.sht == False:
constraints += [w <= self.upperlng * k, w * 1000 >= 0]
elif self.sht == True:
constraints += [
w <= self.upperlng * k,
w >= -self.uppersht * k,
cv.sum(cv.neg(w)) <= self.uppersht * k,
]
else:
constraints = [cv.sum(w) == self.upperlng]
if self.sht == False:
constraints += [w <= self.upperlng, w * 1000 >= 0]
elif self.sht == True:
constraints += [
w <= self.upperlng,
w >= -self.uppersht,
cv.sum(cv.neg(w)) <= self.uppersht,
]
if self.ainequality is not None and self.binequality is not None:
A = np.matrix(self.ainequality)
B = np.matrix(self.binequality)
if obj == "Sharpe":
constraints += [A * w - B * k >= 0]
else:
constraints += [A * w - B >= 0]
# Turnover Constraints
if obj == "Sharpe":
if self.allowTO == True:
constraints += [cv.abs(w - c * k) * 1000 <= self.turnover * k * 1000]
else:
if self.allowTO == True:
constraints += [cv.abs(w - c) * 1000 <= self.turnover * 1000]
# Tracking error Constraints
if obj == "Sharpe":
if self.allowTE == True:
constraints += [TE <= self.TE * k]
else:
if self.allowTE == True:
constraints += [TE <= self.TE]
# Problem risk Constraints
if self.upperdev is not None:
if obj == "Sharpe":
constraints += [risk1_1 <= self.upperdev * k]
else:
constraints += [risk1 <= self.upperdev ** 2]
if self.uppermad is not None:
if obj == "Sharpe":
constraints += [risk2 <= self.uppermad * k / 2]
else:
constraints += [risk2 <= self.uppermad / 2]
madmodel = True
if self.uppersdev is not None:
if obj == "Sharpe":
constraints += [risk3 <= self.uppersdev * k]
else:
constraints += [risk3 <= self.uppersdev]
madmodel = True
if self.upperCVaR is not None:
if obj == "Sharpe":
constraints += [risk4 <= self.upperCVaR * k]
else:
constraints += [risk4 <= self.upperCVaR]
constraints += cvarconstraints
if self.upperwr is not None:
if obj == "Sharpe":
constraints += [-X <= self.upperwr * k]
else:
constraints += [-X <= self.upperwr]
constraints += wrconstraints
if self.upperflpm is not None:
if obj == "Sharpe":
constraints += [risk6 <= self.upperflpm * k]
else:
constraints += [risk6 <= self.upperflpm]
lpmmodel = True
if self.upperslpm is not None:
if obj == "Sharpe":
constraints += [risk7 <= self.upperslpm * k]
else:
constraints += [risk7 <= self.upperslpm]
lpmmodel = True
if self.uppermdd is not None:
if obj == "Sharpe":
constraints += [U[1:] - X1 <= self.uppermdd * k]
else:
constraints += [U[1:] - X1 <= self.uppermdd]
constraints += mddconstraints
drawdown = True
if self.upperadd is not None:
if obj == "Sharpe":
constraints += [risk9 <= self.upperadd * k]
else:
constraints += [risk9 <= self.upperadd]
drawdown = True
if self.upperCDaR is not None:
if obj == "Sharpe":
constraints += [risk10 <= self.upperCDaR * k]
else:
constraints += [risk10 <= self.upperCDaR]
constraints += cdarconstraints
drawdown = True
# Defining risk function
if rm == "MV":
if model != "Classic":
risk = risk1_1
elif model == "Classic":
risk = risk1
elif rm == "MAD":
risk = risk2
madmodel = True
elif rm == "MSV":
risk = risk3
madmodel = True
elif rm == "CVaR":
risk = risk4
if self.upperCVaR is None:
constraints += cvarconstraints
elif rm == "WR":
risk = risk5
if self.upperwr is None:
constraints += wrconstraints
elif rm == "FLPM":
risk = risk6
lpmmodel = True
elif rm == "SLPM":
risk = risk7
lpmmodel = True
elif rm == "MDD":
risk = risk8
drawdown = True
if self.uppermdd is None:
constraints += mddconstraints
elif rm == "ADD":
risk = risk9
drawdown = True
elif rm == "CDaR":
risk = risk10
drawdown = True
if self.upperCDaR is None:
constraints += cdarconstraints
if madmodel == True:
constraints += madconstraints
if lpmmodel == True:
constraints += lpmconstraints
if drawdown == True:
constraints += ddconstraints
# Frontier Variables
portafolio = {}
for i in self.assetslist:
portafolio.update({i: []})
# Optimization Process
# Defining solvers
solvers = [cv.ECOS, cv.SCS, cv.OSQP, cv.CVXOPT, cv.GLPK]
# Defining objective function
if obj == "Sharpe":
if rm != "Classic":
objective = cv.Minimize(risk)
elif rm == "Classic":
objective = cv.Minimize(risk * 1000)
elif obj == "MinRisk":
objective = cv.Minimize(risk)
elif obj == "Utility":
objective = cv.Maximize(ret - l * risk)
elif obj == "MaxRet":
objective = cv.Maximize(ret)
try:
prob = cv.Problem(objective, constraints)
for solver in solvers:
try:
prob.solve(
solver=solver, parallel=True, max_iters=2000, abstol=1e-10
)
except:
pass
if w.value is not None:
break
if obj == "Sharpe":
weights = np.matrix(w.value / k.value).T
else:
weights = np.matrix(w.value).T
if self.sht == False:
weights = np.abs(weights) / np.sum(np.abs(weights))
for j in self.assetslist:
portafolio[j].append(weights[0, self.assetslist.index(j)])
except:
pass
optimum = pd.DataFrame(portafolio, index=["weights"], dtype=np.float64).T
return optimum
def reg(self, X): return 1e10*cp.sum_entries(cp.neg(X)) def __str__(self): return "nonnegative reg"
def eval(self, writer):
td3_results, mql_results = [], []
for i, (test_task_idx, test_buffer) in enumerate(zip(self.task_config.test_tasks, self._test_buffers)):
batch = test_buffer.sample(self._args.batch_size, return_dict=True)
other_batches = [buf.sample(self._args.batch_size, return_dict=True) for buf in self._inner_buffers]
self._env.set_task_idx(test_task_idx)
state, action, next_state, reward = (torch.tensor(batch['obs']).to(self._args.device),
torch.tensor(batch['actions']).to(self._args.device),
torch.tensor(batch['next_obs']).to(self._args.device),
torch.tensor(batch['rewards']).to(self._args.device))
with torch.no_grad():
context_input = torch.cat((state, action, next_state, reward), -1)
context_seq, h_n = self.context_encoder(context_input.unsqueeze(1), self.c0)
context = h_n[-1]
all_test_context = context_seq.squeeze(1)
all_other_context, (state_, action_, next_state_, reward_) = self.get_context_from_batches(other_batches)
all_other_context = all_other_context[torch.randperm(all_other_context.shape[0], device=self._args.device)[:self._args.batch_size]]
labels = np.concatenate((-np.ones((self._args.batch_size)),
np.ones((self._args.batch_size))))
X = torch.cat((all_test_context, all_other_context)).cpu().numpy()
w = cp.Variable((all_test_context.shape[-1]))
obj = cp.Minimize(cp.sum(cp.logistic(cp.neg(cp.multiply(labels, X @ w)))))
prob = cp.Problem(obj)
sol = prob.solve()
w_ = torch.tensor(w.value, device=self._args.device).float()
test_betas = (-all_test_context @ w_).exp()
test_ess = (1/test_betas.shape[0]) * test_betas.sum().pow(2) / test_betas.pow(2).sum()
context_betas = (-all_other_context @ w_).exp()
traj, reward, success = self._rollout_policy(self._env, context)
td3_results.append({'reward': reward, 'success': success, 'task': test_task_idx})
writer.add_scalar(f'TD3_Reward/Task_{test_task_idx}', reward, self.total_it)
writer.add_scalar(f'TD3_Success/Task_{test_task_idx}', success, self.total_it)
actor = copy.deepcopy(self.actor)
critic_target = copy.deepcopy(self.critic_target)
critic = copy.deepcopy(self.critic)
actor_optimizer = torch.optim.Adam(actor.parameters(), lr=3e-4)
critic_optimizer = torch.optim.Adam(critic.parameters(), lr=3e-4)
start_actor_params = [p.clone() for p in actor.parameters()]
start_critic_params = [p.clone() for p in critic.parameters()]
lambda_ = 1 - test_ess
context_ = context
for step in range(self._mql_steps1 + self._mql_steps2):
if step == self._mql_steps1:
traj, reward, success = self._rollout_policy(self._env, context, policy=actor)
writer.add_scalar(f'MQL1_Reward/Task_{test_task_idx}', reward, self.total_it)
writer.add_scalar(f'MQL1_Success/Task_{test_task_idx}', success, self.total_it)
critic_param_loss = sum([(p - p_.detach()).pow(2).sum() for p, p_ in zip(critic.parameters(), start_critic_params)])
actor_param_loss = sum([(p - p_.detach()).pow(2).sum() for p, p_ in zip(actor.parameters(), start_actor_params)])
if step >= self._mql_steps1:
# re-assign state, action, next_state, reward to use data from train buffers
# use w_ to assign weights
# also add regularization to theta
other_batches = [buf.sample(self._args.batch_size, return_dict=True) for buf in self._inner_buffers]
all_other_context, (state, action, next_state, reward) = self.get_context_from_batches(other_batches)
context_betas = (-all_other_context @ w_).exp()
context_ess = (1/context_betas.shape[0]) * context_betas.sum().pow(2) / context_betas.pow(2).sum()
lambda_ = 1 - context_ess
if step == self._mql_steps1 + self._mql_steps2 - 1:
writer.add_scalar(f'Test_Beta/Task_{test_task_idx}', test_betas.mean(), self.total_it)
writer.add_scalar(f'Context_Beta/Task_{test_task_idx}', context_betas.mean(), self.total_it)
writer.add_scalar(f'Test_ESS/Task_{test_task_idx}', test_ess, self.total_it)
writer.add_scalar(f'Context_ESS/Task_{test_task_idx}', context_ess, self.total_it)
else:
lambda_ = 1 - test_ess
not_done = torch.ones((state.shape[0],1)).to(self._args.device)
if context_.shape[0] != state.shape[0]:
context_ = context.repeat(state.shape[0], 1)
with torch.no_grad():
next_action = actor(next_state, context_)
# Compute the target Q value
target_Q1, target_Q2 = critic_target(next_state, next_action, context_)
target_Q = torch.min(target_Q1, target_Q2)
target_Q = reward + not_done * self.discount * target_Q
# Get current Q estimates
current_Q1, current_Q2 = critic(state, action, context_)
# Compute critic loss
critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(
current_Q2, target_Q) + (lambda_ / 2) * critic_param_loss
# Optimize the critic
critic_optimizer.zero_grad()
critic_loss.backward(retain_graph=True)
critic_optimizer.step()
# Compute actor losse
actor_loss = -critic.Q1(state, actor(state, context_), context_).mean() + (lambda_ / 2) * actor_param_loss
# Optimize the actor
actor_optimizer.zero_grad()
actor_loss.backward()
actor_optimizer.step()
for param, target_param in zip(critic.parameters(),
critic_target.parameters()):
target_param.data.copy_(self.tau * param.data +
(1 - self.tau) * target_param.data)
traj, reward, success = self._rollout_policy(self._env, context, policy=actor)
mql_results.append({'reward': reward, 'success': success, 'task': test_task_idx})
writer.add_scalar(f'MQL_Reward/Task_{test_task_idx}', reward, self.total_it)
writer.add_scalar(f'MQL_Success/Task_{test_task_idx}', success, self.total_it)
return td3_results, mql_results
def set_holding_cost(self, weights_new):
self.holding_cost += cp.sum(cp.multiply(self.holding_coeff, cp.neg(weights_new)))
return
for t in np.linspace(0.0, 1.0, num=101):
g = np.where(x.value > t, 1, 0)
v = c.T @ g
m = np.max(A @ g - b)
if m <= 0:
print(f"with t={t}, m={m} and feasible objective val is {v}")
break
## Simple portfolio optimization
pbar = np.loadtxt(open(pbar_file, "rb"), delimiter=",")
S = np.loadtxt(open(S_file, "rb"), delimiter=",")
xu = np.loadtxt(open(xu_file, "rb"), delimiter=",")
wans = np.ones(xu.shape[0])
xu_ret = pbar.T @ xu
xu_rsk = np.sqrt(xu.T @ (S @ xu))
#min risk portfolio with same return as uniform
x = cp.Variable(xu.shape[0])
p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1])
mr_rsk = np.sqrt(p.solve())
#min risk portfolio with same return as uniform but long-only
p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1, 0 <= x])
lo_rsk = np.sqrt(p.solve())
#now limit the total short position to be <= 0.5
p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1, wans.T @ cp.neg(x) <= 0.5])
sc_rsk = np.sqrt(p.solve())
def train_network(X_f,
X_i,
X_s,
Y_i,
dY_i=None,
beta=1e-6,
M=int(5000),
fuse_xxstar=False,
abs_act=False,
sdf=True,
norm=2):
n_f, d = X_f.shape
n_i, d = X_i.shape
X_f = np.append(X_f, np.ones((n_f, 1)), axis=1)
X_i = np.append(X_i, np.ones((n_i, 1)), axis=1)
X_s = np.append(X_s, np.ones((n_i, 1)), axis=1)
d += 1
# Y_i = np.atleast_1d(Y_i.squeeze())
dual_norm = np.inf if norm == 1 else int(1 / (1 - 1 / float(norm)))
## Finite approximation of all possible sign patterns
t0 = time.time()
if n_f + 2 * n_i < 10:
D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar)
else:
D_f, D_i, D_s = random_signed_patterns(X_f,
X_i,
X_s,
M,
fuse_xxstar=fuse_xxstar)
m1 = D_f.shape[1]
print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.')
# Optimal CVX
Uopt0 = cp.Variable((d, 1), value=np.random.randn(d, 1))
Uopt1 = cp.Variable((d, m1), value=np.random.randn(d, m1))
Uopt2 = cp.Variable((d, m1), value=np.random.randn(d, m1))
constraints = []
loss = 0
# Feasible points
if n_f > 0:
ux_f_1 = cp.multiply(D_f, (X_f @ Uopt1))
ux_f_2 = cp.multiply(D_f, (X_f @ Uopt2))
if abs_act:
y_f = X_f @ Uopt0 + cp.sum(ux_f_1 - ux_f_2, axis=1, keepdims=True)
deriv_f = D_f @ (Uopt1 - Uopt2).T + Uopt0.T
else:
y_f = X_f @ Uopt0 + cp.sum(cp.multiply((D_f + 1) / 2,
(X_f @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
deriv_f = ((D_f + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
constraints += [ux_f_1 >= 0, ux_f_2 >= 0, ux_f_1 >= 0, ux_f_2 >= 0]
Y_f_lb = np.max(Y_i - np.linalg.norm(
X_f[None, :, :] - X_i[:, None, :], axis=-1, ord=norm),
axis=0,
keepdims=True).T
if sdf:
loss_f = cp.sum(cp.abs(y_f - Y_f_lb))
else:
loss_f = cp.sum(cp.pos(y_f))
loss = loss + loss_f
lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb))
else:
loss_f = cp.Variable(value=0.)
lipsh_f = cp.Variable(value=0.)
# Infeasible points
if n_i > 0:
ux_i_1 = cp.multiply(D_i, (X_i @ Uopt1))
ux_s_1 = cp.multiply(D_s, (X_s @ Uopt1))
ux_i_2 = cp.multiply(D_i, (X_i @ Uopt2))
ux_s_2 = cp.multiply(D_s, (X_s @ Uopt2))
if abs_act:
y_i = X_i @ Uopt0 + cp.sum(ux_i_1 - ux_i_2, axis=1, keepdims=True)
y_s = X_s @ Uopt0 + cp.sum(ux_s_1 - ux_s_2, axis=1, keepdims=True)
deriv_i = D_i @ (Uopt1 - Uopt2).T + Uopt0.T
deriv_s = D_s @ (Uopt1 - Uopt2).T + Uopt0.T
else:
y_i = X_i @ Uopt0 + cp.sum(cp.multiply((D_i + 1) / 2,
(X_i @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
y_s = X_s @ Uopt0 + cp.sum(cp.multiply((D_s + 1) / 2,
(X_s @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
deriv_i = ((D_i + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
deriv_s = ((D_s + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
constraints += [
ux_i_1 >= 0,
ux_s_1 >= 0,
ux_i_2 >= 0,
ux_s_2 >= 0,
]
loss_i = cp.sum(cp.abs(Y_i - y_i))
loss_s = cp.sum(cp.abs(y_s))
if dY_i is not None:
loss_di = cp.sum(cp.abs(deriv_i[:, :-1] - dY_i))
loss_ds = cp.sum(cp.abs(deriv_s[:, :-1] - dY_i))
else:
loss_di = cp.Variable(value=0., nonneg=True)
loss_ds = cp.Variable(value=0., nonneg=True)
loss = loss + loss_i + loss_s + loss_ds + loss_di
lipsh_i = cp.sum(cp.neg(Y_i - y_i))
else:
loss_i = cp.Variable(value=0.)
loss_s = cp.Variable(value=0.)
lipsh_i = cp.Variable(value=0.)
lipsh_s = cp.Variable(value=0.)
loss_di = cp.Variable(value=0.)
loss_ds = cp.Variable(value=0.)
# Regularization
groupnorm_reg = cp.norm(Uopt0) + cp.mixed_norm(
Uopt1.T, 2, 1) + cp.mixed_norm(Uopt2.T, 2, 1)
if norm == 2:
reg_nrm = cp.norm(Uopt0[:-1]) + cp.mixed_norm(
Uopt1[:-1].T, 2, 1) + cp.mixed_norm(Uopt2[:-1].T, 2, 1)
elif norm == 1:
reg_nrm = cp.Variable(nonneg=True)
for i in range(d - 1):
for s in [-1, 1]:
constraints += [
cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <=
reg_nrm
]
reg = reg_nrm + groupnorm_reg / 100.
# Solution
prob = cp.Problem(cp.Minimize(100 * (loss + beta * reg)), constraints)
t0 = time.time()
options = {} #dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8})
prob.solve(verbose=True, **options)
# prob.solve(solver=cp.SCS, verbose=True)
print(
f'Status: {prob.status}, \n '
f'Value: {prob.value :.2E}, \n '
f'loss_f: {loss_f.value :.2E}, \n '
f'loss_i: {loss_i.value :.2E}, \n '
f'loss_s: {loss_s.value :.2E}, \n '
f'loss_di: {loss_di.value :.2E}, \n '
f'loss_ds: {loss_ds.value :.2E}, \n '
f'Reg: {reg.value : .2E}, \n '
# # f'lipsh_f: {lipsh_f.value :.2E}, \n '
# # f'lipsh_i: {lipsh_i.value :.2E}, \n '
f'Time: {time.time()-t0 :.2f}s')
if prob.status.lower() == 'infeasible':
st()
return None
u0, u1, u2 = torch.tensor(Uopt0.value), torch.tensor(
Uopt1.value), torch.tensor(Uopt2.value)
torch.save(
{
'u1': u1,
'u2': u2,
'u0': u0,
'D_f': torch.tensor(D_f),
'D_i': torch.tensor(D_i),
'D_s': torch.tensor(D_s)
}, 'tmp.csv')
return u0, u1, u2
def test_matrix_multiperiod_portfolio(self):
np.random.seed(1)
k = 10
n = 50
T = 5
borrow_cost = 0.0001
lam = {
"risk": 50,
"borrow": 0.0001,
"norm1_trade": 0.01,
"norm0_trade": 1.0,
}
# Parameters
hat_r = [
cp.Parameter(n, name="hat_r_%s" % str(t)) for t in range(1, T + 1)
]
w_init = cp.Parameter(n, name="w_init")
# F = cp.Parameter((n, k), name="F")
sqrt_SigmaF_FT = cp.Parameter((k, n), name="SigmaF_FT")
sqrt_D = cp.Parameter(n, name="sqrt_D")
# Formulate problem
w = [cp.Variable(n) for t in range(T + 1)]
# Define cost components
cost = 0
constraints = [w[0] == w_init]
for t in range(1, T + 1):
risk_cost = lam["risk"] * (
# cp.quad_form(F.T * w[t], Sigma_F)
cp.sum_squares(sqrt_SigmaF_FT @ w[t]) +
cp.sum_squares(cp.multiply(sqrt_D, w[t])))
holding_cost = lam["borrow"] * cp.sum(borrow_cost * cp.neg(w[t]))
transaction_cost = lam["norm1_trade"] * cp.norm(w[t] - w[t - 1], 1)
cost += (hat_r[t - 1] @ w[t] - risk_cost - holding_cost -
transaction_cost)
constraints += [cp.sum(w[t]) == 1.0]
# Define optimizer
problem = cp.Problem(cp.Maximize(cost), constraints)
m = Optimizer(problem)
self.assertTrue(m._problem.parameters_in_matrices)
# Sample parameters
df_train = sample_portfolio(n, k, N=1000)
# Train and test using pytorch
m.train(df_train,
filter_strategies=True,
parallel=True,
n_train_trials=10,
learner=PYTORCH)
# Assert fewer strategies than training samples
self.assertTrue(len(m.encoding) < len(df_train))
self.assertTrue(len(m.encoding) > 1)
def stabilize_chunk(desiredShot, aspectRatio, noDataFrames, imageSize, fps, crop_factor, apparent_motion, external_boundaries, screen_pos, lambda1=0.002, lambda2=0.0001, zoomSmooth=1.5, lambda3=0.005):
"""From a time sequence of unstabilized frame boxes, compute a stabilized frame.
All parameters are normalized with respect to frame size and time, so that simultaneaously doubling the imageSize and the desiredShot does not change the solution, and neither does using twice as many frames and doubling the fps.
The main differences with the paper are:
- only D, L11 and L13 terms are implemented
- zoomSmooth was added
- the shots are optimized over chunks of shot_s*2 = 10s, and the five first seconds are kept. This makes the problem a lot more tractable.
If a frame in desiredShot goes outside of the original image, it is cropped.
Reference: Gandhi Vineet, Ronfard Remi, Gleicher Michael
Multi-Clip Video Editing from a Single Viewpoint
European Conference on Visual Media Production (CVMP) 2014
http://imagine.inrialpes.fr/people/vgandhi/GRG_CVMP_2014.pdf
Keyword arguments:
desiredShot -- a n x 4 numpy array containing on each line the box as [xmin, ymin, xmax, ymax]
lambda1 -- see eq. (10) in paper
lambda2 -- see eq. (10) in paper
zoomSmooth -- a factor applied on the terms that deal with frame size in the regularization term: raise if the stabilized frame zooms in and out too much
aspectRatio -- the desired output aspect ration (e.g. 16/9.)
noDataFrames -- the list of frames that have no desiredShot information - only regularization is used to stabilize these frames
imageSize -- [xmin, ymin, xmax, ymax] for the original image (typically [0,0,1920,1080] for HD)
fps -- number of frames per seconds in the video - used for normalization
"""
#print "noDataFrames:", noDataFrames
# print(noDataFrames, desiredShot)
# set to
desiredShot[noDataFrames, :] = 0.
imageHeight = float(imageSize[1])
imageWidth = float(imageSize[0])
len_w = imageWidth/2
len_h = imageHeight/2
if imageHeight* aspectRatio < imageWidth:
len_w = round((imageHeight*aspectRatio)/2)
elif imageWidth / aspectRatio < imageHeight:
len_h = round((imageWidth / aspectRatio)/2)
# crop the desiredShot to the image window
# we keep a 1-pixel margin to be sure that constraints can be satisfied
margin = 1
low_x1_flags = desiredShot[:, 0] < (0. + margin)
desiredShot[low_x1_flags, 0] = 0. + margin
low_x2_flags = desiredShot[:, 2] < (0. + margin)
desiredShot[low_x2_flags, 2] = 0. + margin
high_x1_flags = desiredShot[:, 0] > (imageWidth - margin)
desiredShot[high_x1_flags, 0] = imageWidth - margin
high_x2_flags = desiredShot[:, 2] > (imageWidth - margin)
desiredShot[high_x2_flags, 2] = imageWidth - margin
low_y1_flags = desiredShot[:, 1] < (0. + margin)
desiredShot[low_y1_flags, 1] = 0. + margin
low_y2_flags = desiredShot[:, 3] < (0. + margin)
desiredShot[low_y2_flags, 3] = 0. + margin
high_y1_flags = desiredShot[:, 1] > (imageHeight - margin)
desiredShot[high_y1_flags, 1] = imageHeight - margin
high_y2_flags = desiredShot[:, 3] > (imageHeight - margin)
desiredShot[high_y2_flags, 3] = imageHeight - margin
# Make sure that a crop of the given aspectRatio can be contained in imageSize and can contain the desiredShot.
# This may be an issue eg. when doing a 16/9 or a 4/3 movie from 2K.
# else, we must cut the desiredshot on both sides.
for k in range(desiredShot.shape[0]):
if (desiredShot[k, 2] - desiredShot[k, 0]) > (imageHeight * aspectRatio - margin):
xcut = (desiredShot[k, 2] - desiredShot[k, 0]) - \
(imageHeight * aspectRatio - margin)
desiredShot[k, 2] -= xcut / 2
desiredShot[k, 0] += xcut / 2
if (desiredShot[k, 3] - desiredShot[k, 1]) > (imageWidth / aspectRatio - margin):
ycut = (desiredShot[k, 3] - desiredShot[k, 1]) - \
(imageWidth / aspectRatio - margin)
desiredShot[k, 3] -= ycut / 2
desiredShot[k, 1] += ycut / 2
# print("desiredShot:", desiredShot)
# print("noDataFrames:", noDataFrames)
x_center = (desiredShot[:, 0] + desiredShot[:, 2]) / 2.
y_center = (desiredShot[:, 1] + desiredShot[:, 3]) / 2.
# elementwise maximum of each array
half_height_opt = np.maximum((desiredShot[:, 2] - desiredShot[:, 0]) /
aspectRatio, ((desiredShot[:, 3] - desiredShot[:, 1]))) / 2
# smooth x_center y_center and half_height_opt using a binomial filter (Marchand and Marmet 1983)
# eg [1 2 1]/4 or [1 4 6 4 1]/16 (obtained by applying it twice)
# TODO: ignore noDataFrames when smoothing!
x_center_residual = x_center
# binomial_3(x_center_residual)
# binomial_3(x_center_residual)
y_center_residual = y_center
# binomial_3(y_center_residual)
# binomial_3(y_center_residual)
half_height_opt_residual = half_height_opt
# binomial_3(half_height_opt_residual)
# binomial_3(half_height_opt_residual)
half_width = (desiredShot[:, 2] - desiredShot[:, 0]) / 2.
zero_flags = half_width[:] < 0
half_width[zero_flags] = 0.
half_height = (desiredShot[:, 3] - desiredShot[:, 1]) / 2.
zero_flags = half_height[:] < 0
half_height[zero_flags] = 0.
# now trick the constraints so that there are no inner inclusion constraints at noDataFrames
x_center[noDataFrames] = imageWidth / 2.
half_width[noDataFrames] = -imageWidth / 2. # negative on purpose
y_center[noDataFrames] = imageHeight / 2.
half_height[noDataFrames] = -imageHeight / 2. # negative on purpose
half_height_opt[noDataFrames] = imageHeight / 2.
# print(half_height[noDataFrames], noDataFrames, imageHeight)
# print(np.isnan(half_height))
# for i in range(len(desiredShot)):
# if (y_center[i] - half_height[i]) < 0 or (y_center[i] + half_height[i]) > imageHeight:
# print('indice ',i)
external_boundaries[noDataFrames] = [0, 0, 0, 0, 0, 0]
l_tl = []
for t in external_boundaries[:, 4]:
if t == 1:
l_tl.append(0.)
else:
l_tl.append(1.)
tl_inv = np.array(l_tl)
l_tr = []
for t in external_boundaries[:, 5]:
if t == 1:
l_tr.append(0.)
else:
l_tr.append(1.)
tr_inv = np.array(l_tr)
tl_inv[noDataFrames] = 0
tr_inv[noDataFrames] = 0
assert ((x_center - half_width) >=
0).all() and ((x_center + half_width) <= imageWidth).all()
assert ((y_center - half_height) >=
0).all() and ((y_center + half_height) <= imageHeight).all()
n = x_center.size
#print "n:", n
# We split the problem into chunks of fixed duration.
# We compute a subsolution for the current chunk and the next chunk, and ensure continuity for the
# variation (1st derivative) and jerk (3rd derivative) terms.
# Then we only keep the solution for the current chunk and advance.
# compute the opposite of noDataFrames
# normalize with image height
weightsAll = np.ones(n) / imageHeight
weightsAll[noDataFrames] = 0. # do not use residuals on the optimal frame where there's no data
#print "weightsAll:", weightsAll
#print "half_height_opt:", half_height_opt
optimised_xcenter = np.zeros(n)
optimised_ycenter = np.zeros(n)
optimised_height = np.zeros(n)
chunk_s = 5 # size of a chunk in seconds
chunk_n = int(chunk_s * fps) # number of samples in a chunk
full_chunk_n = chunk_n * 2 # number of samples in a subproblem
# starting index for the chunk (also used to check if it is the first chunk)
chunk_start = 0
while chunk_start < n:
chunk_end = min(n, chunk_start + chunk_n)
chunk_size = chunk_end - chunk_start
full_chunk_end = min(n, chunk_start + full_chunk_n)
full_chunk_size = full_chunk_end - chunk_start
# print("chunk:", chunk_start, chunk_end, full_chunk_end)
x = cvx.Variable(full_chunk_size)
y = cvx.Variable(full_chunk_size)
# half height (see sec. 4 in the paper)
h = cvx.Variable(full_chunk_size)
weights = weightsAll[chunk_start:full_chunk_end]
x_center_chunk = x_center[chunk_start:full_chunk_end]
y_center_chunk = y_center[chunk_start:full_chunk_end]
half_height_chunk = half_height[chunk_start:full_chunk_end]
half_width_chunk = half_width[chunk_start:full_chunk_end]
x_center_residual_chunk = x_center[chunk_start:full_chunk_end]
y_center_residual_chunk = y_center[chunk_start:full_chunk_end]
half_height_opt_residual_chunk = half_height_opt_residual[chunk_start:full_chunk_end]
#Vector screen pos
h_vector = screen_pos[chunk_start:full_chunk_end]
#M1 term see paper 4.5
c_x_factor = crop_factor[:, chunk_start:full_chunk_end-1, 0]
c_y_factor = crop_factor[:, chunk_start:full_chunk_end-1, 1]
c_h_factor = crop_factor[:, chunk_start:full_chunk_end-1, 2]
#M2 term see paper 4.5
b_x_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 0]
b_y_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 1]
b_h_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 2]
#E term [xl, xl1, xr, xr1, tl, tr]
tl = external_boundaries[chunk_start:full_chunk_end, 4]
tr = external_boundaries[chunk_start:full_chunk_end, 5]
inv_tl = tl_inv[chunk_start:full_chunk_end]
inv_tr = tr_inv[chunk_start:full_chunk_end]
xl = external_boundaries[chunk_start:full_chunk_end, 0]
xl1 = external_boundaries[chunk_start:full_chunk_end, 1]
xr = external_boundaries[chunk_start:full_chunk_end, 2]
xr1 = external_boundaries[chunk_start:full_chunk_end, 3]
#assert ((x_center_chunk - half_width_chunk) >= 0).all() and ((x_center_chunk + half_width_chunk) <= imageWidth).all()
#assert ((y_center_chunk - half_height_chunk) >= 0).all() and ((y_center_chunk + half_height_chunk) <= imageHeight).all()
# for f in [97, 98, 99, 100]:
# print f, weights[f], x_center[f], y_center[f], half_height_opt[f]
expr = cvx.sum_squares(weights*(x_center_residual_chunk + (0.17*aspectRatio*half_height_opt_residual_chunk*h_vector) - x)) + cvx.sum_squares(weights*(
y_center_residual_chunk - y)) + cvx.sum_squares(weights*(half_height_opt_residual_chunk - h))
expr /= n # normalize by the number of images, get a cost per image
# end of version 1
# version 2:
#dataFrames = np.nonzero(weights)
# expr = cvx.sum_squares(x_center[dataFrames] - x[dataFrames]) + \
# cvx.sum_squares(y_center[dataFrames] - y[dataFrames]) + \
# cvx.sum_squares(half_height_opt[dataFrames] - h[dataFrames]) / (zoomSmooth*zoomSmooth)
# expr /= (imageHeight*imageHeight)*n # normalize by the number of images, get a cost per image
# end of version 2
if lambda1 != 0.:
lambda1Factor = lambda1 * fps / imageHeight
# print("lambda 1 ",lambda1Factor)
if n > 1:
expr += lambda1Factor * \
(cvx.tv(x) + cvx.tv(y) + cvx.tv(h) * zoomSmooth)
# if not the first chunk, add continuity with previous samples
if chunk_start >= 1:
expr += lambda1Factor * (cvx.abs(x[0] - optimised_xcenter[chunk_start - 1]) +
cvx.abs(y[0] - optimised_ycenter[chunk_start - 1]) +
cvx.abs(h[0] - optimised_height[chunk_start - 1]) * zoomSmooth)
if lambda2 != 0.:
lambda2Factor = lambda2 * fps * fps * fps / imageHeight
# print("lambda 2 ",lambda2Factor)
if n > 2:
expr += lambda2Factor * (cvx.norm(x[3:] - 3*x[2:full_chunk_size-1] + 3*x[1:full_chunk_size-2] - x[0:full_chunk_size-3], 1) +
cvx.norm(y[3:] - 3*y[2:full_chunk_size-1] + 3*y[1:full_chunk_size-2] - y[0:full_chunk_size-3], 1) +
cvx.norm(h[3:] - 3*h[2:full_chunk_size-1] + 3*h[1:full_chunk_size-2] - h[0:full_chunk_size-3], 1) * zoomSmooth)
# if not the first chunk, add continuity with previous samples
if chunk_start >= 3 and chunk_size >= 3:
expr += lambda2Factor * ((cvx.abs(x[0] - 3 * optimised_xcenter[chunk_start - 1] + 3 * optimised_xcenter[chunk_start - 2] - optimised_xcenter[chunk_start - 3]) +
cvx.abs(x[1] - 3 * x[0] + 3 * optimised_xcenter[chunk_start - 1] - optimised_xcenter[chunk_start - 2]) +
cvx.abs(x[2] - 3 * x[1] + 3 * x[0] - optimised_xcenter[chunk_start - 1])) +
(cvx.abs(y[0] - 3 * optimised_ycenter[chunk_start - 1] + 3 * optimised_ycenter[chunk_start - 2] - optimised_ycenter[chunk_start - 3]) +
cvx.abs(y[1] - 3 * y[0] + 3 * optimised_ycenter[chunk_start - 1] - optimised_ycenter[chunk_start - 2]) +
cvx.abs(y[2] - 3 * y[1] + 3 * y[0] - optimised_ycenter[chunk_start - 1])) +
(cvx.abs(h[0] - 3 * optimised_height[chunk_start - 1] + 3 * optimised_height[chunk_start - 2] - optimised_height[chunk_start - 3]) +
cvx.abs(h[1] - 3 * h[0] + 3 * optimised_height[chunk_start - 1] - optimised_height[chunk_start - 2]) +
cvx.abs(h[2] - 3 * h[1] + 3 * h[0] - optimised_height[chunk_start - 1])) * zoomSmooth)
if lambda3 != 0.:
lambda3Factor = lambda3 * fps / imageHeight
lambda2Factor = lambda2 * fps * fps * fps / imageHeight
lambdaM = 5
if n > 1:
m1_term = 0
for c_x, c_y, c_h in zip(c_x_factor, c_y_factor, c_h_factor):
m1_term += lambdaM * (cvx.norm(c_x*(x[1:]-x[0:full_chunk_size-1]), 1) +
cvx.norm(c_y*(y[1:]-y[0:full_chunk_size-1]), 1) +
cvx.norm(c_h*(h[1:]-h[0:full_chunk_size-1]), 1) * zoomSmooth)
if chunk_start >= 1:
for c in crop_factor:
c_x = c[chunk_start-1, 0]
c_y = c[chunk_start-1, 1]
c_h = c[chunk_start-1, 2]
m1_term += lambdaM * (cvx.norm(c_x*(x[0]-optimised_xcenter[chunk_start-1]), 1) +
cvx.norm(c_y*(y[0]-optimised_ycenter[chunk_start-1]), 1) +
cvx.norm(c_h*(h[0]-optimised_height[chunk_start-1]), 1) * zoomSmooth)
if n > 1:
m2_term = 0
for b_x, b_y, b_h_g in zip(b_x_factor, b_y_factor, b_h_factor):
b_h = gaussian_filter(b_h_g,sigma=5)
# print(b_x, b_x.dtype)
m2_term += lambdaM * (cvx.neg((b_x-(x[1:]-x[0:full_chunk_size-1]))*b_x) +
cvx.neg((b_y-(y[1:]-y[0:full_chunk_size-1]))*b_y) +
cvx.neg((b_h-(h[1:]-h[0:full_chunk_size-1]))*b_h) * zoomSmooth)
if chunk_start >=1 :
for b in apparent_motion:
b_x = b[chunk_start-1, 0]
b_y = b[chunk_start-1, 0]
b_h = b[chunk_start-1, 0]
m2_term += lambdaM * (cvx.neg((b_x-(x[0]-optimised_xcenter[chunk_start-1]))*b_x) +
cvx.neg((b_y-(y[0]-optimised_ycenter[chunk_start-1]))*b_y) +
cvx.neg((b_h-(h[0]-optimised_height[chunk_start-1]))*b_h) * zoomSmooth)
# expr += m1_term + m2_term
# if n > 1:
# # E out
# expr += lambda3Factor * ((inv_tl * cvx.pos(xl1 - x + aspectRatio * h)) +
# (inv_tr * cvx.pos(x + aspectRatio * h - xr1)) )
#
# # E in
# expr += lambda3Factor * ((tl * cvx.pos(x - aspectRatio * h - xl)) +
# (tr * cvx.pos(xr - x - aspectRatio * h)) )
obj = cvx.Minimize(expr)
#print expr
# print("H=%d, W=%d lambda1=%f lambda2=%f zoomSmooth=%f fps=%f imageHeight=%f" % (
# imageHeight, imageWidth, lambda1, lambda2, zoomSmooth, fps, imageHeight))
# note that the following constraints are tricked (see above) at noDataFrames, using negative values for half_width and half_height
constraints = [h >= 0,
(x - aspectRatio * h) >= 0,
(x - aspectRatio * h) <= (x_center_chunk - half_width_chunk),
(x + aspectRatio * h) >= (x_center_chunk + half_width_chunk),
(x + aspectRatio * h) <= imageWidth,
aspectRatio * h <= len_w,
(y - h) >= 0,
(y - h) <= (y_center_chunk - half_height_chunk),
(y + h) >= (y_center_chunk + half_height_chunk),
(y + h) <= imageHeight,
h <= len_h]
prob = cvx.Problem(obj, constraints)
tryagain = True
tryreason = ""
if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED:
tryagain = False
try:
# ECOS, the default solver, is much better at solving our problems, especially at handling frames where the actor is not visible
# all tolerances are multiplied by 10
result = prob.solve(solver=cvx.ECOS, verbose=False, abstol = 1e-6, reltol = 1e-5, abstol_inacc = 5e-4, reltol_inacc = 5e-4, feastol_inacc = 1e-3)
except cvx.SolverError as e:
tryagain = True
tryreason = str(e)
if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED:
tryagain = False
try:
result = prob.solve(solver=cvx.SCS, verbose=False, max_iters = 2500, eps = 1e-2)
except cvx.SolverError as e:
tryagain = True
tryreason = str(e)
if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED:
tryagain = False
try:
result = prob.solve(solver=cvx.CVXOPT, verbose=False, abstol = 1e-6, reltol = 1e-5, feastol = 1e-6)
except cvx.SolverError as e:
tryagain = True
tryreason = str(e)
if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED:
tryagain = False
try:
result = prob.solve(
solver=cvx.CVXOPT, kktsolver=cvx.ROBUST_KKTSOLVER, verbose=False, abstol = 1e-6, reltol = 1e-5, feastol = 1e-6)
except cvx.SolverError as e:
tryagain = True
tryreason = str(e)
if tryagain:
raise cvx.solverError(tryreason)
if prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED:
raise cvx.SolverError('Problem is infeasible or unbounded')
#raise ValueError('Yeah!')
# print("result=", result, "\n")
if full_chunk_end >= n:
# last chunk - get the full chunk
optimised_xcenter[chunk_start:full_chunk_end] = x.value.reshape(
full_chunk_size)
optimised_ycenter[chunk_start:full_chunk_end] = y.value.reshape(
full_chunk_size)
optimised_height[chunk_start:full_chunk_end] = h.value.reshape(
full_chunk_size)
chunk_start = full_chunk_end
else:
# only get the chunk and advance
optimised_xcenter[chunk_start:chunk_end] = x.value[:chunk_size].reshape(
chunk_size)
optimised_ycenter[chunk_start:chunk_end] = y.value[:chunk_size].reshape(
chunk_size)
optimised_height[chunk_start:chunk_end] = h.value[:chunk_size].reshape(
chunk_size)
chunk_start = chunk_end
return np.vstack([optimised_xcenter - aspectRatio * optimised_height, optimised_ycenter - optimised_height, optimised_xcenter + aspectRatio * optimised_height, optimised_ycenter + optimised_height]).transpose()
