def _estimate(self, t, w_plus, z, value): """Estimate holding costs. Args: t: time of estimate wplus: holdings tau: time to estimate (default=t) """ try: w_plus = w_plus[w_plus.index != self.cash_key] w_plus = w_plus.values except AttributeError: w_plus = w_plus[:-1] # TODO fix when cvxpy pandas ready try: self.expression = cvx.multiply( values_in_time(self.borrow_costs, t), cvx.neg(w_plus)) except TypeError: self.expression = cvx.multiply( values_in_time(self.borrow_costs, t).values, cvx.neg(w_plus)) try: self.expression -= cvx.multiply(values_in_time(self.dividends, t), w_plus) except TypeError: self.expression -= cvx.multiply( values_in_time(self.dividends, t).values, w_plus) return cvx.sum(self.expression), []
def __init__(self, m, n, r, eps=1e-4): cvxopt.glpk.options["msg_lev"] = "GLP_MSG_OFF" self.m = m self.n = n self.r = r self.eps = eps self.last_move = None self.a = cp.Parameter(m) # Adjustments self.C = cp.Parameter((m, m)) # C: Gradient inner products, G^T G self.Ca = cp.Parameter(m) # d_bal^TG self.rhs = cp.Parameter(m) # RHS of constraints for balancing self.alpha = cp.Variable(m) # Variable to optimize obj_bal = cp.Maximize(self.alpha @ self.Ca) # objective for balance constraints_bal = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Simplex self.C @ self.alpha >= self.rhs] self.prob_bal = cp.Problem(obj_bal, constraints_bal) # LP balance obj_dom = cp.Maximize(cp.sum(self.alpha @ self.C)) # obj for descent constraints_res = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Restrict self.alpha @ self.Ca >= -cp.neg(cp.max(self.Ca)), self.C @ self.alpha >= 0] constraints_rel = [self.alpha >= 0, cp.sum(self.alpha) == 1, # Relaxed self.C @ self.alpha >= 0] self.prob_dom = cp.Problem(obj_dom, constraints_res) # LP dominance self.prob_rel = cp.Problem(obj_dom, constraints_rel) # LP dominance self.gamma = 0 # Stores the latest Optimum value of the LP problem self.mu_rl = 0 # Stores the latest non-uniformity
def soiling_seperation_algorithm(observed, iterations=5, weights=None, index_set=None, tau=0.85): if weights is None: weights = np.ones_like(observed) if index_set is None: index_set = ~np.isnan(observed) zero_set = np.zeros(len(observed) - 1, dtype=np.bool) eps = .01 n = len(observed) s1 = cvx.Variable(n) s2 = cvx.Variable(n) s3 = cvx.Variable(n) w = cvx.Parameter(n - 2, nonneg=True) w.value = np.ones(len(observed) - 2) for i in range(iterations): # cvx.norm(cvx.multiply(s3, weights), p=2) \ cost = .1 * cvx.sum(tau * cvx.pos(s3) +(1 - tau) * cvx.neg(s3)) \ + 10 * cvx.norm(cvx.diff(s2, k=2), p=2) \ + .2 * cvx.norm(cvx.multiply(w, cvx.diff(s1, k=2)), p=1) objective = cvx.Minimize(cost) constraints = [ observed[index_set] == s1[index_set] + s2[index_set] + s3[index_set], s2[365:] - s2[:-365] == 0, cvx.sum(s2[:365]) == 0 # s1 <= 1 ] if np.sum(zero_set) > 0: constraints.append(cvx.diff(s1, k=1)[zero_set] == 0) problem = cvx.Problem(objective, constraints) problem.solve(solver='MOSEK') w.value = 1 / (eps + 1e2* np.abs(cvx.diff(s1, k=2).value)) # Reweight the L1 penalty zero_set = np.abs(cvx.diff(s1, k=1).value) <= 5e-5 # Make nearly flat regions exactly flat (sparse 1st diff) return s1.value, s2.value, s3.value
def est_freq_read(L): """Estimate expected genotype frequencies for an individual using maximum likelihood. This is a convex optimization problem i.e. a mixture distribution with fixed components Args ---- L : np.array p x 3 matrix of likelihoods Returns ------- pi_hat : np.array 3 x 1 vector of estimated mixture proportions """ # variable for mixture proportions pi = cvx.Variable(3) # objective function objective = cvx.Minimize(cvx.neg(cvx.sum_entries(cvx.log(L * pi)))) # constraints of mixture proportions constraints = [0 <= pi, pi <= 1, cvx.sum_entries(pi) == 1] # intailize the optimization problem prob = cvx.Problem(objective, constraints) # sovle the problem result = prob.solve() # the optimal value for the mixture proportions pi_hat = pi.value return (pi_hat)
def risk_adj_return(x, s, Fs, Rs, Ds, mus, n, T, L, gamma=1): ''' Compute objective function to portfolio problem ''' obj = 0 CASH = np.eye(1, n, n - 1).T for ii in range(T): if ii == 0: obj += s.T.dot(cvx.neg(x[ii*n:(ii+1)*n]).value) + gamma*(cvx.sum_squares(Fs[ii].T.dot(x[ii*n:(ii+1)*n])).value +\ Rs[ii]*cvx.sum_squares(x[ii*n:(ii+1)*n]).value) - np.dot(mus[ii].T,x[ii*n:(ii+1)*n]) +Ds[0] * cvx.sum_squares(x[:n]).value - (Ds[0] * CASH).T.dot(x[:n]) else: obj += s.T.dot(cvx.neg(x[ii*n:(ii+1)*n]).value) + gamma*(cvx.sum_squares(Fs[ii].T.dot(x[ii*n:(ii+1)*n])).value +\ Rs[ii]*cvx.sum_squares(x[ii*n:(ii+1)*n]).value) - np.dot(mus[ii].T,x[ii*n:(ii+1)*n]) return float(obj + 0.5 * cvx.quad_form(x, L).value)
def get_constr_error(constr): if isinstance(constr, cp.constraints.Equality): error = cp.abs(constr.args[0] - constr.args[1]) elif isinstance(constr, cp.constraints.Inequality): error = cp.pos(constr.args[0] - constr.args[1]) elif isinstance(constr, cp.constraints.PSD): mat = constr.args[0] - constr.args[1] error = cp.neg(cp.lambda_min(mat + mat.T) / 2) return cp.sum(error)
def objective(ii): full_obj = cvx.Minimize( s.T * cvx.neg(Xs[ii]) + gamma * (cvx.sum_squares(f[ii]) + Rs[ii] * cvx.sum_squares(Xs[ii])) - mus[ii].T * Xs[ii] + .5 * LHAT_max_eigval * cvx.sum_squares(Xs[ii] - cvx.reshape(x_prev[ii * n:(ii + 1) * n], (n, ))) + h_k[ii * n:(ii + 1) * n].T * Xs[ii]) return full_obj
def get_constr_error(constr): if isinstance(constr, cvx.constraints.EqConstraint): error = cvx.abs(constr.args[0] - constr.args[1]) elif isinstance(constr, cvx.constraints.LeqConstraint): error = cvx.pos(constr.args[0] - constr.args[1]) elif isinstance(constr, cvx.constraints.PSDConstraint): mat = constr.args[0] - constr.args[1] error = cvx.neg(cvx.lambda_min(mat + mat.T)/2) return cvx.sum_entries(error)
def quantile_reg(Y, X, quantile=0.5, lamb=0.01): y, x = np.array(Y), np.array(X) T, K = X.shape b = cvx.Variable(K) u = y - x * b J = (quantile * cvx.sum_entries(cvx.pos(u)) + (1 - quantile) * cvx.sum_entries(cvx.neg(u))) / T L2 = cvx.norm(b)**2 prob = cvx.Problem(cvx.Minimize(J + lamb * L2)) prob.solve() return np.asarray(b.value).reshape(-1)
def _genModelConstraints(self, x, prepared_constraints, prepared_option): CVXConstraints = [] for iType, iConstraint in prepared_constraints.items(): if iType == "Box": CVXConstraints.extend([ x <= iConstraint["ub"].flatten(), x >= iConstraint["lb"].flatten() ]) elif iType == "LinearIn": CVXConstraints.append( iConstraint["A"] @ x <= iConstraint["b"].flatten()) elif iType == "LinearEq": CVXConstraints.append( iConstraint["Aeq"] @ x == iConstraint["beq"].flatten()) elif iType == "Quadratic": for jSubConstraint in iConstraint: if "X" in jSubConstraint: jSigma = np.dot( np.dot(jSubConstraint["X"], jSubConstraint["F"]), jSubConstraint["X"].T) + np.diag( jSubConstraint["Delta"].flatten()) jSigma = (jSigma + jSigma.T) / 2 elif "Sigma" in jSubConstraint: jSigma = jSubConstraint["Sigma"] CVXConstraints.append( cvx.quad_form(x, jSigma) + jSubConstraint["Mu"].T @ x <= jSubConstraint["q"]) elif iType == "L1": for jSubConstraint in iConstraint: CVXConstraints.append( cvx.norm(x - jSubConstraint["c"].flatten(), p=1) <= jSubConstraint["l"]) elif iType == "Pos": for jSubConstraint in iConstraint: CVXConstraints.append( cvx.pos(x - jSubConstraint["c_pos"].flatten()) <= jSubConstraint["l_pos"]) elif iType == "Neg": for jSubConstraint in iConstraint: CVXConstraints.append( cvx.neg(x - jSubConstraint["c_neg"].flatten()) <= jSubConstraint["l_neg"]) elif iType == "NonZeroNum": raise __QS_Error__("不支持的约束条件: '非零数目约束'!") return CVXConstraints
def finance_example_regular_solve(problem_data): ''' Solves multi-period portfolio optimization problem by forming one large problem, and invoking CVXPY's solve() method. ''' L = problem_data['L'] n = problem_data['n'] m = problem_data['m'] T = problem_data['p'] gamma = problem_data['gamma'] tolerance = problem_data['tol'] mus = problem_data['mus'] Fs = problem_data['Fs'] Rs = problem_data['Rs'] Ds = problem_data['Ds'] s = problem_data['s'] SOLVER = problem_data['SOLVER'] print('Solving problem via CVXPY, no MM.') CASH = np.eye(1, n, n - 1).T.reshape(n, ) #cash vector x = cvx.Variable((n, T)) f = cvx.Variable((m, T)) objs_normal = 0 #Ds[0] * cvx.sum_squares(x[:,0]) - (np.dot(Ds[0],CASH).T * x[:,0]) constraints_normal = [x[:, 0] == CASH, x[:, T - 1] == CASH] for ii in range(T): constraints_normal += [ sum(x[:, ii]) == 1, f[:, ii] == Fs[ii].T * x[:, ii] ] objs_normal += s.T * cvx.neg(x[:, ii]) + gamma * (cvx.sum_squares( f[:, ii]) + Rs[ii] * cvx.sum_squares(x[:, ii])) - (mus[ii].T * x[:, ii]) #Solve problem with CVXPY t_start_reg = time.time() (x_opt, optval) = solve_complete(x, objs_normal, constraints_normal, L, Ds, SOLVER) t_reg = time.time() - t_start_reg return x_opt, optval, t_reg
def _solveMeanVarianceModel(self, nvar, prepared_objective, prepared_constraints, prepared_option): x = cvx.Variable(nvar) Obj = 0 if "f" in prepared_objective: Obj += prepared_objective["f"].T @ x if "X" in prepared_objective: Sigma = np.dot( np.dot(prepared_objective["X"], prepared_objective["F"]), prepared_objective["X"].T) + np.diag( prepared_objective["Delta"].flatten()) Sigma = (Sigma + Sigma.T) / 2 Obj += cvx.quad_form(x, Sigma) elif "Sigma" in prepared_objective: Obj += cvx.quad_form(x, prepared_objective["Sigma"]) if "Mu" in prepared_objective: Obj += prepared_objective["Mu"].T @ x if "lambda1" in prepared_objective: Obj += prepared_objective["lambda1"] * cvx.norm( x - prepared_objective["c"].flatten(), p=1) if "lambda2" in prepared_objective: Obj += prepared_objective["lambda2"] * cvx.pos( x - prepared_objective["c_pos"].flatten()) if "lambda3" in prepared_objective: Obj += prepared_objective["lambda3"] * cvx.neg( x - prepared_objective["c_neg"].flatten()) CVXConstraints = self._genModelConstraints(x, prepared_constraints, prepared_option) self._Model = cvx.Problem(cvx.Minimize(Obj), CVXConstraints) self._x = x self._Model.solve(**prepared_option) Status = (1 if self._Model.status not in ("infeasible", "unbounded") else 0) return (self._x.value, { "Status": Status, "Msg": self._Model.status, "solver_name": self._Model.solver_stats.solver_name, "solve_time": self._Model.solver_stats.solve_time, "setup_time": self._Model.solver_stats.setup_time, "num_iters": self._Model.solver_stats.num_iters })
problem_data['p'] = T problem_data['gamma'] = gamma problem_data['tol'] = tol problem_data['max_iter'] = max_iter problem_data['mus'] = mus problem_data['Fs'] = Fs problem_data['Rs'] = Rs problem_data['Ds'] = Ds problem_data['s'] = s problem_data['SOLVER'] = 'OSQP' print('Using %s as solver.' % problem_data['SOLVER']) '''Solve static problem to get initial points for MM.''' print('Solving static problem.') t_start_static = time.time() x_static = cvx.Variable((n, 1)) obj_static = s.T * cvx.neg(x_static) + gamma * ( cvx.sum_squares(Fs[0].T * x_static) + Rs[0] * cvx.sum_squares(x_static)) - mus[0].T * x_static constraints_static = [sum(x_static) == 1] cvx.Problem(cvx.Minimize(obj_static), constraints_static).solve() t_static = time.time() - t_start_static print('Computation time was %s seconds for the static problem.' % str(t_static)) print('Solve MPO problem using problem.solve().') (x_opt, obj_opt, t_reg) = finance_example_regular_solve(problem_data) print('Time it took for regular solve = %s' % t_reg) '''instantiate x^0 with static problem solution.''' x_prev = cvx.Parameter((n * T, 1)) x_prev.value = np.zeros((n * T, 1)) for ii in range(T):
def backtest(returns, Z_returns, benchmark, means, covs, lev_lim, bottom_sec_limit, upper_sec_limit, shorting_cost, tcost, MAXRISK, kappa=None, bid_ask=None): _, num_assets = returns.shape T, K = Z_returns.shape Zs_time = np.zeros((T - 1, K)) value_strat, value_benchmark = 1, 1 vals_strat, vals_benchmark = [value_strat], [value_benchmark] benchmark_returns = benchmark.loc[Z_returns.index].copy().values.flatten() """ On the fly computing for stratified model policy """ W = [np.zeros(18)] W[0][8] = 1 #vti for date in range(1, T): # if date % 50 == 0: print(date) dt = Z_returns.iloc[date].name.strftime("%Y-%m-%d") node = tuple(Z_returns.iloc[date]) Zs_time[date - 1, :] = [*node] if date == 1: w_prev = W[0].copy() else: w_prev = W[-1].flatten().copy() w = cp.Variable(num_assets) #adding cash asset into returns and covariances SIGMA = covs[node] MU = means[node] roll = 15 #get last 5 days tcs, lagged by one! so this doesnt include today's date tau = np.maximum(bid_ask.loc[:dt].iloc[-(roll + 1):-1].mean().values, 0) / 2 obj = -w @ (MU + 1) + shorting_cost * (kappa @ cp.neg(w)) + tcost * ( cp.abs(w - w_prev)) @ tau cons = [ cp.quad_form(w, SIGMA) <= MAXRISK * 100 * 100, sum(w) == 1, cp.norm1(w) <= lev_lim, bottom_sec_limit * np.ones(num_assets) <= w, w <= upper_sec_limit * np.ones(num_assets), ] prob_sm = cp.Problem(cp.Minimize(obj), cons) prob_sm.solve(verbose=False) returns_date = 1 + returns[date, :] #get TODAY's tc tau_sim = bid_ask.loc[dt].values.flatten() / 2 value_strat *= returns_date @ w.value - (kappa @ cp.neg(w)).value - ( cp.abs(w - w_prev) @ tau_sim).value vals_strat += [value_strat] value_benchmark *= 1 + benchmark_returns[date] vals_benchmark += [value_benchmark] w_prev = w.value.copy() * returns_date W += [w_prev.reshape(-1, 1)] vals = pd.DataFrame(data=np.vstack([vals_strat, vals_benchmark]).T, columns=["policy", "benchmark"], index=Z_returns.index.rename("Date")) Zs_time = pd.DataFrame(data=Zs_time, index=Z_returns.index[1:], columns=Z_returns.columns) #calculate sharpe rr = vals.pct_change() sharpes = np.sqrt(250) * rr.mean() / rr.std() returns = 250 * rr.mean() return vals, Zs_time, W, sharpes, returns
def routing_algorithm(world, robots, mode="random", maximumSeconds=120): """ Route the world using various algorithms. Parameters ---------- world: World object world object as defined in world.py robots: list list of Robot objects that will be altered for the animation mode: str a string that represents the mode we are going to use """ assignments = [[] for _ in range(len(robots))] if mode == "random": """Assign each task to a random robot multiple times to find a distribution.""" tasks = {} for i, hospital in enumerate(world.hospitals): if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[ hospital]['demand2'] != 0: pointer = world.graph.nodes[hospital] tasks[hospital] = np.array( [pointer['demand1'], pointer['demand2']], dtype=float) # Assign robots to tasks until there are none left while len(tasks) > 0: # print("Remaining tasks: ", tasks) for i, robot in enumerate(robots): if len(tasks) == 0: break random_goal = random.choice(list(tasks.keys())) tasks[random_goal] -= np.array( [robot.capacity, robot.capacity]) assignments[i].append( [random_goal, np.array([robot.capacity, robot.capacity])]) if np.all(tasks[random_goal] <= 0.): del tasks[random_goal] elif mode == "hungarian": """ We want a cost matrix of size number_of_robots x no_of_tasks""" number_of_robots = len(robots) tasks = [] for hospital in world.hospitals: if world.graph.nodes[hospital]['demand1'] != 0: tasks.append( (hospital, 0, world.graph.nodes[hospital]['demand1'])) if world.graph.nodes[hospital]['demand2'] != 0: tasks.append( (hospital, 1, world.graph.nodes[hospital]['demand2'])) cost_matrix = np.zeros((number_of_robots, len(tasks))) # So now we have a list of tasks. Each task is a tuple: (hospital, demand type, demand) # We also have a cost matrix of the correct size, now we fill it in with the time cost for i, robot in enumerate(robots): for j, task in enumerate(tasks): #print("robot: {}, task: {}".format(i,j)) path_length = nx.shortest_path_length(world.graph, robot.start_node, task[0], weight='length') # Calculate time taken to go to that hospital time_taken = path_length / robot.speed cost_matrix[i][j] = time_taken robot_ind, task_ind = linear_sum_assignment(cost_matrix) for i, index in enumerate(robot_ind): #print("robot_ind: ", i) task = tasks[task_ind[i]] assignments[i].append([ task[0], np.array([(1 - task[1]) * robots[index].capacity, task[1] * robots[index].capacity]) ]) elif mode == "linear_separate_tasks": # Takes all the tasks as separate tasks, i.e. does not join them together and the robot only delivers one type # of task. # Obtain all the demand and priority for goods 1 and 2 demand = [] tasks = [] types = [] priority = [] for i, hospital in enumerate(world.hospitals): if world.graph.nodes[hospital]['demand1'] != 0: demand.append(world.graph.nodes[hospital]['demand1']) tasks.append(hospital) types.append(0) priority.append(world.graph.nodes[hospital]['priority']) if world.graph.nodes[hospital]['demand2'] != 0: demand.append(world.graph.nodes[hospital]['demand2']) tasks.append(hospital) types.append(1) priority.append(world.graph.nodes[hospital]['priority']) # Obtain the time cost matrix and the robots' capacity. T = np.zeros((len(robots), len(demand))) capacity = np.zeros(len(robots)) for i, robot in enumerate(robots): capacity[i] = robot.capacity for j, task in enumerate(tasks): path_length = nx.shortest_path_length(world.graph, robot.start_node, task, weight='length') time_taken = path_length / robot.speed T[i][j] = time_taken # Change the demand and priority arrays to numpy ones demand = np.array(demand) priority = np.array(priority) * 10000 # Define the optimization problem x = cp.Variable((len(robots), len(demand)), boolean=True) cost = cp.sum(cp.multiply(T, x)) + cp.sum( cp.neg(cp.multiply(cp.matmul(capacity, x) - demand, priority))) objective = cp.Minimize(cost) inequality = [cp.sum(x, axis=1) <= 1] problem = cp.Problem(objective, inequality) problem.solve() # Assign the results to the robots and evaluate the costs nonzero = x.value.nonzero() for index in range(len(nonzero[0])): i = nonzero[1][index] j = nonzero[0][index] robot = robots[j] task = tasks[i] assignments[j].append([ task, np.array([(1 - types[i]) * robot.capacity, types[i] * robot.capacity]) ]) elif mode == "linear_joined_tasks": # As above but allows the robots to deliver two goods at the same time # Obtain all the demand and priority for goods 1 and 2 demand = [] tasks = [] priority = [] for i, hospital in enumerate(world.hospitals): if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[ hospital]['demand2'] != 0: pointer = world.graph.nodes[hospital] demand.append([pointer['demand1'], pointer['demand2']]) tasks.append(hospital) priority.append([pointer['priority'], pointer['priority']]) # Obtain the time cost matrix and the robots' capacity T = np.zeros((len(robots), len(demand))) capacity = np.zeros((len(robots), 2)) for i, robot in enumerate(robots): capacity[i, 0] = robot.capacity capacity[i, 1] = robot.capacity for j, task in enumerate(tasks): path_length = nx.shortest_path_length(world.graph, robot.start_node, task, weight='length') time_taken = path_length / robot.speed T[i][j] = time_taken # Change the demand and priority arrays to numpy ones demand = np.array(demand) priority = np.array(priority) * 10000 # Define the optimization problem x = cp.Variable((len(robots), len(demand)), boolean=True) cost = cp.sum(cp.multiply(T, x)) + cp.sum( cp.neg( cp.multiply( cp.matmul(cp.transpose(x), capacity) - demand, priority))) objective = cp.Minimize(cost) inequality = [cp.sum(x, axis=1) <= 1] problem = cp.Problem(objective, inequality) problem.solve(verbose=True, tm_lim=60000) # Assign the results to the robots and evaluate the costs nonzero = x.value.nonzero() # print(nonzero) for index in range(len(nonzero[0])): i = nonzero[1][index] j = nonzero[0][index] task = tasks[i] assignments[j].append( [task, np.array([capacity[j][0], capacity[j][1]])]) elif mode == "tsm": # Method based on the m-travelling salesmen problem # Obtain all the demand and priority for goods 1 and 2 demand = [] tasks = [] priority = [] for i, hospital in enumerate(world.hospitals): if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[ hospital]['demand2'] != 0: pointer = world.graph.nodes[hospital] demand.append([pointer['demand1'], pointer['demand2']]) tasks.append(hospital) priority.append([pointer['priority'], pointer['priority']]) # Obtain the time cost matrix and the robots' capacity x = [] v = [] z = [] capacities = [] distances = [] constraints = [] costs = [] for i, robot in enumerate(robots): # Create necessary variables x.append( cp.Variable((len(demand) + 1, len(demand) + 1), boolean=True)) v.append(cp.Variable(len(demand) + 1, boolean=True)) z.append(cp.Variable((len(demand), 1))) capacities.append(cp.Variable((len(demand), 2))) # Add constraints constraints.append(cp.sum( x[i], axis=0) == v[i]) # Note if city is moved from constraints.append(cp.sum(x[i], axis=0) == cp.sum( x[i], axis=1)) # Note if city is visited constraints.append(z[i] - cp.transpose(z[i]) + len(demand) * x[i][1:, 1:] <= len(demand) - 1) constraints.append(cp.sum(x[i], axis=1) <= cp.sum( x[i][0, :])) # We start from 0th node constraints.append( cp.sum(capacities[i], axis=0) <= robot.capacity) # Make sure we are not over capacity constraints.append( cp.sum(capacities[i], axis=1) <= 1000 * v[i][1:]) # Check how many units are delivered constraints.append(capacities[i] >= 0) # Fill in the distance matrix distances.append(np.zeros((len(demand) + 1, len(demand) + 1))) for j, task in enumerate([robot.start_node] + tasks): for k, other_task in enumerate([robot.start_node] + tasks): if other_task != task: path_length = nx.shortest_path_length(world.graph, task, other_task, weight='length') distances[i][j][k] = path_length / robot.speed else: distances[i][j][k] = 100000 costs.append(cp.sum(cp.multiply(x[i], distances[i]))) demand = np.array(demand) priority = np.array(priority) * 10000 costs = cp.max(cp.hstack(costs)) + cp.sum( cp.multiply(cp.pos(demand - sum(capacities)), priority)) objective = cp.Minimize(costs) problem = cp.Problem(objective, constraints) problem.solve(verbose=True, solver=cp.CBC, numberThreads=8, logLevel=1, maximumSeconds=maximumSeconds, allowablePercentageGap=10) # Assign the results to the robots and evaluate the costs for i, x_i in enumerate(x): print(f"variable {i + 1}: {x_i.value}") print(f"variable {i + 1}: {np.sum(x_i.value * distances[i])}") print(f"variable {i + 1}: {capacities[i].value}") for i, robot in enumerate(robots): node = np.argmax(x[i].value[0, :]) while node != 0: if np.any(capacities[i].value[node - 1] != 0): task = tasks[node - 1] assignments[i].append( [task, capacities[i].value[node - 1]]) node = np.argmax(x[i].value[node, :]) elif mode == "home": # Home made method allowing to specify the depth of the solution to be considered N # Obtain all the demand and priority for goods 1 and 2 demand = [] tasks = [] priority = [] N = 2 for i, hospital in enumerate(world.hospitals): if world.graph.nodes[hospital]['demand1'] != 0 or world.graph.nodes[ hospital]['demand2'] != 0: pointer = world.graph.nodes[hospital] demand.append([pointer['demand1'], pointer['demand2']]) tasks.append(hospital) priority.append([pointer['priority'], pointer['priority']]) # Obtain the time cost matrix and the robots' capacity x = [[] for _ in range(len(robots))] capacities = [] constraints = [] costs = [] for i, robot in enumerate(robots): # Obtain the time matrix for the robot time_matrix = np.zeros((len(demand), len(demand))) original = np.zeros(len(demand)) for j, task in enumerate(tasks): path_length = nx.shortest_path_length(world.graph, robot.start_node, task, weight='length') original[j] = path_length / robot.speed for k, other_task in enumerate(tasks): if other_task != task: path_length = nx.shortest_path_length(world.graph, task, other_task, weight='length') time_matrix[j][k] = path_length / robot.speed else: time_matrix[j][k] = 0 # Add the initial state current_costs = [] x[i].append(cp.Variable((len(demand), 1), boolean=True)) constraints.append(cp.sum(x[i][0]) <= 1) current_costs.append(cp.sum(cp.multiply(cp.vec(x[i][0]), original))) capacities.append(cp.Variable((len(demand), 2), integer=True)) # Loop over the other future states for n in range(1, N): x[i].append(cp.Variable((len(demand), 1), boolean=True)) current_costs.append( cp.sum( cp.multiply( cp.pos(x[i][n - 1] + cp.transpose(x[i][n]) - 1), time_matrix))) constraints.append(cp.sum(x[i][n]) <= cp.sum(x[i][n - 1])) # Add the capacities constraints constraints.append( cp.sum(capacities[i], axis=0) <= robot.capacity) # Make sure we are not over capacity constraints.append( cp.sum(capacities[i], axis=1) <= 1000 * cp.vec(sum(x[i]))) constraints.append(capacities[i] >= 0) # Add the current costs to the total costs costs.append(sum(current_costs)) demand = np.array(demand) priority = np.array(priority) * 10000 costs = cp.max(cp.hstack(costs)) + cp.sum( cp.multiply(cp.pos(demand - sum(capacities)), priority)) objective = cp.Minimize(costs) problem = cp.Problem(objective, constraints) problem.solve(verbose=True, solver=cp.CBC, logLevel=1, numberThreads=4, maximumSeconds=120, allowablePercentageGap=5) # Assign the results to the robots and evaluate the costs for robot_i, robot in enumerate(x): print(f"ROBOT {robot_i}\n-------") print(f"delivered capacities: \n{capacities[robot_i].value}") print(f"step 0: {0}") for i, x_i in enumerate(robot): print(f"step {i + 1}: {x_i.value.T}") for i, robot in enumerate(robots): visited_tasks = set() robot._current_node = robot.start_node for j, x_i in enumerate(x[i]): index = np.argmax(x_i.value) task = tasks[index] if np.any(capacities[i].value[index] != 0 ) and task not in visited_tasks: visited_tasks.add(task) assignments[i].append([task, capacities[i].value[index]]) robot._current_node = task else: raise NotImplementedError return assignments
def reg(self, X): return 1e10*cp.sum(cp.neg(X)) def __str__(self): return "nonnegative reg"
def reg(self, X): return 1e10 * cp.sum_entries(cp.neg(X))
def train_network(X_f, X_i, X_s, Y_i, beta, M=int(5000), fuse_xxstar=False, abs_act=False, sdf=True, norm=2): n_f, d = X_f.shape n_i, d = X_i.shape X_f = np.append(X_f,np.ones((n_f,1)),axis=1) X_i = np.append(X_i,np.ones((n_i,1)),axis=1) X_s = np.append(X_s,np.ones((n_i,1)),axis=1) d += 1 Y_i = np.atleast_1d(Y_i.squeeze()) dual_norm = np.inf if norm==1 else int(1/(1-1/float(norm))) ## Finite approximation of all possible sign patterns t0 = time.time() if n_f + 2*n_i < 15: D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar) else: D_f, D_i, D_s = random_signed_patterns(X_f, X_i, X_s, M, fuse_xxstar=fuse_xxstar) m1 = D_f.shape[1] print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.') # Optimal CVX Uopt1=cp.Variable((d,m1), value=np.random.randn(d,m1)) Uopt2=cp.Variable((d,m1), value=np.random.randn(d,m1)) constraints = [] loss = 0 # Feasible points if n_f > 0: ux_f_1 = cp.multiply(D_f,(X_f @ Uopt1)) ux_f_2 = cp.multiply(D_f,(X_f @ Uopt2)) y_f = cp.sum(ux_f_1 - ux_f_2,axis=1) if abs_act \ else cp.sum(cp.multiply((D_f+1)/2,(X_f @ (Uopt1 - Uopt2))),axis=1) constraints += [ ux_f_1>=0, ux_f_2>=0 ] Y_f_lb = np.max(Y_i[:, None] - np.linalg.norm(X_f[None, :, :] - X_i[:, None, :], axis=-1, ord=dual_norm), axis=0) if sdf: loss_f = cp.sum(cp.abs(y_f - Y_f_lb)) else: loss_f = cp.sum(cp.pos(y_f)) loss = loss + loss_f lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb)) else: loss_f = cp.Variable(value=0.) lipsh_f = cp.Variable(value=0.) # Infeasible points if n_i > 0: ux_i_1 = cp.multiply(D_i,(X_i @ Uopt1)) ux_s_1 = cp.multiply(D_s,(X_s @ Uopt1)) ux_i_2 = cp.multiply(D_i,(X_i @ Uopt2)) ux_s_2 = cp.multiply(D_s,(X_s @ Uopt2)) if abs_act: y_i = cp.sum(ux_i_1 - ux_i_2,axis=1) y_s = cp.sum(ux_s_1 - ux_s_2,axis=1) if sdf: if norm == 2: constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1] elif norm == 1: constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1] else: y_i = cp.sum(cp.multiply((D_i+1)/2,(X_i @ (Uopt1 - Uopt2))),axis=1) y_s = cp.sum(cp.multiply((D_s+1)/2,(X_s @ (Uopt1 - Uopt2))),axis=1) if sdf: if norm == 2: constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1] elif norm == 1: constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1] constraints += [ ux_i_1>=0, ux_s_1>=0, ux_i_2>=0, ux_s_2>=0, ] loss_i = cp.sum(cp.abs(Y_i - y_i)) loss_s = cp.sum(cp.abs(y_s)) loss = loss + loss_i + loss_s lipsh_i = cp.sum(cp.neg(Y_i - y_i)) else: loss_i = cp.Variable(value=0.) loss_s = cp.Variable(value=0.) lipsh_i = cp.Variable(value=0.) lipsh_s = cp.Variable(value=0.) # Regularization if norm == 2: reg = cp.mixed_norm(Uopt1[:-1].T,2,1) + cp.mixed_norm(Uopt2[:-1].T,2,1) elif norm == 1: reg = cp.Variable(nonneg=True) for i in range(d-1): for s in [-1,1]: constraints += [cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <= reg] # Solution prob=cp.Problem(cp.Minimize(100*(loss + beta * reg)),constraints) t0 = time.time() options = dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8}) prob.solve(solver=cp.MOSEK, verbose=True, **options) # prob.solve(solver=cp.SCS) print(f'Status: {prob.status}, \n ' f'Value: {prob.value :.2E}, \n ' f'loss_f: {loss_f.value :.2E}, \n ' f'loss_i: {loss_i.value :.2E}, \n ' f'loss_s: {loss_s.value :.2E}, \n ' f'Reg: {reg.value : .2E}, \n ' f'lipsh_f: {lipsh_f.value :.2E}, \n ' f'lipsh_i: {lipsh_i.value :.2E}, \n ' f'Time: {time.time()-t0 :.2f}s') if prob.status.lower() == 'infeasible': st() return None u1, u2 = torch.tensor(Uopt1.value), torch.tensor(Uopt2.value) st() return u1, u2
#No (additional) constraints constraints_i = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1] prob_ai = cvx.Problem(obj_a, constraints_i) risk_ai = prob_ai.solve() #Long-only constraints_ii = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1 , x >= 0] prob_aii = cvx.Problem(obj_a, constraints_ii) risk_aii = prob_aii.solve() #Limit on total short position #one = np.matrix(np.ones((n, 1))) constraints_iii = [pbarm.T * x == r_min, cvx.sum_entries(x)== 1] for i in range (n): constraints_iii += [1 * cvx.neg(x[i]) <= 0.5] prob_aiii = cvx.Problem(obj_a, constraints_iii) risk_aiii = prob_aiii.solve() print('Uniform portfolio risk: {}'.format(risk_uniform)) print('No (additional) constraints risk: {}'.format(risk_ai)) print('Long-only risk: {}'.format(risk_aii)) print('Limit on total short position risk: {}'.format(risk_aiii)) # Part (b) Plot optimal risk-return trade-off curves mus = np.logspace(0, 5, n) mean_long = np.zeros(n) std_long = np.zeros(n) mean_totalshort = np.zeros(n) std_totalshort = np.zeros(n) constraints_long = [cvx.sum_entries(x) == 1, x>= 0]
def reg(self, X): return 1e10 * cp.sum(cp.neg(X))
required_return = expected_return(x_unif) x = cp.Variable((n, 1)) sum_to_one_constraint = np.ones((n, 1)).T @ x == 1 required_return_constraint = pbar.T @ x == required_return objective = cp.Minimize(cp.quad_form(x, 2 * S)) min_risk_prob = cp.Problem(objective, [sum_to_one_constraint, required_return_constraint]) min_risk_prob.solve() print("Status = " + str(min_risk_prob.status)) print_portfolio(x.value, "min risk") long_only_constraint = x >= 0 long_only_prob = cp.Problem( objective, [sum_to_one_constraint, required_return_constraint, long_only_constraint]) long_only_prob.solve() print("Status = " + str(long_only_prob.status)) print_portfolio(x.value, "long only") limit_short_constraint = np.ones((n, 1)).T @ cp.neg(x) <= 0.5 limit_short_prob = cp.Problem(objective, [ sum_to_one_constraint, required_return_constraint, limit_short_constraint ]) limit_short_prob.solve() print("Status = " + str(limit_short_prob.status)) print_portfolio(x.value, "limit short")
print("min risk with expected return equals uniform portfolio") x = cp.Variable(n) constraints = \ [ cp.sum(x) == 1, pMean.T @ x == pMean.T @ xU ] prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints) prob.solve() assert prob.status == cp.OPTIMAL print(f"optimal risk {prob.value**0.5}") #%% # What is the risk of a long-only portfolio ЁЭСетк░0? print("x >= 0") prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [x >= 0]) prob.solve() assert prob.status == cp.OPTIMAL print(f"optimal risk {prob.value**0.5}") #%% # What is the risk of a portfolio with a limit on total short position: 1ЁЭСЗ(ЁЭСетИТ)тЙд0.5 , where (ЁЭСетИТ)ЁЭСЦ=max{тИТЁЭСеЁЭСЦ,0} ? print("limit on short") prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [cp.sum(cp.neg(x)) <= 0.5]) prob.solve() assert prob.status == cp.OPTIMAL print(f"optimal risk {prob.value**0.5}")
def optimization( self, model="Classic", rm="MV", obj="Sharpe", rf=0, l=2, hist=True ): r""" This method that calculates the optimum portfolio according to the optimization model selected by the user. The general problem that solves is: .. math:: \begin{align} &\underset{x}{\text{optimize}} & & F(w)\\ &\text{s. t.} & & Aw \geq B\\ & & & \phi_{i}(w) \leq c_{i}\\ \end{align} Where: :math:`F(w)` is the objective function. :math:`Aw \geq B` is a set of linear constraints. :math:`\phi_{i}(w) \leq c_{i}` are constraints on maximum values of several risk measures. Parameters ---------- model : str can be 'Classic', 'BL' or 'FM' The model used for optimize the portfolio. The default is 'Classic'. Posible values are: - 'Classic': use estimates of expected return vector and covariance matrix that depends on historical data. - 'BL': use estimates of expected return vector and covariance matrix based on the Black Litterman model. - 'FM': use estimates of expected return vector and covariance matrix based on a Risk Factor model specified by the user. rm : str, optional The risk measure used to optimze the portfolio. The default is 'MV'. Posible values are: - 'MV': Standard Deviation. - 'MAD': Mean Absolute Deviation. - 'MSV': Semi Standard Deviation. - 'FLPM': First Lower Partial Moment (Omega Ratio). - 'SLPM': Second Lower Partial Moment (Sortino Ratio). - 'CVaR': Conditional Value at Risk. - 'WR': Worst Realization (Minimax) - 'MDD': Maximum Drawdown of uncompounded returns (Calmar Ratio). - 'ADD': Average Drawdown of uncompounded returns. - 'CDaR': Conditional Drawdown at Risk of uncompounded returns. obj : str can be {'MinRisk', 'Utility', 'Sharpe' or 'MaxRet'. Objective function of the optimization model. The default is 'Sharpe'. Posible values are: - 'MinRisk': Minimize the selected risk measure. - 'Utility': Maximize the Utility function :math:`mu w - l \phi_{i}(w)`. - 'Sharpe': Maximize the risk adjusted return ratio based on the selected risk measure. - 'MaxRet': Maximize the expected return of the portfolio. rf : float, optional Risk free rate, must be in the same period of assets returns. The default is 0. l : scalar, optional Risk aversion factor of the 'Utility' objective function. The default is 2. hist : bool, optional Indicate if uses historical or factor estimation of returns to calculate risk measures that depends on scenarios (All except 'MV' risk measure). The default is True. Returns ------- w : DataFrame The weights of optimum portfolio. """ # General model Variables mu = None sigma = None returns = None if model == "Classic": mu = np.matrix(self.mu) sigma = np.matrix(self.cov) returns = np.matrix(self.returns) nav = np.matrix(self.nav) elif model == "FM": mu = np.matrix(self.mu_fm) if hist == False: sigma = np.matrix(self.cov_fm) returns = np.matrix(self.returns_fm) nav = np.matrix(self.nav_fm) elif hist == True: sigma = np.matrix(self.cov) returns = np.matrix(self.returns) nav = np.matrix(self.nav) elif model == "BL": mu = np.matrix(self.mu_bl) if hist == False: sigma = np.matrix(self.cov_bl) elif hist == True: sigma = np.matrix(self.cov) returns = np.matrix(self.returns) nav = np.matrix(self.nav) elif model == "BL_FM": mu = np.matrix(self.mu_bl_fm) if hist == False: sigma = np.matrix(self.cov_bl_fm) returns = np.matrix(self.returns_fm) nav = np.matrix(self.nav_fm) elif hist == True: sigma = np.matrix(self.cov) returns = np.matrix(self.returns) nav = np.matrix(self.nav) # General Model Variables returns = np.matrix(returns) w = cv.Variable((mu.shape[1], 1)) k = cv.Variable((1, 1)) rf0 = cv.Parameter(nonneg=True) rf0.value = rf n = cv.Parameter(nonneg=True) n.value = returns.shape[0] ret = mu * w # MV Model Variables risk1 = cv.quad_form(w, sigma) returns_1 = af.cov_returns(sigma) * 1000 n1 = cv.Parameter(nonneg=True) n1.value = returns_1.shape[0] risk1_1 = cv.norm(returns_1 * w, "fro") / cv.sqrt(n1 - 1) # MAD Model Variables madmodel = False Y = cv.Variable((returns.shape[0], 1)) u = np.matrix(np.ones((returns.shape[0], 1)) * mu) a = returns - u risk2 = cv.sum(Y) / n # madconstraints=[a*w >= -Y, a*w <= Y, Y >= 0] madconstraints = [a * w <= Y, Y >= 0] # Semi Variance Model Variables risk3 = cv.norm(Y, "fro") / cv.sqrt(n - 1) # CVaR Model Variables alpha1 = self.alpha VaR = cv.Variable(1) alpha = cv.Parameter(nonneg=True) alpha.value = alpha1 X = returns * w Z = cv.Variable((returns.shape[0], 1)) risk4 = VaR + 1 / (alpha * n) * cv.sum(Z) cvarconstraints = [Z >= 0, Z >= -X - VaR] # Worst Realization (Minimax) Model Variables M = cv.Variable(1) risk5 = M wrconstraints = [-X <= M] # Lower Partial Moment Variables lpmmodel = False lpm = cv.Variable((returns.shape[0], 1)) lpmconstraints = [lpm >= 0] if obj == "Sharpe": lpmconstraints += [lpm >= rf0 * k - X] else: lpmconstraints += [lpm >= rf0 - X] # First Lower Partial Moment (Omega) Model Variables risk6 = cv.sum(lpm) / n # Second Lower Partial Moment (Sortino) Model Variables risk7 = cv.norm(lpm, "fro") / cv.sqrt(n - 1) # Drawdown Model Variables drawdown = False if obj == "Sharpe": X1 = k + nav * w else: X1 = 1 + nav * w U = cv.Variable((nav.shape[0] + 1, 1)) ddconstraints = [U[1:] >= X1, U[1:] >= U[:-1]] if obj == "Sharpe": ddconstraints += [U[1:] >= k, U[0] == k] else: ddconstraints += [U[1:] >= 1, U[0] == 1] # Maximum Drawdown Model Variables MDD = cv.Variable(1) risk8 = MDD mddconstraints = [MDD >= U[1:] - X1] # Average Drawdown Model Variables risk9 = 1 / n * cv.sum(U[1:] - X1) # Conditional Drawdown Model Variables CDaR = cv.Variable(1) Zd = cv.Variable((nav.shape[0], 1)) risk10 = CDaR + 1 / (alpha * n) * cv.sum(Zd) cdarconstraints = [Zd >= U[1:] - X1 - CDaR, Zd >= 0] # Tracking Error Model Variables c = self.benchweights if self.kindbench == True: bench = np.matrix(returns) * c else: bench = self.benchindex if obj == "Sharpe": TE = cv.norm(returns * w - bench * k, "fro") / cv.sqrt(n - 1) else: TE = cv.norm(returns * w - bench, "fro") / cv.sqrt(n - 1) # Problem aditional linear constraints if obj == "Sharpe": constraints = [ cv.sum(w) == self.upperlng * k, k >= 0, mu * w - rf0 * k == 1, ] if self.sht == False: constraints += [w <= self.upperlng * k, w * 1000 >= 0] elif self.sht == True: constraints += [ w <= self.upperlng * k, w >= -self.uppersht * k, cv.sum(cv.neg(w)) <= self.uppersht * k, ] else: constraints = [cv.sum(w) == self.upperlng] if self.sht == False: constraints += [w <= self.upperlng, w * 1000 >= 0] elif self.sht == True: constraints += [ w <= self.upperlng, w >= -self.uppersht, cv.sum(cv.neg(w)) <= self.uppersht, ] if self.ainequality is not None and self.binequality is not None: A = np.matrix(self.ainequality) B = np.matrix(self.binequality) if obj == "Sharpe": constraints += [A * w - B * k >= 0] else: constraints += [A * w - B >= 0] # Turnover Constraints if obj == "Sharpe": if self.allowTO == True: constraints += [cv.abs(w - c * k) * 1000 <= self.turnover * k * 1000] else: if self.allowTO == True: constraints += [cv.abs(w - c) * 1000 <= self.turnover * 1000] # Tracking error Constraints if obj == "Sharpe": if self.allowTE == True: constraints += [TE <= self.TE * k] else: if self.allowTE == True: constraints += [TE <= self.TE] # Problem risk Constraints if self.upperdev is not None: if obj == "Sharpe": constraints += [risk1_1 <= self.upperdev * k] else: constraints += [risk1 <= self.upperdev ** 2] if self.uppermad is not None: if obj == "Sharpe": constraints += [risk2 <= self.uppermad * k / 2] else: constraints += [risk2 <= self.uppermad / 2] madmodel = True if self.uppersdev is not None: if obj == "Sharpe": constraints += [risk3 <= self.uppersdev * k] else: constraints += [risk3 <= self.uppersdev] madmodel = True if self.upperCVaR is not None: if obj == "Sharpe": constraints += [risk4 <= self.upperCVaR * k] else: constraints += [risk4 <= self.upperCVaR] constraints += cvarconstraints if self.upperwr is not None: if obj == "Sharpe": constraints += [-X <= self.upperwr * k] else: constraints += [-X <= self.upperwr] constraints += wrconstraints if self.upperflpm is not None: if obj == "Sharpe": constraints += [risk6 <= self.upperflpm * k] else: constraints += [risk6 <= self.upperflpm] lpmmodel = True if self.upperslpm is not None: if obj == "Sharpe": constraints += [risk7 <= self.upperslpm * k] else: constraints += [risk7 <= self.upperslpm] lpmmodel = True if self.uppermdd is not None: if obj == "Sharpe": constraints += [U[1:] - X1 <= self.uppermdd * k] else: constraints += [U[1:] - X1 <= self.uppermdd] constraints += mddconstraints drawdown = True if self.upperadd is not None: if obj == "Sharpe": constraints += [risk9 <= self.upperadd * k] else: constraints += [risk9 <= self.upperadd] drawdown = True if self.upperCDaR is not None: if obj == "Sharpe": constraints += [risk10 <= self.upperCDaR * k] else: constraints += [risk10 <= self.upperCDaR] constraints += cdarconstraints drawdown = True # Defining risk function if rm == "MV": if model != "Classic": risk = risk1_1 elif model == "Classic": risk = risk1 elif rm == "MAD": risk = risk2 madmodel = True elif rm == "MSV": risk = risk3 madmodel = True elif rm == "CVaR": risk = risk4 if self.upperCVaR is None: constraints += cvarconstraints elif rm == "WR": risk = risk5 if self.upperwr is None: constraints += wrconstraints elif rm == "FLPM": risk = risk6 lpmmodel = True elif rm == "SLPM": risk = risk7 lpmmodel = True elif rm == "MDD": risk = risk8 drawdown = True if self.uppermdd is None: constraints += mddconstraints elif rm == "ADD": risk = risk9 drawdown = True elif rm == "CDaR": risk = risk10 drawdown = True if self.upperCDaR is None: constraints += cdarconstraints if madmodel == True: constraints += madconstraints if lpmmodel == True: constraints += lpmconstraints if drawdown == True: constraints += ddconstraints # Frontier Variables portafolio = {} for i in self.assetslist: portafolio.update({i: []}) # Optimization Process # Defining solvers solvers = [cv.ECOS, cv.SCS, cv.OSQP, cv.CVXOPT, cv.GLPK] # Defining objective function if obj == "Sharpe": if rm != "Classic": objective = cv.Minimize(risk) elif rm == "Classic": objective = cv.Minimize(risk * 1000) elif obj == "MinRisk": objective = cv.Minimize(risk) elif obj == "Utility": objective = cv.Maximize(ret - l * risk) elif obj == "MaxRet": objective = cv.Maximize(ret) try: prob = cv.Problem(objective, constraints) for solver in solvers: try: prob.solve( solver=solver, parallel=True, max_iters=2000, abstol=1e-10 ) except: pass if w.value is not None: break if obj == "Sharpe": weights = np.matrix(w.value / k.value).T else: weights = np.matrix(w.value).T if self.sht == False: weights = np.abs(weights) / np.sum(np.abs(weights)) for j in self.assetslist: portafolio[j].append(weights[0, self.assetslist.index(j)]) except: pass optimum = pd.DataFrame(portafolio, index=["weights"], dtype=np.float64).T return optimum
def reg(self, X): return 1e10*cp.sum_entries(cp.neg(X)) def __str__(self): return "nonnegative reg"
def eval(self, writer): td3_results, mql_results = [], [] for i, (test_task_idx, test_buffer) in enumerate(zip(self.task_config.test_tasks, self._test_buffers)): batch = test_buffer.sample(self._args.batch_size, return_dict=True) other_batches = [buf.sample(self._args.batch_size, return_dict=True) for buf in self._inner_buffers] self._env.set_task_idx(test_task_idx) state, action, next_state, reward = (torch.tensor(batch['obs']).to(self._args.device), torch.tensor(batch['actions']).to(self._args.device), torch.tensor(batch['next_obs']).to(self._args.device), torch.tensor(batch['rewards']).to(self._args.device)) with torch.no_grad(): context_input = torch.cat((state, action, next_state, reward), -1) context_seq, h_n = self.context_encoder(context_input.unsqueeze(1), self.c0) context = h_n[-1] all_test_context = context_seq.squeeze(1) all_other_context, (state_, action_, next_state_, reward_) = self.get_context_from_batches(other_batches) all_other_context = all_other_context[torch.randperm(all_other_context.shape[0], device=self._args.device)[:self._args.batch_size]] labels = np.concatenate((-np.ones((self._args.batch_size)), np.ones((self._args.batch_size)))) X = torch.cat((all_test_context, all_other_context)).cpu().numpy() w = cp.Variable((all_test_context.shape[-1])) obj = cp.Minimize(cp.sum(cp.logistic(cp.neg(cp.multiply(labels, X @ w))))) prob = cp.Problem(obj) sol = prob.solve() w_ = torch.tensor(w.value, device=self._args.device).float() test_betas = (-all_test_context @ w_).exp() test_ess = (1/test_betas.shape[0]) * test_betas.sum().pow(2) / test_betas.pow(2).sum() context_betas = (-all_other_context @ w_).exp() traj, reward, success = self._rollout_policy(self._env, context) td3_results.append({'reward': reward, 'success': success, 'task': test_task_idx}) writer.add_scalar(f'TD3_Reward/Task_{test_task_idx}', reward, self.total_it) writer.add_scalar(f'TD3_Success/Task_{test_task_idx}', success, self.total_it) actor = copy.deepcopy(self.actor) critic_target = copy.deepcopy(self.critic_target) critic = copy.deepcopy(self.critic) actor_optimizer = torch.optim.Adam(actor.parameters(), lr=3e-4) critic_optimizer = torch.optim.Adam(critic.parameters(), lr=3e-4) start_actor_params = [p.clone() for p in actor.parameters()] start_critic_params = [p.clone() for p in critic.parameters()] lambda_ = 1 - test_ess context_ = context for step in range(self._mql_steps1 + self._mql_steps2): if step == self._mql_steps1: traj, reward, success = self._rollout_policy(self._env, context, policy=actor) writer.add_scalar(f'MQL1_Reward/Task_{test_task_idx}', reward, self.total_it) writer.add_scalar(f'MQL1_Success/Task_{test_task_idx}', success, self.total_it) critic_param_loss = sum([(p - p_.detach()).pow(2).sum() for p, p_ in zip(critic.parameters(), start_critic_params)]) actor_param_loss = sum([(p - p_.detach()).pow(2).sum() for p, p_ in zip(actor.parameters(), start_actor_params)]) if step >= self._mql_steps1: # re-assign state, action, next_state, reward to use data from train buffers # use w_ to assign weights # also add regularization to theta other_batches = [buf.sample(self._args.batch_size, return_dict=True) for buf in self._inner_buffers] all_other_context, (state, action, next_state, reward) = self.get_context_from_batches(other_batches) context_betas = (-all_other_context @ w_).exp() context_ess = (1/context_betas.shape[0]) * context_betas.sum().pow(2) / context_betas.pow(2).sum() lambda_ = 1 - context_ess if step == self._mql_steps1 + self._mql_steps2 - 1: writer.add_scalar(f'Test_Beta/Task_{test_task_idx}', test_betas.mean(), self.total_it) writer.add_scalar(f'Context_Beta/Task_{test_task_idx}', context_betas.mean(), self.total_it) writer.add_scalar(f'Test_ESS/Task_{test_task_idx}', test_ess, self.total_it) writer.add_scalar(f'Context_ESS/Task_{test_task_idx}', context_ess, self.total_it) else: lambda_ = 1 - test_ess not_done = torch.ones((state.shape[0],1)).to(self._args.device) if context_.shape[0] != state.shape[0]: context_ = context.repeat(state.shape[0], 1) with torch.no_grad(): next_action = actor(next_state, context_) # Compute the target Q value target_Q1, target_Q2 = critic_target(next_state, next_action, context_) target_Q = torch.min(target_Q1, target_Q2) target_Q = reward + not_done * self.discount * target_Q # Get current Q estimates current_Q1, current_Q2 = critic(state, action, context_) # Compute critic loss critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss( current_Q2, target_Q) + (lambda_ / 2) * critic_param_loss # Optimize the critic critic_optimizer.zero_grad() critic_loss.backward(retain_graph=True) critic_optimizer.step() # Compute actor losse actor_loss = -critic.Q1(state, actor(state, context_), context_).mean() + (lambda_ / 2) * actor_param_loss # Optimize the actor actor_optimizer.zero_grad() actor_loss.backward() actor_optimizer.step() for param, target_param in zip(critic.parameters(), critic_target.parameters()): target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data) traj, reward, success = self._rollout_policy(self._env, context, policy=actor) mql_results.append({'reward': reward, 'success': success, 'task': test_task_idx}) writer.add_scalar(f'MQL_Reward/Task_{test_task_idx}', reward, self.total_it) writer.add_scalar(f'MQL_Success/Task_{test_task_idx}', success, self.total_it) return td3_results, mql_results
def set_holding_cost(self, weights_new): self.holding_cost += cp.sum(cp.multiply(self.holding_coeff, cp.neg(weights_new))) return
for t in np.linspace(0.0, 1.0, num=101): g = np.where(x.value > t, 1, 0) v = c.T @ g m = np.max(A @ g - b) if m <= 0: print(f"with t={t}, m={m} and feasible objective val is {v}") break ## Simple portfolio optimization pbar = np.loadtxt(open(pbar_file, "rb"), delimiter=",") S = np.loadtxt(open(S_file, "rb"), delimiter=",") xu = np.loadtxt(open(xu_file, "rb"), delimiter=",") wans = np.ones(xu.shape[0]) xu_ret = pbar.T @ xu xu_rsk = np.sqrt(xu.T @ (S @ xu)) #min risk portfolio with same return as uniform x = cp.Variable(xu.shape[0]) p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1]) mr_rsk = np.sqrt(p.solve()) #min risk portfolio with same return as uniform but long-only p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1, 0 <= x]) lo_rsk = np.sqrt(p.solve()) #now limit the total short position to be <= 0.5 p = cp.Problem(cp.Minimize(cp.quad_form(x, S)), [pbar.T @ x == xu_ret, wans.T @ x == 1, wans.T @ cp.neg(x) <= 0.5]) sc_rsk = np.sqrt(p.solve())
def train_network(X_f, X_i, X_s, Y_i, dY_i=None, beta=1e-6, M=int(5000), fuse_xxstar=False, abs_act=False, sdf=True, norm=2): n_f, d = X_f.shape n_i, d = X_i.shape X_f = np.append(X_f, np.ones((n_f, 1)), axis=1) X_i = np.append(X_i, np.ones((n_i, 1)), axis=1) X_s = np.append(X_s, np.ones((n_i, 1)), axis=1) d += 1 # Y_i = np.atleast_1d(Y_i.squeeze()) dual_norm = np.inf if norm == 1 else int(1 / (1 - 1 / float(norm))) ## Finite approximation of all possible sign patterns t0 = time.time() if n_f + 2 * n_i < 10: D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar) else: D_f, D_i, D_s = random_signed_patterns(X_f, X_i, X_s, M, fuse_xxstar=fuse_xxstar) m1 = D_f.shape[1] print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.') # Optimal CVX Uopt0 = cp.Variable((d, 1), value=np.random.randn(d, 1)) Uopt1 = cp.Variable((d, m1), value=np.random.randn(d, m1)) Uopt2 = cp.Variable((d, m1), value=np.random.randn(d, m1)) constraints = [] loss = 0 # Feasible points if n_f > 0: ux_f_1 = cp.multiply(D_f, (X_f @ Uopt1)) ux_f_2 = cp.multiply(D_f, (X_f @ Uopt2)) if abs_act: y_f = X_f @ Uopt0 + cp.sum(ux_f_1 - ux_f_2, axis=1, keepdims=True) deriv_f = D_f @ (Uopt1 - Uopt2).T + Uopt0.T else: y_f = X_f @ Uopt0 + cp.sum(cp.multiply((D_f + 1) / 2, (X_f @ (Uopt1 - Uopt2))), axis=1, keepdims=True) deriv_f = ((D_f + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T constraints += [ux_f_1 >= 0, ux_f_2 >= 0, ux_f_1 >= 0, ux_f_2 >= 0] Y_f_lb = np.max(Y_i - np.linalg.norm( X_f[None, :, :] - X_i[:, None, :], axis=-1, ord=norm), axis=0, keepdims=True).T if sdf: loss_f = cp.sum(cp.abs(y_f - Y_f_lb)) else: loss_f = cp.sum(cp.pos(y_f)) loss = loss + loss_f lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb)) else: loss_f = cp.Variable(value=0.) lipsh_f = cp.Variable(value=0.) # Infeasible points if n_i > 0: ux_i_1 = cp.multiply(D_i, (X_i @ Uopt1)) ux_s_1 = cp.multiply(D_s, (X_s @ Uopt1)) ux_i_2 = cp.multiply(D_i, (X_i @ Uopt2)) ux_s_2 = cp.multiply(D_s, (X_s @ Uopt2)) if abs_act: y_i = X_i @ Uopt0 + cp.sum(ux_i_1 - ux_i_2, axis=1, keepdims=True) y_s = X_s @ Uopt0 + cp.sum(ux_s_1 - ux_s_2, axis=1, keepdims=True) deriv_i = D_i @ (Uopt1 - Uopt2).T + Uopt0.T deriv_s = D_s @ (Uopt1 - Uopt2).T + Uopt0.T else: y_i = X_i @ Uopt0 + cp.sum(cp.multiply((D_i + 1) / 2, (X_i @ (Uopt1 - Uopt2))), axis=1, keepdims=True) y_s = X_s @ Uopt0 + cp.sum(cp.multiply((D_s + 1) / 2, (X_s @ (Uopt1 - Uopt2))), axis=1, keepdims=True) deriv_i = ((D_i + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T deriv_s = ((D_s + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T constraints += [ ux_i_1 >= 0, ux_s_1 >= 0, ux_i_2 >= 0, ux_s_2 >= 0, ] loss_i = cp.sum(cp.abs(Y_i - y_i)) loss_s = cp.sum(cp.abs(y_s)) if dY_i is not None: loss_di = cp.sum(cp.abs(deriv_i[:, :-1] - dY_i)) loss_ds = cp.sum(cp.abs(deriv_s[:, :-1] - dY_i)) else: loss_di = cp.Variable(value=0., nonneg=True) loss_ds = cp.Variable(value=0., nonneg=True) loss = loss + loss_i + loss_s + loss_ds + loss_di lipsh_i = cp.sum(cp.neg(Y_i - y_i)) else: loss_i = cp.Variable(value=0.) loss_s = cp.Variable(value=0.) lipsh_i = cp.Variable(value=0.) lipsh_s = cp.Variable(value=0.) loss_di = cp.Variable(value=0.) loss_ds = cp.Variable(value=0.) # Regularization groupnorm_reg = cp.norm(Uopt0) + cp.mixed_norm( Uopt1.T, 2, 1) + cp.mixed_norm(Uopt2.T, 2, 1) if norm == 2: reg_nrm = cp.norm(Uopt0[:-1]) + cp.mixed_norm( Uopt1[:-1].T, 2, 1) + cp.mixed_norm(Uopt2[:-1].T, 2, 1) elif norm == 1: reg_nrm = cp.Variable(nonneg=True) for i in range(d - 1): for s in [-1, 1]: constraints += [ cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <= reg_nrm ] reg = reg_nrm + groupnorm_reg / 100. # Solution prob = cp.Problem(cp.Minimize(100 * (loss + beta * reg)), constraints) t0 = time.time() options = {} #dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8}) prob.solve(verbose=True, **options) # prob.solve(solver=cp.SCS, verbose=True) print( f'Status: {prob.status}, \n ' f'Value: {prob.value :.2E}, \n ' f'loss_f: {loss_f.value :.2E}, \n ' f'loss_i: {loss_i.value :.2E}, \n ' f'loss_s: {loss_s.value :.2E}, \n ' f'loss_di: {loss_di.value :.2E}, \n ' f'loss_ds: {loss_ds.value :.2E}, \n ' f'Reg: {reg.value : .2E}, \n ' # # f'lipsh_f: {lipsh_f.value :.2E}, \n ' # # f'lipsh_i: {lipsh_i.value :.2E}, \n ' f'Time: {time.time()-t0 :.2f}s') if prob.status.lower() == 'infeasible': st() return None u0, u1, u2 = torch.tensor(Uopt0.value), torch.tensor( Uopt1.value), torch.tensor(Uopt2.value) torch.save( { 'u1': u1, 'u2': u2, 'u0': u0, 'D_f': torch.tensor(D_f), 'D_i': torch.tensor(D_i), 'D_s': torch.tensor(D_s) }, 'tmp.csv') return u0, u1, u2
def test_matrix_multiperiod_portfolio(self): np.random.seed(1) k = 10 n = 50 T = 5 borrow_cost = 0.0001 lam = { "risk": 50, "borrow": 0.0001, "norm1_trade": 0.01, "norm0_trade": 1.0, } # Parameters hat_r = [ cp.Parameter(n, name="hat_r_%s" % str(t)) for t in range(1, T + 1) ] w_init = cp.Parameter(n, name="w_init") # F = cp.Parameter((n, k), name="F") sqrt_SigmaF_FT = cp.Parameter((k, n), name="SigmaF_FT") sqrt_D = cp.Parameter(n, name="sqrt_D") # Formulate problem w = [cp.Variable(n) for t in range(T + 1)] # Define cost components cost = 0 constraints = [w[0] == w_init] for t in range(1, T + 1): risk_cost = lam["risk"] * ( # cp.quad_form(F.T * w[t], Sigma_F) cp.sum_squares(sqrt_SigmaF_FT @ w[t]) + cp.sum_squares(cp.multiply(sqrt_D, w[t]))) holding_cost = lam["borrow"] * cp.sum(borrow_cost * cp.neg(w[t])) transaction_cost = lam["norm1_trade"] * cp.norm(w[t] - w[t - 1], 1) cost += (hat_r[t - 1] @ w[t] - risk_cost - holding_cost - transaction_cost) constraints += [cp.sum(w[t]) == 1.0] # Define optimizer problem = cp.Problem(cp.Maximize(cost), constraints) m = Optimizer(problem) self.assertTrue(m._problem.parameters_in_matrices) # Sample parameters df_train = sample_portfolio(n, k, N=1000) # Train and test using pytorch m.train(df_train, filter_strategies=True, parallel=True, n_train_trials=10, learner=PYTORCH) # Assert fewer strategies than training samples self.assertTrue(len(m.encoding) < len(df_train)) self.assertTrue(len(m.encoding) > 1)
def stabilize_chunk(desiredShot, aspectRatio, noDataFrames, imageSize, fps, crop_factor, apparent_motion, external_boundaries, screen_pos, lambda1=0.002, lambda2=0.0001, zoomSmooth=1.5, lambda3=0.005): """From a time sequence of unstabilized frame boxes, compute a stabilized frame. All parameters are normalized with respect to frame size and time, so that simultaneaously doubling the imageSize and the desiredShot does not change the solution, and neither does using twice as many frames and doubling the fps. The main differences with the paper are: - only D, L11 and L13 terms are implemented - zoomSmooth was added - the shots are optimized over chunks of shot_s*2 = 10s, and the five first seconds are kept. This makes the problem a lot more tractable. If a frame in desiredShot goes outside of the original image, it is cropped. Reference: Gandhi Vineet, Ronfard Remi, Gleicher Michael Multi-Clip Video Editing from a Single Viewpoint European Conference on Visual Media Production (CVMP) 2014 http://imagine.inrialpes.fr/people/vgandhi/GRG_CVMP_2014.pdf Keyword arguments: desiredShot -- a n x 4 numpy array containing on each line the box as [xmin, ymin, xmax, ymax] lambda1 -- see eq. (10) in paper lambda2 -- see eq. (10) in paper zoomSmooth -- a factor applied on the terms that deal with frame size in the regularization term: raise if the stabilized frame zooms in and out too much aspectRatio -- the desired output aspect ration (e.g. 16/9.) noDataFrames -- the list of frames that have no desiredShot information - only regularization is used to stabilize these frames imageSize -- [xmin, ymin, xmax, ymax] for the original image (typically [0,0,1920,1080] for HD) fps -- number of frames per seconds in the video - used for normalization """ #print "noDataFrames:", noDataFrames # print(noDataFrames, desiredShot) # set to desiredShot[noDataFrames, :] = 0. imageHeight = float(imageSize[1]) imageWidth = float(imageSize[0]) len_w = imageWidth/2 len_h = imageHeight/2 if imageHeight* aspectRatio < imageWidth: len_w = round((imageHeight*aspectRatio)/2) elif imageWidth / aspectRatio < imageHeight: len_h = round((imageWidth / aspectRatio)/2) # crop the desiredShot to the image window # we keep a 1-pixel margin to be sure that constraints can be satisfied margin = 1 low_x1_flags = desiredShot[:, 0] < (0. + margin) desiredShot[low_x1_flags, 0] = 0. + margin low_x2_flags = desiredShot[:, 2] < (0. + margin) desiredShot[low_x2_flags, 2] = 0. + margin high_x1_flags = desiredShot[:, 0] > (imageWidth - margin) desiredShot[high_x1_flags, 0] = imageWidth - margin high_x2_flags = desiredShot[:, 2] > (imageWidth - margin) desiredShot[high_x2_flags, 2] = imageWidth - margin low_y1_flags = desiredShot[:, 1] < (0. + margin) desiredShot[low_y1_flags, 1] = 0. + margin low_y2_flags = desiredShot[:, 3] < (0. + margin) desiredShot[low_y2_flags, 3] = 0. + margin high_y1_flags = desiredShot[:, 1] > (imageHeight - margin) desiredShot[high_y1_flags, 1] = imageHeight - margin high_y2_flags = desiredShot[:, 3] > (imageHeight - margin) desiredShot[high_y2_flags, 3] = imageHeight - margin # Make sure that a crop of the given aspectRatio can be contained in imageSize and can contain the desiredShot. # This may be an issue eg. when doing a 16/9 or a 4/3 movie from 2K. # else, we must cut the desiredshot on both sides. for k in range(desiredShot.shape[0]): if (desiredShot[k, 2] - desiredShot[k, 0]) > (imageHeight * aspectRatio - margin): xcut = (desiredShot[k, 2] - desiredShot[k, 0]) - \ (imageHeight * aspectRatio - margin) desiredShot[k, 2] -= xcut / 2 desiredShot[k, 0] += xcut / 2 if (desiredShot[k, 3] - desiredShot[k, 1]) > (imageWidth / aspectRatio - margin): ycut = (desiredShot[k, 3] - desiredShot[k, 1]) - \ (imageWidth / aspectRatio - margin) desiredShot[k, 3] -= ycut / 2 desiredShot[k, 1] += ycut / 2 # print("desiredShot:", desiredShot) # print("noDataFrames:", noDataFrames) x_center = (desiredShot[:, 0] + desiredShot[:, 2]) / 2. y_center = (desiredShot[:, 1] + desiredShot[:, 3]) / 2. # elementwise maximum of each array half_height_opt = np.maximum((desiredShot[:, 2] - desiredShot[:, 0]) / aspectRatio, ((desiredShot[:, 3] - desiredShot[:, 1]))) / 2 # smooth x_center y_center and half_height_opt using a binomial filter (Marchand and Marmet 1983) # eg [1 2 1]/4 or [1 4 6 4 1]/16 (obtained by applying it twice) # TODO: ignore noDataFrames when smoothing! x_center_residual = x_center # binomial_3(x_center_residual) # binomial_3(x_center_residual) y_center_residual = y_center # binomial_3(y_center_residual) # binomial_3(y_center_residual) half_height_opt_residual = half_height_opt # binomial_3(half_height_opt_residual) # binomial_3(half_height_opt_residual) half_width = (desiredShot[:, 2] - desiredShot[:, 0]) / 2. zero_flags = half_width[:] < 0 half_width[zero_flags] = 0. half_height = (desiredShot[:, 3] - desiredShot[:, 1]) / 2. zero_flags = half_height[:] < 0 half_height[zero_flags] = 0. # now trick the constraints so that there are no inner inclusion constraints at noDataFrames x_center[noDataFrames] = imageWidth / 2. half_width[noDataFrames] = -imageWidth / 2. # negative on purpose y_center[noDataFrames] = imageHeight / 2. half_height[noDataFrames] = -imageHeight / 2. # negative on purpose half_height_opt[noDataFrames] = imageHeight / 2. # print(half_height[noDataFrames], noDataFrames, imageHeight) # print(np.isnan(half_height)) # for i in range(len(desiredShot)): # if (y_center[i] - half_height[i]) < 0 or (y_center[i] + half_height[i]) > imageHeight: # print('indice ',i) external_boundaries[noDataFrames] = [0, 0, 0, 0, 0, 0] l_tl = [] for t in external_boundaries[:, 4]: if t == 1: l_tl.append(0.) else: l_tl.append(1.) tl_inv = np.array(l_tl) l_tr = [] for t in external_boundaries[:, 5]: if t == 1: l_tr.append(0.) else: l_tr.append(1.) tr_inv = np.array(l_tr) tl_inv[noDataFrames] = 0 tr_inv[noDataFrames] = 0 assert ((x_center - half_width) >= 0).all() and ((x_center + half_width) <= imageWidth).all() assert ((y_center - half_height) >= 0).all() and ((y_center + half_height) <= imageHeight).all() n = x_center.size #print "n:", n # We split the problem into chunks of fixed duration. # We compute a subsolution for the current chunk and the next chunk, and ensure continuity for the # variation (1st derivative) and jerk (3rd derivative) terms. # Then we only keep the solution for the current chunk and advance. # compute the opposite of noDataFrames # normalize with image height weightsAll = np.ones(n) / imageHeight weightsAll[noDataFrames] = 0. # do not use residuals on the optimal frame where there's no data #print "weightsAll:", weightsAll #print "half_height_opt:", half_height_opt optimised_xcenter = np.zeros(n) optimised_ycenter = np.zeros(n) optimised_height = np.zeros(n) chunk_s = 5 # size of a chunk in seconds chunk_n = int(chunk_s * fps) # number of samples in a chunk full_chunk_n = chunk_n * 2 # number of samples in a subproblem # starting index for the chunk (also used to check if it is the first chunk) chunk_start = 0 while chunk_start < n: chunk_end = min(n, chunk_start + chunk_n) chunk_size = chunk_end - chunk_start full_chunk_end = min(n, chunk_start + full_chunk_n) full_chunk_size = full_chunk_end - chunk_start # print("chunk:", chunk_start, chunk_end, full_chunk_end) x = cvx.Variable(full_chunk_size) y = cvx.Variable(full_chunk_size) # half height (see sec. 4 in the paper) h = cvx.Variable(full_chunk_size) weights = weightsAll[chunk_start:full_chunk_end] x_center_chunk = x_center[chunk_start:full_chunk_end] y_center_chunk = y_center[chunk_start:full_chunk_end] half_height_chunk = half_height[chunk_start:full_chunk_end] half_width_chunk = half_width[chunk_start:full_chunk_end] x_center_residual_chunk = x_center[chunk_start:full_chunk_end] y_center_residual_chunk = y_center[chunk_start:full_chunk_end] half_height_opt_residual_chunk = half_height_opt_residual[chunk_start:full_chunk_end] #Vector screen pos h_vector = screen_pos[chunk_start:full_chunk_end] #M1 term see paper 4.5 c_x_factor = crop_factor[:, chunk_start:full_chunk_end-1, 0] c_y_factor = crop_factor[:, chunk_start:full_chunk_end-1, 1] c_h_factor = crop_factor[:, chunk_start:full_chunk_end-1, 2] #M2 term see paper 4.5 b_x_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 0] b_y_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 1] b_h_factor = apparent_motion[:, chunk_start:full_chunk_end-1, 2] #E term [xl, xl1, xr, xr1, tl, tr] tl = external_boundaries[chunk_start:full_chunk_end, 4] tr = external_boundaries[chunk_start:full_chunk_end, 5] inv_tl = tl_inv[chunk_start:full_chunk_end] inv_tr = tr_inv[chunk_start:full_chunk_end] xl = external_boundaries[chunk_start:full_chunk_end, 0] xl1 = external_boundaries[chunk_start:full_chunk_end, 1] xr = external_boundaries[chunk_start:full_chunk_end, 2] xr1 = external_boundaries[chunk_start:full_chunk_end, 3] #assert ((x_center_chunk - half_width_chunk) >= 0).all() and ((x_center_chunk + half_width_chunk) <= imageWidth).all() #assert ((y_center_chunk - half_height_chunk) >= 0).all() and ((y_center_chunk + half_height_chunk) <= imageHeight).all() # for f in [97, 98, 99, 100]: # print f, weights[f], x_center[f], y_center[f], half_height_opt[f] expr = cvx.sum_squares(weights*(x_center_residual_chunk + (0.17*aspectRatio*half_height_opt_residual_chunk*h_vector) - x)) + cvx.sum_squares(weights*( y_center_residual_chunk - y)) + cvx.sum_squares(weights*(half_height_opt_residual_chunk - h)) expr /= n # normalize by the number of images, get a cost per image # end of version 1 # version 2: #dataFrames = np.nonzero(weights) # expr = cvx.sum_squares(x_center[dataFrames] - x[dataFrames]) + \ # cvx.sum_squares(y_center[dataFrames] - y[dataFrames]) + \ # cvx.sum_squares(half_height_opt[dataFrames] - h[dataFrames]) / (zoomSmooth*zoomSmooth) # expr /= (imageHeight*imageHeight)*n # normalize by the number of images, get a cost per image # end of version 2 if lambda1 != 0.: lambda1Factor = lambda1 * fps / imageHeight # print("lambda 1 ",lambda1Factor) if n > 1: expr += lambda1Factor * \ (cvx.tv(x) + cvx.tv(y) + cvx.tv(h) * zoomSmooth) # if not the first chunk, add continuity with previous samples if chunk_start >= 1: expr += lambda1Factor * (cvx.abs(x[0] - optimised_xcenter[chunk_start - 1]) + cvx.abs(y[0] - optimised_ycenter[chunk_start - 1]) + cvx.abs(h[0] - optimised_height[chunk_start - 1]) * zoomSmooth) if lambda2 != 0.: lambda2Factor = lambda2 * fps * fps * fps / imageHeight # print("lambda 2 ",lambda2Factor) if n > 2: expr += lambda2Factor * (cvx.norm(x[3:] - 3*x[2:full_chunk_size-1] + 3*x[1:full_chunk_size-2] - x[0:full_chunk_size-3], 1) + cvx.norm(y[3:] - 3*y[2:full_chunk_size-1] + 3*y[1:full_chunk_size-2] - y[0:full_chunk_size-3], 1) + cvx.norm(h[3:] - 3*h[2:full_chunk_size-1] + 3*h[1:full_chunk_size-2] - h[0:full_chunk_size-3], 1) * zoomSmooth) # if not the first chunk, add continuity with previous samples if chunk_start >= 3 and chunk_size >= 3: expr += lambda2Factor * ((cvx.abs(x[0] - 3 * optimised_xcenter[chunk_start - 1] + 3 * optimised_xcenter[chunk_start - 2] - optimised_xcenter[chunk_start - 3]) + cvx.abs(x[1] - 3 * x[0] + 3 * optimised_xcenter[chunk_start - 1] - optimised_xcenter[chunk_start - 2]) + cvx.abs(x[2] - 3 * x[1] + 3 * x[0] - optimised_xcenter[chunk_start - 1])) + (cvx.abs(y[0] - 3 * optimised_ycenter[chunk_start - 1] + 3 * optimised_ycenter[chunk_start - 2] - optimised_ycenter[chunk_start - 3]) + cvx.abs(y[1] - 3 * y[0] + 3 * optimised_ycenter[chunk_start - 1] - optimised_ycenter[chunk_start - 2]) + cvx.abs(y[2] - 3 * y[1] + 3 * y[0] - optimised_ycenter[chunk_start - 1])) + (cvx.abs(h[0] - 3 * optimised_height[chunk_start - 1] + 3 * optimised_height[chunk_start - 2] - optimised_height[chunk_start - 3]) + cvx.abs(h[1] - 3 * h[0] + 3 * optimised_height[chunk_start - 1] - optimised_height[chunk_start - 2]) + cvx.abs(h[2] - 3 * h[1] + 3 * h[0] - optimised_height[chunk_start - 1])) * zoomSmooth) if lambda3 != 0.: lambda3Factor = lambda3 * fps / imageHeight lambda2Factor = lambda2 * fps * fps * fps / imageHeight lambdaM = 5 if n > 1: m1_term = 0 for c_x, c_y, c_h in zip(c_x_factor, c_y_factor, c_h_factor): m1_term += lambdaM * (cvx.norm(c_x*(x[1:]-x[0:full_chunk_size-1]), 1) + cvx.norm(c_y*(y[1:]-y[0:full_chunk_size-1]), 1) + cvx.norm(c_h*(h[1:]-h[0:full_chunk_size-1]), 1) * zoomSmooth) if chunk_start >= 1: for c in crop_factor: c_x = c[chunk_start-1, 0] c_y = c[chunk_start-1, 1] c_h = c[chunk_start-1, 2] m1_term += lambdaM * (cvx.norm(c_x*(x[0]-optimised_xcenter[chunk_start-1]), 1) + cvx.norm(c_y*(y[0]-optimised_ycenter[chunk_start-1]), 1) + cvx.norm(c_h*(h[0]-optimised_height[chunk_start-1]), 1) * zoomSmooth) if n > 1: m2_term = 0 for b_x, b_y, b_h_g in zip(b_x_factor, b_y_factor, b_h_factor): b_h = gaussian_filter(b_h_g,sigma=5) # print(b_x, b_x.dtype) m2_term += lambdaM * (cvx.neg((b_x-(x[1:]-x[0:full_chunk_size-1]))*b_x) + cvx.neg((b_y-(y[1:]-y[0:full_chunk_size-1]))*b_y) + cvx.neg((b_h-(h[1:]-h[0:full_chunk_size-1]))*b_h) * zoomSmooth) if chunk_start >=1 : for b in apparent_motion: b_x = b[chunk_start-1, 0] b_y = b[chunk_start-1, 0] b_h = b[chunk_start-1, 0] m2_term += lambdaM * (cvx.neg((b_x-(x[0]-optimised_xcenter[chunk_start-1]))*b_x) + cvx.neg((b_y-(y[0]-optimised_ycenter[chunk_start-1]))*b_y) + cvx.neg((b_h-(h[0]-optimised_height[chunk_start-1]))*b_h) * zoomSmooth) # expr += m1_term + m2_term # if n > 1: # # E out # expr += lambda3Factor * ((inv_tl * cvx.pos(xl1 - x + aspectRatio * h)) + # (inv_tr * cvx.pos(x + aspectRatio * h - xr1)) ) # # # E in # expr += lambda3Factor * ((tl * cvx.pos(x - aspectRatio * h - xl)) + # (tr * cvx.pos(xr - x - aspectRatio * h)) ) obj = cvx.Minimize(expr) #print expr # print("H=%d, W=%d lambda1=%f lambda2=%f zoomSmooth=%f fps=%f imageHeight=%f" % ( # imageHeight, imageWidth, lambda1, lambda2, zoomSmooth, fps, imageHeight)) # note that the following constraints are tricked (see above) at noDataFrames, using negative values for half_width and half_height constraints = [h >= 0, (x - aspectRatio * h) >= 0, (x - aspectRatio * h) <= (x_center_chunk - half_width_chunk), (x + aspectRatio * h) >= (x_center_chunk + half_width_chunk), (x + aspectRatio * h) <= imageWidth, aspectRatio * h <= len_w, (y - h) >= 0, (y - h) <= (y_center_chunk - half_height_chunk), (y + h) >= (y_center_chunk + half_height_chunk), (y + h) <= imageHeight, h <= len_h] prob = cvx.Problem(obj, constraints) tryagain = True tryreason = "" if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED: tryagain = False try: # ECOS, the default solver, is much better at solving our problems, especially at handling frames where the actor is not visible # all tolerances are multiplied by 10 result = prob.solve(solver=cvx.ECOS, verbose=False, abstol = 1e-6, reltol = 1e-5, abstol_inacc = 5e-4, reltol_inacc = 5e-4, feastol_inacc = 1e-3) except cvx.SolverError as e: tryagain = True tryreason = str(e) if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED: tryagain = False try: result = prob.solve(solver=cvx.SCS, verbose=False, max_iters = 2500, eps = 1e-2) except cvx.SolverError as e: tryagain = True tryreason = str(e) if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED: tryagain = False try: result = prob.solve(solver=cvx.CVXOPT, verbose=False, abstol = 1e-6, reltol = 1e-5, feastol = 1e-6) except cvx.SolverError as e: tryagain = True tryreason = str(e) if tryagain or prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED: tryagain = False try: result = prob.solve( solver=cvx.CVXOPT, kktsolver=cvx.ROBUST_KKTSOLVER, verbose=False, abstol = 1e-6, reltol = 1e-5, feastol = 1e-6) except cvx.SolverError as e: tryagain = True tryreason = str(e) if tryagain: raise cvx.solverError(tryreason) if prob.status == cvx.INFEASIBLE or prob.status == cvx.UNBOUNDED: raise cvx.SolverError('Problem is infeasible or unbounded') #raise ValueError('Yeah!') # print("result=", result, "\n") if full_chunk_end >= n: # last chunk - get the full chunk optimised_xcenter[chunk_start:full_chunk_end] = x.value.reshape( full_chunk_size) optimised_ycenter[chunk_start:full_chunk_end] = y.value.reshape( full_chunk_size) optimised_height[chunk_start:full_chunk_end] = h.value.reshape( full_chunk_size) chunk_start = full_chunk_end else: # only get the chunk and advance optimised_xcenter[chunk_start:chunk_end] = x.value[:chunk_size].reshape( chunk_size) optimised_ycenter[chunk_start:chunk_end] = y.value[:chunk_size].reshape( chunk_size) optimised_height[chunk_start:chunk_end] = h.value[:chunk_size].reshape( chunk_size) chunk_start = chunk_end return np.vstack([optimised_xcenter - aspectRatio * optimised_height, optimised_ycenter - optimised_height, optimised_xcenter + aspectRatio * optimised_height, optimised_ycenter + optimised_height]).transpose()