def test_psd_constraints(self):
""" Test positive semi-definite constraints
"""
C = Variable((3, 3))
obj = Maximize(C[0, 2])
constraints = [
diag(C) == 1, C[0, 1] == 0.6, C[1, 2] == -0.3, C == C.T, C >> 0
]
prob = Problem(obj, constraints)
self.assertTrue(FlipObjective().accepts(prob))
p_min = FlipObjective().apply(prob)
self.assertTrue(ConeMatrixStuffing().accepts(p_min[0]))
C = Variable((2, 2))
obj = Maximize(C[0, 1])
constraints = [C == 1, C >> [[2, 0], [0, 2]]]
prob = Problem(obj, constraints)
self.assertTrue(FlipObjective().accepts(prob))
p_min = FlipObjective().apply(prob)
self.assertTrue(ConeMatrixStuffing().accepts(p_min[0]))
C = Variable((2, 2), symmetric=True)
obj = Minimize(C[0, 0])
constraints = [C << [[2, 0], [0, 2]]]
prob, _ = CvxAttr2Constr().apply(Problem(obj, constraints))
self.assertTrue(ConeMatrixStuffing().accepts(prob))
def test_portfolio(self):
n = 10
beta = 0.05
p_bar = 1.5 * np.random.randn(n) + 4
sigma = 2.5 * np.random.randn(n, n)
sigma = sigma.T.dot(sigma)
price = np.random.multivariate_normal(p_bar, sigma, self.N)
# Optimal portfolio.
x = Variable(n)
# ret = sum(price*x)/self.N
ret = p_bar.T * x
constr = [sum(x) == 1, x >= -0.1, prob(price * x <= 0) <= beta]
p = Problem(Maximize(ret), constr)
p.solve(solver="MOSEK")
ret_opt = price.dot(x.value)
print("Optimal portfolio")
print("Expected return:", ret.value)
print("Fraction nonpositive:", np.mean(price.dot(x.value) <= 0))
print("Standard deviation:", np.std(ret_opt))
# Optimal portfolio without loss risk constraint.
constr = [sum(x) == 1, x >= -0.1]
p = Problem(Maximize(ret), constr)
p.solve(solver="MOSEK")
ret_nocc = price.dot(x.value)
print("\nOptimal portfolio without loss risk constraint")
print("Expected return:", ret.value)
print("Fraction nonpositive:", np.mean(price.dot(x.value) <= 0))
print("Standard deviation:", np.std(ret_nocc))
# Uniform portfolio.
x_unif = np.ones(n) / n
ret_unif = price.dot(x_unif)
print("\nUniform portfolio")
print("Expected return:", np.sum(ret_unif) / self.N)
print("Fraction nonpositive:", np.mean(ret_unif <= 0))
# Plot return distributions.
rets = [ret_opt, ret_nocc, ret_unif]
titles = ["Optimal", "Without Loss Constraint", "Uniform"]
for i in range(len(rets)):
plt.subplot(3, 1, i + 1)
sns.distplot(rets[i], hist=False, kde=True, color="blue")
plt.axvline(np.mean(rets[i]), color="red")
plt.xlim(-10, 25)
plt.ylim(0, 0.275)
plt.title(titles[i])
plt.gca().axes.get_yaxis().set_visible(False)
plt.gca().axes.get_xaxis().set_ticks_position("both")
plt.gca().axes.tick_params(axis="x", direction="in")
plt.subplots_adjust(hspace=0.6)
def test_partial_problem(self):
"""Test domain for partial minimization/maximization problems.
"""
for obj in [Minimize((self.a)**-1), Maximize(log(self.a))]:
orig_prob = Problem(obj, [self.x + self.a >= [5, 8]])
# Optimize over nothing.
expr = partial_optimize(orig_prob, dont_opt_vars=[self.x, self.a])
dom = expr.domain
constr = [self.a >= -100, self.x >= 0]
prob = Problem(Minimize(sum(self.x + self.a)), dom + constr)
prob.solve()
self.assertAlmostEqual(prob.value, 13)
assert self.a.value >= 0
assert np.all((self.x + self.a - [5, 8]).value >= -1e-3)
# Optimize over x.
expr = partial_optimize(orig_prob, opt_vars=[self.x])
dom = expr.domain
constr = [self.a >= -100, self.x >= 0]
prob = Problem(Minimize(sum(self.x + self.a)), dom + constr)
prob.solve(solver=cvxpy.ECOS)
self.assertAlmostEqual(prob.value, 0)
assert self.a.value >= -1e-3
self.assertItemsAlmostEqual(self.x.value, [0, 0])
# Optimize over x and a.
expr = partial_optimize(orig_prob, opt_vars=[self.x, self.a])
dom = expr.domain
constr = [self.a >= -100, self.x >= 0]
prob = Problem(Minimize(sum(self.x + self.a)), dom + constr)
prob.solve(solver=cvxpy.ECOS)
self.assertAlmostEqual(self.a.value, -100)
self.assertItemsAlmostEqual(self.x.value, [0, 0])
def mpt_opt(data, gamma_vec):
NUM_SAMPLES = len(gamma_vec)
w_vec_results = [None] * NUM_SAMPLES
ret_results = np.zeros(NUM_SAMPLES)
risk_results = np.zeros(NUM_SAMPLES)
N = len(data)
w_vec = Variable(N)
mu_vec = np.array([np.mean(data[i]) for i in range(N)])
sigma_mat = np.cov(data)
gamma = Parameter(nonneg=True)
ret_val = mu_vec.T * w_vec
risk_val = quad_form(w_vec, sigma_mat) # w^T Sigma w
problem = Problem(Maximize(ret_val - gamma * risk_val),
[sum(w_vec) == 1, w_vec >= 0])
for i, new_gamma in enumerate(gamma_vec):
gamma.value = new_gamma
problem.solve()
w_vec_results[i] = w_vec.value
ret_results[i] = ret_val.value
risk_results[i] = sqrt(risk_val).value
return (w_vec_results, ret_results, risk_results)
def objective(self):
""" """
objective = Maximize(
sum(self.areas @ self.X)
# + sum(self.HE_Joint1 @ self.X + self.HE_Joint2 @ self.X)
+ sum(self.J))
return objective
def test_reduction_basic(self):
# Minimize quantile(y, 0.5) subject to y >= 0.
t = Variable()
constr = [quantile(self.y, 0.5) <= t, self.y >= 0]
p0 = ccprob.Problem(Minimize(t), constr)
p0.solve()
y_epi = self.y.value
obj = quantile(self.y, 0.5)
constr = [self.y >= 0]
p1 = ccprob.Problem(Minimize(obj), constr)
p1.solve()
self.assertAlmostEqual(p1.value, p0.value)
self.assertItemsAlmostEqual(self.y.value, y_epi)
# Maximize quantile(y, 0.75) subject to y <= 5.
constr = [quantile(self.y, 0.75) >= t, self.y <= 5]
p0 = ccprob.Problem(Maximize(t), constr)
p0.solve()
y_epi = self.y.value
obj = quantile(self.y, 0.75)
constr = [self.y <= 5]
p1 = ccprob.Problem(Maximize(obj), constr)
p1.solve()
self.assertAlmostEqual(p1.value, p0.value)
self.assertItemsAlmostEqual(self.y.value, y_epi)
# Minimize quantile(abs(A*x - b), 0.5)
# subject to 0 <= x <= 1,
# quantile(x, 0.25) >= 0.1
b = np.random.randn(self.A.shape[0])
constr = [
quantile(abs(self.A * self.x - b), 0.5) <= t, self.x >= 0,
self.x <= 1,
quantile(self.x, 0.25) >= 0.1
]
p0 = ccprob.Problem(Minimize(t), constr)
p0.solve()
x_epi = self.x.value
obj = quantile(abs(self.A * self.x - b), 0.5)
constr = [self.x >= 0, self.x <= 1, quantile(self.x, 0.25) >= 0.1]
p1 = ccprob.Problem(Minimize(obj), constr)
p1.solve()
self.assertAlmostEqual(p1.value, p0.value)
self.assertItemsAlmostEqual(self.x.value, x_epi)
def maximize_problem(self, solver):
A = np.random.randn(5, 2)
A = np.maximum(A, 0)
b = np.random.randn(5)
b = np.maximum(b, 0)
p = Problem(Maximize(-sum(self.x)), [self.x >= 0, A * self.x <= b])
self.solve_QP(p, solver)
for var in p.variables():
self.assertItemsAlmostEqual([0., 0.], var.value, places=3)
def test_nonneg_constraints_backend(self) -> None:
x = Variable(shape=(2, ), name='x')
objective = Maximize(-4 * x[0] - 5 * x[1])
constr_expr = hstack(
[3 - (2 * x[0] + x[1]), 3 - (x[0] + 2 * x[1]), x[0], x[1]])
constraints = [NonNeg(constr_expr)]
prob = Problem(objective, constraints)
self.assertFalse(ConeMatrixStuffing().accepts(prob))
self.assertTrue(FlipObjective().accepts(prob))
p_min = FlipObjective().apply(prob)
self.assertTrue(ConeMatrixStuffing().accepts(p_min[0]))
def test_flow_control(self):
# Problem data.
m = 40
m_a = 20
n = 22
n_a = 5
n_b = 5
np.random.seed(1)
R = np.vstack((np.hstack((np.round(rand(m_a,n_a)), np.zeros((m_a,n_b)), np.round(rand(m_a,n-n_a-n_b)))),
np.hstack((np.zeros((m-m_a,n_a)), np.round(rand(m-m_a,n_b)), np.round(rand(m-m_a,n-n_a-n_b))))
))
c = 5*rand(m)
# Find optimum directly.
f_star = Variable(n)
prob = Problem(Maximize(sum(sqrt(f_star))), [R*f_star <= c])
prob.solve()
print("True Objective:", prob.value)
print("True Solution:", f_star.value)
# Partition data into two groups with overlap.
R_a = R[:m_a,:n_a]
R_b = R[m_a:,n_a:(n_a + n_b)]
S_a = R[:m_a,(n_a + n_b):]
S_b = R[m_a:,(n_a + n_b):]
c_a = c[:m_a]
c_b = c[m_a:]
n_ab = n - n_a - n_b
# Define separate problem for each group.
f_a = Variable(n_a)
f_b = Variable(n_b)
x = Variable(n_ab)
p_list = [Problem(Maximize(sum(sqrt(f_a)) + 0.5*sum(sqrt(x))), [R_a*f_a + S_a*x <= c_a]),
Problem(Maximize(sum(sqrt(f_b)) + 0.5*sum(sqrt(x))), [R_b*f_b + S_b*x <= c_b])]
probs = Problems(p_list)
# Solve via consensus.
probs.solve(method = "consensus", rho_init = 10, max_iter = 20)
print("Consensus Objective:", -probs.value) # TODO: All problems recast as minimization, so flip sign of objective to compare
print("Consensus Solution:", np.hstack((f_a.value, f_b.value, x.value)))
def main():
x = Variable(1)
y = Variable(1)
constraints = [x >= 0, y >= 0, x+y == 1]
objective = Maximize(square(x) + square(y))
problem = Problem(objective, constraints)
print("problem is DCP:", problem.is_dcp())
print("problem is DCCP:", is_dccp(problem))
problem.solve(method='dccp')
print('solution (x,y): ', x.value, y.value)
def test_lp(self):
n = 10
c = np.random.randn(n)
sigma = np.eye(n)
A = np.random.multivariate_normal(c, sigma, self.N)
x = Variable(n)
obj = c.T * x
constr = [prob(A * x >= 0) <= 0.75, x >= 0, x <= 1]
p = Problem(Maximize(obj), constr)
p.solve(solver="MOSEK")
print("Objective:", p.value)
print("Chance constraint fraction:", np.mean(A.dot(x.value) <= 0))
print("Margin cutoff:", np.percentile(constr[0].margins(), q=75))
def test_partial_problem(self) -> None:
"""Test grad for partial minimization/maximization problems.
"""
for obj in [Minimize((self.a)**-1), Maximize(cp.entr(self.a))]:
prob = Problem(obj, [self.x + self.a >= [5, 8]])
# Optimize over nothing.
expr = partial_optimize(prob,
dont_opt_vars=[self.x, self.a],
solver=cp.ECOS)
self.a.value = None
self.x.value = None
grad = expr.grad
self.assertAlmostEqual(grad[self.a], None)
self.assertAlmostEqual(grad[self.x], None)
# Outside domain.
self.a.value = 1.0
self.x.value = [5, 5]
grad = expr.grad
self.assertAlmostEqual(grad[self.a], None)
self.assertAlmostEqual(grad[self.x], None)
self.a.value = 1
self.x.value = [10, 10]
grad = expr.grad
self.assertAlmostEqual(grad[self.a], obj.args[0].grad[self.a])
self.assertItemsAlmostEqual(grad[self.x].toarray(), [0, 0, 0, 0])
# Optimize over x.
expr = partial_optimize(prob, opt_vars=[self.x], solver=cp.ECOS)
self.a.value = 1
grad = expr.grad
self.assertAlmostEqual(grad[self.a], obj.args[0].grad[self.a] + 0)
# Optimize over a.
fix_prob = Problem(obj, [self.x + self.a >= [5, 8], self.x == 0])
fix_prob.solve(solver=cp.ECOS)
dual_val = fix_prob.constraints[0].dual_variables[0].value
expr = partial_optimize(prob, opt_vars=[self.a], solver=cp.ECOS)
self.x.value = [0, 0]
grad = expr.grad
self.assertItemsAlmostEqual(grad[self.x].toarray(), dual_val)
# Optimize over x and a.
expr = partial_optimize(prob,
opt_vars=[self.x, self.a],
solver=cp.ECOS)
grad = expr.grad
self.assertAlmostEqual(grad, {})
def dgp():
# DGP requires Variables to be declared positive via `pos=True`.
x = Variable(pos=True)
y = Variable(pos=True)
z = Variable(pos=True)
objective_fn = x * y * z
constraints = [
4 * x * y * z + 2 * x * z <= 10, x <= 2 * y, y <= 2 * x, z >= 1
]
problem = Problem(Maximize(objective_fn), constraints)
problem.solve(gp=True)
print("Optimal value: ", problem.value)
print("x: ", x.value)
print("y: ", y.value)
print("z: ", z.value)
def test_card():
"""Test card variable."""
x = Card(5, k=3, M=1)
p = Problem(Maximize(sum(x)), [x <= 1, x >= 0])
result = p.solve(**solve_args)
assert result[0] == approx(3)
for v in np.nditer(x.value):
assert v * (1 - v) == approx(0)
assert x.value.sum() == approx(3)
# Should be equivalent to x == choose.
x = Variable((5, 4))
c = Choose((5, 4), k=4)
b = Boolean((5, 4))
p = cp.Problem(Minimize(sum(1 - x) + sum(x)), [x == c, x == b])
result = p.solve(**solve_args)
assert result[0] == approx(20)
for v in np.nditer(x.value):
assert v * (1 - v) == approx(0)
def test_card(self):
x = Card(5, k=3)
p = Problem(Maximize(sum(x)), [x <= 1, x >= 0])
result = p.solve(method="admm")
self.assertAlmostEqual(result, 3)
for v in np.nditer(x.value):
self.assertAlmostEqual(v * (1 - v), 0)
self.assertAlmostEqual(x.value.sum(), 3)
#should be equivalent to x == choose
x = Variable(5, 4)
c = Card(5, 4, k=4)
b = Boolean(5, 4)
p = Problem(Minimize(sum(1 - x) + sum(x)), [x == c, x == b])
result = p.solve(method="admm", solver=CVXOPT)
self.assertAlmostEqual(result, 20)
for i in range(x.size[0]):
for j in range(x.size[1]):
v = x.value[i, j]
self.assertAlmostEqual(v * (1 - v), 0)
def test_simple_problem(self):
# Create problem data.
n = numpy.random.randint(1, 10)
eta = 0.95
num_samples = 10
c = numpy.random.rand(n, 1)
mu = numpy.zeros(n)
Sigma = numpy.eye(n)
a = NormalRandomVariable(mu, Sigma)
b = numpy.random.randn()
# Create and solve optimization problem.
x = Variable(n)
p = Problem(
Maximize(x.T * c),
[prob(max_entries(x.T * a - b) >= 0, num_samples) <= 1 - eta])
p.solve()
self.assert_feas(p)
def compute_node_be_curvature(g, n=None, solver=None, solver_options={}, verbose=False):
if n is not None:
if g.degree[n] > 0:
dgamma2 = construct_dgamma2(g, n, verbose)
gammax = construct_gammax(g, n)
dim_b1 = gammax.shape[0]
dim_b2 = dgamma2.shape[0]
dim_s2 = dgamma2.shape[0] - gammax.shape[0]
if verbose:
print('dim dgamma2 {0}; dim gammax {1}'.format(dim_b2, dim_b1))
gammax_ext = np.block([
[gammax, np.zeros((dim_b1, dim_s2))],
[np.zeros((dim_b1, dim_s2)).T, np.zeros((dim_s2, dim_s2))],
])
a = Parameter((dim_b2, dim_b2), value=dgamma2)
b = Parameter((dim_b2, dim_b2), value=gammax_ext)
kappa = Variable()
constraints = [(a - kappa * b >> 0)]
objective = Maximize(kappa)
prob = Problem(objective, constraints)
if verbose:
print(prob.status)
prob.solve(solver=solver, **solver_options)
if verbose:
print(prob.status)
return prob.value
else:
return 0
else:
r = {n: compute_node_be_curvature(g, n, solver=solver, solver_options=solver_options, verbose=verbose) for n in g.nodes()}
return r
# s.t. v >=0 , sum(v) = 1, P*v >= t*1
# Input data
n = 12
m = 12
P = cvxopt.normal(n, m)
# Variables for two players
x = Variable(n)
y = Variable(m)
t1 = Variable()
t2 = Variable()
# Note in one case we are maximizing; in the other we are minimizing
objective1 = Minimize(t1)
objective2 = Maximize(t2)
constraints1 = [x >= 0, sum(x) == 1, P.T * x <= t1]
constraints2 = [y >= 0, sum(y) == 1, P * y >= t2]
p1 = Problem(objective1, constraints1)
p2 = Problem(objective2, constraints2)
# Optimal strategy for Player 1
print('Computing the optimal strategy for player 1 ... ')
result1 = p1.solve()
print('Done!')
# Optimal strategy for Player 2
print('Computing the optimal strategy for player 2 ... ')
result2 = p2.solve()
def as_objective(self, maximize=True):
return Maximize(cvx.sum(self.mat() @ self.action))
def main():
argparser = argparse.ArgumentParser(
description=
"Solves for the optimal set of queries to attempt in order to "
"maximize coverage during a symbolic execution.")
argparser.add_argument(
'explorationgraph',
help="The serialized exploration graph on which to solve the best "
"scheduling strategy.")
argparser.add_argument('-d',
'--debug',
help="Enables debugging output.",
action="store_true")
argparser.add_argument(
'-v',
'--verbose',
help=
"Enables verbose output. Debugging output includes verbose output.",
action='store_true')
args = argparser.parse_args()
debuglogging = args.debug
verboselogging = args.verbose
if debuglogging:
loglevel = logging.DEBUG
elif verboselogging:
loglevel = logging.INFO
else:
loglevel = logging.WARNING
FORMAT = '%(asctime)s - %(levelname)s : %(message)s'
logging.basicConfig(stream=sys.stderr,
level=loglevel,
format=FORMAT,
datefmt='%m/%d/%Y %I:%M:%S %p')
graph = pickle.load(open(args.explorationgraph, 'rb'))
root = graph.root_constraint
exploration_timeout = 0.1
maximize_code = True
maximize_branch = False
assert (maximize_code != maximize_branch)
ilp_constraints = []
# Create constraint variables
path_constraints = {} # {Constraint.id : ilp_constraint_id}
for path_constraint in all_path_constraints(root):
path_constraints[path_constraint.id] = len(path_constraints)
ilp_path_constraints = Int(len(path_constraints))
for ilp_path_constraint in ilp_path_constraints:
ilp_constraints.append(ilp_path_constraint <= 1)
ilp_constraints.append(ilp_path_constraint >= 0)
# Create branch coverage variables
if maximize_branch:
branch_coverage = {
} # {(filename, arc_origin, arc_dest) : ilp_branch_id}
for path_constraint in all_path_constraints(root):
for filename, arcs in path_constraint.branches_covered.items():
for arc in arcs:
arc_origin, arc_dest = arc
if (filename, arc_origin, arc_dest) not in branch_coverage:
branch_coverage[(filename, arc_origin,
arc_dest)] = len(branch_coverage)
ilp_branches = Int(len(branch_coverage))
for ilp_branch in ilp_branches:
ilp_constraints.append(ilp_branch <= 1)
ilp_constraints.append(ilp_branch >= 0)
# Create code coverage variables
if maximize_code:
code_coverage = {} # {(filename, line): ilp_code_id}
for path_constraint in all_path_constraints(root):
for filename, lines in path_constraint.lines_covered.items():
for line in lines:
if (filename, line) not in code_coverage:
code_coverage[(filename, line)] = len(code_coverage)
ilp_code_lines = Int(len(code_coverage))
for ilp_code_line in ilp_code_lines:
ilp_constraints.append(ilp_code_line <= 1)
ilp_constraints.append(ilp_code_line >= 0)
# Calculate response times
response_times = {} # {Constraint.id : response_time}
for path_constraint in all_path_constraints(root):
if path_constraint.parent is not None and path_constraint.parent.inputs == path_constraint.inputs:
response_times[path_constraint.id] = 0.
else:
response_times[path_constraint.id] = path_constraint.solving_time
# Constraint 1: Total solve time
total_solve_time = 0
for path_constraint_id, ilp_path_constraint_id in path_constraints.items():
total_solve_time += ilp_path_constraints[
ilp_path_constraint_id] * response_times[path_constraint_id]
ilp_constraints.append(total_solve_time <= exploration_timeout)
# Constraint 2: Seed input
ilp_constraints.append(ilp_path_constraints[0] == 1)
# Constraint 3: Branch discovery
for path_constraint in all_path_constraints(root):
parent = path_constraint.parent
if parent is not None:
# The constraint is only in the schedule if its parent is in the schedule
ilp_constraints.append(
ilp_path_constraints[path_constraints[path_constraint.id]] <=
ilp_path_constraints[path_constraints[parent.id]])
if extract_model(parent) == extract_model(path_constraint):
ilp_constraints.append(
ilp_path_constraints[path_constraints[path_constraint.id]]
== ilp_path_constraints[path_constraints[parent.id]])
# Constraint 4: Coverage
## A path constraint is "covering" a branch if the input that discovers the path constraint covers the branch.
if maximize_branch:
for branch, ilp_branch_var in branch_coverage.items():
covering_path_constraints = 0
for path_constraint in all_path_constraints(root):
filename, arc_origin, arc_dest = branch
if (arc_origin,
arc_dest) in path_constraint.branches_covered.get(
filename, set()):
covering_path_constraints += ilp_path_constraints[
path_constraints[path_constraint.id]]
assert (type(covering_path_constraints) != int)
ilp_constraints.append(
ilp_branches[ilp_branch_var] <= covering_path_constraints)
if maximize_code:
for code_line, ilp_code_line_var in code_coverage.items():
covering_path_constraints = 0
for path_constraint in all_path_constraints(root):
filename, line = code_line
if line in path_constraint.lines_covered.get(filename, set()):
covering_path_constraints += ilp_path_constraints[
path_constraints[path_constraint.id]]
assert (type(covering_path_constraints) != int)
ilp_constraints.append(
ilp_code_lines[ilp_code_line_var] <= covering_path_constraints)
if maximize_branch:
objective = Maximize(
sum_entries(ilp_branches)) # Maximize branch coverage
if maximize_code:
objective = Maximize(
sum_entries(ilp_code_lines)) # Maximize code coverage
problem = Problem(objective, ilp_constraints)
problem.solve()
# Correctness assertions
## All constraints are 0 or 1 indicator variables
for ilp_path_constraint_var in path_constraints.values():
assert (0 <= round(ilp_path_constraints[ilp_path_constraint_var].value)
<= 1)
## All branch coverage variables are 0 or 1
if maximize_branch:
for ilp_branch_var in branch_coverage.values():
assert (0 <= round(ilp_branches[ilp_branch_var].value) <= 1)
## All code coverage variables are 0 or 1
if maximize_code:
for ilp_code_line_var in code_coverage.values():
assert (0 <= round(ilp_code_lines[ilp_code_line_var].value) <= 1)
## Initial values are in the schedule and they have a response time of 0
assert (round(ilp_path_constraints[path_constraints[0]].value) == 1)
assert (response_times[path_constraints[0]] == 0)
## Constraints are discovered if used
for path_constraint_id, ilp_path_constraint_var in path_constraints.items(
):
if round(ilp_path_constraints[ilp_path_constraint_var].value) == 1:
for path_constraint in all_path_constraints(root):
if path_constraint.id == path_constraint_id and path_constraint.parent is not None:
assert (round(ilp_path_constraints[path_constraints[
path_constraint.parent.id]].value) == 1)
## If input is used in constraint, then all constraints from that input are used as well
for path_constraint_id, ilp_path_constraint_var in path_constraints.items(
):
if round(ilp_path_constraints[ilp_path_constraint_var].value) == 1:
path_constraint_input = None
for path_constraint in all_path_constraints(root):
if path_constraint_id == path_constraint.id:
path_constraint_input = extract_model(path_constraint)
assert (path_constraint_input is not None
or path_constraint_id == 0)
for path_constraint in all_path_constraints(root):
if extract_model(path_constraint) == path_constraint_input:
assert (round(ilp_path_constraints[path_constraints[
path_constraint.id]].value) == 1)
## Branch coverage comes from discovered constraints
if maximize_branch:
for branch, ilp_branch_var in branch_coverage.items():
if round(ilp_branches[ilp_branch_var].value) == 1:
filename, arc_origin, arc_dest = branch
assert (any(
(arc_origin, arc_dest) in
path_constraint.branches_covered.get(filename, set())
for path_constraint in all_path_constraints(root)
if round(ilp_path_constraints[path_constraints[
path_constraint.id]].value) == 1))
## Code coverage comes from discovered constraints
if maximize_code:
for code_line, ilp_code_line_var in code_coverage.items():
if round(ilp_code_lines[ilp_code_line_var].value) == 1:
filename, line = code_line
assert (any(
line in path_constraint.lines_covered.get(filename, set())
for path_constraint in all_path_constraints(root)
if round(ilp_path_constraints[path_constraints[
path_constraint.id]].value) == 1))
optimal_constraint_ids = set() # {Constraint.id}
optimal_inputs = {} # {inputs : solving_time}
optimal_branches = {
} # {(source_file: str) : {(origin_line: int, dest_line: int)}}
optimal_lines = {} # {(source_file: str) : {line: int}}
for path_constraint_id, var_index in path_constraints.items():
if bool(round(ilp_path_constraints[var_index].value)):
optimal_constraint_ids.add(path_constraint_id)
for path_constraint in all_path_constraints(root):
if path_constraint.id in optimal_constraint_ids and not extract_model(
path_constraint) is None:
optimal_inputs[extract_model(
path_constraint)] = path_constraint.solving_time
for file, branches in path_constraint.branches_covered.items():
if file in optimal_branches:
optimal_branches[file] |= branches
else:
optimal_branches[file] = set(branches)
for file, lines in path_constraint.lines_covered.items():
if file in optimal_lines:
optimal_lines[file] |= lines
else:
optimal_lines[file] = set(lines)
print("Solver CPU: {} seconds".format(sum(optimal_inputs.values())))
print("Path coverage: {} paths".format(len(optimal_inputs)))
print("Line coverage: {} lines".format(
sum(len(lines) for file, lines in optimal_lines.items())))
print("Branch coverage: {} branches".format(
sum(len(branches) for file, branches in optimal_branches.items())))
objective1 = Minimize( norm ( A*x - b , p) )
p1 = Problem(objective1, [])
print('Computing the optimal solution of problem 1... ')
opt1 = p1.solve()
# Reformulation 1
x = Variable(n)
y = Variable(m)
objective2 = Minimize ( norm( y, p ) )
p2 = Problem(objective2, [ A*x - b == y ])
print('Computing the optimal solution of problem 2... ')
opt2 = p2.solve()
# Dual of reformulation 1
nu = Variable(m)
objective3 = Maximize( b.T * nu )
p3 = Problem(objective3, [ norm( nu, q) <= 1, A.T*nu == 0 ])
print('Computing the optimal solution of problem 3... ')
opt3 = p3.solve()
# Reformulation 2
x = Variable(n)
y = Variable(m)
objective4 = Minimize( 0.5*square( norm(y, p) ) )
p4 = Problem(objective4, [ A*x - b == y ] )
print('Computing the optimal solution of problem 4... ')
opt4 = math.sqrt(2*p4.solve())
# Dual of reformulation 2
nu = Variable(m)
objective5 = Maximize( -0.5*square( norm(nu,q) ) + b.T*nu )
for i in range(COMMODITIES):
source,sink = r.sample(nodes, 2)
# Only count accumulation of a single commodity.
commodity_vec = cvxopt.matrix(0,(COMMODITIES,1))
commodity_vec[i] = 1
# Initialize the source.
source.capacity = commodity_vec*Variable()
sources.append(source)
# Initialize the sink.
sink.capacity = commodity_vec*Variable()
sinks.append(sink)
# Construct edges.
edges = []
for n1,n2,capacity in data[g.EDGES_KEY]:
edges.append(MultiEdge(capacity))
edges[-1].connect(nodes[n1], nodes[n2])
# Construct the problem.
objective = Maximize(sum(sum(s.accumulation() for s in sinks)))
constraints = []
for o in nodes + edges:
constraints += o.constraints()
p = Problem(objective, constraints)
result = p.solve()
print("Objective value = %s" % result)
# Show how the flow for each commodity.
for i,s in enumerate(sinks):
accumulation = sum(f.value[i] for f in s.edge_flows)
print("Accumulation of commodity %s = %s" % (i, accumulation))
def test_scalar_lp(self):
"""Test scalar LP problems.
"""
for solver in self.solvers:
p = Problem(Minimize(3 * self.a), [self.a >= 2])
self.assertTrue(ConeMatrixStuffing().accepts(p))
result = p.solve(solver.name())
p_new = ConeMatrixStuffing().apply(p)
result_new = p_new[0].solve(solver.name())
self.assertAlmostEqual(result, result_new)
sltn = solver.solve(p_new[0], False, False, {})
self.assertAlmostEqual(sltn.opt_val, result)
inv_sltn = ConeMatrixStuffing().invert(sltn, p_new[1])
self.assertAlmostEqual(inv_sltn.opt_val, result)
self.assertAlmostEqual(inv_sltn.primal_vars[self.a.id],
self.a.value)
# TODO: Maximize
p = Problem(Minimize(-3 * self.a + self.b),
[self.a <= 2, self.b == self.a, self.b <= 5])
result = p.solve(solver.name())
self.assertTrue(ConeMatrixStuffing().accepts(p))
p_new = ConeMatrixStuffing().apply(p)
sltn = solver.solve(p_new[0], False, False, {})
self.assertAlmostEqual(sltn.opt_val, result)
inv_sltn = ConeMatrixStuffing().invert(sltn, p_new[1])
self.assertAlmostEqual(inv_sltn.opt_val, result)
self.assertAlmostEqual(inv_sltn.primal_vars[self.a.id],
self.a.value)
self.assertAlmostEqual(inv_sltn.primal_vars[self.b.id],
self.b.value)
# With a constant in the objective.
p = Problem(Minimize(3 * self.a - self.b + 100), [
self.a >= 2, self.b + 5 * self.c - 2 == self.a,
self.b <= 5 + self.c
])
self.assertTrue(ConeMatrixStuffing().accepts(p))
result = p.solve(solver.name())
p_new = ConeMatrixStuffing().apply(p)
sltn = solver.solve(p_new[0], False, False, {})
self.assertAlmostEqual(sltn.opt_val, result - 100)
inv_sltn = ConeMatrixStuffing().invert(sltn, p_new[1])
self.assertAlmostEqual(inv_sltn.opt_val, result)
self.assertAlmostEqual(inv_sltn.primal_vars[self.a.id],
self.a.value)
self.assertAlmostEqual(inv_sltn.primal_vars[self.b.id],
self.b.value)
# Unbounded problems.
# TODO: Maximize
p = Problem(Minimize(-self.a), [self.a >= 2])
self.assertTrue(ConeMatrixStuffing().accepts(p))
try:
result = p.solve(solver.name())
except SolverError: # Gurobi fails on this one
return
p_new = ConeMatrixStuffing().apply(p)
self.assertTrue(solver.accepts(p_new[0]))
sltn = solver.solve(p_new[0], False, False, {})
self.assertAlmostEqual(sltn.opt_val, result)
inv_sltn = ConeMatrixStuffing().invert(sltn, p_new[1])
self.assertAlmostEqual(inv_sltn.opt_val, result)
# Infeasible problems.
p = Problem(Maximize(self.a), [self.a >= 2, self.a <= 1])
result = p.solve(solver.name())
self.assertTrue(FlipObjective().accepts(p))
p_min = FlipObjective().apply(p)
self.assertTrue(ConeMatrixStuffing().accepts(p_min[0]))
p_new = ConeMatrixStuffing().apply(p_min[0])
result_new = p_new[0].solve(solver.name())
self.assertAlmostEqual(result, -result_new)
self.assertTrue(solver.accepts(p_new[0]))
sltn = solver.solve(p_new[0], False, False, {})
self.assertAlmostEqual(sltn.opt_val, -result)
inv_sltn = ConeMatrixStuffing().invert(sltn, p_new[1])
self.assertAlmostEqual(inv_sltn.opt_val, -result)
inv_flipped_sltn = FlipObjective().invert(inv_sltn, p_min[1])
self.assertAlmostEqual(inv_flipped_sltn.opt_val, result)
from cvxpy import Maximize, Parameter, Problem, Variable, norm2, square
num_assets = 100
num_factors = 20
mu = cvxopt.exp(cvxopt.normal(num_assets))
F = cvxopt.normal(num_assets, num_factors)
D = cvxopt.spdiag(cvxopt.uniform(num_assets))
x = Variable(num_assets)
gamma = Parameter(nonneg=True)
expected_return = mu.T * x
variance = square(norm2(F.T * x)) + square(norm2(D * x))
# construct portfolio optimization problem *once*
p = Problem(Maximize(expected_return - gamma * variance),
[sum(x) == 1, x >= 0])
# encapsulate the allocation function
def allocate(gamma_value):
gamma.value = gamma_value
p.solve()
w = x.value
expected_return, risk = mu.T * w, w.T * (F * F.T + D * D) * w
return (expected_return[0], math.sqrt(risk[0]))
# create a pool of workers and a grid of gamma values
pool = Pool(processes=4)
gammas = numpy.logspace(-1, 2, num=100)
a1 = cvxopt.matrix([2, 1], (2, 1))
a2 = cvxopt.matrix([2, -1], (2, 1))
a3 = cvxopt.matrix([-1, 2], (2, 1))
a4 = cvxopt.matrix([-1, -2], (2, 1))
b = cvxopt.matrix(1, (4, 1))
constraints = [
a1.T * center + np.linalg.norm(a1, 2) * radius <= b[0],
a2.T * center + np.linalg.norm(a2, 2) * radius <= b[1],
a3.T * center + np.linalg.norm(a3, 2) * radius <= b[2],
a4.T * center + np.linalg.norm(a4, 2) * radius <= b[3]
]
# objective
objective = Maximize(radius)
p = Problem(objective, constraints)
# The optimal objective is returned by p.solve().
result = p.solve()
# The optimal value
print(radius.value)
print(center.value)
# Convert to 1D array.
center_val = np.asarray(center.value[:, 0])
# Now let's plot it
x = np.linspace(-2, 2, 256, endpoint=True)
theta = np.linspace(0, 2 * np.pi, 100)
# plot the constraints
def create_schedule(n_days,
inventory_start,
n_totes_washed_start,
pars=None,
do_plot=True,
verbose=True):
"""
Demo an optimal supply chain scheduling with variable
labor costs, and the concept of totes that hold a number of
products. Totes need to be cleaned on a regular basis.
:param pars: parameters from create_default_params
:param do_plot: True if you want a plot created (default)
:return: None
"""
if pars is None:
pars = create_default_params()
days = np.arange(n_days)
print 'creating demand'
demand = create_demand(days)
labor_costs = get_labor_costs(days, pars)
# define variables which keep track of
# production, inventory and number of totes washed per day
print 'defining variables'
production = Variable(n_days)
sales = Variable(n_days)
#inventory = Variable(n_days)
inventory = Integer(n_days, M=100000)
print 'here'
n_totes_washed = Variable(n_days)
print 'calculating costs and profit'
# calculate when the totes that were washed become dirty again
shift_matrix = mu.time_shift_matrix(n_days, pars['days_until_cleaning'])
n_totes_become_dirty = (shift_matrix * n_totes_washed)[:n_days]
# calculate the number of clean totes on any day
cum_matrix = mu.cumulative_matrix(n_days)
n_washed_totes_available = n_totes_washed_start \
+ cum_matrix*(n_totes_washed - n_totes_become_dirty)
print 'calculating total cost'
# Minimize total cost which is
# sum of labor costs, storage costs and washing costs
total_cost = production.T*labor_costs + \
pars['storage_cost'] * sum(inventory) + \
pars['washing_tote_cost'] * sum(n_totes_washed)
total_profit = pars['sales_price'] * sum(sales) - total_cost
print 'defining objective'
objective = Maximize(total_profit)
# Subject to these constraints
constraints = make_constraints(production, sales, inventory, pars,
n_washed_totes_available, n_totes_washed,
demand, inventory_start)
# define the problem and solve it
problem = Problem(objective, constraints)
solver = 'cvxpy'
print 'solving with: %s' % solver
start = time()
problem.solve(verbose=verbose)
finish = time()
run_time = finish - start
print 'Solve time: %s seconds' % run_time
print "Status: %s" % problem.status
if problem.status == 'infeasible':
print "Problem is infeasible, no solution found"
return
n_items_sold = sum(sales.value)
total_cost = problem.value
total_washing_cost = pars['washing_tote_cost'] * sum(n_totes_washed.value)
total_labor_cost = (production.T * labor_costs).value
total_storage_cost = sum(inventory.value) * pars['storage_cost']
total_cost_per_item = problem.value / n_items_sold
print "Total cost: %s" % total_cost
print "Total labor cost: %s" % total_labor_cost
print "Total washing cost: %s" % total_washing_cost
print "Total storage cost: %s" % total_storage_cost
print "Total cost/item: %s" % total_cost_per_item
print "Total profit: %s" % total_profit.value
if do_plot:
plot_variables(days, production, inventory, sales, demand,
n_washed_totes_available)
plt.clf()
# Read a graph from a file.
f = open(g.FILE, 'r')
data = pickle.load(f)
f.close()
# Construct incidence matrix and capacities vector.
node_count = data[g.NODE_COUNT_KEY]
edges = data[g.EDGES_KEY]
E = 2 * len(edges)
A = cvxopt.matrix(0, (node_count, E + 2), tc='d')
c = cvxopt.matrix(1000, (E, 1), tc='d')
for i, (n1, n2, capacity) in enumerate(edges):
A[n1, 2 * i] = -1
A[n2, 2 * i] = 1
A[n1, 2 * i + 1] = 1
A[n2, 2 * i + 1] = -1
c[2 * i] = capacity
c[2 * i + 1] = capacity
# Add source.
A[0, E] = 1
# Add sink.
A[-1, E + 1] = -1
# Construct the problem.
flows = Variable(E)
source = Variable()
sink = Variable()
p = Problem(Maximize(source),
[A * vstack([flows, source, sink]) == 0, 0 <= flows, flows <= c])
result = p.solve()
print(result)
def Optimization(alpha_prediction,
date,
tdays,
engine_quant,
engine_price_vol,
stocklist,
df_benchmark_weight_indus,
style_factor,
indus_factor,
t=2,
Tc=0.003,
style_max=0.1,
style_min=-0.1,
indus_pct=0.05):
'''
Optimizatin to get the optimal stock weights
Calculate the weight everyday, but the transaction cost is according to period t's turnover
从stocklist股票池（剔除停牌，上市没多久的股票）中筛选股票，
并添加条件：剔除ST、涨跌停（大于9.5%或小于-9.5%）
Params:
alpha_prediction:
pd.DataFrame, idnex: stock_id, columns: periods, 1,2,3,4,5
date:
string, like '%Y%m%d'
stocklist:
list, 1) listed before dategap from date, and has not been delisted till date
b.Delisting == '19000101' means still on trade
2) on date, Turnover > 0
df_benchmark_weight_indus:
pd.DataFrame, multiindex, trade_date and stock_id, columns: weight and industry_citic
style_factor:
pd.DataFrame, index: stock_id, columns: 9 style factor name
indus_factor:
pd.DataFrame, index: stock_id, columns: industry name
t:
int, Prediction Period
Tc:
transaction cost, double cost
style_max:
float, the maximum value of style factor exposure
style_min:
float, the minimum value of style factor exposure
indus_pct:
relative percent fo industry weight
'''
stocklist_st_limit = get_stocklist_st_limit(date, engine_quant)
stk_pool = sorted(list(set(stocklist) - set(stocklist_st_limit)))
stk_pool = sorted(
list(set(alpha_prediction.index.tolist()) & set(stk_pool)))
w = Variable(len(stk_pool))
date_last = get_tradeday_before(date, tdays, timelong=t)
w_last = get_weight(date_last, tdays, engine_price_vol)
if len(w_last) == 0:
w_last = pd.Series(0, index=stk_pool)
w_last = w_last.reindex(stk_pool).fillna(0)
ret = np.mat(alpha_prediction[t].reindex(stk_pool)) * w
cost = Tc * sum_entries(abs(w - w_last.values)) / 2
objective = Maximize(ret - cost)
w_benchmark = df_benchmark_weight_indus.loc[
df_benchmark_weight_indus.index.levels[0].asof(date)]['weight']
w_benchmark = w_benchmark.reindex(stk_pool).fillna(0) #这种做法有待商榷
w_benchmark = w_benchmark / w_benchmark.sum()
X_style = style_factor.reindex(stk_pool)
style_constraint = (w - w_benchmark.values).T * np.mat(X_style)
w_benchmark_industry = df_benchmark_weight_indus.loc[
df_benchmark_weight_indus.index.levels[0].asof(date)]
w_benchmark_industry = w_benchmark_industry.groupby(
'industry_citic')['weight'].sum()
w_benchmark_industry = w_benchmark_industry.reindex(
indus_factor.columns).fillna(0)
w_benchmark_industry = w_benchmark_industry / w_benchmark_industry.sum()
X_indus = indus_factor.reindex(stk_pool)
indus_constraint = w.T * np.mat(X_indus)
constraints = [
sum_entries(w) == 1, w >= 0, style_constraint <= np.mat(
[style_max for i in range(X_style.shape[1])]), style_constraint >=
np.mat([style_min for i in range(X_style.shape[1])]),
indus_constraint <= np.mat(w_benchmark_industry * (1 + indus_pct)),
indus_constraint >= np.mat(w_benchmark_industry * (1 - indus_pct))
]
prob = Problem(objective, constraints)
prob.solve()
weight = pd.Series(w.value.A1, index=stk_pool, name='weight')
return weight
return self.forward.constraints() + self.backward.constraints()
if __name__ == "__main__":
# Read a graph from a file.
f = open(FILE, 'r')
data = pickle.load(f)
f.close()
# Construct nodes.
node_count = data[NODE_COUNT_KEY]
nodes = [Node() for i in range(node_count)]
# Add source.
nodes[0].accumulation = Variable()
# Add sink.
nodes[-1].accumulation = Variable()
# Construct edges.
edges = []
for n1, n2, capacity in data[EDGES_KEY]:
edges.append(LeakyUndirected(capacity))
edges[-1].connect(nodes[n1], nodes[n2])
# Construct the problem.
constraints = []
for o in nodes + edges:
constraints += o.constraints()
p = Problem(Maximize(nodes[-1].accumulation), constraints)
result = p.solve()
print(result)
def objective(self):
objective = Maximize(cvx.sum(self.areas * self.X))
return objective
