def compute(self):
"""Performs the Layer patching."""
patch_start = timer()
layer = self.network.layers[self.layer_index]
if isinstance(layer, FullyConnectedLayer):
weights = layer.weights.numpy().copy()
biases = layer.biases.numpy().copy()
elif isinstance(layer, Conv2DLayer):
weights = layer.filter_weights.numpy().copy()
biases = layer.biases.numpy().copy()
else:
raise NotImplementedError
model = Model()
if self.gurobi_timelimit is not None:
model.Params.TimeLimit = self.gurobi_timelimit
if self.gurobi_crossover != -1:
model.Params.Crossover = self.gurobi_crossover
model.Params.Method = 2
# Adding variables...
lb, ub = self.delta_bounds
weight_deltas = model.addVars(weights.flatten().size, lb=lb, ub=ub).select()
bias_deltas = model.addVars(biases.size, lb=lb, ub=ub).select()
all_deltas = weight_deltas + bias_deltas
if self.constraint_type == "hard":
soft_constraint_bounds = []
elif self.constraint_type == "linf":
soft_constraint_bounds = [model.addVar(
lb=self.soft_constraint_slack_lb,
ub=self.soft_constraint_slack_ub,
vtype=self.soft_constraint_slack_type)]
elif self.constraint_type == "l1":
out_dims = self.network.compute(self.inputs[:1]).shape[1]
soft_constraint_bounds = model.addVars(
len(self.inputs) * (out_dims - 1),
lb=self.soft_constraint_slack_lb,
ub=self.soft_constraint_slack_ub,
vtype=self.soft_constraint_slack_type).select()
else:
raise NotImplementedError
# Adding constraints...
jacobian_compute_time = 0.
for batch_start in tqdm(range(0, len(self.inputs), self.batch_size)):
batch_slice = slice(batch_start, batch_start + self.batch_size)
batch_labels = self.labels[batch_slice]
jacobian_start = timer()
A_batch, b_batch = self.network_jacobian(batch_slice)
jacobian_compute_time += (timer() - jacobian_start)
out_dims = A_batch.shape[1]
assert out_dims == b_batch.shape[1]
variables = None
if self.constraint_type == "l1":
variables = weight_deltas + bias_deltas + bounds
weight_softs = 1.
if self.soft_constraint_slack_type == GRB.BINARY:
weight_softs = 10.
full_As, full_bs = [], []
bounds_slice = slice(batch_slice.start * (out_dims - 1),
batch_slice.stop * (out_dims - 1))
constraint_bounds_batch = soft_constraint_bounds[bounds_slice]
for i, (A, b, label) in enumerate(zip(A_batch, b_batch, batch_labels)):
other_labels = [l for l in range(out_dims) if l != label]
# A[label]x + b[label] >= A[other]x + b[other]
# (A[label] - A[other])x >= b[other] - b[label]
As = np.expand_dims(A[label], 0) - A[other_labels]
bs = (b[other_labels] - np.expand_dims(b[label], 0))
bs += self.constraint_buffer
if self.constraint_type == "linf":
As = np.concatenate((As, weight_softs * np.ones((As.shape[0], 1))), axis=1)
elif self.constraint_type == "l1":
bounds = constraint_bounds_batch[
i*(out_dims-1):((i+1)*(out_dims-1))]
As = np.concatenate((As, weight_softs * np.eye(len(bounds))), axis=1)
full_As.append(As)
full_bs.extend(bs)
model.addMConstr(np.concatenate(tuple(full_As), axis=0),
variables, '>', full_bs)
# Specifying objective...
objective = 0.
if self.delta_l1_weight != 0.:
# Penalize the L_1 norm. To do this, we must add variables which
# represent the absolute value of each of the deltas. The approach
# used here is described at:
# https://optimization.mccormick.northwestern.edu/index.php/Optimization_with_absolute_values
n_vars = len(all_deltas)
abs_ub = max(abs(lb), abs(ub))
variable_abs = model.addVars(n_vars, lb=0., ub=abs_ub).select()
n_vars += n_vars
A = sparse.diags([1., -1.], [0, (n_vars // 2)],
shape=(n_vars // 2, n_vars),
dtype=np.float, format="lil")
b = np.zeros(n_vars // 2)
model.addMConstr(A, all_deltas + variable_abs, '<', b)
A = sparse.diags([-1., -1.], [0, (n_vars // 2)],
shape=(n_vars // 2, n_vars),
dtype=np.float, format="lil")
model.addMConstr(A, all_deltas + variable_abs, '<', b)
# Then the objective we use is just the L_1 norm. TODO: maybe we
# should wait until the end to weight this so the coefficients
# aren't too small?
if self.normalize_objective:
weight = self.delta_l1_weight / len(variable_abs)
else:
weight = self.delta_l1_weight
objective += (weight * sum(variable_abs))
if self.delta_linf_weight != 0.:
# Penalize the L_infty norm. We use a similar approach, except it
# only takes one additional variable. For some reason it throws an
# error if I use just addVar here.
l_inf = model.addVar(lb=0., ub=max(abs(lb), abs(ub)))
n_vars = len(weight_deltas) + len(bias_deltas) + 1
A = sparse.eye(n_vars - 1, n_vars, dtype=np.float, format="lil")
A[:, -1] = -1.
b = np.zeros(n_vars - 1)
model.addMConstr(A, all_deltas + [l_inf], '<', b)
A = sparse.diags([-1.], shape=(n_vars - 1, n_vars),
dtype=np.float, format="lil")
A[:, -1] = -1.
model.addMConstr(A, all_deltas + [l_inf], '<', b)
# L_inf objective.
objective += (self.delta_linf_weight * l_inf)
# Soft constraint weight.
if self.normalize_objective:
weight = self.soft_constraints_weight / max(len(soft_constraint_bounds), 1)
else:
weight = self.soft_constraints_weight
objective += weight * sum(soft_constraint_bounds)
model.setObjective(objective, GRB.MINIMIZE)
# Solving...
gurobi_start = timer()
model.update()
model.optimize()
gurobi_solve_time = (timer() - gurobi_start)
self.timing = dict({
"jacobian": jacobian_compute_time,
"solver": gurobi_solve_time,
})
self.timing["did_timeout"] = (model.status == GRB.TIME_LIMIT)
if model.status != GRB.OPTIMAL:
print("Not optimal!")
print("Model status:", model.status)
self.timing["total"] = (timer() - patch_start)
return None
# Extracting weights...
weights += np.asarray([d.X for d in weight_deltas]).reshape(weights.shape)
biases += np.asarray([d.X for d in bias_deltas]).reshape(biases.shape)
# Returning a patched network!
if isinstance(layer, FullyConnectedLayer):
patched_layer = FullyConnectedLayer(weights.copy(), biases.copy())
else:
patched_layer = Conv2DLayer(layer.window_data, weights.copy(), biases.copy())
patched = self.construct_patched(patched_layer)
self.timing["total"] = (timer() - patch_start)
return patched
def gurobi_solve_qp(P,
q,
G=None,
h=None,
A=None,
b=None,
initvals=None,
verbose: bool = False) -> Optional[ndarray]:
"""
Solve a Quadratic Program defined as:
.. math::
\\begin{split}\\begin{array}{ll}
\\mbox{minimize} &
\\frac{1}{2} x^T P x + q^T x \\\\
\\mbox{subject to}
& G x \\leq h \\\\
& A x = h
\\end{array}\\end{split}
using `Gurobi <http://www.gurobi.com/>`_.
Parameters
----------
P : array, shape=(n, n)
Primal quadratic cost matrix.
q : array, shape=(n,)
Primal quadratic cost vector.
G : array, shape=(m, n)
Linear inequality constraint matrix.
h : array, shape=(m,)
Linear inequality constraint vector.
A : array, shape=(meq, n), optional
Linear equality constraint matrix.
b : array, shape=(meq,), optional
Linear equality constraint vector.
initvals : array, shape=(n,), optional
Warm-start guess vector (not used).
verbose : bool, optional
Set to `True` to print out extra information.
Returns
-------
x : array, shape=(n,)
Solution to the QP, if found, otherwise ``None``.
"""
if initvals is not None:
warn("Gurobi: warm-start values given but they will be ignored")
model = Model()
if not verbose: # optionally turn off solver output
model.setParam("OutputFlag", 0)
num_vars = P.shape[0]
x = model.addMVar(num_vars,
lb=-GRB.INFINITY,
ub=GRB.INFINITY,
vtype=GRB.CONTINUOUS)
if A is not None: # include equality constraints
model.addMConstr(A, x, GRB.EQUAL, b)
if G is not None: # include inequality constraints
model.addMConstr(G, x, GRB.LESS_EQUAL, h)
objective = 0.5 * (x @ P @ x) + q @ x
model.setObjective(objective, sense=GRB.MINIMIZE)
model.optimize()
status = model.status
if status not in (GRB.OPTIMAL, GRB.SUBOPTIMAL):
return None
return array(x.X)
