def compute(self): """Performs the Layer patching.""" patch_start = timer() layer = self.network.layers[self.layer_index] if isinstance(layer, FullyConnectedLayer): weights = layer.weights.numpy().copy() biases = layer.biases.numpy().copy() elif isinstance(layer, Conv2DLayer): weights = layer.filter_weights.numpy().copy() biases = layer.biases.numpy().copy() else: raise NotImplementedError model = Model() if self.gurobi_timelimit is not None: model.Params.TimeLimit = self.gurobi_timelimit if self.gurobi_crossover != -1: model.Params.Crossover = self.gurobi_crossover model.Params.Method = 2 # Adding variables... lb, ub = self.delta_bounds weight_deltas = model.addVars(weights.flatten().size, lb=lb, ub=ub).select() bias_deltas = model.addVars(biases.size, lb=lb, ub=ub).select() all_deltas = weight_deltas + bias_deltas if self.constraint_type == "hard": soft_constraint_bounds = [] elif self.constraint_type == "linf": soft_constraint_bounds = [model.addVar( lb=self.soft_constraint_slack_lb, ub=self.soft_constraint_slack_ub, vtype=self.soft_constraint_slack_type)] elif self.constraint_type == "l1": out_dims = self.network.compute(self.inputs[:1]).shape[1] soft_constraint_bounds = model.addVars( len(self.inputs) * (out_dims - 1), lb=self.soft_constraint_slack_lb, ub=self.soft_constraint_slack_ub, vtype=self.soft_constraint_slack_type).select() else: raise NotImplementedError # Adding constraints... jacobian_compute_time = 0. for batch_start in tqdm(range(0, len(self.inputs), self.batch_size)): batch_slice = slice(batch_start, batch_start + self.batch_size) batch_labels = self.labels[batch_slice] jacobian_start = timer() A_batch, b_batch = self.network_jacobian(batch_slice) jacobian_compute_time += (timer() - jacobian_start) out_dims = A_batch.shape[1] assert out_dims == b_batch.shape[1] variables = None if self.constraint_type == "l1": variables = weight_deltas + bias_deltas + bounds weight_softs = 1. if self.soft_constraint_slack_type == GRB.BINARY: weight_softs = 10. full_As, full_bs = [], [] bounds_slice = slice(batch_slice.start * (out_dims - 1), batch_slice.stop * (out_dims - 1)) constraint_bounds_batch = soft_constraint_bounds[bounds_slice] for i, (A, b, label) in enumerate(zip(A_batch, b_batch, batch_labels)): other_labels = [l for l in range(out_dims) if l != label] # A[label]x + b[label] >= A[other]x + b[other] # (A[label] - A[other])x >= b[other] - b[label] As = np.expand_dims(A[label], 0) - A[other_labels] bs = (b[other_labels] - np.expand_dims(b[label], 0)) bs += self.constraint_buffer if self.constraint_type == "linf": As = np.concatenate((As, weight_softs * np.ones((As.shape[0], 1))), axis=1) elif self.constraint_type == "l1": bounds = constraint_bounds_batch[ i*(out_dims-1):((i+1)*(out_dims-1))] As = np.concatenate((As, weight_softs * np.eye(len(bounds))), axis=1) full_As.append(As) full_bs.extend(bs) model.addMConstr(np.concatenate(tuple(full_As), axis=0), variables, '>', full_bs) # Specifying objective... objective = 0. if self.delta_l1_weight != 0.: # Penalize the L_1 norm. To do this, we must add variables which # represent the absolute value of each of the deltas. The approach # used here is described at: # https://optimization.mccormick.northwestern.edu/index.php/Optimization_with_absolute_values n_vars = len(all_deltas) abs_ub = max(abs(lb), abs(ub)) variable_abs = model.addVars(n_vars, lb=0., ub=abs_ub).select() n_vars += n_vars A = sparse.diags([1., -1.], [0, (n_vars // 2)], shape=(n_vars // 2, n_vars), dtype=np.float, format="lil") b = np.zeros(n_vars // 2) model.addMConstr(A, all_deltas + variable_abs, '<', b) A = sparse.diags([-1., -1.], [0, (n_vars // 2)], shape=(n_vars // 2, n_vars), dtype=np.float, format="lil") model.addMConstr(A, all_deltas + variable_abs, '<', b) # Then the objective we use is just the L_1 norm. TODO: maybe we # should wait until the end to weight this so the coefficients # aren't too small? if self.normalize_objective: weight = self.delta_l1_weight / len(variable_abs) else: weight = self.delta_l1_weight objective += (weight * sum(variable_abs)) if self.delta_linf_weight != 0.: # Penalize the L_infty norm. We use a similar approach, except it # only takes one additional variable. For some reason it throws an # error if I use just addVar here. l_inf = model.addVar(lb=0., ub=max(abs(lb), abs(ub))) n_vars = len(weight_deltas) + len(bias_deltas) + 1 A = sparse.eye(n_vars - 1, n_vars, dtype=np.float, format="lil") A[:, -1] = -1. b = np.zeros(n_vars - 1) model.addMConstr(A, all_deltas + [l_inf], '<', b) A = sparse.diags([-1.], shape=(n_vars - 1, n_vars), dtype=np.float, format="lil") A[:, -1] = -1. model.addMConstr(A, all_deltas + [l_inf], '<', b) # L_inf objective. objective += (self.delta_linf_weight * l_inf) # Soft constraint weight. if self.normalize_objective: weight = self.soft_constraints_weight / max(len(soft_constraint_bounds), 1) else: weight = self.soft_constraints_weight objective += weight * sum(soft_constraint_bounds) model.setObjective(objective, GRB.MINIMIZE) # Solving... gurobi_start = timer() model.update() model.optimize() gurobi_solve_time = (timer() - gurobi_start) self.timing = dict({ "jacobian": jacobian_compute_time, "solver": gurobi_solve_time, }) self.timing["did_timeout"] = (model.status == GRB.TIME_LIMIT) if model.status != GRB.OPTIMAL: print("Not optimal!") print("Model status:", model.status) self.timing["total"] = (timer() - patch_start) return None # Extracting weights... weights += np.asarray([d.X for d in weight_deltas]).reshape(weights.shape) biases += np.asarray([d.X for d in bias_deltas]).reshape(biases.shape) # Returning a patched network! if isinstance(layer, FullyConnectedLayer): patched_layer = FullyConnectedLayer(weights.copy(), biases.copy()) else: patched_layer = Conv2DLayer(layer.window_data, weights.copy(), biases.copy()) patched = self.construct_patched(patched_layer) self.timing["total"] = (timer() - patch_start) return patched
def gurobi_solve_qp(P, q, G=None, h=None, A=None, b=None, initvals=None, verbose: bool = False) -> Optional[ndarray]: """ Solve a Quadratic Program defined as: .. math:: \\begin{split}\\begin{array}{ll} \\mbox{minimize} & \\frac{1}{2} x^T P x + q^T x \\\\ \\mbox{subject to} & G x \\leq h \\\\ & A x = h \\end{array}\\end{split} using `Gurobi <http://www.gurobi.com/>`_. Parameters ---------- P : array, shape=(n, n) Primal quadratic cost matrix. q : array, shape=(n,) Primal quadratic cost vector. G : array, shape=(m, n) Linear inequality constraint matrix. h : array, shape=(m,) Linear inequality constraint vector. A : array, shape=(meq, n), optional Linear equality constraint matrix. b : array, shape=(meq,), optional Linear equality constraint vector. initvals : array, shape=(n,), optional Warm-start guess vector (not used). verbose : bool, optional Set to `True` to print out extra information. Returns ------- x : array, shape=(n,) Solution to the QP, if found, otherwise ``None``. """ if initvals is not None: warn("Gurobi: warm-start values given but they will be ignored") model = Model() if not verbose: # optionally turn off solver output model.setParam("OutputFlag", 0) num_vars = P.shape[0] x = model.addMVar(num_vars, lb=-GRB.INFINITY, ub=GRB.INFINITY, vtype=GRB.CONTINUOUS) if A is not None: # include equality constraints model.addMConstr(A, x, GRB.EQUAL, b) if G is not None: # include inequality constraints model.addMConstr(G, x, GRB.LESS_EQUAL, h) objective = 0.5 * (x @ P @ x) + q @ x model.setObjective(objective, sense=GRB.MINIMIZE) model.optimize() status = model.status if status not in (GRB.OPTIMAL, GRB.SUBOPTIMAL): return None return array(x.X)