def get_var_at_point(self, x, var_key): """ Computes the variables prefixed by var_key at a given point. This does not modify any state (makes a copy of the gurobi model prior to changing it) ARGS: x: np.array - np array of the requisite input shape reset: bool - if True, we remove the input constraints and update RETURNS: var (as a numpy array), but also modifies the model """ model = self.model.copy() x_var_namer = utils.build_var_namer('x') constr_namer = lambda x_var_name: 'fixed::' + x_var_name for i in range(len(self.get_vars('x'))): x_var = model.getVarByName(x_var_namer(i)) constr_name = constr_namer(x_var.varName) constr = model.getConstrByName(constr_name) if constr is not None: model.remove(constr) model.addConstr(x_var == x[i], name=constr_name) model.setObjective(0) model.update() model.optimize() output_namer = utils.build_var_namer(var_key) output_num = len(self.get_vars(var_key)) return [model.getVarByName(output_namer(i)).X for i in range(output_num)]
def add_backprop_switch_layer_mip(layer_no, squire, input_key, relu_key, output_key): """ Encodes the funtion output_key[i] := 0 if sign_key[i] == 0 input_key[i] if sign_key != 0 where 0 <= input_key[i] <= squire.backprop_pos_bounds[i] """ network = squire.network model = squire.model switchbox = squire.get_ith_switch_box(layer_no - 1) switch_inbox = squire.get_ith_backward_box(layer_no) post_switch_vars = [] post_switch_namer = utils.build_var_namer(output_key) post_switch_namer = utils.build_var_namer(output_key) # First add variables for idx, val in enumerate(switchbox): name = post_switch_namer(idx) if val < 0: # Switch is always off lb = ub = 0.0 elif val > 0: # Switch is always on lb, ub = switch_inbox[idx] else: # Switch uncertain lb = min([0, switch_inbox[idx][0]]) ub = max([0, switch_inbox[idx][1]]) post_switch_vars.append(model.addVar(lb=lb, ub=ub, name=name)) squire.set_vars(output_key, post_switch_vars) # And then add constraints relu_vars = squire.get_vars(relu_key) pre_switch_vars = squire.get_vars(input_key) for idx, val in enumerate(switchbox): relu_var = relu_vars.get(idx) pre_switch_var = pre_switch_vars[idx] post_switch_var = post_switch_vars[idx] if val < 0: continue elif val > 0: model.addConstr(post_switch_var == pre_switch_var) continue else: # In this case, the relu is uncertain and we need to encode # 4 constraints. This depends on the backprop low or high though bp_lo, bp_hi = switch_inbox[idx] lo_bar = min([bp_lo, 0]) hi_bar = max([bp_hi, 0]) not_relu_var = (1 - relu_var) model.addConstr(post_switch_var <= pre_switch_var - lo_bar * not_relu_var) model.addConstr(post_switch_var >= pre_switch_var - hi_bar * not_relu_var) model.addConstr(post_switch_var >= lo_bar * relu_var) model.addConstr(post_switch_var <= hi_bar * relu_var) model.update()
def add_abs_layer(squire, hyperbox_bounds, input_key, sign_key, output_key): """ Encodes the absolute value as gurobi models: - creates variables keyed by output_key that are the absolute value of input_key (sign_key are integer variables to control the nonconvexity of input_key) - conservative sign bounds based on preact 0th layer backprop bounds control signs """ network = squire.network model = squire.model tolerance = 1e-6 input_vars = squire.get_vars(input_key) output_namer = utils.build_var_namer(output_key) sign_namer = utils.build_var_namer(sign_key) output_vars = [] sign_vars = {} for i, input_var in enumerate(input_vars): lb, ub = hyperbox_bounds[i] output_name = output_namer(i) # always positive if lb >= 0: output_var = model.addVar(lb=-tolerance, name=output_name) model.addConstr(output_var == input_var) # always negative elif ub <= 0: output_var = model.addVar(lb=-tolerance, name=output_name) model.addConstr(output_var == -input_var) # could be positive or negative else: output_var = model.addVar(lb=- tolerance, ub=max([abs(lb), ub]) + tolerance, name=output_name) sign_var = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY, name=sign_namer(i)) #model.addConstr(output_var >= 0) model.addConstr(output_var >= input_var - tolerance) model.addConstr(output_var >= -input_var - tolerance) model.addConstr(output_var <= input_var - 2 * lb * (1 - sign_var) + tolerance) model.addConstr(output_var <= -input_var + 2 * ub * sign_var + tolerance) sign_vars[i] = sign_var output_vars.append(output_var) model.update() squire.set_vars(output_key, output_vars) squire.set_vars(sign_key, sign_vars)
def add_first_backprop_layer(squire, input_key, output_key): """ Encodes the backprop of the first linear layer. All the variables will be constant, and dependent upon the c_vector """ network = squire.network model = squire.model backprop_vars = squire.pre_bounds.gurobi_backprop_domain(squire, input_key) output_vars = [] output_var_namer = utils.build_var_namer(output_key) if isinstance(network.fcs[-1], nn.Linear): weight = utils.as_numpy(network.fcs[-1].weight).T for i in range(network.fcs[-1].in_features): output_vars.append(model.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY, name=output_var_namer(i))) model.addConstr(output_vars[i] == gb.LinExpr(weight[i], backprop_vars)) else: for i in range(len(backprop_vars)): output_vars.append(model.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY, name=output_var_namer(i))) model.addConstr(output_vars[i] == backprop_vars[i]) squire.set_vars(output_key, output_vars) model.update()
def add_abs_layer_relu(squire, hyperbox_bounds, input_key, sign_key, output_key): network = squire.network model = squire.model input_vars = squire.get_vars(input_key) output_namer = utils.build_var_namer(output_key) pos_sign_namer = utils.build_var_namer(sign_key + 'POS') neg_sign_namer = utils.build_var_namer(sign_key + 'NEG') output_vars = [] sign_vars = {} for i, input_var in enumerate(input_vars): lb, ub = hyperbox_bounds[i] output_name = output_namer(i) if lb >= 0: output_var = model.addVar(lb=0, name=output_name) model.addConstr(output_var == input_var) elif ub <= 0: output_var = model.addVar(lb=0, name=output_name) model.addConstr(output_var == -input_var) else: pos_sign = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY, name=pos_sign_namer(i)) neg_sign = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY, name=neg_sign_namer(i)) # ADD TWO RELU CONSTRAINTS # -- positive relu constraint pos_term = model.addVar(lb=0) model.addConstr(pos_term >= 0) model.addConstr(pos_term >= input_var) model.addConstr(pos_term <= ub * pos_sign) model.addConstr(pos_term <= input_var - lb * (1 - pos_sign)) neg_term = model.addVar(lb=0) # range is [-ub, -lb] model.addConstr(neg_term >= 0) model.addConstr(neg_term >= -input_var) model.addConstr(neg_term <= -lb * neg_sign) model.addConstr(neg_term <= -input_var + ub * (1 - neg_sign)) output_var = model.addVar(lb=0, name=output_name) model.addConstr(output_var == neg_term + pos_term) output_vars.append(output_var) model.update() squire.set_vars(output_key, output_vars)
def encode_as_gurobi_model(self, squire, key): model = squire.model namer = utils.build_var_namer(key) gb_vars = [] for i, (lb, ub) in enumerate(self): gb_vars.append(model.addVar(lb=lb, ub=ub, name=namer(i))) squire.set_vars(key, gb_vars) squire.update() return gb_vars
def lp_ify_model(self, tighter_relu=False): """ Converts this model to a linear program. If tighter_relu is True, we add convex upper envelope constraints for all ReLU's, otherwise we just change binary variables to continous ones. RETURNS: gurobi model object (does not change self at all) """ self.model.update() model_clone = self.model.copy() for var in model_clone.getVars(): if var.VType == gb.GRB.BINARY: var.VType = gb.GRB.CONTINUOUS var.LB = 0.0 var.UB = 1.0 model_clone.update() if not tighter_relu: # If we don't do the tight relu return model_clone # For each ReLU variable, collect it's pre/post inputs and the # bounds relu_regex = r'^relu_\d+$' for key in self.var_dict: if re.match(relu_regex, key) is None: continue suffix = key.split('_')[1] bounds = self.get_ith_relu_box(int(suffix) - 1) pre_relu_namer = utils.build_var_namer('fc_%s_pre' % suffix) post_relu_namer = utils.build_var_namer('fc_%s_post' % suffix) for idx in self.var_dict[key]: pre_var = model_clone.getVarByName(pre_relu_namer(idx)) post_var = model_clone.getVarByName(post_relu_namer(idx)) lo, hi = bounds[idx] pre_var.LB = lo pre_var.UB = hi assert (lo < 0 < hi) model_clone.addConstr(post_var <= hi * pre_var / (hi - lo) - hi * lo / (hi - lo) ) model_clone.update() return model_clone
def set_l1_objective(squire, abs_key): """ Sets the objective for the sum of the abs_keys (absolute value has already been encoded by abs_layer) """ abs_vars = squire.get_vars(abs_key) namer = utils.build_var_namer('l1_obj') obj_var = squire.model.addVar(name=namer(0)) squire.model.addConstr(sum(abs_vars) == obj_var) squire.model.setObjective(obj_var, gb.GRB.MAXIMIZE) squire.set_vars('l1_obj', [obj_var]) squire.update()
def add_relu_layer_mip(layer_no, squire, input_key, sign_key, output_key): network = squire.network model = squire.model post_relu_vars = [] relu_vars = {} # keyed by neuron # (int) post_relu_namer = utils.build_var_namer(output_key) relu_namer = utils.build_var_namer(sign_key) input_box = squire.get_ith_relu_box(layer_no) for i, (low, high) in enumerate(input_box): post_relu_name = post_relu_namer(i) relu_name = relu_namer(i) if high <= 0: post_relu_vars.append(model.addVar(lb=0.0, ub=0.0, name=post_relu_name)) else: pre_relu = squire.get_vars(input_key)[i] post_relu_vars.append(model.addVar(lb=low, ub=high, name=post_relu_name)) post_relu = post_relu_vars[-1] if low >= 0: model.addConstr(post_relu == pre_relu) continue else: relu_var = model.addVar(lb=0.0, ub=1.0, vtype=gb.GRB.BINARY, name=relu_name) relu_vars[i] = relu_var # relu(x) >= 0 and relu(x) >= x model.addConstr(post_relu >= 0.0) model.addConstr(post_relu >= pre_relu) # relu(x) <= u * a model.addConstr(post_relu <= high * relu_var) # relu(x) <= pre_relu - l(1-a) model.addConstr(post_relu <= pre_relu - low * (1 - relu_var)) model.update() squire.var_dict[output_key] = post_relu_vars squire.var_dict[sign_key] = relu_vars
def build_input_constraints(squire, var_key): # If domain is a hyperbox, can cover with lb/ub in var constructor var_namer = utils.build_var_namer(var_key) model = squire.model input_domain = squire.pre_bounds.input_domain input_vars = [] if isinstance(input_domain, Hyperbox): for i, (lb, ub) in enumerate(input_domain): input_vars.append(model.addVar(lb=lb, ub=ub, name=var_namer(i))) else: raise NotImplementedError("Only hyperboxes allowed for now!") model.update() squire.set_vars(var_key, input_vars)
def find_feasible_from_signs(self, sign_configs, input_hbox=None): """ Finds a feasible differentiable point that has the given ReLU configs. """ # First check shapes are okay: assert len(sign_configs) == self.num_relus assert all([ len(sign_configs[i]) == self.layer_sizes[i + 1] for i in range(self.num_relus) ]) # Then build a gurobi model and add constraints for each layer with utils.silent(): model = gb.Model() # Add input keys: input_key = 'input' input_namer = utils.build_var_namer(input_key) input_vars = [] for i in range(self.layer_sizes[0]): if input_hbox is not None: lb, ub = input_hbox[i] else: lb, ub = -gb.GRB.INFINITY, gb.GRB.INFINITY input_vars.append(model.addVar(lb=lb, ub=ub, name=input_namer(i))) slack_var = model.addVar(lb=0, name='slack') # And then iteratively add layers lin_vars = input_vars for i in range(self.num_relus): lin_vars = self._add_layer_to_gurobi_model(i, model, lin_vars, slack_var, sign_configs[i]) # Add the objective to maximize and then solve model.setObjective(slack_var, gb.GRB.MAXIMIZE) model.update() model.optimize() # And handle the outputs if model.Status in [3, 4]: return None else: return { 'slack': model.getObjective().getValue(), 'x': np.array([v.X for v in input_vars]), 'model': model }
def set_linf_objective(squire, box_range, abs_key, maxint_key): """ Sets the objective for the MAX of the abs_keys where box_range is a hyperbox for the max of these variables before the absolute value is applied ARGS: squire: gurobi squire object which holds the model box_range : Hyperbox bounding the values of abs_key variables before the abs_key : string that points to the continuous variables that represent the absolute value of some other variable maxint_key : string that will refer to the integer variables names """ model = squire.model abs_vars = squire.get_vars(abs_key) ubs = np.maximum(box_range.box_hi, abs(box_range.box_low)) lbs = np.maximum(box_range.box_low, 0) l_max = max(lbs) relevant_idxs = [_ for _ in range(len(ubs)) if _ >= l_max] top_two = sorted(relevant_idxs, key=lambda el: -ubs[el])[:2] max_var = model.addVar(lb=l_max, ub=ubs[top_two[0]]) maxint_namer = utils.build_var_namer(maxint_key) maxint_vars = {} if len(relevant_idxs) == 1: print("ONLY 1 THING TO MAXIMIZE") model.addConstr(max_var == abs_vars[relevant_idxs[0]]) model.setObjective(max_var, gb.GRB.MAXIMIZE) squire.update() else: for idx in relevant_idxs: if idx == top_two[0]: u_max = ubs[top_two[1]] else: u_max = ubs[top_two[0]] maxint_var = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY, name=maxint_namer(idx)) maxint_vars[idx] = maxint_var model.addConstr(max_var >= abs_vars[idx]) model.addConstr(max_var <= abs_vars[idx] + (1 - maxint_var) * (u_max - lbs[idx])) model.addConstr(1 == sum(list(maxint_vars.values()))) model.setObjective(max_var, gb.GRB.MAXIMIZE) squire.set_vars(maxint_key, maxint_vars) squire.update()
def add_linear_layer_mip(layer_no, squire, input_key, output_key): network = squire.network model = squire.model fc_layer = network.fcs[layer_no] fc_weight = utils.as_numpy(fc_layer.weight) if network.bias: fc_bias = utils.as_numpy(fc_layer.bias) else: fc_bias = np.zeros(fc_layer.out_features) input_vars = squire.get_vars(input_key) var_namer = utils.build_var_namer(output_key) pre_relu_vars = [model.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY, name=var_namer(i)) for i in range(fc_layer.out_features)] squire.set_vars(output_key, pre_relu_vars) model.addConstrs((pre_relu_vars[i] == gb.LinExpr(fc_weight[i], input_vars) + fc_bias[i]) for i in range(fc_layer.out_features)) model.update() return
def add_backprop_linear_layer(layer_no, squire, input_key, output_key): """ Encodes the backprop version of a linear layer """ network = squire.network model = squire.model output_vars = [] output_var_namer = utils.build_var_namer(output_key) fc_layer = network.fcs[layer_no] fc_weight = utils.as_numpy(fc_layer.weight) backprop_bounds = squire.get_ith_backward_box(layer_no) input_vars = squire.get_vars(input_key) for i in range(fc_layer.in_features): output_var = model.addVar(lb=backprop_bounds[i][0], ub=backprop_bounds[i][1], name=output_var_namer(i)) weight_col = fc_weight[:, i] model.addConstr(output_var == gb.LinExpr(weight_col, input_vars)) output_vars.append(output_var) model.update() squire.set_vars(output_key, output_vars)
def gurobi_backprop_domain(self, squire, key): """ Adds variables representing feasible points in the backprop_domain to the gurobi model. These are based on the c_vector and not the backprop domain ARGS: squire : gurobiSquire object - holds the model key: string - key for the new variables to be added RETURNS: gurobipy Variables[] - list of variables added to gurobi """ VALID_C_NAMES = [ 'crossLipschitz', # m-choose-2, convex hull w/ simplex 'targetCrossLipschitz', # m-1, convex hull w/simplex 'trueCrossLipschitz', # m-choose-2, MIP 'trueTargetCrossLipschitz', #m-1, MIP 'l1Ball1' # C can be in the l1 ball of norm 1 ] assert utils.arraylike(self.c_vector) or self.c_vector in VALID_C_NAMES model = squire.model namer = utils.build_var_namer(key) # HANDLE HYPERBOX CASE if isinstance(self.c_vector, Hyperbox): return self.c_vector.encode_as_gurobi_model(squire, key) # HANDLE FIXED C-VECTOR CASE gb_vars = [] if utils.arraylike(self.c_vector): for i, el in enumerate(self.c_vector): gb_vars.append(model.addVar(lb=el, ub=el, name=namer(i))) squire.set_vars(key, gb_vars) squire.update() return gb_vars # HANDLE STRING CASES (CROSS LIPSCHITZ, l1ball) output_dim = self.network.layer_sizes[-1] if self.c_vector == 'l1Ball1': l1_ball = L1Ball.make_unit_ball(output_dim) l1_ball.encode_as_gurobi_model(squire, key) return squire.get_vars(key) if self.c_vector == 'crossLipschitz': # --- HANDLE CROSS LIPSCHITZ CASE gb_vars = [ model.addVar(lb=-1.0, ub=1.0, name=namer(i)) for i in range(output_dim) ] pos_vars = [ model.addVar(lb=0.0, ub=1.0) for i in range(output_dim) ] neg_vars = [ model.addVar(lb=0.0, ub=1.0) for i in range(output_dim) ] model.addConstr(sum(pos_vars) <= 1) model.addConstr(sum(neg_vars) <= 1) model.addConstr(sum(neg_vars) <= sum(pos_vars)) if self.c_vector == 'trueCrossLipschitz': # --- HANDLE TRUE CROSS LIPSCHITZ CASE gb_vars = [ model.addVar(lb=-1.0, ub=1.0, name=namer(i)) for i in range(output_dim) ] pos_vars = [ model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY) for i in range(output_dim) ] neg_vars = [ model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY) for i in range(output_dim) ] model.addConstr(sum(pos_vars) == 1) model.addConstr(sum(neg_vars) == 1) for i in range(output_dim): model.addConstr(gb_vars[i] == pos_vars[i] - neg_vars[i]) if self.c_vector == 'targetCrossLipschitz': network = squire.network center = squire.pre_bounds.input_domain.get_center() label = network.classify_np(center) label_less_vars = [] for i in range(output_dim): if i == label: gb_vars.append(model.addVar(lb=1.0, ub=1.0, name=namer(i))) else: new_var = model.addVar(lb=-1.0, ub=0.0, name=namer(i)) label_less_vars.append(new_var) gb_vars.append(new_var) model.addConstr(sum(label_less_vars) >= -1.0) if self.c_vector == 'trueTargetCrossLipschitz': network = squire.network center = squire.pre_bounds.input_domain.get_center() label = network.classify_np(center) int_vars = [] for i in range(output_dim): if i == label: gb_vars.append(model.addVar(lb=1.0, ub=1.0, name=namer(i))) else: gb_vars.append(model.addVar(lb=-1.0, ub=0.0, name=namer(i))) int_vars.append( model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY)) model.addConstr(gb_vars[-1] == -int_vars[-1]) model.addConstr(sum(int_vars) <= 1) squire.set_vars(key, gb_vars) squire.update() return gb_vars