def get_var_at_point(self, x, var_key):
""" Computes the variables prefixed by var_key at a given point.
This does not modify any state (makes a copy of the gurobi model
prior to changing it)
ARGS:
x: np.array - np array of the requisite input shape
reset: bool - if True, we remove the input constraints and update
RETURNS:
var (as a numpy array), but also modifies the model
"""
model = self.model.copy()
x_var_namer = utils.build_var_namer('x')
constr_namer = lambda x_var_name: 'fixed::' + x_var_name
for i in range(len(self.get_vars('x'))):
x_var = model.getVarByName(x_var_namer(i))
constr_name = constr_namer(x_var.varName)
constr = model.getConstrByName(constr_name)
if constr is not None:
model.remove(constr)
model.addConstr(x_var == x[i], name=constr_name)
model.setObjective(0)
model.update()
model.optimize()
output_namer = utils.build_var_namer(var_key)
output_num = len(self.get_vars(var_key))
return [model.getVarByName(output_namer(i)).X
for i in range(output_num)]
def add_backprop_switch_layer_mip(layer_no, squire,
input_key, relu_key, output_key):
"""
Encodes the funtion
output_key[i] := 0 if sign_key[i] == 0
input_key[i] if sign_key != 0
where 0 <= input_key[i] <= squire.backprop_pos_bounds[i]
"""
network = squire.network
model = squire.model
switchbox = squire.get_ith_switch_box(layer_no - 1)
switch_inbox = squire.get_ith_backward_box(layer_no)
post_switch_vars = []
post_switch_namer = utils.build_var_namer(output_key)
post_switch_namer = utils.build_var_namer(output_key)
# First add variables
for idx, val in enumerate(switchbox):
name = post_switch_namer(idx)
if val < 0:
# Switch is always off
lb = ub = 0.0
elif val > 0:
# Switch is always on
lb, ub = switch_inbox[idx]
else:
# Switch uncertain
lb = min([0, switch_inbox[idx][0]])
ub = max([0, switch_inbox[idx][1]])
post_switch_vars.append(model.addVar(lb=lb, ub=ub, name=name))
squire.set_vars(output_key, post_switch_vars)
# And then add constraints
relu_vars = squire.get_vars(relu_key)
pre_switch_vars = squire.get_vars(input_key)
for idx, val in enumerate(switchbox):
relu_var = relu_vars.get(idx)
pre_switch_var = pre_switch_vars[idx]
post_switch_var = post_switch_vars[idx]
if val < 0:
continue
elif val > 0:
model.addConstr(post_switch_var == pre_switch_var)
continue
else:
# In this case, the relu is uncertain and we need to encode
# 4 constraints. This depends on the backprop low or high though
bp_lo, bp_hi = switch_inbox[idx]
lo_bar = min([bp_lo, 0])
hi_bar = max([bp_hi, 0])
not_relu_var = (1 - relu_var)
model.addConstr(post_switch_var <= pre_switch_var -
lo_bar * not_relu_var)
model.addConstr(post_switch_var >= pre_switch_var -
hi_bar * not_relu_var)
model.addConstr(post_switch_var >= lo_bar * relu_var)
model.addConstr(post_switch_var <= hi_bar * relu_var)
model.update()
def add_abs_layer(squire, hyperbox_bounds,
input_key, sign_key, output_key):
""" Encodes the absolute value as gurobi models:
- creates variables keyed by output_key that are the
absolute value of input_key (sign_key are integer variables
to control the nonconvexity of input_key)
- conservative sign bounds based on preact 0th layer backprop
bounds control signs
"""
network = squire.network
model = squire.model
tolerance = 1e-6
input_vars = squire.get_vars(input_key)
output_namer = utils.build_var_namer(output_key)
sign_namer = utils.build_var_namer(sign_key)
output_vars = []
sign_vars = {}
for i, input_var in enumerate(input_vars):
lb, ub = hyperbox_bounds[i]
output_name = output_namer(i)
# always positive
if lb >= 0:
output_var = model.addVar(lb=-tolerance, name=output_name)
model.addConstr(output_var == input_var)
# always negative
elif ub <= 0:
output_var = model.addVar(lb=-tolerance, name=output_name)
model.addConstr(output_var == -input_var)
# could be positive or negative
else:
output_var = model.addVar(lb=- tolerance,
ub=max([abs(lb), ub]) + tolerance,
name=output_name)
sign_var = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY,
name=sign_namer(i))
#model.addConstr(output_var >= 0)
model.addConstr(output_var >= input_var - tolerance)
model.addConstr(output_var >= -input_var - tolerance)
model.addConstr(output_var <= input_var - 2 * lb * (1 - sign_var) + tolerance)
model.addConstr(output_var <= -input_var + 2 * ub * sign_var + tolerance)
sign_vars[i] = sign_var
output_vars.append(output_var)
model.update()
squire.set_vars(output_key, output_vars)
squire.set_vars(sign_key, sign_vars)
def add_first_backprop_layer(squire, input_key, output_key):
""" Encodes the backprop of the first linear layer.
All the variables will be constant, and dependent upon the
c_vector
"""
network = squire.network
model = squire.model
backprop_vars = squire.pre_bounds.gurobi_backprop_domain(squire, input_key)
output_vars = []
output_var_namer = utils.build_var_namer(output_key)
if isinstance(network.fcs[-1], nn.Linear):
weight = utils.as_numpy(network.fcs[-1].weight).T
for i in range(network.fcs[-1].in_features):
output_vars.append(model.addVar(lb=-gb.GRB.INFINITY,
ub=gb.GRB.INFINITY,
name=output_var_namer(i)))
model.addConstr(output_vars[i] == gb.LinExpr(weight[i], backprop_vars))
else:
for i in range(len(backprop_vars)):
output_vars.append(model.addVar(lb=-gb.GRB.INFINITY,
ub=gb.GRB.INFINITY,
name=output_var_namer(i)))
model.addConstr(output_vars[i] == backprop_vars[i])
squire.set_vars(output_key, output_vars)
model.update()
def add_abs_layer_relu(squire, hyperbox_bounds,
input_key, sign_key, output_key):
network = squire.network
model = squire.model
input_vars = squire.get_vars(input_key)
output_namer = utils.build_var_namer(output_key)
pos_sign_namer = utils.build_var_namer(sign_key + 'POS')
neg_sign_namer = utils.build_var_namer(sign_key + 'NEG')
output_vars = []
sign_vars = {}
for i, input_var in enumerate(input_vars):
lb, ub = hyperbox_bounds[i]
output_name = output_namer(i)
if lb >= 0:
output_var = model.addVar(lb=0, name=output_name)
model.addConstr(output_var == input_var)
elif ub <= 0:
output_var = model.addVar(lb=0, name=output_name)
model.addConstr(output_var == -input_var)
else:
pos_sign = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY,
name=pos_sign_namer(i))
neg_sign = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY,
name=neg_sign_namer(i))
# ADD TWO RELU CONSTRAINTS
# -- positive relu constraint
pos_term = model.addVar(lb=0)
model.addConstr(pos_term >= 0)
model.addConstr(pos_term >= input_var)
model.addConstr(pos_term <= ub * pos_sign)
model.addConstr(pos_term <= input_var - lb * (1 - pos_sign))
neg_term = model.addVar(lb=0)
# range is [-ub, -lb]
model.addConstr(neg_term >= 0)
model.addConstr(neg_term >= -input_var)
model.addConstr(neg_term <= -lb * neg_sign)
model.addConstr(neg_term <= -input_var + ub * (1 - neg_sign))
output_var = model.addVar(lb=0, name=output_name)
model.addConstr(output_var == neg_term + pos_term)
output_vars.append(output_var)
model.update()
squire.set_vars(output_key, output_vars)
def encode_as_gurobi_model(self, squire, key):
model = squire.model
namer = utils.build_var_namer(key)
gb_vars = []
for i, (lb, ub) in enumerate(self):
gb_vars.append(model.addVar(lb=lb, ub=ub, name=namer(i)))
squire.set_vars(key, gb_vars)
squire.update()
return gb_vars
def lp_ify_model(self, tighter_relu=False):
""" Converts this model to a linear program.
If tighter_relu is True, we add convex upper envelope constraints
for all ReLU's, otherwise we just change binary variables to
continous ones.
RETURNS:
gurobi model object (does not change self at all)
"""
self.model.update()
model_clone = self.model.copy()
for var in model_clone.getVars():
if var.VType == gb.GRB.BINARY:
var.VType = gb.GRB.CONTINUOUS
var.LB = 0.0
var.UB = 1.0
model_clone.update()
if not tighter_relu: # If we don't do the tight relu
return model_clone
# For each ReLU variable, collect it's pre/post inputs and the
# bounds
relu_regex = r'^relu_\d+$'
for key in self.var_dict:
if re.match(relu_regex, key) is None:
continue
suffix = key.split('_')[1]
bounds = self.get_ith_relu_box(int(suffix) - 1)
pre_relu_namer = utils.build_var_namer('fc_%s_pre' % suffix)
post_relu_namer = utils.build_var_namer('fc_%s_post' % suffix)
for idx in self.var_dict[key]:
pre_var = model_clone.getVarByName(pre_relu_namer(idx))
post_var = model_clone.getVarByName(post_relu_namer(idx))
lo, hi = bounds[idx]
pre_var.LB = lo
pre_var.UB = hi
assert (lo < 0 < hi)
model_clone.addConstr(post_var <= hi * pre_var / (hi - lo)
- hi * lo / (hi - lo) )
model_clone.update()
return model_clone
def set_l1_objective(squire, abs_key):
""" Sets the objective for the sum of the abs_keys
(absolute value has already been encoded by abs_layer)
"""
abs_vars = squire.get_vars(abs_key)
namer = utils.build_var_namer('l1_obj')
obj_var = squire.model.addVar(name=namer(0))
squire.model.addConstr(sum(abs_vars) == obj_var)
squire.model.setObjective(obj_var, gb.GRB.MAXIMIZE)
squire.set_vars('l1_obj', [obj_var])
squire.update()
def add_relu_layer_mip(layer_no, squire, input_key,
sign_key, output_key):
network = squire.network
model = squire.model
post_relu_vars = []
relu_vars = {} # keyed by neuron # (int)
post_relu_namer = utils.build_var_namer(output_key)
relu_namer = utils.build_var_namer(sign_key)
input_box = squire.get_ith_relu_box(layer_no)
for i, (low, high) in enumerate(input_box):
post_relu_name = post_relu_namer(i)
relu_name = relu_namer(i)
if high <= 0:
post_relu_vars.append(model.addVar(lb=0.0, ub=0.0,
name=post_relu_name))
else:
pre_relu = squire.get_vars(input_key)[i]
post_relu_vars.append(model.addVar(lb=low, ub=high,
name=post_relu_name))
post_relu = post_relu_vars[-1]
if low >= 0:
model.addConstr(post_relu == pre_relu)
continue
else:
relu_var = model.addVar(lb=0.0, ub=1.0, vtype=gb.GRB.BINARY,
name=relu_name)
relu_vars[i] = relu_var
# relu(x) >= 0 and relu(x) >= x
model.addConstr(post_relu >= 0.0)
model.addConstr(post_relu >= pre_relu)
# relu(x) <= u * a
model.addConstr(post_relu <= high * relu_var)
# relu(x) <= pre_relu - l(1-a)
model.addConstr(post_relu <= pre_relu - low * (1 - relu_var))
model.update()
squire.var_dict[output_key] = post_relu_vars
squire.var_dict[sign_key] = relu_vars
def build_input_constraints(squire, var_key):
# If domain is a hyperbox, can cover with lb/ub in var constructor
var_namer = utils.build_var_namer(var_key)
model = squire.model
input_domain = squire.pre_bounds.input_domain
input_vars = []
if isinstance(input_domain, Hyperbox):
for i, (lb, ub) in enumerate(input_domain):
input_vars.append(model.addVar(lb=lb, ub=ub, name=var_namer(i)))
else:
raise NotImplementedError("Only hyperboxes allowed for now!")
model.update()
squire.set_vars(var_key, input_vars)
def find_feasible_from_signs(self, sign_configs, input_hbox=None):
""" Finds a feasible differentiable point that has the given
ReLU configs.
"""
# First check shapes are okay:
assert len(sign_configs) == self.num_relus
assert all([
len(sign_configs[i]) == self.layer_sizes[i + 1]
for i in range(self.num_relus)
])
# Then build a gurobi model and add constraints for each layer
with utils.silent():
model = gb.Model()
# Add input keys:
input_key = 'input'
input_namer = utils.build_var_namer(input_key)
input_vars = []
for i in range(self.layer_sizes[0]):
if input_hbox is not None:
lb, ub = input_hbox[i]
else:
lb, ub = -gb.GRB.INFINITY, gb.GRB.INFINITY
input_vars.append(model.addVar(lb=lb, ub=ub, name=input_namer(i)))
slack_var = model.addVar(lb=0, name='slack')
# And then iteratively add layers
lin_vars = input_vars
for i in range(self.num_relus):
lin_vars = self._add_layer_to_gurobi_model(i, model, lin_vars,
slack_var,
sign_configs[i])
# Add the objective to maximize and then solve
model.setObjective(slack_var, gb.GRB.MAXIMIZE)
model.update()
model.optimize()
# And handle the outputs
if model.Status in [3, 4]:
return None
else:
return {
'slack': model.getObjective().getValue(),
'x': np.array([v.X for v in input_vars]),
'model': model
}
def set_linf_objective(squire, box_range, abs_key, maxint_key):
""" Sets the objective for the MAX of the abs_keys where
box_range is a hyperbox for the max of these variables before
the absolute value is applied
ARGS:
squire: gurobi squire object which holds the model
box_range : Hyperbox bounding the values of abs_key variables before
the
abs_key : string that points to the continuous variables that
represent the absolute value of some other variable
maxint_key : string that will refer to the integer variables names
"""
model = squire.model
abs_vars = squire.get_vars(abs_key)
ubs = np.maximum(box_range.box_hi, abs(box_range.box_low))
lbs = np.maximum(box_range.box_low, 0)
l_max = max(lbs)
relevant_idxs = [_ for _ in range(len(ubs)) if _ >= l_max]
top_two = sorted(relevant_idxs, key=lambda el: -ubs[el])[:2]
max_var = model.addVar(lb=l_max, ub=ubs[top_two[0]])
maxint_namer = utils.build_var_namer(maxint_key)
maxint_vars = {}
if len(relevant_idxs) == 1:
print("ONLY 1 THING TO MAXIMIZE")
model.addConstr(max_var == abs_vars[relevant_idxs[0]])
model.setObjective(max_var, gb.GRB.MAXIMIZE)
squire.update()
else:
for idx in relevant_idxs:
if idx == top_two[0]:
u_max = ubs[top_two[1]]
else:
u_max = ubs[top_two[0]]
maxint_var = model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY,
name=maxint_namer(idx))
maxint_vars[idx] = maxint_var
model.addConstr(max_var >= abs_vars[idx])
model.addConstr(max_var <= abs_vars[idx] +
(1 - maxint_var) * (u_max - lbs[idx]))
model.addConstr(1 == sum(list(maxint_vars.values())))
model.setObjective(max_var, gb.GRB.MAXIMIZE)
squire.set_vars(maxint_key, maxint_vars)
squire.update()
def add_linear_layer_mip(layer_no, squire,
input_key, output_key):
network = squire.network
model = squire.model
fc_layer = network.fcs[layer_no]
fc_weight = utils.as_numpy(fc_layer.weight)
if network.bias:
fc_bias = utils.as_numpy(fc_layer.bias)
else:
fc_bias = np.zeros(fc_layer.out_features)
input_vars = squire.get_vars(input_key)
var_namer = utils.build_var_namer(output_key)
pre_relu_vars = [model.addVar(lb=-gb.GRB.INFINITY, ub=gb.GRB.INFINITY,
name=var_namer(i))
for i in range(fc_layer.out_features)]
squire.set_vars(output_key, pre_relu_vars)
model.addConstrs((pre_relu_vars[i] == gb.LinExpr(fc_weight[i], input_vars) + fc_bias[i])
for i in range(fc_layer.out_features))
model.update()
return
def add_backprop_linear_layer(layer_no, squire,
input_key, output_key):
""" Encodes the backprop version of a linear layer """
network = squire.network
model = squire.model
output_vars = []
output_var_namer = utils.build_var_namer(output_key)
fc_layer = network.fcs[layer_no]
fc_weight = utils.as_numpy(fc_layer.weight)
backprop_bounds = squire.get_ith_backward_box(layer_no)
input_vars = squire.get_vars(input_key)
for i in range(fc_layer.in_features):
output_var = model.addVar(lb=backprop_bounds[i][0],
ub=backprop_bounds[i][1],
name=output_var_namer(i))
weight_col = fc_weight[:, i]
model.addConstr(output_var == gb.LinExpr(weight_col, input_vars))
output_vars.append(output_var)
model.update()
squire.set_vars(output_key, output_vars)
def gurobi_backprop_domain(self, squire, key):
""" Adds variables representing feasible points in the
backprop_domain to the gurobi model. These are based on the
c_vector and not the backprop domain
ARGS:
squire : gurobiSquire object - holds the model
key: string - key for the new variables to be added
RETURNS:
gurobipy Variables[] - list of variables added to gurobi
"""
VALID_C_NAMES = [
'crossLipschitz', # m-choose-2, convex hull w/ simplex
'targetCrossLipschitz', # m-1, convex hull w/simplex
'trueCrossLipschitz', # m-choose-2, MIP
'trueTargetCrossLipschitz', #m-1, MIP
'l1Ball1' # C can be in the l1 ball of norm 1
]
assert utils.arraylike(self.c_vector) or self.c_vector in VALID_C_NAMES
model = squire.model
namer = utils.build_var_namer(key)
# HANDLE HYPERBOX CASE
if isinstance(self.c_vector, Hyperbox):
return self.c_vector.encode_as_gurobi_model(squire, key)
# HANDLE FIXED C-VECTOR CASE
gb_vars = []
if utils.arraylike(self.c_vector):
for i, el in enumerate(self.c_vector):
gb_vars.append(model.addVar(lb=el, ub=el, name=namer(i)))
squire.set_vars(key, gb_vars)
squire.update()
return gb_vars
# HANDLE STRING CASES (CROSS LIPSCHITZ, l1ball)
output_dim = self.network.layer_sizes[-1]
if self.c_vector == 'l1Ball1':
l1_ball = L1Ball.make_unit_ball(output_dim)
l1_ball.encode_as_gurobi_model(squire, key)
return squire.get_vars(key)
if self.c_vector == 'crossLipschitz':
# --- HANDLE CROSS LIPSCHITZ CASE
gb_vars = [
model.addVar(lb=-1.0, ub=1.0, name=namer(i))
for i in range(output_dim)
]
pos_vars = [
model.addVar(lb=0.0, ub=1.0) for i in range(output_dim)
]
neg_vars = [
model.addVar(lb=0.0, ub=1.0) for i in range(output_dim)
]
model.addConstr(sum(pos_vars) <= 1)
model.addConstr(sum(neg_vars) <= 1)
model.addConstr(sum(neg_vars) <= sum(pos_vars))
if self.c_vector == 'trueCrossLipschitz':
# --- HANDLE TRUE CROSS LIPSCHITZ CASE
gb_vars = [
model.addVar(lb=-1.0, ub=1.0, name=namer(i))
for i in range(output_dim)
]
pos_vars = [
model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY)
for i in range(output_dim)
]
neg_vars = [
model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY)
for i in range(output_dim)
]
model.addConstr(sum(pos_vars) == 1)
model.addConstr(sum(neg_vars) == 1)
for i in range(output_dim):
model.addConstr(gb_vars[i] == pos_vars[i] - neg_vars[i])
if self.c_vector == 'targetCrossLipschitz':
network = squire.network
center = squire.pre_bounds.input_domain.get_center()
label = network.classify_np(center)
label_less_vars = []
for i in range(output_dim):
if i == label:
gb_vars.append(model.addVar(lb=1.0, ub=1.0, name=namer(i)))
else:
new_var = model.addVar(lb=-1.0, ub=0.0, name=namer(i))
label_less_vars.append(new_var)
gb_vars.append(new_var)
model.addConstr(sum(label_less_vars) >= -1.0)
if self.c_vector == 'trueTargetCrossLipschitz':
network = squire.network
center = squire.pre_bounds.input_domain.get_center()
label = network.classify_np(center)
int_vars = []
for i in range(output_dim):
if i == label:
gb_vars.append(model.addVar(lb=1.0, ub=1.0, name=namer(i)))
else:
gb_vars.append(model.addVar(lb=-1.0, ub=0.0,
name=namer(i)))
int_vars.append(
model.addVar(lb=0, ub=1, vtype=gb.GRB.BINARY))
model.addConstr(gb_vars[-1] == -int_vars[-1])
model.addConstr(sum(int_vars) <= 1)
squire.set_vars(key, gb_vars)
squire.update()
return gb_vars
