def test_abs(number):
input = Constant(number, '', 0, 0)
delta = Variable(0, 0, '', 'd', 'Int')
delta.update_bounds(0, 1)
output = Variable(0, 0, '', 'a')
vars = [delta, output]
constrs = [Abs(input, output, delta)]
model = create_gurobi_model(vars, constrs)
model.optimize()
res = model.getVarByName('a_0_0').X
return vars, constrs, res
def check_equivalence_layer(self, layer_idx):
opt_vars = []
opt_constrs = []
if layer_idx == 0:
opt_vars += self.input_layer.get_outvars()[:]
else:
a_outs = self.a_layers[layer_idx - 1].get_outvars()[:]
b_outs = self.b_layers[layer_idx - 1].get_outvars()[:]
opt_vars += a_outs + b_outs
# at this stage we assume the previous layers to be equivalent
for avar, bvar in zip(a_outs, b_outs):
opt_constrs += [Linear(avar, bvar)]
bounds = []
for i, (a_var, a_constr, b_var, b_constr) in enumerate(
zip(self.a_layers[layer_idx].get_optimization_vars(),
self.a_layers[layer_idx].get_optimization_constraints(),
self.b_layers[layer_idx].get_optimization_vars(),
self.b_layers[layer_idx].get_optimization_constraints())):
diff = Variable(layer_idx, i, 'E', 'diff')
diff_constr = Linear(Sum([a_var, Neg(b_var)]), diff)
if i == 1:
pretty_print(opt_vars + [a_var, b_var, diff],
opt_constrs + [a_constr, b_constr, diff_constr])
lb, ub = self.optimize_variable(
diff, opt_vars + [a_var, b_var, diff],
opt_constrs + [a_constr, b_constr, diff_constr])
diff.update_bounds(lb, ub)
bounds.append((lb, ub))
return bounds
def encode_equivalence_layer(outs1, outs2, mode='diff_zero'):
def one_hot_comparison(oh1, oh2, net, layer, row, desired='different'):
'''
Compares two one-hot vectors and returns constraints that can only be satisfied,
if the vectors are equal/different
:param oh1: one-hot vector
:param oh2: one-hot vector
:param net: netPrefix
:param layer: layer of the net, in which this operation takes place
:param row: row of the net, in which this operation takes place
:param desired: keyword
different - the constraints can only be satisfied, if the vectors are different
equal - the constraints can only be satisfied, if the vectors are equal
:return: a tuple of (deltas, diffs, constraints) where constraints are as described above and deltas, diffs
are variables used in these constraints
'''
# requires that oh_i are one-hot vectors
oh_deltas = []
oh_diffs = []
oh_constraints = []
desired_result = 1
if desired == 'different':
desired_result = 1
elif desired == 'equal':
desired_result = 0
terms = []
x = 1
for i, (oh1, oh2) in enumerate(zip(oh1, oh2)):
constant = Constant(x, net, layer, row)
terms.append(Multiplication(constant, oh1))
terms.append(Neg(Multiplication(constant, oh2)))
x *= 2
sumvar = Variable(layer, row, net, 's', 'Int')
oh_constraints.append(Linear(Sum(terms), sumvar))
delta_gt = Variable(layer, row, net, 'dg', 'Int')
delta_lt = Variable(layer, row, net, 'dl', 'Int')
zero = Constant(0, net, layer, row)
oh_constraints.append(Gt_Int(sumvar, zero, delta_gt))
oh_constraints.append(Gt_Int(zero, sumvar, delta_lt))
oh_constraints.append(
Geq(Sum([delta_lt, delta_gt]),
Constant(desired_result, net, layer, row)))
oh_deltas.append(delta_gt)
oh_deltas.append(delta_lt)
oh_diffs.append(sumvar)
return oh_deltas, oh_diffs, oh_constraints
def number_comparison(n1, n2, net, layer, row, epsilon=0):
'''
Compares two arbitrary numbers and returns constraints, s.t. one of the deltas is equal to 1, if the numbers
are not equal
:param n1: number
:param n2: number
:param net: netPrefix
:param layer: layer of the net, in which this operation takes place
:param row: row of the net, in which this operation takes place
:return: a tuple of (deltas, diffs, constraints) where constraints are as described above and deltas, diffs
are variables used in these constraints
'''
v_deltas = []
v_diffs = []
v_constraints = []
delta_gt = Variable(layer, row, net, 'dg', 'Int')
delta_lt = Variable(layer + 1, row, net, 'dl', 'Int')
if epsilon > 0:
eps = Constant(epsilon, net, layer + 1, row)
diff_minus_eps = Variable(layer, row, net, 'x_m')
diff_plus_eps = Variable(layer, row, net, 'x_p')
v_constraints.append(
Linear(Sum([n2, Neg(n1), Neg(eps)]), diff_minus_eps))
v_constraints.append(Linear(Sum([n2, Neg(n1), eps]),
diff_plus_eps))
v_constraints.append(Greater_Zero(diff_minus_eps, delta_gt))
v_constraints.append(Greater_Zero(Neg(diff_plus_eps), delta_lt))
v_diffs.append(diff_minus_eps)
v_diffs.append(diff_plus_eps)
else:
diff = Variable(layer, row, net, 'x')
v_constraints.append(Linear(Sum([n1, Neg(n2)]), diff))
v_constraints.append(Greater_Zero(diff, delta_gt))
v_constraints.append(Greater_Zero(Neg(diff), delta_lt))
v_diffs.append(diff)
v_deltas.append(delta_gt)
v_deltas.append(delta_lt)
#v_constraints.append(Geq(Sum(v_deltas), Constant(desired_result, net, layer + 1, row)))
return v_deltas, v_diffs, v_constraints
deltas = []
diffs = []
constraints = []
if mode == 'diff_zero' or mode.startswith('epsilon_'):
eps = 0
if mode.startswith('epsilon_'):
eps = float(mode.split('_')[-1])
for i, (out1, out2) in enumerate(zip(outs1, outs2)):
n_deltas, n_diffs, n_constraints = number_comparison(out1,
out2,
'E',
0,
i,
epsilon=eps)
deltas += n_deltas
diffs += n_diffs
constraints += n_constraints
constraints.append(Geq(Sum(deltas), Constant(1, 'E', 1, 0)))
elif mode in [
'optimize_diff', 'optimize_diff_manhattan',
'optimize_diff_chebyshev'
]:
for i, (out1, out2) in enumerate(zip(outs1, outs2)):
diff_i = Variable(0, i, 'E', 'diff')
constraints.append(Linear(Sum([out1, Neg(out2)]), diff_i))
diffs.append(diff_i)
# will continue to be either optimize_diff_manhattan or ..._chebyshev
if mode.startswith('optimize_diff_'):
abs_vals = []
for i, diff in enumerate(diffs):
abs_val_i = Variable(0, i, 'E', 'abs_d')
abs_vals.append(abs_val_i)
delta_i = Variable(0, i, 'E', 'd', 'Int')
delta_i.update_bounds(0, 1)
deltas.append(delta_i)
constraints.append(Abs(diff, abs_val_i, delta_i))
diffs.append(abs_vals)
if mode == 'optimize_diff_manhattan':
norm = Variable(1, 0, 'E', 'norm')
constraints.append(Linear(Sum(abs_vals), norm))
diffs.append(norm)
elif mode == 'optimize_diff_chebyshev':
partial_matrix, partial_vars, partial_constrs = encode_partial_layer(
1, abs_vals, 1, 'E')
diffs.append(partial_vars)
constraints.append(partial_constrs)
deltas.append(partial_matrix)
context_constraints = []
if fc.use_context_groups:
# partial_vars = ([E_y_ij, ...] + [E_o_1_0])
context_constraints.append(
TopKGroup(partial_vars[-1], abs_vals, 1))
constraints.append(context_constraints)
# only for interface to norm optimization, otherwise would have to optimize E_o_1_0
norm = Variable(1, 0, 'E', 'norm')
constraints.append(Linear(partial_vars[-1], norm))
diffs.append(norm)
elif mode == 'diff_one_hot':
# requires that outs_i are the pi_1_js in of the respective permutation matrices
# or input to this layer are one-hot vectors
deltas, diffs, constraints = one_hot_comparison(outs1,
outs2,
'E',
0,
0,
desired='different')
elif mode.startswith('ranking_top_'):
# assumes outs1 = one-hot vector with maximum output of NN1
# outs2 = (one-hot biggest, one-hot 2nd biggest, ...) of NN2
k = int(mode.split('_')[-1])
for i in range(k):
k_deltas, k_diffs, k_constraints = one_hot_comparison(
outs1, outs2[i], 'E', 0, i, desired='different')
deltas += k_deltas
diffs += k_diffs
constraints += k_constraints
elif mode.startswith('one_ranking_top_'):
# assumes outs1 = permutation matrix of NN1
# outs2 = outputs of NN1
k = int(mode.split('_')[-1])
matrix = outs1
ordered2 = [Variable(0, i, 'E', 'o') for i in range(len(outs2))]
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'E', matrix,
ordered2)
order_constrs = []
deltas = []
for i in range(k, len(outs2)):
delta_i = Variable(0, i, 'E', 'd', type='Int')
deltas.append(delta_i)
# o_1 < o_i <--> d = 1
# 0 < o_i - o_1 <--> d = 1
order_constrs.append(
Greater_Zero(Sum([ordered2[i], Neg(ordered2[0])]), delta_i))
order_constrs.append(Geq(Sum(deltas), Constant(1, 'E', 0, 0)))
constraints = mat_constrs + order_constrs
diffs = res_vars + ordered2
elif mode.startswith('optimize_ranking_top_'):
k = int(mode.split('_')[-1])
matrix = outs1
ordered2 = [Variable(0, i, 'E', 'o') for i in range(len(outs2))]
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'E', matrix,
ordered2)
order_constrs = []
diffs = []
for i in range(k, len(outs2)):
diff_i = Variable(0, i, 'E', 'diff')
diffs.append(diff_i)
order_constrs.append(
Linear(Sum([ordered2[i], Neg(ordered2[0])]), diff_i))
constraints = mat_constrs + order_constrs
deltas = res_vars + ordered2
elif mode.startswith('partial_top_'):
# assumes outs1 = [partial matrix, set-var] of NN1
# assumes outs2 = outputs of NN2
partial_matrix = outs1[0]
one_hot_vec = partial_matrix[0]
set_var = outs1[1]
top = Variable(0, 0, 'E', 'top')
# one_hot_vec and top need to be enclosed in [], so that indexing in binmult_matrix works
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'E',
[one_hot_vec], [top])
order_constrs = []
for i in range(len(outs2)):
order_constrs.append(
Impl(set_var[i], 0, Sum([outs2[i], Neg(top)]),
Constant(0, 'E', 0, 0)))
constraints = mat_constrs + order_constrs
deltas = res_vars
diffs = [top]
elif mode.startswith('optimize_partial_top_'):
# assumes outs1 = [partial matrix, set-var] of NN1
# assumes outs2 = outputs of NN2
partial_matrix = outs1[0]
one_hot_vec = partial_matrix[0]
set_var = outs1[1]
top = Variable(0, 0, 'E', 'top')
# one_hot_vec and top need to be enclosed in [], so that indexing in binmult_matrix works
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'E',
[one_hot_vec], [top])
order_constrs = []
diffs = [Variable(0, i, 'E', 'diff') for i in range(len(outs2))]
order_constrs.append(
IndicatorToggle(
set_var, 0,
[Sum([outs2[i], Neg(top)]) for i in range(len(outs2))], diffs))
max_diff_vec = [
Variable(1, i, 'E', 'pi', 'Int') for i in range(len(diffs))
]
max_diff = Variable(1, 0, 'E', 'max_diff')
res_vars2, mat_constrs2 = encode_binmult_matrix(
diffs, 1, 'Emax', [max_diff_vec], [max_diff])
for diff in diffs:
order_constrs.append(Geq(max_diff, diff))
diffs.append(max_diff)
constraints = mat_constrs + order_constrs + mat_constrs2
deltas = res_vars + [top] + max_diff_vec + res_vars2
elif mode.startswith('one_hot_partial_top_'):
k = int(mode.split('_')[-1])
# assumes outs1 = one hot vector of NN1
# assumes outs2 = output of NN2
one_hot_vec = outs1
top = Variable(0, 0, 'E', 'top')
# one_hot_vec and top need to be enclosed in [], so that indexing in binmult_matrix works
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'Eoh',
[one_hot_vec], [top])
partial_matrix, partial_vars, partial_constrs = encode_partial_layer(
k, outs2, 1, 'E')
context_constraints = []
if fc.use_context_groups:
context_constraints.append(ExtremeGroup(top, outs2))
# partial_vars = ([E_y_ij, ...] + [E_o_1_0, E_o_1_1, ..., E_o_1_(k-1)])
for i in range(1, k + 1):
context_constraints.append(
TopKGroup(partial_vars[i - (k + 1)], outs2, i))
diff = Variable(0, k, 'E', 'diff')
diff_constr = Linear(Sum([partial_vars[-1], Neg(top)]), diff)
deltas = [top] + res_vars + partial_matrix + partial_vars
diffs = [diff]
constraints = mat_constrs + partial_constrs + context_constraints + [
diff_constr
]
elif mode == 'one_hot_diff':
# assumes outs1 = one hot vector of NN1
# assumes outs2 = output of NN2
one_hot_vec = outs1
top = Variable(0, 0, 'E', 'top')
# one_hot_vec and top need to be enclosed in [], so that indexing in binmult_matrix works
res_vars, mat_constrs = encode_binmult_matrix(outs2, 0, 'E',
[one_hot_vec], [top])
diffs = [Variable(0, i, 'E', 'diff') for i in range(len(outs2))]
diff_constrs = [
Linear(Sum([out, Neg(top)]), diff)
for out, diff in zip(outs2, diffs)
]
deltas = [top] + res_vars
constraints = mat_constrs + diff_constrs
else:
raise ValueError('There is no \'' + mode +
'\' keyword for parameter mode')
return deltas, diffs, constraints