def encode_krelu_cons(nn, man, element, offset, layerno, length, lbi, ubi,
relu_groups, need_pop, domain):
if (need_pop):
relu_groups.pop()
lbi = np.asarray(lbi, dtype=np.double)
ubi = np.asarray(ubi, dtype=np.double)
candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
candidate_vars = sorted(candidate_vars,
key=lambda i: ubi[i] - lbi[i],
reverse=True)
candidate_vars = grouping_heuristic(candidate_vars, lbi, ubi)
relucons = []
tdim = ElinaDim(offset + length)
if domain == 'refinezono':
element = dn.add_dimensions(man, element, offset + length, 1)
krelu_args = []
if config.dyn_krelu and candidate_vars:
limit3relucalls = 500
firstk = math.sqrt(6 * limit3relucalls / len(candidate_vars))
firstk = int(min(firstk, len(candidate_vars)))
if (layerno < 3):
firstk = 1
else:
firstk = 5 #int(max(1,firstk))
print("firstk ", firstk)
if firstk > 3:
while candidate_vars:
headlen = min(firstk, len(candidate_vars))
head = candidate_vars[:headlen]
candidate_vars = candidate_vars[headlen:]
if len(head) <= 3:
krelu_args.append(head)
else:
for arg in itertools.combinations(head, 3):
krelu_args.append(arg)
klist = ([3] if
(config.use_3relu) else []) + ([2] if
(config.use_2relu) else []) + [1]
for k in klist:
while len(candidate_vars) >= k:
krelu_args.append(candidate_vars[:k])
candidate_vars = candidate_vars[k:]
Krelu.man = man
Krelu.element = element
Krelu.tdim = tdim
Krelu.length = length
Krelu.layerno = layerno
Krelu.offset = offset
Krelu.domain = domain
start = time.time()
with multiprocessing.Pool(config.numproc) as pool:
krelu_results = pool.map(make_krelu_obj, krelu_args)
for krelu_inst in krelu_results:
relucons.append(krelu_inst)
end = time.time()
if config.debug:
print('krelu time spent: ' + str(end - start))
if domain == 'refinezono':
element = dn.remove_dimensions(man, element, offset + length, 1)
relu_groups.append(relucons)
return
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi,
ubi, relu_groups, need_pop, domain,
activation_type):
import deepzono_nodes as dn
if (need_pop):
relu_groups.pop()
last_conv = -1
is_conv = False
for i in range(nn.numlayer):
if nn.layertypes[i] == 'Conv':
last_conv = i
is_conv = True
lbi = np.asarray(lbi, dtype=np.double)
ubi = np.asarray(ubi, dtype=np.double)
candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
candidate_vars_areas = {
var: -lbi[var] * ubi[var]
for var in candidate_vars
}
# Sort vars by descending area
candidate_vars = sorted(candidate_vars,
key=lambda var: -candidate_vars_areas[var])
# Use sparse heuristic to select args (uncomment to use)
KAct_args = sparse_heuristic_with_cutoff(candidate_vars,
candidate_vars_areas)
relucons = []
# print("UBI ",ubi)
tdim = ElinaDim(offset + length)
if domain == 'refinezono':
element = dn.add_dimensions(man, element, offset + length, 1)
KAct.man = man
KAct.element = element
KAct.tdim = tdim
KAct.length = length
KAct.layerno = layerno
KAct.offset = offset
KAct.domain = domain
KAct.type = activation_type
# klist = ([3] if (config.use_3relu) else []) + ([2] if (config.use_2relu) else []) + [1]
# for k in klist:
# while len(candidate_vars) >= k:
# KAct_args.append(candidate_vars[:k])
# candidate_vars = candidate_vars[k:]
#KAct.man = man
#KAct.element = element
#KAct.tdim = tdim
#KAct.length = length
#KAct.layerno = layerno
#KAct.offset = offset
#KAct.domain = domain
start = time.time()
if domain == 'refinezono':
with multiprocessing.Pool(config.numproc) as pool:
cdd_hrepr_array = pool.map(get_ineqs_zono, KAct_args)
else:
# KAct_results = []
total_size = 0
listCoeff = [-2, -1, 0, 1, 2]
for varsid in KAct_args:
if len(varsid) == 1:
size = len(listCoeff)**len(varsid) - 1
else:
size = len(listCoeff)**len(varsid) - (3**len(varsid))
total_size = total_size + size
linexpr0 = elina_linexpr0_array_alloc(total_size)
# HERE we should define our expressions
i = -1
for varsid in KAct_args:
for coeffs in itertools.product(listCoeff, repeat=len(varsid)):
i = i + 1
if len(varsid) == 1:
if coeffs[0] == 0:
i = i - 1
continue
else:
if all((abs(c) == 2 or c == 0) for c in
coeffs): #ELE for redundant constraints with 2,1
i = i - 1
continue
linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
upper_bound = get_upper_bound_for_linexpr0(man, element, linexpr0,
total_size, layerno)
i = -1
cdd_hrepr_array = []
bound_val = []
for varsid in KAct_args:
cdd_hrepr = []
for coeffs in itertools.product(listCoeff, repeat=len(varsid)):
i = i + 1
if len(varsid) == 1:
if coeffs[0] == 0:
i = i - 1
continue
else:
if all((abs(c) == 2 or c == 0) for c in
coeffs): # ELE for redundant constraints with 2,1
i = i - 1
continue
cdd_hrepr.append([upper_bound[i]] + [-c for c in coeffs])
bound_val.append(upper_bound[i]) # ELE
# print("UPPER BOUND ", upper_bound[i], "COEFF ", coeffs)
# if len(varsid)>1:
# print("LB ", lbi[varsid[0]],lbi[varsid[1]], "UB ", ubi[varsid[0]], ubi[varsid[1]])
cdd_hrepr_array.append(cdd_hrepr)
with multiprocessing.Pool(
config.numproc
) as pool: # here is entering in 'get_orthant_points'
KAct_results = pool.map(make_KAct_obj, cdd_hrepr_array) #, lbi, ubi)
# KAct_results.append(make_KAct_obj(KAct_args[i]))
# bound_expr_list = []
gid = 0
lower_bound_expr = {}
upper_bound_expr = {}
for KAct_inst in KAct_results:
varsid = KAct_args[gid]
KAct_inst.varsid = varsid
# For now disabling since in the experiments updating expression bounds makes results worse.
# compute_expr_bounds(KAct_inst, varsid, lower_bound_expr, upper_bound_expr, lbi, ubi)
# print("VARSID ",varsid)
# bound_expr_list.append(KAct_expr(lower_bound_expr, upper_bound_expr, varsid))
relucons.append(KAct_inst)
gid = gid + 1
end = time.time()
if config.debug:
print('KAct time spent: ' + str(end - start))
if domain == 'refinezono':
element = dn.remove_dimensions(man, element, offset + length, 1)
relu_groups.append(relucons)
return lower_bound_expr, upper_bound_expr
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi, ubi, constraint_groups, need_pop, domain, activation_type):
import deepzono_nodes as dn
if need_pop:
constraint_groups.pop()
lbi = np.asarray(lbi, dtype=np.double)
ubi = np.asarray(ubi, dtype=np.double)
if activation_type == "ReLU":
kact_args = sparse_heuristic_with_cutoff(length, lbi, ubi)
else:
kact_args = sparse_heuristic_curve(length, lbi, ubi, activation_type == "Sigmoid")
kact_cons = []
tdim = ElinaDim(offset+length)
if domain == 'refinezono':
element = dn.add_dimensions(man,element,offset+length,1)
KAct.man = man
KAct.element = element
KAct.tdim = tdim
KAct.length = length
KAct.layerno = layerno
KAct.offset = offset
KAct.domain = domain
KAct.type = activation_type
start = time.time()
if domain == 'refinezono':
with multiprocessing.Pool(config.numproc) as pool:
input_hrep_array = pool.map(get_ineqs_zono, kact_args)
else:
total_size = 0
for varsid in kact_args:
size = 3**len(varsid) - 1
total_size = total_size + size
linexpr0 = elina_linexpr0_array_alloc(total_size)
i = 0
for varsid in kact_args:
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
i = i + 1
upper_bound = get_upper_bound_for_linexpr0(man,element,linexpr0, total_size, layerno)
i=0
input_hrep_array = []
for varsid in kact_args:
input_hrep = []
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
input_hrep.append([upper_bound[i]] + [-c for c in coeffs])
i = i + 1
input_hrep_array.append(input_hrep)
with multiprocessing.Pool(config.numproc) as pool:
kact_results = pool.map(make_kactivation_obj, input_hrep_array)
gid = 0
for inst in kact_results:
varsid = kact_args[gid]
inst.varsid = varsid
kact_cons.append(inst)
gid = gid+1
end = time.time()
if config.debug:
print('kactivation time spent: ' + str(end-start))
if domain == 'refinezono':
element = dn.remove_dimensions(man, element, offset+length, 1)
constraint_groups.append(kact_cons)
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi,
ubi, relu_groups, need_pop, domain,
activation_type):
import deepzono_nodes as dn
if (need_pop):
relu_groups.pop()
last_conv = -1
is_conv = False
for i in range(nn.numlayer):
if nn.layertypes[i] == 'Conv':
last_conv = i
is_conv = True
lbi = np.asarray(lbi, dtype=np.double)
ubi = np.asarray(ubi, dtype=np.double)
candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
candidate_vars_areas = {
var: -lbi[var] * ubi[var]
for var in candidate_vars
}
# Sort vars by descending area
candidate_vars = sorted(candidate_vars,
key=lambda var: -candidate_vars_areas[var])
# Use sparse heuristic to select args (uncomment to use)
krelu_args = sparse_heuristic_with_cutoff(candidate_vars,
candidate_vars_areas)
relucons = []
#print("UBI ",ubi)
tdim = ElinaDim(offset + length)
if domain == 'refinezono':
element = dn.add_dimensions(man, element, offset + length, 1)
#krelu_args = []
#if config.dyn_krelu and candidate_vars:
# limit3relucalls = 500
# firstk = math.sqrt(6*limit3relucalls/len(candidate_vars))
# firstk = int(min(firstk, len(candidate_vars)))
# if is_conv and layerno < last_conv:
# firstk = 1
# else:
# firstk = 5#int(max(1,firstk))
# print("firstk ",firstk)
# if firstk>3:
# while candidate_vars:
# headlen = min(firstk, len(candidate_vars))
# head = candidate_vars[:headlen]
# candidate_vars = candidate_vars[headlen:]
# if len(head)<=3:
# krelu_args.append(head)
# else:
# for arg in itertools.combinations(head, 3):
# krelu_args.append(arg)
#klist = ([3] if (config.use_3relu) else []) + ([2] if (config.use_2relu) else []) + [1]
#for k in klist:
# while len(candidate_vars) >= k:
# krelu_args.append(candidate_vars[:k])
# candidate_vars = candidate_vars[k:]
Krelu.man = man
Krelu.element = element
Krelu.tdim = tdim
Krelu.length = length
Krelu.layerno = layerno
Krelu.offset = offset
Krelu.domain = domain
start = time.time()
if domain == 'refinezono':
with multiprocessing.Pool(config.numproc) as pool:
cdd_hrepr_array = pool.map(get_ineqs_zono, krelu_args)
else:
# krelu_results = []
total_size = 0
for varsid in krelu_args:
size = 3**len(varsid) - 1
total_size = total_size + size
linexpr0 = elina_linexpr0_array_alloc(total_size)
i = 0
for varsid in krelu_args:
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
i = i + 1
upper_bound = get_upper_bound_for_linexpr0(man, element, linexpr0,
total_size, layerno)
i = 0
cdd_hrepr_array = []
for varsid in krelu_args:
cdd_hrepr = []
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
cdd_hrepr.append([upper_bound[i]] + [-c for c in coeffs])
#print("UPPER BOUND ", upper_bound[i], "COEFF ", coeffs)
#if len(varsid)>1:
# print("LB ", lbi[varsid[0]],lbi[varsid[1]], "UB ", ubi[varsid[0]], ubi[varsid[1]])
i = i + 1
cdd_hrepr_array.append(cdd_hrepr)
with multiprocessing.Pool(config.numproc) as pool:
krelu_results = pool.map(make_krelu_obj, cdd_hrepr_array)
# krelu_results.append(make_krelu_obj(krelu_args[i]))
#bound_expr_list = []
gid = 0
lower_bound_expr = {}
upper_bound_expr = {}
for krelu_inst in krelu_results:
varsid = krelu_args[gid]
krelu_inst.varsid = varsid
# For now disabling since in the experiments updating expression bounds makes results worse.
# compute_expr_bounds(krelu_inst, varsid, lower_bound_expr, upper_bound_expr, lbi, ubi)
#print("VARSID ",varsid)
#bound_expr_list.append(Krelu_expr(lower_bound_expr, upper_bound_expr, varsid))
relucons.append(krelu_inst)
gid = gid + 1
end = time.time()
if config.debug:
print('krelu time spent: ' + str(end - start))
if domain == 'refinezono':
element = dn.remove_dimensions(man, element, offset + length, 1)
relu_groups.append(relucons)
return lower_bound_expr, upper_bound_expr
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi, ubi, constraint_groups, need_pop, domain, activation_type):
import deepzono_nodes as dn
if need_pop:
constraint_groups.pop()
lbi = np.asarray(lbi, dtype=np.double)
ubi = np.asarray(ubi, dtype=np.double)
candidate_vars = [i for i in range(length) if lbi[i] < 0 < ubi[i]]
candidate_vars_areas = {var: -lbi[var] * ubi[var] for var in candidate_vars}
# Sort vars by descending area
candidate_vars = sorted(candidate_vars, key=lambda var: -candidate_vars_areas[var])
# Use sparse heuristic to select args
kact_args = sparse_heuristic_with_cutoff(candidate_vars, candidate_vars_areas)
print("krelu: n", config.sparse_n,
"splitting zero", len(candidate_vars),
"number of args", len(kact_args))
kact_cons = []
tdim = ElinaDim(offset+length)
if domain == 'refinezono':
element = dn.add_dimensions(man,element,offset+length,1)
KAct.man = man
KAct.element = element
KAct.tdim = tdim
KAct.length = length
KAct.layerno = layerno
KAct.offset = offset
KAct.domain = domain
KAct.type = activation_type
start = time.time()
if domain == 'refinezono':
with multiprocessing.Pool(config.numproc) as pool:
input_hrep_array = pool.map(get_ineqs_zono, kact_args)
else:
total_size = 0
for varsid in kact_args:
size = 3**len(varsid) - 1
total_size = total_size + size
linexpr0 = elina_linexpr0_array_alloc(total_size)
i = 0
for varsid in kact_args:
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
i = i + 1
upper_bound = get_upper_bound_for_linexpr0(man,element,linexpr0, total_size, layerno)
i=0
input_hrep_array = []
for varsid in kact_args:
input_hrep = []
for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
if all(c == 0 for c in coeffs):
continue
input_hrep.append([upper_bound[i]] + [-c for c in coeffs])
i = i + 1
input_hrep_array.append(input_hrep)
with multiprocessing.Pool(config.numproc) as pool:
kact_results = pool.map(make_kactivation_obj, input_hrep_array)
gid = 0
for inst in kact_results:
varsid = kact_args[gid]
inst.varsid = varsid
kact_cons.append(inst)
gid = gid+1
end = time.time()
if config.debug:
print('kactivation time spent: ' + str(end-start))
if domain == 'refinezono':
element = dn.remove_dimensions(man, element, offset+length, 1)
constraint_groups.append(kact_cons)