def refine_activation_with_solver_bounds(nn, self, man, element, nlb, nub, relu_groups, timeout_lp, timeout_milp, use_default_heuristic, domain): """ refines the relu transformer Arguments --------- self : Object will be a DeepzonoNode or DeeppolyNode man : ElinaManagerPtr manager which is responsible for element element : ElinaAbstract0Ptr the element in which the results after affine transformation are stored nlb: list of list of doubles contains the lower bounds for all neurons upto layer layerno nub: list of list of doubles contains the upper bounds for all neurons upto layer layerno use_milp: bool array whether to use milp or lp for refinement Return ------ the updated abstract element """ layerno = nn.calc_layerno() predecessor_index = nn.predecessors[layerno + 1][0] - 1 if domain == 'deepzono': offset, length = self.abstract_information else: offset = 0 length = get_num_neurons_in_layer(man, element, predecessor_index) lbi = nlb[predecessor_index] ubi = nub[predecessor_index] first_FC = -1 timeout = timeout_milp for i in range(nn.numlayer): if nn.layertypes[i] == 'FC': first_FC = i break if nn.activation_counter == 0: if domain == 'deepzono': encode_kactivation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono', nn.layertypes[layerno]) if nn.layertypes[layerno] == 'ReLU': element = relu_zono_layerwise(man, True, element, offset, length, use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': element = sigmoid_zono_layerwise(man, True, element, offset, length) else: element = tanh_zono_layerwise(man, True, element, offset, length) return element else: lower_bound_expr, upper_bound_expr = encode_kactivation_cons( nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly', nn.layertypes[layerno]) if nn.layertypes[layerno] == 'ReLU': handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': handle_sigmoid_layer(*self.get_arguments(man, element)) else: handle_tanh_layer(*self.get_arguments(man, element)) #if config.refine_neurons == True: update_activation_expr_bounds(man, element, layerno, lower_bound_expr, upper_bound_expr, lbi, ubi) else: if predecessor_index == first_FC: use_milp = 1 else: use_milp = 0 timeout = timeout_lp use_milp = use_milp and config.use_milp candidate_vars = [] for i in range(length): if ((lbi[i] < 0 and ubi[i] > 0) or (lbi[i] > 0)): candidate_vars.append(i) #TODO handle residual layers here if config.refine_neurons == True: resl, resu, indices = get_bounds_for_layer_with_milp( nn, nn.specLB, nn.specUB, predecessor_index, predecessor_index, length, nlb, nub, relu_groups, use_milp, candidate_vars, timeout) nlb[predecessor_index] = resl nub[predecessor_index] = resu lbi = nlb[predecessor_index] ubi = nub[predecessor_index] if domain == 'deepzono': encode_activation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono', nn.layertypes[layerno]) if config.refine_neurons == True: j = 0 for i in range(length): if ((j < len(indices)) and (i == indices[j])): element = relu_zono_refined(man, True, element, i + offset, resl[i], resu[i]) j = j + 1 else: element = relu_zono(man, True, element, i + offset) return element else: element = relu_zono_layerwise(man, True, element, offset, length, use_default_heuristic) return element else: if config.refine_neurons == True: for j in indices: update_bounds_for_neuron(man, element, predecessor_index, j, resl[j], resu[j]) lower_bound_expr, upper_bound_expr = encode_kactivation_cons( nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly', nn.layertypes[layerno]) if nn.layertypes[layerno] == 'ReLU': handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': handle_sigmoid_layer(*self.get_arguments(man, element)) else: handle_tanh_layer(*self.get_arguments(man, element)) update_activation_expr_bounds(man, element, layerno, lower_bound_expr, upper_bound_expr, lbi, ubi)
def refine_activation_with_solver_bounds(nn, self, man, element, nlb, nub, relu_groups, timeout_lp, timeout_milp, use_default_heuristic, domain, K=3, s=-2, use_milp=False, approx=True): """ refines the relu transformer Arguments --------- self : Object will be a DeepzonoNode or DeeppolyNode man : ElinaManagerPtr manager which is responsible for element element : ElinaAbstract0Ptr the element in which the results after affine transformation are stored nlb: list of list of doubles contains the lower bounds for all neurons upto layer layerno nub: list of list of doubles contains the upper bounds for all neurons upto layer layerno use_milp: bool array whether to use milp or lp for refinement Return ------ the updated abstract element """ layerno = nn.calc_layerno() predecessor_index = nn.predecessors[layerno+1][0] - 1 if domain == 'deepzono': offset, length = self.abstract_information else: offset = 0 length = get_num_neurons_in_layer(man, element, predecessor_index) lbi = nlb[predecessor_index] ubi = nub[predecessor_index] second_FC = -2 timeout = timeout_milp affine_layers = np.array([x=="Conv" or x=="FC" for x in nn.layertypes]).nonzero()[0] for i in range(nn.numlayer): if nn.layertypes[i] == 'Conv': if second_FC == -2: second_FC = -1 if nn.layertypes[i] == 'FC': if second_FC == -2: second_FC = -1 else: second_FC = i break if nn.activation_counter==0: if domain=='deepzono': encode_kactivation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono', nn.layertypes[layerno], K=K, s=s, approx=approx) if nn.layertypes[layerno] == 'ReLU': element = relu_zono_layerwise(man,True,element,offset, length, use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': element = sigmoid_zono_layerwise(man, True, element, offset, length) else: element = tanh_zono_layerwise(man, True, element, offset, length) return element else: encode_kactivation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly', nn.layertypes[layerno], K=K, s=s, approx=approx) if nn.layertypes[layerno] == 'ReLU': handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': handle_sigmoid_layer(*self.get_arguments(man, element), use_default_heuristic) else: handle_tanh_layer(*self.get_arguments(man, element), use_default_heuristic) else: if 0<sum((affine_layers>=second_FC).__and__(predecessor_index>=affine_layers))<=config.n_milp_refine: #predecessor_index >= second_FC :#and domain=="deepzono" and predecessor_index>second_FC: use_milp_temp = use_milp else: use_milp_temp = 0 timeout = timeout_lp # use_milp = use_milp and config.use_milp candidate_vars = [] for i in range(length): if((lbi[i]<0 and ubi[i]>0) or (lbi[i]>0)): candidate_vars.append(i) #TODO handle residual layers here if config.refine_neurons==True and nn.layertypes[predecessor_index]=="FC": start = time.time() resl, resu, indices = get_bounds_for_layer_with_milp(nn, nn.specLB, nn.specUB, predecessor_index, predecessor_index, length, nlb, nub, relu_groups, use_milp_temp, candidate_vars, timeout) end = time.time() if config.debug: print(f"Refinement of bounds time: {end-start:.3f}. MILP used: {use_milp_temp}. {len(indices)} bounds refined.") nlb[predecessor_index] = resl nub[predecessor_index] = resu lbi = nlb[predecessor_index] ubi = nub[predecessor_index] if domain == 'deepzono': encode_kactivation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono', nn.layertypes[layerno]) if config.refine_neurons==True and nn.layertypes[predecessor_index]=="FC": j = 0 for i in range(length): if((j < len(indices)) and (i==indices[j])): element = relu_zono_refined(man,True, element,i+offset, resl[i],resu[i]) j=j+1 else: element = relu_zono(man,True,element,i+offset) return element else: element = relu_zono_layerwise(man,True,element,offset, length, use_default_heuristic) return element else: if config.refine_neurons and nn.layertypes[predecessor_index]=="FC": for j in indices: update_bounds_for_neuron(man,element,predecessor_index,j,resl[j],resu[j]) encode_kactivation_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly', nn.layertypes[layerno], K=K, s=s, approx=approx) if nn.layertypes[layerno] == 'ReLU': handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) elif nn.layertypes[layerno] == 'Sigmoid': handle_sigmoid_layer(*self.get_arguments(man, element), use_default_heuristic) else: handle_tanh_layer(*self.get_arguments(man, element), use_default_heuristic)