def transformer(self, nn, man, element,nlb, nub, relu_groups, refine, timeout_lp, timeout_milp, use_default_heuristic, testing): """ transforms element with ffn_matmult_zono Arguments --------- man : ElinaManagerPtr man to which element belongs element : ElinaAbstract0Ptr abstract element onto which the transformer gets applied Return ------ output : ElinaAbstract0Ptr abstract element after the transformer """ offset, old_length = self.abstract_information man, destructive, element, start_offset, weights, num_vars, expr_offset, expr_size = self.get_arguments(man, element) element = ffn_matmult_zono(man, destructive, element, start_offset, weights, self.bias, num_vars, expr_offset, expr_size) #if self.refine == 'True': # refine_after_affine(self, man, element, nlb, nub) lbi, ubi = add_bounds(man, element, nlb, nub, self.output_length, offset+old_length) # print("num candidates here ", num_candidates) if config.use_2relu or config.use_3relu or config.dyn_krelu: encode_krelu_cons(nn, man, element, start_offset, nn.ffn_counter + nn.conv_counter, num_vars, lbi, ubi, relu_groups, False, 'refinezono') else: relu_groups.append([]) nlb.append(lbi) nub.append(ubi) nn.last_layer = 'Affine' if testing: return remove_dimensions(man, element, offset, old_length), nlb[-1], nub[-1] return remove_dimensions(man, element, offset, old_length)
def calc_bounds(man, element, nn, nlb, nub, relu_groups, is_refine_layer=False, destroy=True, use_krelu=False): layerno = nn.calc_layerno() bounds = box_for_layer(man, element, layerno) num_neurons = get_num_neurons_in_layer(man, element, layerno) itv = [bounds[i] for i in range(num_neurons)] lbi = [x.contents.inf.contents.val.dbl for x in itv] ubi = [x.contents.sup.contents.val.dbl for x in itv] if is_refine_layer: nlb.append(lbi) nub.append(ubi) if use_krelu: encode_krelu_cons(nn, man, element, 0, layerno, num_neurons, lbi, ubi, relu_groups, False, 'refinepoly') if destroy: elina_interval_array_free(bounds, num_neurons) return lbi, ubi return layerno, bounds, num_neurons, lbi, ubi
def transformer(self, nn, man, element,nlb, nub, relu_groups, refine, timeout_lp, timeout_milp): """ transforms element with ffn_matmult_zono Arguments --------- man : ElinaManagerPtr man to which element belongs element : ElinaAbstract0Ptr abstract element onto which the transformer gets applied Return ------ output : ElinaAbstract0Ptr abstract element after the transformer """ offset, old_length = self.abstract_information man, destructive, element, start_offset, weights, num_vars, expr_offset, expr_size = self.get_arguments(man, element) element = ffn_matmult_zono(man, destructive, element, start_offset, weights, self.bias, num_vars, expr_offset, expr_size) #if self.refine == 'True': # refine_after_affine(self, man, element, nlb, nub) dimension = elina_abstract0_dimension(man,element) var_in_element = dimension.intdim + dimension.realdim bounds = elina_abstract0_to_box(man,element) lbi = [] ubi = [] for i in range(num_vars): inf = bounds[i+start_offset].contents.inf sup = bounds[i+start_offset].contents.sup lbi.append(inf.contents.val.dbl) ubi.append(sup.contents.val.dbl) candidate_vars = [] widths = [] for i in range(num_vars): if (lbi[i] < 0 and ubi[i] > 0): candidate_vars.append(i) widths.append(ubi[i] - lbi[i]) widths = np.asarray(widths) num_candidates = len(candidate_vars) # print("num candidates here ", num_candidates) sorted_width_indices = np.argsort(widths) encode_krelu_cons(nn, man, element, start_offset, nn.ffn_counter + nn.conv_counter, num_vars, lbi, ubi, relu_groups, False, 'refinezono') nlb.append(lbi) nub.append(ubi) elina_interval_array_free(bounds,var_in_element) nn.last_layer = 'Affine' return remove_dimensions(man, element, offset, old_length)
def transformer(self, nn, man, element, nlb, nub, relu_groups, refine, timeout_lp, timeout_milp, use_area_heuristic, testing): """ transformer for any intermediate fully connected layer with relu Arguments --------- man : ElinaManagerPtr man to which element belongs element : ElinaAbstract0Ptr abstract element onto which the transformer gets applied Return ------ output : ElinaAbstract0Ptr abstract element after the transformer """ ffn_handle_intermediate_relu_layer(man, element, *self.get_arguments(), use_area_heuristic) layerno, bounds, num_neurons, lbi, ubi = calc_bounds(man, element, nn, nlb, nub, relu_groups, is_refine_layer=True, destroy = False) candidate_vars = [i for i, (l, u) in enumerate(zip(lbi, ubi)) if l<0 and u>0] #print("lbi ", timeout_milp, "ubi ", timeout_lp) if refine: if layerno <= 1: use_milp = config.use_milp else: use_milp = 0 if use_milp: timeout = timeout_milp else: timeout = timeout_lp if nn.is_ffn(): resl, resu, indices = get_bounds_for_layer_with_milp(nn, nn.specLB, nn.specUB, layerno, layerno, num_neurons, nlb, nub, relu_groups, use_milp, candidate_vars, timeout) for j in indices: update_bounds_for_neuron(man,element,layerno,j,resl[j],resu[j]) nlb[-1] = resl nub[-1] = resu encode_krelu_cons(nn, man, element, 0, layerno, num_neurons, lbi, ubi, relu_groups, False, 'refinepoly') elina_interval_array_free(bounds,num_neurons) nn.ffn_counter+=1 if testing: return element, nlb[-1], nub[-1] return element
def transformer(self, nn, man, element, nlb, nub, relu_groups, refine, timeout_lp, timeout_milp, use_area_heuristic, testing): """ transformer for a convolutional layer, if that layer is an intermediate of the network Arguments --------- man : ElinaManagerPtr man to which element belongs element : ElinaAbstract0Ptr abstract element onto which the transformer gets applied Return ------ output : ElinaAbstract0Ptr abstract element after the transformer """ if(self.has_relu): conv_handle_intermediate_relu_layer(man, element, *self.get_arguments(), use_area_heuristic) else: conv_handle_intermediate_affine_layer(man, element, *self.get_arguments(), use_area_heuristic) layerno, bounds, num_neurons, lbi, ubi = calc_bounds(man, element, nn, nlb, nub, relu_groups, is_refine_layer=True, destroy=False) candidate_vars = [i for i, (l, u) in enumerate(zip(lbi, ubi)) if l<0 and u>0] if(refine): use_milp = config.use_milp if use_milp: timeout = timeout_milp else: timeout = timeout_lp #numconvslayers = sum('Conv2D' in l for l in nn.layertypes) #if numconvslayers-nn.conv_counter <= 1: if nn.is_ffn(): resl, resu, indices = get_bounds_for_layer_with_milp(nn, nn.specLB, nn.specUB, num_neurons, nlb, nub, relu_groups, use_milp, candidate_vars, timeout) nlb[-1] = resl nub[-1] = resu for j in indices: update_bounds_for_neuron(man,element,layerno,j,resl[j],resu[j]) encode_krelu_cons(nn, man, element, 0, layerno, num_neurons, lbi, ubi, relu_groups, False, 'refinepoly') elina_interval_array_free(bounds,num_neurons) nn.conv_counter+=1 if testing: return element, nlb[-1], nub[-1] return element
def refine_relu_with_solver_bounds(nn, self, man, element, nlb, nub, relu_groups, timeout_lp, timeout_milp, use_default_heuristic, domain): """ refines the relu transformer Arguments --------- self : Object will be a DeepzonoNode or DeeppolyNode man : ElinaManagerPtr manager which is responsible for element element : ElinaAbstract0Ptr the element in which the results after affine transformation are stored nlb: list of list of doubles contains the lower bounds for all neurons upto layer layerno nub: list of list of doubles contains the upper bounds for all neurons upto layer layerno use_milp: bool array whether to use milp or lp for refinement Return ------ the updated abstract element """ layerno = nn.calc_layerno() predecessor_index = nn.predecessors[layerno + 1][0] - 1 if domain == 'deepzono': offset, length = self.abstract_information else: offset = 0 length = get_num_neurons_in_layer(man, element, predecessor_index) lbi = nlb[predecessor_index] ubi = nub[predecessor_index] first_FC = -1 timeout = timeout_milp for i in range(nn.numlayer): if nn.layertypes[i] == 'FC': first_FC = i break if nn.activation_counter == 0: if domain == 'deepzono': encode_krelu_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono') element = relu_zono_layerwise(man, True, element, offset, length, use_default_heuristic) return element else: lower_bound_expr, upper_bound_expr = encode_krelu_cons( nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly') handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) #if config.refine_neurons == True: update_relu_expr_bounds(man, element, layerno, lower_bound_expr, upper_bound_expr, lbi, ubi) else: if predecessor_index == first_FC: use_milp = 1 else: use_milp = 0 timeout = timeout_lp use_milp = use_milp and config.use_milp candidate_vars = [] for i in range(length): if ((lbi[i] < 0 and ubi[i] > 0) or (lbi[i] > 0)): candidate_vars.append(i) #TODO handle residual layers here if config.refine_neurons == True: resl, resu, indices = get_bounds_for_layer_with_milp( nn, nn.specLB, nn.specUB, predecessor_index, predecessor_index, length, nlb, nub, relu_groups, use_milp, candidate_vars, timeout) nlb[predecessor_index] = resl nub[predecessor_index] = resu lbi = nlb[predecessor_index] ubi = nub[predecessor_index] if domain == 'deepzono': encode_krelu_cons(nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinezono') if config.refine_neurons == True: j = 0 for i in range(length): if ((j < len(indices)) and (i == indices[j])): element = relu_zono_refined(man, True, element, i + offset, resl[i], resu[i]) j = j + 1 else: element = relu_zono(man, True, element, i + offset) return element else: element = relu_zono_layerwise(man, True, element, offset, length, use_default_heuristic) return element else: if config.refine_neurons == True: for j in indices: update_bounds_for_neuron(man, element, predecessor_index, j, resl[j], resu[j]) lower_bound_expr, upper_bound_expr = encode_krelu_cons( nn, man, element, offset, predecessor_index, length, lbi, ubi, relu_groups, False, 'refinepoly') handle_relu_layer(*self.get_arguments(man, element), use_default_heuristic) update_relu_expr_bounds(man, element, layerno, lower_bound_expr, upper_bound_expr, lbi, ubi)
def refine_relu_with_solver_bounds(nn, self, man, element, nlb, nub, relu_groups, timeout_lp, timeout_milp, use_default_heuristic): """ refines the relu transformer Arguments --------- self : Object will be a DeepzonoNode, but could be any object man : ElinaManagerPtr manager which is responsible for element element : ElinaAbstract0Ptr the element in which the results after affine transformation are stored nlb: list of list of doubles contains the lower bounds for all neurons upto layer layerno nub: list of list of doubles contains the upper bounds for all neurons upto layer layerno use_milp: bool array whether to use milp or lp for refinement Return ------ the updated abstract element """ offset, length = self.abstract_information layerno = nn.calc_layerno() lbi = nlb[-1] ubi = nub[-1] if layerno==0 or nn.last_layer=='Conv2D': element = relu_zono_layerwise(man,True,element,offset, length, use_default_heuristic) else: is_conv = False timeout = timeout_milp for i in range(nn.numlayer): if nn.layertypes[i] == 'Conv2D': is_conv = True break if is_conv==True: use_milp = 1 else: if layerno<=3: use_milp = 1 else: use_milp = 0 timeout = timeout_lp use_milp = use_milp and config.use_milp candidate_vars = [] for i in range(length): if((lbi[i]<0 and ubi[i]>0) or (lbi[i]>0)): candidate_vars.append(i) #TODO handle residual layers here resl, resu, indices = get_bounds_for_layer_with_milp(nn, nn.specLB, nn.specUB, layerno, layerno, length, nlb, nub, relu_groups, use_milp, candidate_vars, timeout) nlb[-1] = resl nub[-1] = resu lbi = nlb[layerno] ubi = nub[layerno] if config.use_2relu or config.use_3relu or config.dyn_krelu: encode_krelu_cons(nn, man, element, offset, layerno, length, lbi, ubi, relu_groups, True, 'refinezono') j = 0 for i in range(length): if((j < len(indices)) and (i==indices[j])): element = relu_zono_refined(man,True, element,i+offset, resl[i],resu[i]) j=j+1 else: element = relu_zono(man,True,element,i+offset) return element