Exemple #1
0
def encode_krelu_cons(nn, man, element, offset, layerno, length, lbi, ubi,
                      relu_groups, need_pop, domain):

    if (need_pop):
        relu_groups.pop()

    lbi = np.asarray(lbi, dtype=np.double)
    ubi = np.asarray(ubi, dtype=np.double)

    candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
    candidate_vars = sorted(candidate_vars,
                            key=lambda i: ubi[i] - lbi[i],
                            reverse=True)
    candidate_vars = grouping_heuristic(candidate_vars, lbi, ubi)

    relucons = []

    tdim = ElinaDim(offset + length)
    if domain == 'refinezono':
        element = dn.add_dimensions(man, element, offset + length, 1)

    krelu_args = []

    if config.dyn_krelu and candidate_vars:
        limit3relucalls = 500
        firstk = math.sqrt(6 * limit3relucalls / len(candidate_vars))
        firstk = int(min(firstk, len(candidate_vars)))
        if (layerno < 3):
            firstk = 1
        else:
            firstk = 5  #int(max(1,firstk))
        print("firstk ", firstk)
        if firstk > 3:
            while candidate_vars:
                headlen = min(firstk, len(candidate_vars))
                head = candidate_vars[:headlen]
                candidate_vars = candidate_vars[headlen:]
                if len(head) <= 3:
                    krelu_args.append(head)
                else:
                    for arg in itertools.combinations(head, 3):
                        krelu_args.append(arg)

    klist = ([3] if
             (config.use_3relu) else []) + ([2] if
                                            (config.use_2relu) else []) + [1]

    for k in klist:
        while len(candidate_vars) >= k:
            krelu_args.append(candidate_vars[:k])
            candidate_vars = candidate_vars[k:]
    Krelu.man = man
    Krelu.element = element
    Krelu.tdim = tdim
    Krelu.length = length
    Krelu.layerno = layerno
    Krelu.offset = offset
    Krelu.domain = domain

    start = time.time()
    with multiprocessing.Pool(config.numproc) as pool:
        krelu_results = pool.map(make_krelu_obj, krelu_args)
    for krelu_inst in krelu_results:
        relucons.append(krelu_inst)
    end = time.time()

    if config.debug:
        print('krelu time spent: ' + str(end - start))
    if domain == 'refinezono':
        element = dn.remove_dimensions(man, element, offset + length, 1)

    relu_groups.append(relucons)

    return
Exemple #2
0
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi,
                            ubi, relu_groups, need_pop, domain,
                            activation_type):
    import deepzono_nodes as dn
    if (need_pop):
        relu_groups.pop()

    last_conv = -1
    is_conv = False
    for i in range(nn.numlayer):
        if nn.layertypes[i] == 'Conv':
            last_conv = i
            is_conv = True

    lbi = np.asarray(lbi, dtype=np.double)
    ubi = np.asarray(ubi, dtype=np.double)

    candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
    candidate_vars_areas = {
        var: -lbi[var] * ubi[var]
        for var in candidate_vars
    }
    # Sort vars by descending area
    candidate_vars = sorted(candidate_vars,
                            key=lambda var: -candidate_vars_areas[var])

    # Use sparse heuristic to select args (uncomment to use)
    KAct_args = sparse_heuristic_with_cutoff(candidate_vars,
                                             candidate_vars_areas)

    relucons = []
    # print("UBI ",ubi)
    tdim = ElinaDim(offset + length)
    if domain == 'refinezono':
        element = dn.add_dimensions(man, element, offset + length, 1)

    KAct.man = man
    KAct.element = element
    KAct.tdim = tdim
    KAct.length = length
    KAct.layerno = layerno
    KAct.offset = offset
    KAct.domain = domain
    KAct.type = activation_type

    # klist = ([3] if (config.use_3relu) else []) + ([2] if (config.use_2relu) else []) + [1]
    # for k in klist:
    #    while len(candidate_vars) >= k:
    #        KAct_args.append(candidate_vars[:k])
    #        candidate_vars = candidate_vars[k:]
    #KAct.man = man
    #KAct.element = element
    #KAct.tdim = tdim
    #KAct.length = length
    #KAct.layerno = layerno
    #KAct.offset = offset
    #KAct.domain = domain

    start = time.time()
    if domain == 'refinezono':
        with multiprocessing.Pool(config.numproc) as pool:
            cdd_hrepr_array = pool.map(get_ineqs_zono, KAct_args)
    else:
        #    KAct_results = []
        total_size = 0
        listCoeff = [-2, -1, 0, 1, 2]
        for varsid in KAct_args:
            if len(varsid) == 1:
                size = len(listCoeff)**len(varsid) - 1
            else:
                size = len(listCoeff)**len(varsid) - (3**len(varsid))
            total_size = total_size + size

        linexpr0 = elina_linexpr0_array_alloc(total_size)
        # HERE we should define our expressions
        i = -1
        for varsid in KAct_args:
            for coeffs in itertools.product(listCoeff, repeat=len(varsid)):
                i = i + 1
                if len(varsid) == 1:
                    if coeffs[0] == 0:
                        i = i - 1
                        continue
                else:
                    if all((abs(c) == 2 or c == 0) for c in
                           coeffs):  #ELE for redundant constraints with 2,1
                        i = i - 1
                        continue

                linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
        upper_bound = get_upper_bound_for_linexpr0(man, element, linexpr0,
                                                   total_size, layerno)
        i = -1
        cdd_hrepr_array = []
        bound_val = []
        for varsid in KAct_args:
            cdd_hrepr = []
            for coeffs in itertools.product(listCoeff, repeat=len(varsid)):
                i = i + 1
                if len(varsid) == 1:
                    if coeffs[0] == 0:
                        i = i - 1
                        continue
                else:
                    if all((abs(c) == 2 or c == 0) for c in
                           coeffs):  # ELE for redundant constraints with 2,1
                        i = i - 1
                        continue
                cdd_hrepr.append([upper_bound[i]] + [-c for c in coeffs])
                bound_val.append(upper_bound[i])  # ELE
            #     print("UPPER BOUND ", upper_bound[i], "COEFF ", coeffs)
            # if len(varsid)>1:
            #     print("LB ", lbi[varsid[0]],lbi[varsid[1]], "UB ", ubi[varsid[0]], ubi[varsid[1]])
            cdd_hrepr_array.append(cdd_hrepr)

    with multiprocessing.Pool(
            config.numproc
    ) as pool:  # here is entering in 'get_orthant_points'
        KAct_results = pool.map(make_KAct_obj, cdd_hrepr_array)  #, lbi, ubi)

    #        KAct_results.append(make_KAct_obj(KAct_args[i]))
    # bound_expr_list = []
    gid = 0
    lower_bound_expr = {}
    upper_bound_expr = {}
    for KAct_inst in KAct_results:
        varsid = KAct_args[gid]
        KAct_inst.varsid = varsid
        # For now disabling since in the experiments updating expression bounds makes results worse.
        # compute_expr_bounds(KAct_inst, varsid, lower_bound_expr, upper_bound_expr, lbi, ubi)
        # print("VARSID ",varsid)
        # bound_expr_list.append(KAct_expr(lower_bound_expr, upper_bound_expr, varsid))
        relucons.append(KAct_inst)
        gid = gid + 1
    end = time.time()

    if config.debug:
        print('KAct time spent: ' + str(end - start))
    if domain == 'refinezono':
        element = dn.remove_dimensions(man, element, offset + length, 1)

    relu_groups.append(relucons)

    return lower_bound_expr, upper_bound_expr
Exemple #3
0
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi, ubi, constraint_groups, need_pop, domain, activation_type):
    import deepzono_nodes as dn
    if need_pop:
        constraint_groups.pop()

    lbi = np.asarray(lbi, dtype=np.double)
    ubi = np.asarray(ubi, dtype=np.double)

    if activation_type == "ReLU":
        kact_args = sparse_heuristic_with_cutoff(length, lbi, ubi)
    else:
        kact_args = sparse_heuristic_curve(length, lbi, ubi, activation_type == "Sigmoid")

    kact_cons = []
    tdim = ElinaDim(offset+length)
    if domain == 'refinezono':
        element = dn.add_dimensions(man,element,offset+length,1)

    KAct.man = man
    KAct.element = element
    KAct.tdim = tdim
    KAct.length = length
    KAct.layerno = layerno
    KAct.offset = offset
    KAct.domain = domain
    KAct.type = activation_type

    start = time.time()

    if domain == 'refinezono':
        with multiprocessing.Pool(config.numproc) as pool:
            input_hrep_array = pool.map(get_ineqs_zono, kact_args)
    else:
        total_size = 0
        for varsid in kact_args:
            size = 3**len(varsid) - 1
            total_size = total_size + size

        linexpr0 = elina_linexpr0_array_alloc(total_size)
        i = 0
        for varsid in kact_args:
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue

                linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
                i = i + 1
        upper_bound = get_upper_bound_for_linexpr0(man,element,linexpr0, total_size, layerno)
        i=0
        input_hrep_array = []
        for varsid in kact_args:
            input_hrep = []
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue
                input_hrep.append([upper_bound[i]] + [-c for c in coeffs])
                i = i + 1
            input_hrep_array.append(input_hrep)

    with multiprocessing.Pool(config.numproc) as pool:
        kact_results = pool.map(make_kactivation_obj, input_hrep_array)

    gid = 0
    for inst in kact_results:
        varsid = kact_args[gid]
        inst.varsid = varsid
        kact_cons.append(inst)
        gid = gid+1
    end = time.time()

    if config.debug:
        print('kactivation time spent: ' + str(end-start))
    if domain == 'refinezono':
        element = dn.remove_dimensions(man, element, offset+length, 1)

    constraint_groups.append(kact_cons)
Exemple #4
0
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi,
                            ubi, relu_groups, need_pop, domain,
                            activation_type):
    import deepzono_nodes as dn
    if (need_pop):
        relu_groups.pop()

    last_conv = -1
    is_conv = False
    for i in range(nn.numlayer):
        if nn.layertypes[i] == 'Conv':
            last_conv = i
            is_conv = True

    lbi = np.asarray(lbi, dtype=np.double)
    ubi = np.asarray(ubi, dtype=np.double)

    candidate_vars = [i for i in range(length) if lbi[i] < 0 and ubi[i] > 0]
    candidate_vars_areas = {
        var: -lbi[var] * ubi[var]
        for var in candidate_vars
    }
    # Sort vars by descending area
    candidate_vars = sorted(candidate_vars,
                            key=lambda var: -candidate_vars_areas[var])

    # Use sparse heuristic to select args (uncomment to use)
    krelu_args = sparse_heuristic_with_cutoff(candidate_vars,
                                              candidate_vars_areas)

    relucons = []
    #print("UBI ",ubi)
    tdim = ElinaDim(offset + length)
    if domain == 'refinezono':
        element = dn.add_dimensions(man, element, offset + length, 1)

    #krelu_args = []
    #if config.dyn_krelu and candidate_vars:
    #    limit3relucalls = 500
    #    firstk = math.sqrt(6*limit3relucalls/len(candidate_vars))
    #    firstk = int(min(firstk, len(candidate_vars)))
    #    if is_conv and layerno < last_conv:
    #        firstk = 1
    #    else:
    #        firstk = 5#int(max(1,firstk))
    #    print("firstk ",firstk)
    #    if firstk>3:
    #        while candidate_vars:
    #            headlen = min(firstk, len(candidate_vars))
    #            head = candidate_vars[:headlen]
    #            candidate_vars = candidate_vars[headlen:]
    #            if len(head)<=3:
    #               krelu_args.append(head)
    #            else:
    #                for arg in itertools.combinations(head, 3):
    #                    krelu_args.append(arg)

    #klist = ([3] if (config.use_3relu) else []) + ([2] if (config.use_2relu) else []) + [1]
    #for k in klist:
    #    while len(candidate_vars) >= k:
    #        krelu_args.append(candidate_vars[:k])
    #        candidate_vars = candidate_vars[k:]
    Krelu.man = man
    Krelu.element = element
    Krelu.tdim = tdim
    Krelu.length = length
    Krelu.layerno = layerno
    Krelu.offset = offset
    Krelu.domain = domain

    start = time.time()
    if domain == 'refinezono':
        with multiprocessing.Pool(config.numproc) as pool:
            cdd_hrepr_array = pool.map(get_ineqs_zono, krelu_args)
    else:
        #    krelu_results = []
        total_size = 0
        for varsid in krelu_args:
            size = 3**len(varsid) - 1
            total_size = total_size + size

        linexpr0 = elina_linexpr0_array_alloc(total_size)
        i = 0
        for varsid in krelu_args:
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue

                linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
                i = i + 1
        upper_bound = get_upper_bound_for_linexpr0(man, element, linexpr0,
                                                   total_size, layerno)
        i = 0
        cdd_hrepr_array = []
        for varsid in krelu_args:
            cdd_hrepr = []
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue
                cdd_hrepr.append([upper_bound[i]] + [-c for c in coeffs])
                #print("UPPER BOUND ", upper_bound[i], "COEFF ", coeffs)
                #if len(varsid)>1:
                #    print("LB ", lbi[varsid[0]],lbi[varsid[1]], "UB ", ubi[varsid[0]], ubi[varsid[1]])
                i = i + 1
            cdd_hrepr_array.append(cdd_hrepr)

    with multiprocessing.Pool(config.numproc) as pool:
        krelu_results = pool.map(make_krelu_obj, cdd_hrepr_array)

    #        krelu_results.append(make_krelu_obj(krelu_args[i]))
    #bound_expr_list = []
    gid = 0
    lower_bound_expr = {}
    upper_bound_expr = {}
    for krelu_inst in krelu_results:
        varsid = krelu_args[gid]
        krelu_inst.varsid = varsid
        # For now disabling since in the experiments updating expression bounds makes results worse.
        # compute_expr_bounds(krelu_inst, varsid, lower_bound_expr, upper_bound_expr, lbi, ubi)
        #print("VARSID ",varsid)
        #bound_expr_list.append(Krelu_expr(lower_bound_expr, upper_bound_expr, varsid))
        relucons.append(krelu_inst)
        gid = gid + 1
    end = time.time()

    if config.debug:
        print('krelu time spent: ' + str(end - start))
    if domain == 'refinezono':
        element = dn.remove_dimensions(man, element, offset + length, 1)

    relu_groups.append(relucons)

    return lower_bound_expr, upper_bound_expr
Exemple #5
0
def encode_kactivation_cons(nn, man, element, offset, layerno, length, lbi, ubi, constraint_groups, need_pop, domain, activation_type):
    import deepzono_nodes as dn
    if need_pop:
        constraint_groups.pop()

    lbi = np.asarray(lbi, dtype=np.double)
    ubi = np.asarray(ubi, dtype=np.double)

    candidate_vars = [i for i in range(length) if lbi[i] < 0 < ubi[i]]
    candidate_vars_areas = {var: -lbi[var] * ubi[var] for var in candidate_vars}
    # Sort vars by descending area
    candidate_vars = sorted(candidate_vars, key=lambda var: -candidate_vars_areas[var])

    # Use sparse heuristic to select args
    kact_args = sparse_heuristic_with_cutoff(candidate_vars, candidate_vars_areas)

    print("krelu: n", config.sparse_n,
          "splitting zero", len(candidate_vars),
          "number of args", len(kact_args))

    kact_cons = []
    tdim = ElinaDim(offset+length)
    if domain == 'refinezono':
        element = dn.add_dimensions(man,element,offset+length,1)

    KAct.man = man
    KAct.element = element
    KAct.tdim = tdim
    KAct.length = length
    KAct.layerno = layerno
    KAct.offset = offset
    KAct.domain = domain
    KAct.type = activation_type

    start = time.time()

    if domain == 'refinezono':
        with multiprocessing.Pool(config.numproc) as pool:
            input_hrep_array = pool.map(get_ineqs_zono, kact_args)
    else:
        total_size = 0
        for varsid in kact_args:
            size = 3**len(varsid) - 1
            total_size = total_size + size

        linexpr0 = elina_linexpr0_array_alloc(total_size)
        i = 0
        for varsid in kact_args:
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue

                linexpr0[i] = generate_linexpr0(offset, varsid, coeffs)
                i = i + 1
        upper_bound = get_upper_bound_for_linexpr0(man,element,linexpr0, total_size, layerno)
        i=0
        input_hrep_array = []
        for varsid in kact_args:
            input_hrep = []
            for coeffs in itertools.product([-1, 0, 1], repeat=len(varsid)):
                if all(c == 0 for c in coeffs):
                    continue
                input_hrep.append([upper_bound[i]] + [-c for c in coeffs])
                i = i + 1
            input_hrep_array.append(input_hrep)

    with multiprocessing.Pool(config.numproc) as pool:
        kact_results = pool.map(make_kactivation_obj, input_hrep_array)

    gid = 0
    for inst in kact_results:
        varsid = kact_args[gid]
        inst.varsid = varsid
        kact_cons.append(inst)
        gid = gid+1
    end = time.time()

    if config.debug:
        print('kactivation time spent: ' + str(end-start))
    if domain == 'refinezono':
        element = dn.remove_dimensions(man, element, offset+length, 1)

    constraint_groups.append(kact_cons)