def naughty_brute_force(): for size in irange(1, 6): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin') all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) g.graph['name'] = str(i) _, _, _, tear_set, _ = bb2_solve(g, set(irange(size))) assert opt == len(tear_set) #to_pdf(g, rowp, colp) #print([t[0] for t in _worst_cases]) #print('Len:', len(_worst_cases)) #_worst_cases.sort(key=sort_patterns) # for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1): # msg = 'Index: ' + g.graph['name'] # fname = '{0:03d}a'.format(i) # to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname) # msg = 'OPT = {}, BT: {}'.format(ub, explored) # fname = '{0:03d}b'.format(i) # to_pdf(g, rowp, colp, msg, fname) #_worst_cases[:] = [ ] print_timestamp() print()
def to_bipartite_from_test_string(mat_str): # Unaware of the optional opt in the tuple (dm_decomp does not have opt) rows = mat_str[0].split() cols_rowwise = [line.split() for line in mat_str[1].splitlines()] # check rows for typos eqs = set(rows) assert len(eqs) == len(rows), (sorted(eqs), sorted(rows)) assert len(rows) == len(cols_rowwise) # check cols for typos all_cols = set(chain.from_iterable(cols for cols in cols_rowwise)) both_row_and_col = sorted(eqs & all_cols) assert not both_row_and_col, both_row_and_col # check cols for duplicates for r, cols in izip(rows, cols_rowwise): dups = duplicates(cols) assert not dups, 'Duplicate column IDs {} in row {}'.format(dups, r) #print(rows) #print(cols_rowwise) g = nx.Graph() g.add_nodes_from(rows) g.add_nodes_from(all_cols) g.add_edges_from( (r, c) for r, cols in izip(rows, cols_rowwise) for c in cols) assert is_bipartite_node_set(g, eqs) return g, eqs
def get_t_profiles(graphs, sols): # cost, rowp = sols[i] list_g_rowp = list(izip(graphs, (s[1] for s in sols))) t_profiles = [t_profile(subg, rowp) for subg, rowp in list_g_rowp] assert len(graphs) == len(t_profiles) eq_costs = [profile[-1] for profile in t_profiles] assert all(s[0] == eq_cost[1] for s, eq_cost in izip(sols, eq_costs)) return t_profiles
def get_t_profiles(graphs, sols): # The t profile is the number of torn variables when the equation was # eliminated, listed in the order of the elimination. # cost, rowp = sols[i] list_g_rowp = list(izip(graphs, (s[1] for s in sols))) t_profiles = [t_profile(subg, rowp) for subg, rowp in list_g_rowp] assert len(graphs) == len(t_profiles) eq_costs = [profile[-1] for profile in t_profiles] assert all(s[0] == eq_cost[1] for s, eq_cost in izip(sols, eq_costs)) return t_profiles
def solve_pieces(g, remaining_eqs, seen_asm, indent): graphs = list(connected_component_subgraphs(g)) subeqs = [{n for n in subg if n in remaining_eqs} for subg in graphs] #print(subeqs) assert len(graphs) > 1 sols = list(_solve_problem(subg, seqs, seen_asm, indent=indent) for subg, seqs in izip(graphs, subeqs)) # cost, rowp = sols[i] total_cost = sum(t[0] for t in sols) c_minus_r = [len(subg)-2*len(seqs) for subg, seqs in izip(graphs, subeqs)] keys = [(c_m_r, cost, min(seqs)) for c_m_r, (cost, _), seqs in izip(c_minus_r, sols, subeqs)] elims = list(chain.from_iterable(sols[i][1] for i in argsort(keys))) return total_cost, elims, get_t_profiles(graphs, sols)
def __add_arities(operators, arity): opcodes = operators[::2] dups = duplicates(opcodes) assert not dups, dups already_added = set(opcodes) & set(ARITY) assert not already_added, already_added ARITY.update(izip(opcodes, repeat(arity)))
def create_difficult_pattern(size): '''The eq ids go from 0..size-1, the column ids from size..2*size-1. A pathological pattern, resulting in many ties: | x x | | x x | | x x | | x x x x | | x x x x | | x x x x | ''' assert size % 2 == 0, size rows, cols = list(irange(size)), list(irange(size, 2 * size)) g = Graph() half_size = size // 2 # build upper half for i in irange(half_size): g.add_edges_from(((i, size + 2 * i), (i, size + 2 * i + 1))) # build lower half for i in irange(half_size, size): k = 2 * (i - half_size) vrs = [size + v % size for v in irange(k, k + 4)] g.add_edges_from(izip(repeat(i), vrs)) assert is_bipartite_node_set(g, rows) assert is_bipartite_node_set(g, cols) #to_pdf(g, rows, cols, '', str(size)) #plot_dm_decomp(g, size) return g
def _gen_blocks(problem): # Generates: (cblk, cons), (vblk, vrs), where cons and vrs are generators: # (int id, int blk_id), and the block ids (cblk, vblk, blk_id) are 1 based. segs = problem.segments # Get [(id, blk_id)] from the S segments con_blk_ids, var_blk_ids = segs.con_blocks, segs.var_blocks n_cons, n_vars = problem.nl_header.n_cons, problem.nl_header.n_vars # Assumptions: each con / var is in one of the blocks, the block ids are # contiguous and 1-based in the .nl, there are equally many con & var blocks # after padding with an empty row / col block at the ends if necessary assert len(con_blk_ids) == n_cons, (len(con_blk_ids), n_cons) assert len(var_blk_ids) == n_vars, (len(var_blk_ids), n_vars) def blocks(iterable): # iterable: [(id, block_id)] def by_block_id(tup): return tup[1] return _groupby(iterable, by_block_id) # constraints and variables grouped by blocks cblkid_cid, vblkid_vid = blocks(con_blk_ids), blocks(var_blk_ids) # pad with with zero rows or columns at the ends if necessary to have equal # number of blocks if cblkid_cid[0][0] == 2: # First var block has no cons, add an empty one cblkid_cid.insert(0, (1, [])) if vblkid_vid[-1][ 0] == cblkid_cid[-1][0] - 1: # Last con block has no vars vblkid_vid.append((cblkid_cid[-1][0], [])) cblks = {blk for blk, _ in cblkid_cid} assert sorted(cblks) == list(irange(1, len(cblks) + 1)), sorted(cblks) vblks = {blk for blk, _ in vblkid_vid} assert sorted(vblks) == list(irange(1, len(vblks) + 1)) assert len(cblks) == len(vblks), (len(cblks), len(vblks)) return izip(cblkid_cid, vblkid_vid)
def pairwise(iterable): '''A generator object is returned. [] pairwise: [] [1] pairwise: [] [1,2,3] pairwise: [(1, 2), (2, 3)].''' a, b = itertools.tee(iterable) next(b, None) return izip(a, b)
def add_linear(g, lin_part, t_counter): indices, coefficients = lin_part nzero_coeffs = tuple(c for c in coefficients if c != '0') if not nzero_coeffs: return children = ['v%d' % i for i, c in izip(indices, coefficients) if c != '0'] return add_with_children(g, 't%d' % next(t_counter), ntype.lin_comb, children, coeffs=nzero_coeffs)
def naughty_brute_force(): for size in irange(1, 7): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) opts = marshal_load('data/all_bips/opt_n' + str(size) + '.bin') all_edgelists = marshal_load('data/all_bips/bip_n' + str(size) + '.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) g.graph['name'] = str(i) _, _, _, tear_set, _ = solve_problem(g, set(irange(size))) assert opt == len(tear_set) print('Done with size', size) print_timestamp() print()
def naughty_brute_force(): for size in irange(3, 6): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) # serialized in test_utils, but optimums can be serialized here #opts = [ ] #opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin') #all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin') opts = marshal_load('data/bip_filt/opt_n' + str(size) + '.bin') all_edgelists = marshal_load('data/bip_filt/filt_n' + str(size) + '.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() #for edgelist in all_edgelists: for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): assert len(edgelist) % 2 == 0 g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) assert len(g) == 2 * size g.graph['name'] = str(i) _, _, _, tear_set, _ = solve_problem(g, set(irange(size))) assert opt == len(tear_set) #--- #solve_problem(g, set(irange(size))) #--- #opt = len(solve_problem(g, set(irange(size)))[3]) #opts.append(opt) #--- #to_pdf(g, rowp, colp) #assert len(opts) == len(all_edgelists) #marshal_dump(opts, '/tmp/opt_n'+str(size)+'.bin') print([t[0] for t in _worst_cases]) #print('Len:', len(_worst_cases)) _worst_cases.sort(key=sort_patterns) # for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1): # msg = 'Index: ' + g.graph['name'] # fname = '{0:03d}a'.format(i) # to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname) # msg = 'OPT = {}, BT: {}'.format(ub, explored) # fname = '{0:03d}b'.format(i) # to_pdf(g, rowp, colp, msg, fname) _worst_cases[:] = [] print_timestamp() print()
def solve_pieces(g, remaining_eqs, seen_asm, indent): graphs = list(connected_component_subgraphs(g)) subeqs = [{n for n in subg if n in remaining_eqs} for subg in graphs] #print(subeqs) assert len(graphs) > 1 # Recursion here: sols = list(solve_problem_impl(subg, seqs, seen_asm, indent=indent+1) \ for subg, seqs in izip(graphs, subeqs)) # cost, rowp = sols[i] total_cost = sum(t[0] for t in sols) # We chain together the pieces again, we order the pieces in such a way that # the pieces are "pushed" towards the bottom left corner of the matrix. # c_minus_r = (number of columns) - (number of rows) # Ties broken as follows: easy first, then lower row label first. # Note that this sorting is not necessary for correctness; it is for # esthetic reasons only. See notes Jan 07, 2016. c_minus_r = [len(subg)-2*len(seqs) for subg, seqs in izip(graphs, subeqs)] keys = [(c_m_r, cost, min(seqs)) for c_m_r, (cost, _), seqs in izip(c_minus_r, sols, subeqs)] elims = list(chain.from_iterable(sols[i][1] for i in argsort(keys))) # The t profile is the number of torn variables when the equation was # eliminated, listed in the order of the elimination. # Compare also with apply_exclusion_rule_on_pieces return total_cost, elims, get_t_profiles(graphs, sols)
def create_coomat(n_rows, n_cols, rng): g = raw_rnd_bipartite(n_rows, n_cols, rng.randint(0, 2**32)) # rows, cols = [], [] for r in irange(n_rows): for c in g[r]: rows.append(r) cols.append(c - n_rows) # n_nonzeros = g.number_of_edges() values = [rng.randint(1, 9) for _ in irange(n_nonzeros)] for r, c, v in izip(rows, cols, values): g[r][c + n_rows]['weight'] = v # return g, rows, cols, values
def recompute_tears_matching(g, eqs, forbidden, rowp, colp): # rowp and colp must belong to a valid spiked form assert len(rowp) == len(colp) tears, match = [], {} r_index = {name: i for i, name in enumerate(n for n in rowp)} first_elem = [min(r_index[r] for r in g[c]) for c in colp] for i, (r, c) in enumerate(izip(rowp, colp)): first_nonzero = first_elem[i] assert first_nonzero <= i, (first_nonzero, i ) # on or above the diagonal above_diagonal = first_nonzero < i if above_diagonal or (r, c) in forbidden: tears.append(c) else: # on the diagonal and allowed match[r] = c return tears, match
def fix_accumulating_error(index, x, r, sol, var_ids, var_order): # Sanity check first: The subproblem's solution x_sol must be sane! indices = [var_order[v] for v in var_ids] x_sol = x[indices] assert np.isfinite(x_sol).all(), (x_sol, x) # VA27 failed? # Do the actual work: fix the accumulating error by setting the values in # x to their true values, retrieved from sol. It should not change too much. x_before = np.copy(x) for i, v in izip(indices, var_ids): x[i] = sol[v] if not np.isclose(x_before, x, equal_nan=True).all(): warning( 'Significant deviation from the solution! Multiplicity or singularity?' ) warning(x_before) warning(x) evaluate(index, x, r)
def fix_empty_segments(segments_w_header): # The assumption is that each header is followed by a segment. However, # if the segment is empty, then the header is followed by an another header. new_segments_with_header = [ ] for headers, segment in as_pairs(segments_w_header): n_headers = len(headers) assert n_headers if n_headers == 1: # Everything is fine, just one header with a segment new_segments_with_header.append((headers[0], segment)) else: # Insert fake empty segments after each header segments = [tuple() for _ in irange(n_headers-1)] segments.append(segment) for h, s in izip(headers, segments): new_segments_with_header.append((h, s)) return new_segments_with_header
def lin_comb(g, n): args = tuple(g.predecessors(n)) coeffs = g.node[n]['coeffs'] assert len(args) == len(coeffs) terms = [] for v_i, c in izip(args, coeffs): if c == '1': v = ' + %s' % v_i elif c == '-1': v = ' - %s' % v_i elif c == '0': v = '' else: v = ' + %s*%s' % ('(%s)' % c if c[0] == '-' else c, v_i) terms.append(v) PLUS_SIGN = ' + ' if terms[0].startswith(PLUS_SIGN): terms[0] = terms[0][len(PLUS_SIGN):] return ''.join(terms)
def lin_comb(g, n, seen): # t_i = sum c_k t_k # for each k: # u_k (+)= c_k * u_i args = tuple(g.predecessors(n)) coeffs = g.node[n]['coeffs'] assert len(args) == len(coeffs) u_i = get_u(n) assert u_i in seen, u_i for t_k, c_k in izip(args, coeffs): if t_k[0] == 'n': continue u_k = get_u(t_k) if c_k == '1.0': rhs = u_i elif c_k == '-1.0': rhs = '-' + u_i elif c_k == '0.0': rhs = '0.0' else: rhs = '%s * %s' % ('(%s)' % c_k if c_k[0] == '-' else c_k, u_i) _assign(u_k, seen, rhs)
def as_pairs(itr): # [1, 2, 3, 4] -> (1, 2), (3, 4) return izip(islice(itr, 0, None, 2), islice(itr, 1, None, 2))