def create_difficult_pattern(size): '''The eq ids go from 0..size-1, the column ids from size..2*size-1. A pathological pattern, resulting in many ties: | x x | | x x | | x x | | x x x x | | x x x x | | x x x x | ''' assert size % 2 == 0, size rows, cols = list(irange(size)), list(irange(size, 2 * size)) g = Graph() half_size = size // 2 # build upper half for i in irange(half_size): g.add_edges_from(((i, size + 2 * i), (i, size + 2 * i + 1))) # build lower half for i in irange(half_size, size): k = 2 * (i - half_size) vrs = [size + v % size for v in irange(k, k + 4)] g.add_edges_from(izip(repeat(i), vrs)) assert is_bipartite_node_set(g, rows) assert is_bipartite_node_set(g, cols) #to_pdf(g, rows, cols, '', str(size)) #plot_dm_decomp(g, size) return g
def hessenberg(rows, cols, values, n_rows, n_cols, tie_breaking): 'Tie breaking options: MIN_FIRST, MAX_FIRST, IGNORE' assert tie_breaking in ('IGNORE', 'MIN_FIRST', 'MAX_FIRST'), tie_breaking # The col IDs in cols are shifted by n_rows, must undo later g, eqs, _ = coo_matrix_to_bipartite(rows, cols, values, (n_rows, n_cols)) if tie_breaking != 'IGNORE': # Relabel the rows such that they are ordered by weight row_weights = get_row_weights(g, n_rows) reverse = True if tie_breaking == 'MAX_FIRST' else False row_pos = argsort(row_weights, reverse) mapping = {n: i for i, n in enumerate(row_pos)} # eqs = set(mapping[eq] for eq in eqs) g = partial_relabel(g, mapping) # rperm, cperm, _, _, _, _ = to_hessenberg_form(g, eqs) # Finally, shift the colp such that it is a permutation of 0 .. n_cols-1 cperm = [c - n_rows for c in cperm] # if tie_breaking != 'IGNORE': rperm = [row_pos[r] for r in rperm] # rowp, colp = get_inverse_perm(rperm, cperm) assert sorted(rowp) == list(irange(n_rows)) assert sorted(colp) == list(irange(n_cols)) return rowp, colp
def naughty_brute_force(): for size in irange(1, 6): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin') all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) g.graph['name'] = str(i) _, _, _, tear_set, _ = bb2_solve(g, set(irange(size))) assert opt == len(tear_set) #to_pdf(g, rowp, colp) #print([t[0] for t in _worst_cases]) #print('Len:', len(_worst_cases)) #_worst_cases.sort(key=sort_patterns) # for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1): # msg = 'Index: ' + g.graph['name'] # fname = '{0:03d}a'.format(i) # to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname) # msg = 'OPT = {}, BT: {}'.format(ub, explored) # fname = '{0:03d}b'.format(i) # to_pdf(g, rowp, colp, msg, fname) #_worst_cases[:] = [ ] print_timestamp() print()
def _gen_blocks(problem): # Generates: (cblk, cons), (vblk, vrs), where cons and vrs are generators: # (int id, int blk_id), and the block ids (cblk, vblk, blk_id) are 1 based. segs = problem.segments # Get [(id, blk_id)] from the S segments con_blk_ids, var_blk_ids = segs.con_blocks, segs.var_blocks n_cons, n_vars = problem.nl_header.n_cons, problem.nl_header.n_vars # Assumptions: each con / var is in one of the blocks, the block ids are # contiguous and 1-based in the .nl, there are equally many con & var blocks # after padding with an empty row / col block at the ends if necessary assert len(con_blk_ids) == n_cons, (len(con_blk_ids), n_cons) assert len(var_blk_ids) == n_vars, (len(var_blk_ids), n_vars) def blocks(iterable): # iterable: [(id, block_id)] def by_block_id(tup): return tup[1] return _groupby(iterable, by_block_id) # constraints and variables grouped by blocks cblkid_cid, vblkid_vid = blocks(con_blk_ids), blocks(var_blk_ids) # pad with with zero rows or columns at the ends if necessary to have equal # number of blocks if cblkid_cid[0][0] == 2: # First var block has no cons, add an empty one cblkid_cid.insert(0, (1, [])) if vblkid_vid[-1][ 0] == cblkid_cid[-1][0] - 1: # Last con block has no vars vblkid_vid.append((cblkid_cid[-1][0], [])) cblks = {blk for blk, _ in cblkid_cid} assert sorted(cblks) == list(irange(1, len(cblks) + 1)), sorted(cblks) vblks = {blk for blk, _ in vblkid_vid} assert sorted(vblks) == list(irange(1, len(vblks) + 1)) assert len(cblks) == len(vblks), (len(cblks), len(vblks)) return izip(cblkid_cid, vblkid_vid)
def build_lp(g, feasible_sol): from gurobipy import LinExpr, GRB, Model #, setParam # We introduce an integer index per node ID. Sometimes we will use this # index, and sometimes the node ID; this makes the code a bit messy. i_nodeid = {i: n for i, n in enumerate(g)} n_nodes = len(i_nodeid) model = Model() # The lower half of the n^2 binary variables, the rest: y_{j,i} = 1-y_{i,j} y = {(i, j): model.addVar(vtype=GRB.BINARY) for i, j in combinations(irange(n_nodes), 2)} model.update() # The triangle inequalities for i, j, k in combinations(irange(n_nodes), 3): lhs = LinExpr([(1, y[(i, j)]), (1, y[(j, k)]), (-1, y[(i, k)])]) model.addConstr(lhs, GRB.LESS_EQUAL, 1.0) lhs = LinExpr([(-1, y[(i, j)]), (-1, y[(j, k)]), (1, y[(i, k)])]) model.addConstr(lhs, GRB.LESS_EQUAL, 0.0) # The objective shift = 0 c = [[0] * n_nodes for _ in irange(n_nodes)] indices = ((i, j) for i, j in product(irange(n_nodes), repeat=2) if g.has_edge(i_nodeid[j], i_nodeid[i])) for j, k in indices: w = g[i_nodeid[k]][i_nodeid[j]]['weight'] if k < j: c[k][j] += w elif k > j: c[j][k] -= w shift += w obj = [(c[i][j], y[(i, j)]) for i, j in combinations(irange(n_nodes), 2)] model.setObjective(LinExpr(obj), GRB.MINIMIZE) model.setAttr('ObjCon', shift) set_start_point(g, model, y, i_nodeid, feasible_sol) model.update() return model, y
def lexicographical(row_ids, col_ids, sort_rows, sort_cols): dups = duplicates(row_ids) + duplicates(col_ids) if dups: return {'error_msg': 'Duplicate identifiers: {}'.format(dups)} rpart = argsort(row_ids) if sort_rows else list(irange(len(row_ids))) cpart = argsort(col_ids) if sort_cols else list(irange(len(col_ids))) rowp, colp = get_inverse_perm(rpart, cpart) return _pack(rowp, colp)
def create_diagonal_matrix(n, rng): n_nodes = 2 * n g = nx.Graph() eqs = list(irange(n)) vrs = list(irange(n, n_nodes)) g.add_nodes_from(eqs, bipartite=0) g.add_nodes_from(vrs, bipartite=1) g.add_edges_from(zip(eqs, vrs)) return _finalize(g, n, n, rng)
def bipartite_from_empty_matrix(shape): n_rows, n_cols = shape assert n_rows >= 0 and n_cols >= 0 r_nodes = list(irange(n_rows)) cols = [] g = Graph() g.add_nodes_from(r_nodes) g.add_nodes_from(irange(n_rows, n_rows + n_cols)) assert len(g) == n_rows + n_cols return g, set(r_nodes), cols
def coo_matrix_to_bipartite(rows, cols, values, shape): check_coordinate_format(rows, cols, values, shape) # See _check_coordinate_format in rpc_api too n_rows, n_cols = shape r_nodes = list(irange(n_rows)) # relabel the columns, the caller must undo it later cols = [c + n_rows for c in cols] g = Graph() g.add_nodes_from(r_nodes) g.add_nodes_from(irange(n_rows, n_rows + n_cols)) assert len(g) == n_rows + n_cols g.add_edges_from(izip(rows, cols, ({'weight': int(v)} for v in values))) return g, set(r_nodes), cols
def solve(g, n_eqs, func): print('------------------------------------------------------------------') print('Solving problem of size', n_eqs) msg = 'Size: ' + str(n_eqs) fname = '{0:03d}a'.format(n_eqs) to_pdf(g, list(irange(n_eqs)), irange(n_eqs, 2*n_eqs), msg, fname) # res = func(g, set(irange(n_eqs))) # print('Explored', res.explored, 'nodes') msg = 'OPT = {}, BT: {}'.format(res.ub, res.explored) fname = '{0:03d}b'.format(n_eqs) to_pdf(g, res.rowp, res.colp, msg, fname) print_timestamp()
def create_coomat(n_rows, n_cols, rng): g = raw_rnd_bipartite(n_rows, n_cols, rng.randint(0, 2**32)) # rows, cols = [], [] for r in irange(n_rows): for c in g[r]: rows.append(r) cols.append(c - n_rows) # n_nonzeros = g.number_of_edges() values = [rng.randint(1, 9) for _ in irange(n_nonzeros)] for r, c, v in izip(rows, cols, values): g[r][c + n_rows]['weight'] = v # return g, rows, cols, values
def difficult(size): print('Solving patterns leading to many ties (backtracking) of size', size) msg = 'Size: ' + str(size) fname = '{0:03d}a'.format(size) g = create_difficult_pattern(size) to_pdf(g, list(irange(size)), irange(size, 2 * size), msg, fname) # solve_problem(g, set(irange(size))) # explored, g, _, rowp, colp, ub = _worst_cases[0] msg = 'OPT = {}, BT: {}'.format(ub, explored) fname = '{0:03d}b'.format(size) to_pdf(g, rowp, colp, msg, fname) _worst_cases[:] = [] print_timestamp()
def naughty_brute_force(): for size in irange(1, 7): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) opts = marshal_load('data/all_bips/opt_n' + str(size) + '.bin') all_edgelists = marshal_load('data/all_bips/bip_n' + str(size) + '.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) g.graph['name'] = str(i) _, _, _, tear_set, _ = solve_problem(g, set(irange(size))) assert opt == len(tear_set) print('Done with size', size) print_timestamp() print()
def recover_order(model, y, n_nodes): # Port of the OPL script: Admittedly ugly and inefficient # order[j in 1..n] = sum(i in 1..j-1) x[i,j] + sum(k in j+1..n) (1-x[j,k])+1; order = [-1] * n_nodes for j in irange(n_nodes): # counting the number of nodes preceeding j s = 0 for i in irange(0, j): s += int(round(y[(i, j)].X)) for i in irange(j + 1, n_nodes): s += int(round((1 - y[(j, i)].X))) order[j] = s # elim_order[k in 1..n] = sum(i in 1..n) i*(order[i]==k); print(order) elim_order = [-1] * n_nodes for k in irange(n_nodes): # the permutation realizing `order` elim_order[k] = sum(i if order[i] == k else 0 for i in irange(n_nodes)) print(elim_order) return elim_order
def naughty_brute_force(): for size in irange(3, 6): print_timestamp() print('Testing (naughty) bipartite graphs of size', size) # serialized in test_utils, but optimums can be serialized here #opts = [ ] #opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin') #all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin') opts = marshal_load('data/bip_filt/opt_n' + str(size) + '.bin') all_edgelists = marshal_load('data/bip_filt/filt_n' + str(size) + '.bin') print('Loaded', len(all_edgelists), 'graphs') print_timestamp() #for edgelist in all_edgelists: for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)): assert len(edgelist) % 2 == 0 g = Graph() g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2])) assert len(g) == 2 * size g.graph['name'] = str(i) _, _, _, tear_set, _ = solve_problem(g, set(irange(size))) assert opt == len(tear_set) #--- #solve_problem(g, set(irange(size))) #--- #opt = len(solve_problem(g, set(irange(size)))[3]) #opts.append(opt) #--- #to_pdf(g, rowp, colp) #assert len(opts) == len(all_edgelists) #marshal_dump(opts, '/tmp/opt_n'+str(size)+'.bin') print([t[0] for t in _worst_cases]) #print('Len:', len(_worst_cases)) _worst_cases.sort(key=sort_patterns) # for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1): # msg = 'Index: ' + g.graph['name'] # fname = '{0:03d}a'.format(i) # to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname) # msg = 'OPT = {}, BT: {}'.format(ub, explored) # fname = '{0:03d}b'.format(i) # to_pdf(g, rowp, colp, msg, fname) _worst_cases[:] = [] print_timestamp() print()
def test_rpc(n_rows, n_cols, seed): rng = Random(seed) # g, rows, cols, values = create_coomat(n_rows, n_cols, rng) # #print('Input:') #dbg_show_coomat(rows, cols, values, (n_rows, n_cols)) # #--------------------------------------------------------------------------- result = hessenberg(rows, cols, values, n_rows, n_cols, 'IGNORE') check(result, n_rows, n_cols, values) # #--------------------------------------------------------------------------- result = hessenberg(rows, cols, values, n_rows, n_cols, 'MIN_FIRST') compare = lambda c1, w1, c2, w2: (c1, w1) <= (c2, w2) check_hessenberg_tie_breaking(g, n_rows, n_cols, values, result, compare) # #--------------------------------------------------------------------------- result = hessenberg(rows, cols, values, n_rows, n_cols, 'MAX_FIRST') compare = lambda c1, w1, c2, w2: (c1, w2) <= (c2, w1) check_hessenberg_tie_breaking(g, n_rows, n_cols, values, result, compare) # #--------------------------------------------------------------------------- result = fine_dulmage_mendelsohn(rows, cols, values, n_rows, n_cols, upper=False, minimize=True) check(result, n_rows, n_cols, values) # #--------------------------------------------------------------------------- torn_rows, torn_cols = [], [] # if n_rows: torn_rows = rng.sample(irange(n_rows), rng.randint(0, n_rows - 1)) # if n_cols: torn_cols = rng.sample(irange(n_cols), rng.randint(0, n_cols - 1)) # result = tearing_hand_guided(rows, cols, values, n_rows, n_cols, torn_rows, torn_cols) check(result, n_rows, n_cols, values)
def set_start_point(g_orig, model, y, i_nodeid, feasible_sol): # FIXME It assumes that we only have a single SCC elims, _cost = feasible_solution( g_orig) if feasible_sol is None else feasible_sol g = g_orig.copy() g.remove_edges_from(elims) order = {n: i for i, n in enumerate(topological_sort(g))} for i, j in combinations(irange(len(i_nodeid)), 2): pos_a = order[i_nodeid[i]] pos_b = order[i_nodeid[j]] y[(i, j)].start = 0 if pos_a < pos_b else 1
def create_block_pattern(n_blocks): '''The eq ids go from 0..size-1, the column ids from size..2*size-1. A pathological pattern, resulting in many ties: | x x x | | x x x | | x x x x x | | x x x | | x x x x x | | x x x | | x x x | ''' size = 2 * n_blocks + 1 g = Graph() g.add_nodes_from(irange(2 * size)) for i in irange(0, size - 2, 2): eqs = [i, i + 1, i + 2] j = i + size vrs = [j, j + 1, j + 2] g.add_edges_from(product(eqs, vrs)) #print('Nodes:', g.nodes()) #print('Edges:', sorted(g.edges())) assert is_bipartite_node_set(g, set(irange(size))) return g, size
def fold_constants(expr_tree): # Side effect: converts all ntype.NUM to float and then back to str. tree = deepcopy(expr_tree) nodes_in_postorder = irange(1, len(tree) + 1) for n in nodes_in_postorder: fold_if_possible(tree, n) # If nothing happened, just return the input if len(tree) == len(expr_tree): return expr_tree # Undo str -> float conversion on numbers for n, d in tree.nodes_iter(data=True): if d['kind'] == ntype.NUM: d['value'] = str(d['value']) # Relabel nodes to restore invariant: node ids are 1..n in post-order return convert_node_labels_to_integers(tree, 1, 'sorted')
def _assert_block_lower_hessenberg_form(problem): # Either returns True or raises AssertionError Jrows = get_J_rowwise(problem) seen_vars = set() for (cblk, cons), (vblk, vrs) in _gen_blocks(problem): assert cblk == vblk var_set = {i for i, _ in vrs} deps = set(chain.from_iterable(Jrows[i] for i, _ in cons)) deps -= var_set deps -= seen_vars assert not deps, (problem.name, sorted(deps), cblk, cons, vrs) seen_vars |= var_set unseen_vars = set(irange(problem.nl_header.n_vars)) unseen_vars -= seen_vars assert not unseen_vars, sorted(unseen_vars) return True
def fix_empty_segments(segments_w_header): # The assumption is that each header is followed by a segment. However, # if the segment is empty, then the header is followed by an another header. new_segments_with_header = [ ] for headers, segment in as_pairs(segments_w_header): n_headers = len(headers) assert n_headers if n_headers == 1: # Everything is fine, just one header with a segment new_segments_with_header.append((headers[0], segment)) else: # Insert fake empty segments after each header segments = [tuple() for _ in irange(n_headers-1)] segments.append(segment) for h, s in izip(headers, segments): new_segments_with_header.append((h, s)) return new_segments_with_header
def to_bipart_w_weights(cols_rowwise, vals_rowwise): '''Returns the tuple of: g, eqs, mapping (a list) to undo the row permutation by weight, and the row weights in the same order as in the input. This function does not receive the row identifiers but makes up new ones: 0, 1, ..., n_rows-1.''' n_rows = len(cols_rowwise) rows = list(irange(n_rows)) row_weights = [sum(vals, 0.0) for vals in vals_rowwise] row_pos = argsort(row_weights) #print('Row weights: ', row_weights) #print('Row position:', row_pos) g = Graph() g.add_nodes_from(rows) # Empty rows are allowed (but not empty columns) # Apply row permutation row_pos edges = ((i, c) for i, r in enumerate(row_pos) for c in cols_rowwise[r]) g.add_edges_from(edges) assert is_bipartite_node_set(g, rows) # Same ID for both a col and a row? return g, set(rows), row_pos, row_weights
def plot_dm_decomp(g, size): rows, cols = [], [] for u, v in g.edges_iter(irange(size)): rows.append(u) cols.append(v - size) values = [1] * len(rows) shape = (size, size) g_dup, eqs, vrs = coo_matrix_to_bipartite(rows, cols, values, shape) assert is_bipartite_node_set(g_dup, eqs) assert is_bipartite_node_set(g_dup, vrs) from dm_decomp import blt_with_tearing msg = 'Size: {}'.format(size) blt_with_tearing(rows, cols, values, shape, [size - 1], [0], show=True, name=msg)
def try_neighborhood(sc, running_cost, elims): sccs_to_process = [ sc ] rejected = { } print('\n*** Looking for sub-SCCs that are safe to eliminate ***\n') for cutoff in irange(1, BFS_CUTOFF+1): dirty = sccs_to_process sccs_to_process = [ ] while dirty: scc = dirty.pop() progressed, cost, new_sccs, rejected_relaxations = \ try_each_node(scc, elims, cutoff) if progressed: running_cost += cost dirty.extend(new_sccs) else: rejected.update( rejected_relaxations ) sccs_to_process.append(scc) print('-----------------------------------------------------------') return sccs_to_process, running_cost
def hessenberg_to_spike(g, eqs, forbidden, rowp, colp): assert len(g) == 2 * len(eqs), 'Non-square matrix' assert maxmatch_len(g, eqs) == len(eqs), 'Structurally singular matrix' partition = _get_partition(g, rowp, colp) new_colp, stack = [], [] for rs, cs in partition: # Prefer forbidden and higher column count as spikes candids = sorted(cs, key=lambda c: ((rs[0], c) in forbidden, len(g[c]))) m, n = len(rs), len(cs) if m <= n: new_colp.extend(candids[:m]) stack.extend(candids[m:]) else: new_colp.extend(cs) for _ in irange(m - n): new_colp.append(stack.pop()) assert not stack, stack assert sorted(new_colp) == sorted(colp), new_colp #from plot_ordering import _plot_bipartite #_plot_bipartite(g, forbidden, rowp, new_colp, 'spiked') return new_colp
def digraph_to_undirected_bipartite(dig): # The system is artificially made square #plot(dig, prog='sfdp') # We will introduce fake nodes (equations) to make the system square nodeid = count() eq_id = { n : [next(nodeid) for _ in irange(dig.node[n]['weight'])] \ for n in sorted(dig) } # edgeid = count() var_id = {e: 'x%d' % next(edgeid) for e in sorted(dig.edges_iter())} # Build the bipartite graph: edges of dig become the var node set, and # each var is connected to its equation(s). g = Graph(name=dig.graph['name']) for u, v in dig.edges_iter(): var = var_id[(u, v)] append_eqs_for_var(g, eq_id[u], var) append_eqs_for_var(g, eq_id[v], var) #plot(g, prog='sfdp') eqs = set() for eq_list in six.itervalues(eq_id): eqs.update(eq_list) forbidden = set() return g, eqs, forbidden
def interpret(body, counter): segment = list(body) stack = [] evaluated = ('n', 'v', 't') while segment: token = segment.pop() kind = token[0] if kind in evaluated or token.isdigit(): stack.append(token) else: assert kind == 'o', kind assert token in NAME, token arity = ARITY[token] if not arity: # n-ary operators have their arity as the first argument arity = int(stack.pop()) assert len(stack) >= arity args = [stack.pop() for _ in irange(arity)] tmp = 't%d' % next(counter) #print(tmp, '=', NAME[token], ' '.join(args)) yield tmp, NAME[token], args stack.append(tmp) assert len(stack) == 1, stack assert stack[0][0] in evaluated, stack yield stack[0]
def random(n_rows, n_cols): rowp, colp = list(irange(n_rows)), list(irange(n_cols)) shuffle(rowp) shuffle(colp) return _pack(rowp, colp)
def gen_r_c_color(rows, cols, rowp, colp, colors): to_str = {1: 'black', 2: 'red', 3: 'gray'} for k in irange(0, len(colors)): yield rowp[rows[k]], colp[cols[k]], to_str[colors[k]]
def get_row_weights(g, n_rows): return [sum(d['weight'] for d in itervalues(g[r])) for r in irange(n_rows)]