def create_difficult_pattern(size):
'''The eq ids go from 0..size-1, the column ids from size..2*size-1.
A pathological pattern, resulting in many ties:
| x x |
| x x |
| x x |
| x x x x |
| x x x x |
| x x x x | '''
assert size % 2 == 0, size
rows, cols = list(irange(size)), list(irange(size, 2 * size))
g = Graph()
half_size = size // 2
# build upper half
for i in irange(half_size):
g.add_edges_from(((i, size + 2 * i), (i, size + 2 * i + 1)))
# build lower half
for i in irange(half_size, size):
k = 2 * (i - half_size)
vrs = [size + v % size for v in irange(k, k + 4)]
g.add_edges_from(izip(repeat(i), vrs))
assert is_bipartite_node_set(g, rows)
assert is_bipartite_node_set(g, cols)
#to_pdf(g, rows, cols, '', str(size))
#plot_dm_decomp(g, size)
return g
def hessenberg(rows, cols, values, n_rows, n_cols, tie_breaking):
'Tie breaking options: MIN_FIRST, MAX_FIRST, IGNORE'
assert tie_breaking in ('IGNORE', 'MIN_FIRST', 'MAX_FIRST'), tie_breaking
# The col IDs in cols are shifted by n_rows, must undo later
g, eqs, _ = coo_matrix_to_bipartite(rows, cols, values, (n_rows, n_cols))
if tie_breaking != 'IGNORE':
# Relabel the rows such that they are ordered by weight
row_weights = get_row_weights(g, n_rows)
reverse = True if tie_breaking == 'MAX_FIRST' else False
row_pos = argsort(row_weights, reverse)
mapping = {n: i for i, n in enumerate(row_pos)}
#
eqs = set(mapping[eq] for eq in eqs)
g = partial_relabel(g, mapping)
#
rperm, cperm, _, _, _, _ = to_hessenberg_form(g, eqs)
# Finally, shift the colp such that it is a permutation of 0 .. n_cols-1
cperm = [c - n_rows for c in cperm]
#
if tie_breaking != 'IGNORE':
rperm = [row_pos[r] for r in rperm]
#
rowp, colp = get_inverse_perm(rperm, cperm)
assert sorted(rowp) == list(irange(n_rows))
assert sorted(colp) == list(irange(n_cols))
return rowp, colp
def naughty_brute_force():
for size in irange(1, 6):
print_timestamp()
print('Testing (naughty) bipartite graphs of size', size)
opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin')
all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin')
print('Loaded', len(all_edgelists), 'graphs')
print_timestamp()
for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)):
g = Graph()
g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2]))
g.graph['name'] = str(i)
_, _, _, tear_set, _ = bb2_solve(g, set(irange(size)))
assert opt == len(tear_set)
#to_pdf(g, rowp, colp)
#print([t[0] for t in _worst_cases])
#print('Len:', len(_worst_cases))
#_worst_cases.sort(key=sort_patterns)
# for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1):
# msg = 'Index: ' + g.graph['name']
# fname = '{0:03d}a'.format(i)
# to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname)
# msg = 'OPT = {}, BT: {}'.format(ub, explored)
# fname = '{0:03d}b'.format(i)
# to_pdf(g, rowp, colp, msg, fname)
#_worst_cases[:] = [ ]
print_timestamp()
print()
def _gen_blocks(problem):
# Generates: (cblk, cons), (vblk, vrs), where cons and vrs are generators:
# (int id, int blk_id), and the block ids (cblk, vblk, blk_id) are 1 based.
segs = problem.segments
# Get [(id, blk_id)] from the S segments
con_blk_ids, var_blk_ids = segs.con_blocks, segs.var_blocks
n_cons, n_vars = problem.nl_header.n_cons, problem.nl_header.n_vars
# Assumptions: each con / var is in one of the blocks, the block ids are
# contiguous and 1-based in the .nl, there are equally many con & var blocks
# after padding with an empty row / col block at the ends if necessary
assert len(con_blk_ids) == n_cons, (len(con_blk_ids), n_cons)
assert len(var_blk_ids) == n_vars, (len(var_blk_ids), n_vars)
def blocks(iterable):
# iterable: [(id, block_id)]
def by_block_id(tup):
return tup[1]
return _groupby(iterable, by_block_id)
# constraints and variables grouped by blocks
cblkid_cid, vblkid_vid = blocks(con_blk_ids), blocks(var_blk_ids)
# pad with with zero rows or columns at the ends if necessary to have equal
# number of blocks
if cblkid_cid[0][0] == 2: # First var block has no cons, add an empty one
cblkid_cid.insert(0, (1, []))
if vblkid_vid[-1][
0] == cblkid_cid[-1][0] - 1: # Last con block has no vars
vblkid_vid.append((cblkid_cid[-1][0], []))
cblks = {blk for blk, _ in cblkid_cid}
assert sorted(cblks) == list(irange(1, len(cblks) + 1)), sorted(cblks)
vblks = {blk for blk, _ in vblkid_vid}
assert sorted(vblks) == list(irange(1, len(vblks) + 1))
assert len(cblks) == len(vblks), (len(cblks), len(vblks))
return izip(cblkid_cid, vblkid_vid)
def build_lp(g, feasible_sol):
from gurobipy import LinExpr, GRB, Model #, setParam
# We introduce an integer index per node ID. Sometimes we will use this
# index, and sometimes the node ID; this makes the code a bit messy.
i_nodeid = {i: n for i, n in enumerate(g)}
n_nodes = len(i_nodeid)
model = Model()
# The lower half of the n^2 binary variables, the rest: y_{j,i} = 1-y_{i,j}
y = {(i, j): model.addVar(vtype=GRB.BINARY)
for i, j in combinations(irange(n_nodes), 2)}
model.update()
# The triangle inequalities
for i, j, k in combinations(irange(n_nodes), 3):
lhs = LinExpr([(1, y[(i, j)]), (1, y[(j, k)]), (-1, y[(i, k)])])
model.addConstr(lhs, GRB.LESS_EQUAL, 1.0)
lhs = LinExpr([(-1, y[(i, j)]), (-1, y[(j, k)]), (1, y[(i, k)])])
model.addConstr(lhs, GRB.LESS_EQUAL, 0.0)
# The objective
shift = 0
c = [[0] * n_nodes for _ in irange(n_nodes)]
indices = ((i, j) for i, j in product(irange(n_nodes), repeat=2)
if g.has_edge(i_nodeid[j], i_nodeid[i]))
for j, k in indices:
w = g[i_nodeid[k]][i_nodeid[j]]['weight']
if k < j:
c[k][j] += w
elif k > j:
c[j][k] -= w
shift += w
obj = [(c[i][j], y[(i, j)]) for i, j in combinations(irange(n_nodes), 2)]
model.setObjective(LinExpr(obj), GRB.MINIMIZE)
model.setAttr('ObjCon', shift)
set_start_point(g, model, y, i_nodeid, feasible_sol)
model.update()
return model, y
def lexicographical(row_ids, col_ids, sort_rows, sort_cols):
dups = duplicates(row_ids) + duplicates(col_ids)
if dups:
return {'error_msg': 'Duplicate identifiers: {}'.format(dups)}
rpart = argsort(row_ids) if sort_rows else list(irange(len(row_ids)))
cpart = argsort(col_ids) if sort_cols else list(irange(len(col_ids)))
rowp, colp = get_inverse_perm(rpart, cpart)
return _pack(rowp, colp)
def create_diagonal_matrix(n, rng):
n_nodes = 2 * n
g = nx.Graph()
eqs = list(irange(n))
vrs = list(irange(n, n_nodes))
g.add_nodes_from(eqs, bipartite=0)
g.add_nodes_from(vrs, bipartite=1)
g.add_edges_from(zip(eqs, vrs))
return _finalize(g, n, n, rng)
def bipartite_from_empty_matrix(shape):
n_rows, n_cols = shape
assert n_rows >= 0 and n_cols >= 0
r_nodes = list(irange(n_rows))
cols = []
g = Graph()
g.add_nodes_from(r_nodes)
g.add_nodes_from(irange(n_rows, n_rows + n_cols))
assert len(g) == n_rows + n_cols
return g, set(r_nodes), cols
def coo_matrix_to_bipartite(rows, cols, values, shape):
check_coordinate_format(rows, cols, values, shape)
# See _check_coordinate_format in rpc_api too
n_rows, n_cols = shape
r_nodes = list(irange(n_rows))
# relabel the columns, the caller must undo it later
cols = [c + n_rows for c in cols]
g = Graph()
g.add_nodes_from(r_nodes)
g.add_nodes_from(irange(n_rows, n_rows + n_cols))
assert len(g) == n_rows + n_cols
g.add_edges_from(izip(rows, cols, ({'weight': int(v)} for v in values)))
return g, set(r_nodes), cols
def solve(g, n_eqs, func):
print('------------------------------------------------------------------')
print('Solving problem of size', n_eqs)
msg = 'Size: ' + str(n_eqs)
fname = '{0:03d}a'.format(n_eqs)
to_pdf(g, list(irange(n_eqs)), irange(n_eqs, 2*n_eqs), msg, fname)
#
res = func(g, set(irange(n_eqs)))
#
print('Explored', res.explored, 'nodes')
msg = 'OPT = {}, BT: {}'.format(res.ub, res.explored)
fname = '{0:03d}b'.format(n_eqs)
to_pdf(g, res.rowp, res.colp, msg, fname)
print_timestamp()
def create_coomat(n_rows, n_cols, rng):
g = raw_rnd_bipartite(n_rows, n_cols, rng.randint(0, 2**32))
#
rows, cols = [], []
for r in irange(n_rows):
for c in g[r]:
rows.append(r)
cols.append(c - n_rows)
#
n_nonzeros = g.number_of_edges()
values = [rng.randint(1, 9) for _ in irange(n_nonzeros)]
for r, c, v in izip(rows, cols, values):
g[r][c + n_rows]['weight'] = v
#
return g, rows, cols, values
def difficult(size):
print('Solving patterns leading to many ties (backtracking) of size', size)
msg = 'Size: ' + str(size)
fname = '{0:03d}a'.format(size)
g = create_difficult_pattern(size)
to_pdf(g, list(irange(size)), irange(size, 2 * size), msg, fname)
#
solve_problem(g, set(irange(size)))
#
explored, g, _, rowp, colp, ub = _worst_cases[0]
msg = 'OPT = {}, BT: {}'.format(ub, explored)
fname = '{0:03d}b'.format(size)
to_pdf(g, rowp, colp, msg, fname)
_worst_cases[:] = []
print_timestamp()
def naughty_brute_force():
for size in irange(1, 7):
print_timestamp()
print('Testing (naughty) bipartite graphs of size', size)
opts = marshal_load('data/all_bips/opt_n' + str(size) + '.bin')
all_edgelists = marshal_load('data/all_bips/bip_n' + str(size) +
'.bin')
print('Loaded', len(all_edgelists), 'graphs')
print_timestamp()
for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)):
g = Graph()
g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2]))
g.graph['name'] = str(i)
_, _, _, tear_set, _ = solve_problem(g, set(irange(size)))
assert opt == len(tear_set)
print('Done with size', size)
print_timestamp()
print()
def recover_order(model, y, n_nodes):
# Port of the OPL script: Admittedly ugly and inefficient
# order[j in 1..n] = sum(i in 1..j-1) x[i,j] + sum(k in j+1..n) (1-x[j,k])+1;
order = [-1] * n_nodes
for j in irange(n_nodes): # counting the number of nodes preceeding j
s = 0
for i in irange(0, j):
s += int(round(y[(i, j)].X))
for i in irange(j + 1, n_nodes):
s += int(round((1 - y[(j, i)].X)))
order[j] = s
# elim_order[k in 1..n] = sum(i in 1..n) i*(order[i]==k);
print(order)
elim_order = [-1] * n_nodes
for k in irange(n_nodes): # the permutation realizing `order`
elim_order[k] = sum(i if order[i] == k else 0 for i in irange(n_nodes))
print(elim_order)
return elim_order
def naughty_brute_force():
for size in irange(3, 6):
print_timestamp()
print('Testing (naughty) bipartite graphs of size', size)
# serialized in test_utils, but optimums can be serialized here
#opts = [ ]
#opts = marshal_load('data/all_bips/opt_n'+str(size)+'.bin')
#all_edgelists = marshal_load('data/all_bips/bip_n'+str(size)+'.bin')
opts = marshal_load('data/bip_filt/opt_n' + str(size) + '.bin')
all_edgelists = marshal_load('data/bip_filt/filt_n' + str(size) +
'.bin')
print('Loaded', len(all_edgelists), 'graphs')
print_timestamp()
#for edgelist in all_edgelists:
for i, (edgelist, opt) in enumerate(izip(all_edgelists, opts)):
assert len(edgelist) % 2 == 0
g = Graph()
g.add_edges_from(e for e in izip(edgelist[::2], edgelist[1::2]))
assert len(g) == 2 * size
g.graph['name'] = str(i)
_, _, _, tear_set, _ = solve_problem(g, set(irange(size)))
assert opt == len(tear_set)
#---
#solve_problem(g, set(irange(size)))
#---
#opt = len(solve_problem(g, set(irange(size)))[3])
#opts.append(opt)
#---
#to_pdf(g, rowp, colp)
#assert len(opts) == len(all_edgelists)
#marshal_dump(opts, '/tmp/opt_n'+str(size)+'.bin')
print([t[0] for t in _worst_cases])
#print('Len:', len(_worst_cases))
_worst_cases.sort(key=sort_patterns)
# for i, (explored, g, _, rowp, colp, ub) in enumerate(_worst_cases, 1):
# msg = 'Index: ' + g.graph['name']
# fname = '{0:03d}a'.format(i)
# to_pdf(g, list(irange(size)), irange(size, 2*size), msg, fname)
# msg = 'OPT = {}, BT: {}'.format(ub, explored)
# fname = '{0:03d}b'.format(i)
# to_pdf(g, rowp, colp, msg, fname)
_worst_cases[:] = []
print_timestamp()
print()
def test_rpc(n_rows, n_cols, seed):
rng = Random(seed)
#
g, rows, cols, values = create_coomat(n_rows, n_cols, rng)
#
#print('Input:')
#dbg_show_coomat(rows, cols, values, (n_rows, n_cols))
#
#---------------------------------------------------------------------------
result = hessenberg(rows, cols, values, n_rows, n_cols, 'IGNORE')
check(result, n_rows, n_cols, values)
#
#---------------------------------------------------------------------------
result = hessenberg(rows, cols, values, n_rows, n_cols, 'MIN_FIRST')
compare = lambda c1, w1, c2, w2: (c1, w1) <= (c2, w2)
check_hessenberg_tie_breaking(g, n_rows, n_cols, values, result, compare)
#
#---------------------------------------------------------------------------
result = hessenberg(rows, cols, values, n_rows, n_cols, 'MAX_FIRST')
compare = lambda c1, w1, c2, w2: (c1, w2) <= (c2, w1)
check_hessenberg_tie_breaking(g, n_rows, n_cols, values, result, compare)
#
#---------------------------------------------------------------------------
result = fine_dulmage_mendelsohn(rows,
cols,
values,
n_rows,
n_cols,
upper=False,
minimize=True)
check(result, n_rows, n_cols, values)
#
#---------------------------------------------------------------------------
torn_rows, torn_cols = [], []
#
if n_rows:
torn_rows = rng.sample(irange(n_rows), rng.randint(0, n_rows - 1))
#
if n_cols:
torn_cols = rng.sample(irange(n_cols), rng.randint(0, n_cols - 1))
#
result = tearing_hand_guided(rows, cols, values, n_rows, n_cols, torn_rows,
torn_cols)
check(result, n_rows, n_cols, values)
def set_start_point(g_orig, model, y, i_nodeid, feasible_sol):
# FIXME It assumes that we only have a single SCC
elims, _cost = feasible_solution(
g_orig) if feasible_sol is None else feasible_sol
g = g_orig.copy()
g.remove_edges_from(elims)
order = {n: i for i, n in enumerate(topological_sort(g))}
for i, j in combinations(irange(len(i_nodeid)), 2):
pos_a = order[i_nodeid[i]]
pos_b = order[i_nodeid[j]]
y[(i, j)].start = 0 if pos_a < pos_b else 1
def create_block_pattern(n_blocks):
'''The eq ids go from 0..size-1, the column ids from size..2*size-1.
A pathological pattern, resulting in many ties:
| x x x |
| x x x |
| x x x x x |
| x x x |
| x x x x x |
| x x x |
| x x x | '''
size = 2 * n_blocks + 1
g = Graph()
g.add_nodes_from(irange(2 * size))
for i in irange(0, size - 2, 2):
eqs = [i, i + 1, i + 2]
j = i + size
vrs = [j, j + 1, j + 2]
g.add_edges_from(product(eqs, vrs))
#print('Nodes:', g.nodes())
#print('Edges:', sorted(g.edges()))
assert is_bipartite_node_set(g, set(irange(size)))
return g, size
def fold_constants(expr_tree):
# Side effect: converts all ntype.NUM to float and then back to str.
tree = deepcopy(expr_tree)
nodes_in_postorder = irange(1, len(tree) + 1)
for n in nodes_in_postorder:
fold_if_possible(tree, n)
# If nothing happened, just return the input
if len(tree) == len(expr_tree):
return expr_tree
# Undo str -> float conversion on numbers
for n, d in tree.nodes_iter(data=True):
if d['kind'] == ntype.NUM:
d['value'] = str(d['value'])
# Relabel nodes to restore invariant: node ids are 1..n in post-order
return convert_node_labels_to_integers(tree, 1, 'sorted')
def _assert_block_lower_hessenberg_form(problem):
# Either returns True or raises AssertionError
Jrows = get_J_rowwise(problem)
seen_vars = set()
for (cblk, cons), (vblk, vrs) in _gen_blocks(problem):
assert cblk == vblk
var_set = {i for i, _ in vrs}
deps = set(chain.from_iterable(Jrows[i] for i, _ in cons))
deps -= var_set
deps -= seen_vars
assert not deps, (problem.name, sorted(deps), cblk, cons, vrs)
seen_vars |= var_set
unseen_vars = set(irange(problem.nl_header.n_vars))
unseen_vars -= seen_vars
assert not unseen_vars, sorted(unseen_vars)
return True
def fix_empty_segments(segments_w_header):
# The assumption is that each header is followed by a segment. However,
# if the segment is empty, then the header is followed by an another header.
new_segments_with_header = [ ]
for headers, segment in as_pairs(segments_w_header):
n_headers = len(headers)
assert n_headers
if n_headers == 1:
# Everything is fine, just one header with a segment
new_segments_with_header.append((headers[0], segment))
else:
# Insert fake empty segments after each header
segments = [tuple() for _ in irange(n_headers-1)]
segments.append(segment)
for h, s in izip(headers, segments):
new_segments_with_header.append((h, s))
return new_segments_with_header
def to_bipart_w_weights(cols_rowwise, vals_rowwise):
'''Returns the tuple of: g, eqs, mapping (a list) to undo the row
permutation by weight, and the row weights in the same order as in the
input. This function does not receive the row identifiers but makes up
new ones: 0, 1, ..., n_rows-1.'''
n_rows = len(cols_rowwise)
rows = list(irange(n_rows))
row_weights = [sum(vals, 0.0) for vals in vals_rowwise]
row_pos = argsort(row_weights)
#print('Row weights: ', row_weights)
#print('Row position:', row_pos)
g = Graph()
g.add_nodes_from(rows) # Empty rows are allowed (but not empty columns)
# Apply row permutation row_pos
edges = ((i, c) for i, r in enumerate(row_pos) for c in cols_rowwise[r])
g.add_edges_from(edges)
assert is_bipartite_node_set(g, rows) # Same ID for both a col and a row?
return g, set(rows), row_pos, row_weights
def plot_dm_decomp(g, size):
rows, cols = [], []
for u, v in g.edges_iter(irange(size)):
rows.append(u)
cols.append(v - size)
values = [1] * len(rows)
shape = (size, size)
g_dup, eqs, vrs = coo_matrix_to_bipartite(rows, cols, values, shape)
assert is_bipartite_node_set(g_dup, eqs)
assert is_bipartite_node_set(g_dup, vrs)
from dm_decomp import blt_with_tearing
msg = 'Size: {}'.format(size)
blt_with_tearing(rows,
cols,
values,
shape, [size - 1], [0],
show=True,
name=msg)
def try_neighborhood(sc, running_cost, elims):
sccs_to_process = [ sc ]
rejected = { }
print('\n*** Looking for sub-SCCs that are safe to eliminate ***\n')
for cutoff in irange(1, BFS_CUTOFF+1):
dirty = sccs_to_process
sccs_to_process = [ ]
while dirty:
scc = dirty.pop()
progressed, cost, new_sccs, rejected_relaxations = \
try_each_node(scc, elims, cutoff)
if progressed:
running_cost += cost
dirty.extend(new_sccs)
else:
rejected.update( rejected_relaxations )
sccs_to_process.append(scc)
print('-----------------------------------------------------------')
return sccs_to_process, running_cost
def hessenberg_to_spike(g, eqs, forbidden, rowp, colp):
assert len(g) == 2 * len(eqs), 'Non-square matrix'
assert maxmatch_len(g, eqs) == len(eqs), 'Structurally singular matrix'
partition = _get_partition(g, rowp, colp)
new_colp, stack = [], []
for rs, cs in partition:
# Prefer forbidden and higher column count as spikes
candids = sorted(cs,
key=lambda c: ((rs[0], c) in forbidden, len(g[c])))
m, n = len(rs), len(cs)
if m <= n:
new_colp.extend(candids[:m])
stack.extend(candids[m:])
else:
new_colp.extend(cs)
for _ in irange(m - n):
new_colp.append(stack.pop())
assert not stack, stack
assert sorted(new_colp) == sorted(colp), new_colp
#from plot_ordering import _plot_bipartite
#_plot_bipartite(g, forbidden, rowp, new_colp, 'spiked')
return new_colp
def digraph_to_undirected_bipartite(dig):
# The system is artificially made square
#plot(dig, prog='sfdp')
# We will introduce fake nodes (equations) to make the system square
nodeid = count()
eq_id = { n : [next(nodeid) for _ in irange(dig.node[n]['weight'])] \
for n in sorted(dig) }
#
edgeid = count()
var_id = {e: 'x%d' % next(edgeid) for e in sorted(dig.edges_iter())}
# Build the bipartite graph: edges of dig become the var node set, and
# each var is connected to its equation(s).
g = Graph(name=dig.graph['name'])
for u, v in dig.edges_iter():
var = var_id[(u, v)]
append_eqs_for_var(g, eq_id[u], var)
append_eqs_for_var(g, eq_id[v], var)
#plot(g, prog='sfdp')
eqs = set()
for eq_list in six.itervalues(eq_id):
eqs.update(eq_list)
forbidden = set()
return g, eqs, forbidden
def interpret(body, counter):
segment = list(body)
stack = []
evaluated = ('n', 'v', 't')
while segment:
token = segment.pop()
kind = token[0]
if kind in evaluated or token.isdigit():
stack.append(token)
else:
assert kind == 'o', kind
assert token in NAME, token
arity = ARITY[token]
if not arity: # n-ary operators have their arity as the first argument
arity = int(stack.pop())
assert len(stack) >= arity
args = [stack.pop() for _ in irange(arity)]
tmp = 't%d' % next(counter)
#print(tmp, '=', NAME[token], ' '.join(args))
yield tmp, NAME[token], args
stack.append(tmp)
assert len(stack) == 1, stack
assert stack[0][0] in evaluated, stack
yield stack[0]
def random(n_rows, n_cols):
rowp, colp = list(irange(n_rows)), list(irange(n_cols))
shuffle(rowp)
shuffle(colp)
return _pack(rowp, colp)
def gen_r_c_color(rows, cols, rowp, colp, colors):
to_str = {1: 'black', 2: 'red', 3: 'gray'}
for k in irange(0, len(colors)):
yield rowp[rows[k]], colp[cols[k]], to_str[colors[k]]
def get_row_weights(g, n_rows):
return [sum(d['weight'] for d in itervalues(g[r])) for r in irange(n_rows)]
