def __init__(self, graph):
self.backend = pylp.GurobiBackend()
#self.backend = pylp.ScipBackend()
self.backend.initialize(graph.num_nodes, pylp.VariableType.Binary)
self.objective = pylp.LinearObjective(graph.num_nodes)
for n in range(graph.num_nodes):
self.objective.set_coefficient(n, graph.costs[n])
self.backend.set_objective(self.objective)
self.constraints = pylp.LinearConstraints()
for conflict in graph.conflicts:
constraint = pylp.LinearConstraint()
for n in conflict:
constraint.set_coefficient(n, 1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
self.constraints.add(constraint)
for equal_sum_constraint in graph.equal_sum_constraints:
nodes1 = equal_sum_constraint[0]
nodes2 = equal_sum_constraint[1]
constraint = pylp.LinearConstraint()
for n in nodes1:
constraint.set_coefficient(n, 1)
for n in nodes2:
constraint.set_coefficient(n, -1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(0)
self.constraints.add(constraint)
self.backend.set_constraints(self.constraints)
def __init__(self, graph):
assert (graph.shape[0] == graph.shape[1])
self.num_nodes = graph.shape[0]
# lookup tables from ILP vars to edges and vice versa
self.var_to_edge = {}
self.edge_to_var = {}
var = 0
for (u, v) in zip(graph.nonzero()[0], graph.nonzero()[1]):
self.var_to_edge[var] = (u, v)
self.edge_to_var[(u, v)] = var
var += 1
# initialze ILP solver and problem
self.num_variables = var
self.ilp_solver = pylp.GurobiBackend()
self.ilp_solver.initialize(self.num_variables,
pylp.VariableType.Binary)
self.objective = pylp.LinearObjective(self.num_variables)
self.constraints = pylp.LinearConstraints()
self.graph = graph
self.min_cost = 0
for i in range(self.num_variables):
(u, v) = self.var_to_edge[i]
cost = graph[u, v]
self.min_cost = min(cost, self.min_cost)
self.objective.set_coefficient(i, cost)
self.ilp_solver.set_objective(self.objective)
def _create_solver(self):
self.solver = pylp.LinearSolver(self.num_vars,
pylp.VariableType.Binary,
preference=pylp.Preference.Gurobi)
self.solver.set_objective(self.objective)
all_constraints = pylp.LinearConstraints()
for c in self.main_constraints + self.pin_constraints:
all_constraints.add(c)
self.solver.set_constraints(all_constraints)
self.solver.set_num_threads(1)
self.solver.set_timeout(120)
def update_objective(self, parameters, selected_key):
self.parameters = parameters
self.selected_key = selected_key
self._set_objective()
self.solver.set_objective(self.objective)
self.pinned_edges = {}
self.pin_constraints = []
self._add_pin_constraints()
all_constraints = pylp.LinearConstraints()
for c in self.main_constraints + self.pin_constraints:
all_constraints.add(c)
self.solver.set_constraints(all_constraints)
def gurobi_installed_with_license():
try:
solver = pylp.create_linear_solver(pylp.Preference.Gurobi)
solver.initialize(1, pylp.VariableType.Binary)
objective = pylp.LinearObjective(1)
objective.set_coefficient(1, 1)
solver.set_objective(objective)
constraints = pylp.LinearConstraints()
solver.set_constraints(constraints)
solution, message = solver.solve()
success = True
except RuntimeError:
success = False
return pytest.mark.skipif(not success, reason="Requires Gurobi License")
def match(costs):
n = costs.shape[0]
m = costs.shape[1]
num_variables = n * m
objective = pylp.LinearObjective(num_variables)
for i in range(n):
for j in range(m):
objective.set_coefficient(i * m + j, costs[i, j])
constraints = pylp.LinearConstraints()
for i in range(n):
sum_to_one = pylp.LinearConstraint()
for j in range(m):
sum_to_one.set_coefficient(i * m + j, 1.0)
sum_to_one.set_relation(pylp.Relation.LessEqual)
sum_to_one.set_value(1.0)
constraints.add(sum_to_one)
for j in range(m):
sum_to_one = pylp.LinearConstraint()
for i in range(n):
sum_to_one.set_coefficient(i * m + j, 1.0)
sum_to_one.set_relation(pylp.Relation.LessEqual)
sum_to_one.set_value(1.0)
constraints.add(sum_to_one)
solver = pylp.GurobiBackend()
solver.initialize(num_variables, pylp.VariableType.Binary)
solver.set_objective(objective)
solver.set_constraints(constraints)
solution = pylp.Solution()
solver.solve(solution)
matches = []
for i in range(n):
for j in range(m):
if solution[i * m + j] > 0.5:
matches.append((i, j))
return matches
def __init__(
self,
graph: nx.Graph,
tree: nx.DiGraph,
node_match_costs: Iterable[Tuple[Node, Node, float]],
edge_match_costs: Iterable[Tuple[Edge, Edge, float]],
use_gurobi: bool = True,
create_constraints: bool = True,
timeout: float = 600,
):
if isinstance(graph, nx.DiGraph):
self.undirected_graph = graph.to_undirected()
# TODO: support graph as nx.DiGraph
# note: some operations such as graph.neighbors only return successors
# breaking some constraints.
self.graph = self.undirected_graph.to_directed(graph)
elif isinstance(graph, nx.Graph):
self.undirected_graph = graph
self.graph = graph.to_directed()
self.tree = tree
# TODO: do we even need this constraint? Couldn't we do arbitrary
# subgraph matching. The only difference between ours and the solution
# in the thesis we based this off of is matching "chains"
assert nx.is_directed_acyclic_graph(self.tree), (
"cannot match an arbitrary source to an arbitrary target. " +
"target graph should be a DAG.")
self.use_gurobi = use_gurobi
self.timeout = timeout
self.node_match_costs = list(node_match_costs)
self.edge_match_costs = list(edge_match_costs)
self.objective = None
self.constraints = pylp.LinearConstraints()
self.__create_inidicators()
if create_constraints:
self.__create_constraints()
self.__create_objective()
def __create_constraints(self):
"""
constraints based on the implementation described here:
https://hal.archives-ouvertes.fr/hal-00726076/document
pg 11
Pierre Le Bodic, Pierre Héroux, Sébastien Adam, Yves Lecourtier. An integer
linear program for substitution-tolerant subgraph isomorphism and its use for
symbol spotting in technical drawings.
Pattern Recognition, Elsevier, 2012, 45 (12), pp.4214-4224. ffhal-00726076
"""
self.num_constraints = 0
all_constraints = get_constraints(self.graph, self.tree,
self.node_match_costs,
self.edge_match_costs)
self.constraints = pylp.LinearConstraints()
for key, (constraints, relation, value) in all_constraints.items():
self.num_constraints += len(constraints)
logger.debug(f"{key}: {len(constraints)}")
self.add_constraint(constraints, relation, value, key)
def find_optimal_split(synapse_ids,
superset_by_synapse_id,
nt_by_synapse_id,
neurotransmitters,
supersets,
train_fraction=0.8,
ensure_non_empty=True):
"""Find optimal synapse split per neurotransmitter
and synapse superset (e.g. hemi lineage/neuron/brain region)
Args:
synapse_ids (List):
Synapse ids to consider
superset_by_synapse_id (dict):
Mapping from each synapse_id in synapse_ids to its
associated superset.
nt_by_synapse_id (dict):
Mapping from each synapse_id in synapse_ids to
its associated nt.
neurotransmitters (List of tuples):
List of neurotransmitters to consider.
supersets (List of objects):
List of supersets to consider.
train_fraction (float):
Fraction of synapses to assign to training
"""
# Constuct combined dict:
synapses_by_superset_and_nt = {(ss, nt): []
for ss in supersets
for nt in neurotransmitters}
synapse_ids_by_nt = {nt: [] for nt in neurotransmitters}
for synapse_id in synapse_ids:
ss = superset_by_synapse_id[synapse_id]
nt = nt_by_synapse_id[synapse_id]
synapses_by_superset_and_nt[(ss, nt)].append(synapse_id)
synapse_ids_by_nt[nt].append(synapse_id)
# find optimal 80/20 split
num_variables = 0
train_indicators = {}
target = {}
sum_synapses = {}
slack_u = {}
slack_l = {}
constraints = pylp.LinearConstraints()
# for each NT:
for nt in neurotransmitters:
# compute target value: target_NT
target[nt] = int(train_fraction * len(synapse_ids_by_nt[nt]))
# let s_NT be sum of synapses in training for NT
sum_synapses[nt] = num_variables
num_variables += 1
# measure distance to target_NT: d_NT = s_NT - target_NT
sum_constraint = pylp.LinearConstraint()
# for each HL:
for ss in supersets:
# add indicator for using (HL, NT) in train: i_HL_NT
i = num_variables
num_variables += 1
train_indicators[(ss, nt)] = i
# i_HL_NT * #_of_synapses...
sum_constraint.set_coefficient(
i, len(synapses_by_superset_and_nt[(ss, nt)]))
# ... - s_NT = 0
sum_constraint.set_coefficient(sum_synapses[nt], -1)
sum_constraint.set_relation(pylp.Relation.Equal)
sum_constraint.set_value(0)
constraints.add(sum_constraint)
# add two slack variables for s_NT:
slack_u[nt] = num_variables
num_variables += 1
slack_l[nt] = num_variables
num_variables += 1
# su_NT ≥ d_NT = s_NT - target_NT su_NT ≥ 0
# target_NT ≥ d_NT - su_NT = s_NT - su_NT su_NT ≥ 0
# sl_NT ≥ -d_NT = target_NT - s_NT sl_NT ≥ 0
# -target_NT ≥ -d_NT - sl_NT = -s_NT - sl_NT sl_NT ≥ 0
slack_constraint_u = pylp.LinearConstraint()
slack_constraint_l = pylp.LinearConstraint()
slack_constraint_u_0 = pylp.LinearConstraint()
slack_constraint_l_0 = pylp.LinearConstraint()
slack_constraint_u.set_coefficient(sum_synapses[nt], 1)
slack_constraint_u.set_coefficient(slack_u[nt], -1)
slack_constraint_u.set_relation(pylp.Relation.LessEqual)
slack_constraint_u.set_value(target[nt])
slack_constraint_u_0.set_coefficient(slack_u[nt], 1)
slack_constraint_u_0.set_relation(pylp.Relation.GreaterEqual)
slack_constraint_u_0.set_value(0)
slack_constraint_l.set_coefficient(sum_synapses[nt], -1)
slack_constraint_l.set_coefficient(slack_l[nt], -1)
slack_constraint_l.set_relation(pylp.Relation.LessEqual)
slack_constraint_l.set_value(-target[nt])
slack_constraint_l_0.set_coefficient(slack_l[nt], 1)
slack_constraint_l_0.set_relation(pylp.Relation.GreaterEqual)
slack_constraint_l_0.set_value(0)
constraints.add(slack_constraint_u)
constraints.add(slack_constraint_l)
constraints.add(slack_constraint_u_0)
constraints.add(slack_constraint_l_0)
# ensure that either all or none of the NTs per hemi-lineages are used for
# training
for ss in supersets:
prev_nt = None
for nt in neurotransmitters:
if prev_nt is not None:
joint_constraint = pylp.LinearConstraint()
joint_constraint.set_coefficient(
train_indicators[(ss, prev_nt)], 1)
joint_constraint.set_coefficient(train_indicators[(ss, nt)],
-1)
joint_constraint.set_relation(pylp.Relation.Equal)
joint_constraint.set_value(0)
constraints.add(joint_constraint)
prev_nt = nt
# Ensure that at least one superset is in test for each nt
if ensure_non_empty:
for nt in neurotransmitters:
non_zero_constraint = pylp.LinearConstraint()
non_one_constraint = pylp.LinearConstraint()
# Compute number of supersets in nt:
non_zero_ss = 0
for ss in supersets:
if synapses_by_superset_and_nt[(ss, nt)]:
non_zero_ss += 1
# If there are more than one superset
# require that the number of supersets
# in test and train is at least 1
if non_zero_ss > 1:
for ss in supersets:
if synapses_by_superset_and_nt[(ss, nt)]:
non_zero_constraint.set_coefficient(
train_indicators[(ss, nt)], 1)
non_one_constraint.set_coefficient(
train_indicators[(ss, nt)], 1)
non_zero_constraint.set_relation(pylp.Relation.GreaterEqual)
non_zero_constraint.set_value(1)
non_one_constraint.set_relation(pylp.Relation.LessEqual)
non_one_constraint.set_value(non_zero_ss - 1)
constraints.add(non_zero_constraint)
constraints.add(non_one_constraint)
# add sl_NT + su_NT to objective
objective = pylp.LinearObjective(num_variables)
for nt in neurotransmitters:
objective.set_coefficient(slack_u[nt], 1. / target[nt])
objective.set_coefficient(slack_l[nt], 1. / target[nt])
variable_types = pylp.VariableTypeMap()
for nt in neurotransmitters:
variable_types[slack_u[nt]] = pylp.VariableType.Integer
variable_types[slack_l[nt]] = pylp.VariableType.Integer
variable_types[sum_synapses[nt]] = pylp.VariableType.Integer
solver = pylp.create_linear_solver(pylp.Preference.Gurobi)
solver.initialize(num_variables, pylp.VariableType.Binary, variable_types)
solver.set_objective(objective)
solver.set_constraints(constraints)
solution, msg = solver.solve()
print(msg)
train_synapses_by_ss = {}
test_synapses_by_ss = {}
for nt in neurotransmitters:
print(nt,
float(solution[sum_synapses[nt]]) / len(synapse_ids_by_nt[nt]),
"% ", solution[sum_synapses[nt]], '/',
len(synapse_ids_by_nt[nt]), '(', target[nt], ')')
for ss in supersets:
if len(synapses_by_superset_and_nt[(ss, nt)]) > 0:
if solution[train_indicators[(ss, nt)]] > 0.5:
if ss in list(train_synapses_by_ss):
pass
else:
train_synapses_by_ss[ss] = []
train_synapses_by_ss[ss].extend(
synapses_by_superset_and_nt[(ss, nt)])
print('+', ss)
else:
if ss in list(test_synapses_by_ss):
pass
else:
test_synapses_by_ss[ss] = []
test_synapses_by_ss[ss].extend(
synapses_by_superset_and_nt[(ss, nt)])
print('-', ss)
return train_synapses_by_ss, test_synapses_by_ss
def initialize(self):
logger.info("Creating Indicators...")
start_time = time.time()
self.__create_indicators()
logger.info("...took %s seconds" % (time.time() - start_time))
logger.info("Getting continuation constraints...")
start_time = time.time()
self.__get_continuation_constraints()
logger.info("...took %s seconds" % (time.time() - start_time))
logger.info("Setting objective...")
start_time = time.time()
self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi)
self.backend.initialize(self.n_triplets, pylp.VariableType.Binary)
self.backend.set_num_threads(1)
self.objective = pylp.LinearObjective(self.n_triplets)
self.constraints = pylp.LinearConstraints()
for t in self.triplets.keys():
self.objective.set_coefficient(t, self.get_cost(t))
constraint = pylp.LinearConstraint()
if t in self.t_selected:
constraint.set_coefficient(t, 1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(1)
self.constraints.add(constraint)
self.backend.set_objective(self.objective)
logger.info("...took %s seconds" % (time.time() - start_time))
logger.info("Setting center conflicts...")
start_time = time.time()
for conflict in self.t_center_conflicts:
constraint = pylp.LinearConstraint()
all_solved = True
for t in conflict:
if not t in self.t_solved:
all_solved = False
constraint.set_coefficient(t, 1)
if not all_solved:
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
self.constraints.add(constraint)
logger.info("...took %s seconds" % (time.time() - start_time))
logger.info("Setting continuation constraints...")
start_time = time.time()
for continuation_constraint in self.continuation_constraints:
t_l = continuation_constraint["t_l"]
t_r = continuation_constraint["t_r"]
all_solved = np.all([t in self.t_solved for t in (t_l + t_r)])
if not all_solved:
constraint = pylp.LinearConstraint()
for t in t_l:
constraint.set_coefficient(t, 1)
for t in t_r:
constraint.set_coefficient(t, -1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(0)
self.constraints.add(constraint)
logger.info("...took %s seconds" % (time.time() - start_time))
logger.info("Setting must pick one constraints...")
start_time = time.time()
for must_pick_one in self.e_selected + self.v_selected:
constraint = pylp.LinearConstraint()
if must_pick_one:
for t in must_pick_one:
constraint.set_coefficient(t, 1)
constraint.set_relation(pylp.Relation.GreaterEqual)
constraint.set_value(1)
self.constraints.add(constraint)
self.backend.set_constraints(self.constraints)
logger.info("...took %s seconds" % (time.time() - start_time))
def __init__(self,
g1,
distance_factor,
orientation_factor,
start_edge_prior,
comb_angle_factor,
vertex_selection_cost,
backend="Gurobi"):
if backend == "Gurobi":
logger.info("Use Gurobi backend")
self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi)
elif backend == "Scip":
logger.info("Use Scip backend")
self.backend = pylp.create_linear_solver(pylp.Preference.Scip)
else:
raise NotImplementedError("Choose between Gurobi or Scip backend")
g1.reindex_edges_save()
self.g1 = g1
self.distance_factor = distance_factor
self.orientation_factor = orientation_factor
self.start_edge_prior = start_edge_prior
self.comb_angle_factor = comb_angle_factor
self.vertex_selection_cost = vertex_selection_cost
self.vertex_cost = g1.get_vertex_cost()
self.edge_cost = g1.get_edge_cost(distance_factor, orientation_factor,
start_edge_prior)
self.edge_combination_cost, self.edges_to_middle =\
g1.get_edge_combination_cost(comb_angle_factor=comb_angle_factor,
return_edges_to_middle=True)
self.n_vertices = g1.get_number_of_vertices()
self.n_dummy = self.n_vertices
self.n_edges = g1.get_number_of_edges() + self.n_dummy
self.n_comb_edges = len(self.edge_combination_cost)
# Variables are vertices, edges and combination of egdes
self.n_variables = self.n_vertices + self.n_edges + self.n_comb_edges
self.backend.initialize(self.n_variables, pylp.VariableType.Binary)
self.objective = pylp.LinearObjective(self.n_variables)
"""
Set costs
"""
binary_id = 0
# Add one variable for each vertex (selection cost only for mt's)
self.vertex_to_binary = {}
self.binary_to_vertex = {}
for v in g1.get_vertex_iterator():
self.objective.set_coefficient(binary_id,
self.vertex_selection_cost +\
self.vertex_cost[v])
self.vertex_to_binary[v] = binary_id
self.binary_to_vertex[binary_id] = v
binary_id += 1
assert (binary_id == self.n_vertices)
# Add one variable for each edge
self.edge_to_binary = {}
self.binary_to_edge = {}
for e in g1.get_edge_iterator():
self.objective.set_coefficient(binary_id, self.edge_cost[e])
self.edge_to_binary[e] = binary_id
self.binary_to_edge[binary_id] = e
binary_id += 1
# Add one dummy edge for each vertex
self.dummy_to_binary = {}
self.binary_to_dummy = {}
for v in g1.get_vertex_iterator():
self.objective.set_coefficient(binary_id,
self.edge_cost[G1.START_EDGE])
self.dummy_to_binary[v] = binary_id
self.binary_to_dummy[binary_id] = v
binary_id += 1
assert (binary_id == self.n_vertices + self.n_edges)
# Add one variable for each combination of edges:
self.comb_to_binary = {}
self.binary_to_comb = {}
for ee, cost in self.edge_combination_cost.iteritems():
self.objective.set_coefficient(binary_id, cost)
self.comb_to_binary[ee] = binary_id
self.binary_to_comb[binary_id] = ee
binary_id += 1
assert (binary_id == self.n_variables)
self.backend.set_objective(self.objective)
"""
Constraints
"""
self.constraints = pylp.LinearConstraints()
# Edge selection implies vertex selection:
for e in g1.get_edge_iterator():
v0 = e.source()
v1 = e.target()
constraint = pylp.LinearConstraint()
constraint.set_coefficient(self.edge_to_binary[e], 2)
constraint.set_coefficient(self.vertex_to_binary[v0], -1)
constraint.set_coefficient(self.vertex_to_binary[v1], -1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(0)
self.constraints.add(constraint)
# Vertex selection implies 2 edges:
for v in g1.get_vertex_iterator():
incident_edges = g1.get_incident_edges(v)
constraint = pylp.LinearConstraint()
constraint.set_coefficient(self.vertex_to_binary[v], 2)
constraint.set_coefficient(self.dummy_to_binary[v], -1)
for e in incident_edges:
constraint.set_coefficient(self.edge_to_binary[e], -1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(0)
self.constraints.add(constraint)
# Combination of 2 edges implies edges and vice versa:
for ee in self.edge_combination_cost.keys():
e0 = ee[0]
e1 = ee[1]
assert (e0 != G1.START_EDGE or e1 != G1.START_EDGE)
if e0 == G1.START_EDGE:
middle_vertex = self.edges_to_middle[ee]
b0 = self.dummy_to_binary[middle_vertex]
b1 = self.edge_to_binary[e1]
elif e1 == G1.START_EDGE:
middle_vertex = self.edges_to_middle[ee]
b0 = self.edge_to_binary[e0]
b1 = self.dummy_to_binary[middle_vertex]
else:
b0 = self.edge_to_binary[e0]
b1 = self.edge_to_binary[e1]
constraint = pylp.LinearConstraint()
constraint.set_coefficient(self.comb_to_binary[ee], 2)
constraint.set_coefficient(b0, -1)
constraint.set_coefficient(b1, -1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(0)
self.constraints.add(constraint)
# Edges implies combination:
constraint = pylp.LinearConstraint()
constraint.set_coefficient(b0, 1)
constraint.set_coefficient(b1, 1)
constraint.set_coefficient(self.comb_to_binary[ee], -1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
self.constraints.add(constraint)
# Add partner constraints:
for v in g1.get_vertex_iterator():
partner = g1.get_partner(v)
if partner != -1:
if v < partner:
constraint = pylp.LinearConstraint()
constraint.set_coefficient(self.vertex_to_binary[v], 1)
constraint.set_coefficient(self.vertex_to_binary[partner],
1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
self.constraints.add(constraint)
self.backend.set_constraints(self.constraints)
def match(costs, no_match_cost):
''' Arguments:
costs (``dict`` from ``tuple`` of ids to ``float``):
A dictionary from a pair of edges to the cost of matching
those edges together. Assumes edges with no provided
cost cannot be matched together.
no_match_cost (``float``):
The cost of not matching an edge with anything.
'''
edge_ids_x = set()
edge_ids_y = set()
for id_x, id_y in costs.keys():
edge_ids_x.add(id_x)
edge_ids_y.add(id_y)
edge_ids_x = sorted(edge_ids_x)
edge_ids_y = sorted(edge_ids_y)
no_match_edge_x = max(edge_ids_x) + 1
no_match_edge_y = max(edge_ids_y) + 1
edge_ids_x.append(no_match_edge_x)
edge_ids_y.append(no_match_edge_y)
# default cost must be high enough that it will always choose to
# not match two edges rather than match them if there
# is no cost given for the pair
default_cost = 2 * no_match_cost + 1
n = len(edge_ids_x)
m = len(edge_ids_y)
num_variables = n * m
objective = pylp.LinearObjective(num_variables)
for i, id_x in enumerate(edge_ids_x):
for j, id_y in enumerate(edge_ids_y):
key = (id_x, id_y)
coeff_index = i * m + j
if key in costs:
objective.set_coefficient(coeff_index, costs[key])
elif id_x == no_match_edge_x or id_y == no_match_edge_y:
if id_x == no_match_edge_x and id_y == no_match_edge_y:
continue
objective.set_coefficient(coeff_index, no_match_cost)
else:
objective.set_coefficient(coeff_index, default_cost)
constraints = pylp.LinearConstraints()
for i in range(n - 1):
sum_to_one = pylp.LinearConstraint()
for j in range(m):
sum_to_one.set_coefficient(i * m + j, 1.0)
sum_to_one.set_relation(pylp.Relation.Equal)
sum_to_one.set_value(1.0)
constraints.add(sum_to_one)
for j in range(m - 1):
sum_to_one = pylp.LinearConstraint()
for i in range(n):
sum_to_one.set_coefficient(i * m + j, 1.0)
sum_to_one.set_relation(pylp.Relation.Equal)
sum_to_one.set_value(1.0)
constraints.add(sum_to_one)
solver = pylp.LinearSolver(num_variables,
pylp.VariableType.Binary,
preference=pylp.Preference.Gurobi)
solver.set_objective(objective)
solver.set_constraints(constraints)
solver.set_num_threads(1)
solver.set_timeout(240)
logger.debug("start solving (num vars %d, num constr. %d, num costs %d)",
num_variables, len(constraints), len(costs))
start = time.time()
solution, message = solver.solve()
end = time.time()
logger.debug("solving took %f seconds", end - start)
logger.debug("solver message: %s", message)
sol_cost = solution.get_value()
logger.debug("solution cost: %s", sol_cost)
matches = []
for i, id_x in enumerate(edge_ids_x):
if id_x == no_match_edge_x:
continue
for j, id_y in enumerate(edge_ids_y):
if id_y == no_match_edge_y:
continue
if solution[i * m + j] > 0.5:
matches.append((id_x, id_y))
return matches, sol_cost
def __init__(self, g2, backend="Gurobi"):
self.g2_vertices_N = g2.get_number_of_vertices()
if backend == "Gurobi":
logger.info("Use Gurobi backend")
self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi)
elif backend == "Scip":
logger.info("Use Scip backend")
self.backend = pylp.create_linear_solver(pylp.Preference.Scip)
else:
raise NotImplementedError("Choose between Gurobi or Scip backend")
self.backend.initialize(self.g2_vertices_N, pylp.VariableType.Binary)
self.backend.set_num_threads(1)
self.objective = pylp.LinearObjective(self.g2_vertices_N)
#pylp.set_log_level(pylp.LogLevel.Debug)
g2_vertex_index_map = g2.get_vertex_index_map()
self.constraints = pylp.LinearConstraints()
for v in g2.get_vertex_iterator():
self.objective.set_coefficient(g2_vertex_index_map[v],
g2.get_cost(v))
constraint = pylp.LinearConstraint()
if g2.get_forced(v):
constraint.set_coefficient(v, 1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(1)
self.constraints.add(constraint)
self.backend.set_objective(self.objective)
for conflict in g2.get_conflicts():
constraint = pylp.LinearConstraint()
all_solved = True
for v in conflict:
if not g2.get_solved(v):
all_solved = False
constraint.set_coefficient(v, 1)
if not all_solved:
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
self.constraints.add(constraint)
for sum_constraint in g2.get_sum_constraints():
vertices_1 = sum_constraint[0]
vertices_2 = sum_constraint[1]
constraint = pylp.LinearConstraint()
for v in vertices_1:
constraint.set_coefficient(v, 1)
for v in vertices_2:
constraint.set_coefficient(v, -1)
all_solved = True
for v in vertices_1 + vertices_2:
if not g2.get_solved(v):
all_solved = False
break
if not all_solved:
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(0)
self.constraints.add(constraint)
for must_pick_one in g2.get_must_pick_one():
constraint = pylp.LinearConstraint()
if must_pick_one:
for v in must_pick_one:
constraint.set_coefficient(v, 1)
constraint.set_relation(pylp.Relation.GreaterEqual)
constraint.set_value(1)
self.constraints.add(constraint)
self.backend.set_constraints(self.constraints)
def __init__(self, graph_in, classA, classB, classC):
self.graph = graph_in
#n_vertices = graph.get_num_vertices()
self.backend = pylp.create_linear_solver(pylp.Preference.Scip)
#self.graph = graph
#vertices = graph.vertices
NUM_LEAVES = len(self.graph.nodes[0]['cost'])
NUM_GRAPH_NODES = len(self.graph.nodes)
NUM_GRAPH_EDGES = len(self.graph.edges)
NUM_SOLVER_COEFF = NUM_GRAPH_NODES * NUM_LEAVES + NUM_GRAPH_EDGES
GRAPH_NODES = list(self.graph.nodes)
GRAPH_EDGES = list(self.graph.edges)
self.backend.initialize(NUM_SOLVER_COEFF, pylp.VariableType.Binary)
# backend.set_num_threads(1)
objective = pylp.LinearObjective(NUM_SOLVER_COEFF)
print('adding objective node coefficients')
#add variables to objective per vertex
id = 0
self.vertex_id = []
self.vertex_class_id = []
for v in GRAPH_NODES:
leaf_id = []
class_id = []
for leaf in range(NUM_LEAVES):
objective.set_coefficient(
id, -1 * self.graph.nodes[v]['cost'][leaf])
leaf_id.append(id)
id += 1
# add indicator variables for class A,B,C as well
for c in range(3):
objective.set_coefficient(id, 1)
class_id.append(id)
id += 1
self.vertex_id.append(leaf_id)
self.vertex_class_id.append(class_id)
assert (id == NUM_GRAPH_NODES * NUM_LEAVES)
print('adding objective edge coefficients')
self.edge_nodes = []
self.obj_edge_id = []
for e in GRAPH_EDGES:
objective.set_coefficient(id, 1)
self.edge_nodes.append(e)
self.obj_edge_id.append(id)
id += 1
assert (id == NUM_SOLVER_COEFF)
# #add variables to objective per vertex
# binary_id = 0
# vertex_to_binary = {}
# binary_to_vertex = {}
# for v in g1.get_vertex_iterator():
# objective.set_coefficient(binary_id,graph[v])
#
# vertex_to_binary[v] = binary_id
# binary_to_vertex[binary_id] = v
# binary_id += 1
#
# assert (binary_id == n_vertices)
# #add variables to objective per edge
# edge_to_binary = {}
# binary_to_edge = {}
# for e in g1.get_edge_iterator():
# objective.set_coefficient(binary_id,
# edge_cost[e])
# edge_to_binary[e] = binary_id
# binary_to_edge[binary_id] = e
# binary_id += 1
# assert(binary_id == n_vertices + n_edges)
#
# self.backend.set_objective(objective)
print('adding constraints')
self.constraints = pylp.LinearConstraints()
#add node constraints: binary, one leaf per node
for v in self.vertex_id:
constraintV = pylp.LinearConstraint()
for l in v:
constraint1 = pylp.LinearConstraint()
constraint2 = pylp.LinearConstraint()
constraint1.set_coefficient(l, 1)
constraint1.set_relation(pylp.Relation.GreaterEqual)
constraint1.set_value(0)
constraint2.set_coefficient(l, 1)
constraint2.set_relation(pylp.Relation.LessEqual)
constraint2.set_value(1)
constraintV.set_coefficient(l, 1)
self.constraints.add(constraint1)
self.constraints.add(constraint2)
constraintV.set_relation(pylp.Relation.Equal)
constraintV.set_value(1)
self.constraints.add(constraintV)
#add node constraints: binary, one class per node (redundant?)
for v in self.vertex_class_id:
constraintV = pylp.LinearConstraint()
for l in v:
constraint1 = pylp.LinearConstraint()
constraint2 = pylp.LinearConstraint()
constraint1.set_coefficient(l, 1)
constraint1.set_relation(pylp.Relation.GreaterEqual)
constraint1.set_value(0)
constraint2.set_coefficient(l, 1)
constraint2.set_relation(pylp.Relation.LessEqual)
constraint2.set_value(1)
constraintV.set_coefficient(l, 1)
self.constraints.add(constraint1)
self.constraints.add(constraint2)
constraintV.set_relation(pylp.Relation.Equal)
constraintV.set_value(1)
self.constraints.add(constraintV)
#add node constraints: node leaf sets node class
for v in self.vertex_id:
constraintA = pylp.LinearConstraint()
for a in classA:
constraintA.set_coefficient(v[a], 1)
constraintA.set_coefficient(self.vertex_class_id[0], -1)
constraintA.set_relation(pylp.Relation.Equal)
constraintA.set_value(0)
constraintB = pylp.LinearConstraint()
for b in classB:
constraintB.set_coefficient(v[b], 1)
constraintB.set_coefficient(self.vertex_class_id[1], -1)
constraintB.set_relation(pylp.Relation.Equal)
constraintB.set_value(0)
constraintC = pylp.LinearConstraint()
for c in classC:
constraintC.set_coefficient(v[c], 1)
constraintC.set_coefficient(self.vertex_class_id[2], -1)
constraintC.set_relation(pylp.Relation.Equal)
constraintC.set_value(0)
self.constraints.add(constraintA)
self.constraints.add(constraintB)
self.constraints.add(constraintC)
#add edge constraints: edge can't span class A(lumen) to class C(cytosol)
#TODO: double check
for e in GRAPH_EDGES:
constraint_orderF = pylp.LinearConstraint()
constraint_orderR = pylp.LinearConstraint()
start_idx = GRAPH_NODES.index(e[0])
end_idx = GRAPH_NODES.index(e[1])
constraint_orderF.add_coefficient(
self.vertex_class_id[start_idx][0], 1)
constraint_orderF.add_coefficient(self.vertex_class_id[end_idx][2],
1)
constraint_orderF.add_relation(pylp.Relation.Equal)
constraint_orderF.add_value(0)
self.constraints.add(constraint_orderF)
constraint_orderR.add_coefficient(
self.vertex_class_id[start_idx][2], 1)
constraint_orderR.add_coefficient(self.vertex_class_id[end_idx][0],
1)
constraint_orderR.add_relation(pylp.Relation.Equal)
constraint_orderR.add_value(0)
self.constraints.add(constraint_orderR)
#TODO: add constraint edges with different classes are cut
# #Edge selection implies vertex selection
# for e in g1.get_edge_iterator():
# v0 = e.source()
# v1 = e.target()
#
# #TODO: XOR logic?
# constraint = pylp.LinearConstraint()
# constraint.set_coefficient(edge_to_binary[e], 2)
# constraint.set_coefficient(vertex_to_binary[v0], -1)
# constraint.set_coefficient(vertex_to_binary[v1], -1)
# constraint.set_relation(pylp.Relation.LessEqual)
# constraint.set_value(0)
self.backend.set_objective(objective)
self.backend.set_constraints(self.constraints)
def match_components(nodes_x,
nodes_y,
edges_xy,
node_labels_x,
node_labels_y,
allow_many_to_many=False,
edge_costs=None,
no_match_costs=0,
edge_conflicts=None):
'''Match nodes from X to nodes from Y by selecting candidate edges x <-> y,
such that the split/merge error induced from the labels for X and Y is
minimized.
Example::
X: Y:
1 a
| |
2 b
| \
3 h c
| | |
4 C i d
| | |
5 j e
| /
6 f
| |
7 g
A B
1-7: nodes in X labelled A; a-g: nodes in Y labelled B; h-j: nodes in Y
labelled C.
Assuming that all nodes in X can be matched to all nodes in Y in the same
line (``edges_xy`` would be (1, a), (2, b), (3, h), (3, c), and so on), the
solution would be to match:
1 - a
2 - b
3 - c
4 - d
5 - e
6 - f
7 - g
h, i, and j would remain unmatched, since matching them would incur a split
error of A into B and C.
Args:
nodes_x, nodes_y (array-like of ``int``):
A list of IDs of set X and Y, respectively.
edges_xy (array-like of tuple):
A list of tuples ``(id_x, id_y)`` of matching edges to chose from.
node_labels_x, node_labels_y (``dict``):
A dictionary from IDs to labels.
allow_many_to_many (``bool``, optional):
If ``True``, allow that one node in X can match to multiple nodes
in Y and vice versa. Default is ``False``.
edge_costs (array-like of ``float``, optional):
If given, defines a preference for selecting edges from
``edges_xy`` by contributing costs ``edge_costs[i]`` for edge
``edges_xy[i]``.
The edge costs form a secondary objective, i.e., the matching is
still performed to minimize the total number of errors (splits,
merges, FPs, and FNs). However, for matching problems where several
solutions exist with the same number of errors, the edge costs
define a preference (e.g., by favouring matches between nodes that
are spatially close, if the edge costs represent distances).
See also ``no_match_costs``.
no_match_costs (``float``, optional):
A cost for not matching a node either in X or Y. Complementary to
``edge_costs``.
edge_conflicts(``list of lists of tuples (id_x, id_y)`` of edges_xy, optional):
Each list in edge conflicts should contain edges_xy that are in conflict
with each other. That is for each set of edges edge_conflicts[i] only one edge
is picked.
Returns:
(label_matches, node_matches, num_splits, num_merges, num_fps, num_fns)
``label_matches``: A list of tuples ``(label_x, label_y)`` of labels
that got matched.
``node_matches``: A list of tuples ``(id_x, id_y)`` of nodes that got
matched. Subset of ``edges_xy``.
``num_splits``, ``num_merges``, ...: The number of label splits,
merges, false positives (unmatched in X), and false negatives
(unmatched in Y).
'''
if edge_costs is None and no_match_costs != 0:
edge_costs = [0] * len(edges_xy)
num_vars = 0
# add "no match in X" and "no match in Y" dummy nodes
no_match_node = max(nodes_x + nodes_y) + 1
no_match_label = max(max(node_labels_x.keys()), max(
node_labels_y.keys())) + 1
node_labels_x = dict(node_labels_x)
node_labels_y = dict(node_labels_y)
node_labels_x.update({no_match_node: no_match_label})
node_labels_y.update({no_match_node: no_match_label})
labels_x = set(node_labels_x.values())
labels_y = set(node_labels_y.values())
# add additional edges to dummy nodes
edges_xy += [(n, no_match_node) for n in nodes_x]
edges_xy += [(no_match_node, n) for n in nodes_y]
# create indicator for each matching edge
edge_indicators = {}
edges_by_node_x = {}
edges_by_node_y = {}
for edge in edges_xy:
edge_indicators[edge] = num_vars
num_vars += 1
u, v = edge
if u not in edges_by_node_x:
edges_by_node_x[u] = []
if v not in edges_by_node_y:
edges_by_node_y[v] = []
edges_by_node_x[u].append(edge)
edges_by_node_y[v].append(edge)
# Require that each node matches to exactly one (or at least one, depending
# on the allow_many_to_many parameter) other node. Dummy nodes can match to
# any number.
constraints = pylp.LinearConstraints()
for nodes, edges_by_node in zip([nodes_x, nodes_y],
[edges_by_node_x, edges_by_node_y]):
for node in nodes:
if node == no_match_node:
continue
constraint = pylp.LinearConstraint()
for edge in edges_by_node[node]:
constraint.set_coefficient(edge_indicators[edge], 1)
if allow_many_to_many:
constraint.set_relation(pylp.Relation.GreaterEqual)
else:
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(1)
constraints.add(constraint)
# add indicators for label matches
label_indicators = {}
edges_by_label_pair = {}
for edge in edges_xy:
label_pair = node_labels_x[edge[0]], node_labels_y[edge[1]]
if label_pair not in label_indicators:
label_indicators[label_pair] = num_vars
num_vars += 1
if label_pair not in edges_by_label_pair:
edges_by_label_pair[label_pair] = []
edges_by_label_pair[label_pair].append(edge)
label_indicators[(no_match_label, no_match_label)] = num_vars
num_vars += 1
# couple label indicators to edge indicators
for label_pair, edges in edges_by_label_pair.items():
# y == 1 <==> sum(x1, ..., xn) > 0
#
# y - sum(x1, ..., xn) <= 0
# sum(x1, ..., xn) - n*y <= 0
constraint1 = pylp.LinearConstraint()
constraint2 = pylp.LinearConstraint()
constraint1.set_coefficient(label_indicators[label_pair], 1)
constraint2.set_coefficient(label_indicators[label_pair], -len(edges))
for edge in edges:
constraint1.set_coefficient(edge_indicators[edge], -1)
constraint2.set_coefficient(edge_indicators[edge], 1)
constraint1.set_relation(pylp.Relation.LessEqual)
constraint2.set_relation(pylp.Relation.LessEqual)
constraint1.set_value(0)
constraint2.set_value(0)
constraints.add(constraint1)
constraints.add(constraint2)
if edge_conflicts is not None:
for conflict in edge_conflicts:
constraint = pylp.LinearConstraint()
for edge in conflict:
constraint.set_coefficient(edge_indicators[tuple(edge)], 1)
constraint.set_relation(pylp.Relation.LessEqual)
constraint.set_value(1)
constraints.add(constraint)
# pin no-match pair indicator to 1
constraint = pylp.LinearConstraint()
no_match_indicator = label_indicators[(no_match_label, no_match_label)]
constraint.set_coefficient(no_match_indicator, 1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(1)
constraints.add(constraint)
# add integer for splits
# splits = sum of all label pair indicators - n
# with n number of labels in x (including no-match)
# sum - splits = n
splits = num_vars
num_vars += 1
constraint = pylp.LinearConstraint()
for _, label_indicator in label_indicators.items():
constraint.set_coefficient(label_indicator, 1)
constraint.set_coefficient(splits, -1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(len(labels_x))
constraints.add(constraint)
# add integer for merges
merges = num_vars
num_vars += 1
constraint = pylp.LinearConstraint()
for _, label_indicator in label_indicators.items():
constraint.set_coefficient(label_indicator, 1)
constraint.set_coefficient(merges, -1)
constraint.set_relation(pylp.Relation.Equal)
constraint.set_value(len(labels_y))
constraints.add(constraint)
# set objective
objective = pylp.LinearObjective(num_vars)
objective.set_coefficient(splits, 1)
objective.set_coefficient(merges, 1)
min_edge_cost = None
if edge_costs is not None:
edge_costs, no_match_costs = normalize_matching_costs(
len(nodes_x), len(nodes_y), edge_costs, no_match_costs)
edge_costs += [no_match_costs] * (len(nodes_x) + len(nodes_y))
min_edge_cost = min(edge_costs)
for edge, cost in zip(edges_xy, edge_costs):
objective.set_coefficient(edge_indicators[edge], cost)
# solve
logger.debug("Added %d constraints", len(constraints))
for i in range(len(constraints)):
logger.debug(constraints[i])
logger.debug("Creating linear solver")
solver = pylp.create_linear_solver(pylp.Preference.Any)
variable_types = pylp.VariableTypeMap()
variable_types[splits] = pylp.VariableType.Integer
variable_types[merges] = pylp.VariableType.Integer
if min_edge_cost is None:
logger.debug("Set optimality gap to zero")
solver.set_optimality_gap(0.0, True)
else:
logger.debug("Set optimality gap to lowest edge cost")
epsilon = 10**(-4)
solver.set_optimality_gap(max([min_edge_cost - epsilon, 0.0]), True)
logger.debug("Initializing solver with %d variables", num_vars)
solver.initialize(num_vars, pylp.VariableType.Binary, variable_types)
logger.debug("Setting objective")
solver.set_objective(objective)
logger.debug("Setting constraints")
solver.set_constraints(constraints)
logger.debug("Solving...")
solution, message = solver.solve()
logger.debug("Solver returned: %s", message)
if 'NOT' in message:
raise RuntimeError("No optimal solution found...")
# get label matches
label_matches = []
for label_pair, label_indicator in label_indicators.items():
if no_match_node not in label_pair:
if solution[label_indicator] > 0.5:
label_matches.append(label_pair)
# get node matches
node_matches = [
e for e in edges_xy
if solution[edge_indicators[e]] > 0.5 and no_match_node not in e
]
# get error counts
num_splits = solution[splits]
num_merges = solution[merges]
num_fps = 0
num_fns = 0
for label_pair, label_indicator in label_indicators.items():
if label_pair[0] == no_match_label:
num_fps += solution[label_indicator]
if label_pair[1] == no_match_label:
num_fns += solution[label_indicator]
num_fps -= 1
num_fns -= 1
num_splits -= num_fps
num_merges -= num_fns
return (label_matches, node_matches, num_splits, num_merges, num_fps,
num_fns)
