def __init__(self, graph): self.backend = pylp.GurobiBackend() #self.backend = pylp.ScipBackend() self.backend.initialize(graph.num_nodes, pylp.VariableType.Binary) self.objective = pylp.LinearObjective(graph.num_nodes) for n in range(graph.num_nodes): self.objective.set_coefficient(n, graph.costs[n]) self.backend.set_objective(self.objective) self.constraints = pylp.LinearConstraints() for conflict in graph.conflicts: constraint = pylp.LinearConstraint() for n in conflict: constraint.set_coefficient(n, 1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) self.constraints.add(constraint) for equal_sum_constraint in graph.equal_sum_constraints: nodes1 = equal_sum_constraint[0] nodes2 = equal_sum_constraint[1] constraint = pylp.LinearConstraint() for n in nodes1: constraint.set_coefficient(n, 1) for n in nodes2: constraint.set_coefficient(n, -1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(0) self.constraints.add(constraint) self.backend.set_constraints(self.constraints)
def simple_solver_test(self, preference): num_vars = 10 special_var = 5 solver = pylp.LinearSolver( num_vars, pylp.VariableType.Binary, { special_var: pylp.VariableType.Continuous }, preference) objective = pylp.LinearObjective() for i in range(num_vars): objective.set_coefficient(i, 1.0) objective.set_coefficient(special_var, 0.5) constraint = pylp.LinearConstraint() for i in range(num_vars): constraint.set_coefficient(i, 1.0) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1.0) solver.set_objective(objective) solver.add_constraint(constraint) solution, msg = solver.solve() self.assertEqual(solution[5], 1)
def add_constraint(self, constraints, relation, value, key): for constraint in constraints: c = pylp.LinearConstraint() for match_indicator, weight in constraint: c.set_coefficient(match_indicator, weight) c.set_relation(getattr(pylp.Relation, relation)) c.set_value(value) self.constraints.add(c)
def _add_cell_cycle_constraints(self): # If an edge is selected, the division and child indicators are # linked. Let e=(u,v) be an edge linking node u at time t + 1 to v in # time t. # Constraints: # child(u) + selected(e) - split(v) <= 1 # split(v) + selected(e) - child(u) <= 1 for e in self.graph.edges(): # if e is selected, u and v have to be selected u, v = e ind_e = self.edge_selected[e] split_v = self.node_split[v] child_u = self.node_child[u] link_constraint_1 = pylp.LinearConstraint() link_constraint_1.set_coefficient(child_u, 1) link_constraint_1.set_coefficient(ind_e, 1) link_constraint_1.set_coefficient(split_v, -1) link_constraint_1.set_relation(pylp.Relation.LessEqual) link_constraint_1.set_value(1) self.main_constraints.append(link_constraint_1) link_constraint_2 = pylp.LinearConstraint() link_constraint_2.set_coefficient(split_v, 1) link_constraint_2.set_coefficient(ind_e, 1) link_constraint_2.set_coefficient(child_u, -1) link_constraint_2.set_relation(pylp.Relation.LessEqual) link_constraint_2.set_value(1) self.main_constraints.append(link_constraint_2) # Every selected node must be a split, child or continuation # (exclusively). If a node is not selected, all the cell cycle # indicators should be zero. # Constraint for each node: # split + child + continuation - selected = 0 for node in self.graph.nodes(): cycle_set_constraint = pylp.LinearConstraint() cycle_set_constraint.set_coefficient(self.node_split[node], 1) cycle_set_constraint.set_coefficient(self.node_child[node], 1) cycle_set_constraint.set_coefficient(self.node_continuation[node], 1) cycle_set_constraint.set_coefficient(self.node_selected[node], -1) cycle_set_constraint.set_relation(pylp.Relation.Equal) cycle_set_constraint.set_value(0) self.main_constraints.append(cycle_set_constraint)
def match(costs): n = costs.shape[0] m = costs.shape[1] num_variables = n * m objective = pylp.LinearObjective(num_variables) for i in range(n): for j in range(m): objective.set_coefficient(i * m + j, costs[i, j]) constraints = pylp.LinearConstraints() for i in range(n): sum_to_one = pylp.LinearConstraint() for j in range(m): sum_to_one.set_coefficient(i * m + j, 1.0) sum_to_one.set_relation(pylp.Relation.LessEqual) sum_to_one.set_value(1.0) constraints.add(sum_to_one) for j in range(m): sum_to_one = pylp.LinearConstraint() for i in range(n): sum_to_one.set_coefficient(i * m + j, 1.0) sum_to_one.set_relation(pylp.Relation.LessEqual) sum_to_one.set_value(1.0) constraints.add(sum_to_one) solver = pylp.GurobiBackend() solver.initialize(num_variables, pylp.VariableType.Binary) solver.set_objective(objective) solver.set_constraints(constraints) solution = pylp.Solution() solver.solve(solution) matches = [] for i in range(n): for j in range(m): if solution[i * m + j] > 0.5: matches.append((i, j)) return matches
def add_cycle_constraint(self, cycle): constraint = pylp.LinearConstraint() for variable in [self.edge_to_var[e] for e in cycle]: constraint.set_coefficient(variable, 1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(len(cycle) - 1) self.constraints.add(constraint)
def _add_pin_constraints(self): for e in self.graph.edges(): if self.selected_key in self.graph.edges[e]: selected = self.graph.edges[e][self.selected_key] self.pinned_edges[e] = selected ind_e = self.edge_selected[e] constraint = pylp.LinearConstraint() constraint.set_coefficient(ind_e, 1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1 if selected else 0) self.pin_constraints.append(constraint)
def add_path_constraint(self, node_path): path_constraint = pylp.LinearConstraint() for i in range(len(node_path) - 1): try: idx = self.edge_nodes.index((node_path[i], node_path[i + 1])) except: idx = self.edge_nodes.index((node_path[i + 1], node_path[i])) #edge_idx = np.argwhere(self.edge_nodes == (node_path[i],node_path[i+1])) #edge_id = self.edge_id[edge_idx] edge_id = self.obj_edge_id[idx] path_constraint.set_coefficient(edge_id, 1) path_constraint.set_relation(pylp.Relation.NotEqual) path_constraint.set_value(2) self.constraints.add(path_constraint) self.backend.set_constraints(self.constraints)
def _add_edge_constraints(self): logger.debug("setting edge constraints") for e in self.graph.edges(): # if e is selected, u and v have to be selected u, v = e ind_e = self.edge_selected[e] ind_u = self.node_selected[u] ind_v = self.node_selected[v] constraint = pylp.LinearConstraint() constraint.set_coefficient(ind_e, 2) constraint.set_coefficient(ind_u, -1) constraint.set_coefficient(ind_v, -1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(0) self.main_constraints.append(constraint) logger.debug("set edge constraint %s", constraint)
def enforce_expected_assignments( self, expected_assignments: Iterable[Tuple[Matchable, Matchable]]): """ Force a specific set of matchings to be made in any solution. """ expected_assignment_constraint = pylp.LinearConstraint() failed_matchings = [] num_assignments = 0 for match_a, match_b in expected_assignments: match_ind = self.g2t(match_a, match_b) logger.debug( f"Enforcing match {(match_a, match_b)} with match index {match_ind}" ) if match_ind is None: # This is most likely an edge matching since edges don't have an # explicit indicator for None matchings if match_a in self.g2t_match_indicators and match_b is None: # if there is no None match available, then this must be an edge matching # enfore unmatched edge for match, ind in self.g2ts(match_a).items(): expected_assignment_constraint.set_coefficient(ind, -1) elif match_a in self.g2t_match_indicators and match_b is not None: logger.debug( f"Matching {(match_a, match_b)} is impossible!") failed_matchings.append((match_a, match_b)) elif match_a not in self.g2t_match_indicators: logger.debug( f"Attempting to enforce matching for {match_a} which has no possible matches!" ) failed_matchings.append((match_a, match_b)) else: expected_assignment_constraint.set_coefficient(match_ind, 1) num_assignments += 1 expected_assignment_constraint.set_relation(pylp.Relation.Equal) expected_assignment_constraint.set_value(num_assignments) self.constraints.add(expected_assignment_constraint) logger.warning( f"{len(failed_matchings)} of {len(expected_assignments)} were not enforcable!" )
def initialize(self): logger.info("Creating Indicators...") start_time = time.time() self.__create_indicators() logger.info("...took %s seconds" % (time.time() - start_time)) logger.info("Getting continuation constraints...") start_time = time.time() self.__get_continuation_constraints() logger.info("...took %s seconds" % (time.time() - start_time)) logger.info("Setting objective...") start_time = time.time() self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi) self.backend.initialize(self.n_triplets, pylp.VariableType.Binary) self.backend.set_num_threads(1) self.objective = pylp.LinearObjective(self.n_triplets) self.constraints = pylp.LinearConstraints() for t in self.triplets.keys(): self.objective.set_coefficient(t, self.get_cost(t)) constraint = pylp.LinearConstraint() if t in self.t_selected: constraint.set_coefficient(t, 1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1) self.constraints.add(constraint) self.backend.set_objective(self.objective) logger.info("...took %s seconds" % (time.time() - start_time)) logger.info("Setting center conflicts...") start_time = time.time() for conflict in self.t_center_conflicts: constraint = pylp.LinearConstraint() all_solved = True for t in conflict: if not t in self.t_solved: all_solved = False constraint.set_coefficient(t, 1) if not all_solved: constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) self.constraints.add(constraint) logger.info("...took %s seconds" % (time.time() - start_time)) logger.info("Setting continuation constraints...") start_time = time.time() for continuation_constraint in self.continuation_constraints: t_l = continuation_constraint["t_l"] t_r = continuation_constraint["t_r"] all_solved = np.all([t in self.t_solved for t in (t_l + t_r)]) if not all_solved: constraint = pylp.LinearConstraint() for t in t_l: constraint.set_coefficient(t, 1) for t in t_r: constraint.set_coefficient(t, -1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(0) self.constraints.add(constraint) logger.info("...took %s seconds" % (time.time() - start_time)) logger.info("Setting must pick one constraints...") start_time = time.time() for must_pick_one in self.e_selected + self.v_selected: constraint = pylp.LinearConstraint() if must_pick_one: for t in must_pick_one: constraint.set_coefficient(t, 1) constraint.set_relation(pylp.Relation.GreaterEqual) constraint.set_value(1) self.constraints.add(constraint) self.backend.set_constraints(self.constraints) logger.info("...took %s seconds" % (time.time() - start_time))
def _add_inter_frame_constraints(self, t): '''Linking constraints from t to t+1.''' logger.debug("setting inter-frame constraints for frame %d", t) # Every selected node has exactly one selected edge to the previous and # one or two to the next frame. This includes the special "appear" and # "disappear" edges. for node in self.graph.cells_by_frame(t): # we model this as three constraints: # sum(prev) - node = 0 # exactly one prev edge, # iff node selected # sum(next) - 2*node <= 0 # at most two next edges # -sum(next) + node <= 0 # at least one next, iff node selected constraint_prev = pylp.LinearConstraint() constraint_next_1 = pylp.LinearConstraint() constraint_next_2 = pylp.LinearConstraint() # sum(prev) # all neighbors in previous frame pinned_to_1 = [] for edge in self.graph.prev_edges(node): constraint_prev.set_coefficient(self.edge_selected[edge], 1) if edge in self.pinned_edges and self.pinned_edges[edge]: pinned_to_1.append(edge) if len(pinned_to_1) > 1: raise RuntimeError( "Node %d has more than one prev edge pinned: %s" % (node, pinned_to_1)) # plus "appear" constraint_prev.set_coefficient(self.node_appear[node], 1) # sum(next) for edge in self.graph.next_edges(node): constraint_next_1.set_coefficient(self.edge_selected[edge], 1) constraint_next_2.set_coefficient(self.edge_selected[edge], -1) # plus "disappear" constraint_next_1.set_coefficient(self.node_disappear[node], 1) constraint_next_2.set_coefficient(self.node_disappear[node], -1) # node constraint_prev.set_coefficient(self.node_selected[node], -1) constraint_next_1.set_coefficient(self.node_selected[node], -2) constraint_next_2.set_coefficient(self.node_selected[node], 1) # relation, value constraint_prev.set_relation(pylp.Relation.Equal) constraint_next_1.set_relation(pylp.Relation.LessEqual) constraint_next_2.set_relation(pylp.Relation.LessEqual) constraint_prev.set_value(0) constraint_next_1.set_value(0) constraint_next_2.set_value(0) self.main_constraints.append(constraint_prev) self.main_constraints.append(constraint_next_1) self.main_constraints.append(constraint_next_2) logger.debug("set inter-frame constraints:\t%s\n\t%s\n\t%s", constraint_prev, constraint_next_1, constraint_next_2) # Ensure that the split indicator is set for every cell that splits # into two daughter cells. for node in self.graph.cells_by_frame(t): # I.e., each node with two forwards edges is a split node. # Constraint 1 # sum(forward edges) - split <= 1 # sum(forward edges) > 1 => split == 1 # Constraint 2 # sum(forward edges) - 2*split >= 0 # sum(forward edges) <= 1 => split == 0 constraint_1 = pylp.LinearConstraint() constraint_2 = pylp.LinearConstraint() # sum(forward edges) for edge in self.graph.next_edges(node): constraint_1.set_coefficient(self.edge_selected[edge], 1) constraint_2.set_coefficient(self.edge_selected[edge], 1) # -[2*]split constraint_1.set_coefficient(self.node_split[node], -1) constraint_2.set_coefficient(self.node_split[node], -2) constraint_1.set_relation(pylp.Relation.LessEqual) constraint_2.set_relation(pylp.Relation.GreaterEqual) constraint_1.set_value(1) constraint_2.set_value(0) self.main_constraints.append(constraint_1) self.main_constraints.append(constraint_2) logger.debug("set split-indicator constraints:\n\t%s\n\t%s", constraint_1, constraint_2)
def __init__(self, g1, distance_factor, orientation_factor, start_edge_prior, comb_angle_factor, vertex_selection_cost, backend="Gurobi"): if backend == "Gurobi": logger.info("Use Gurobi backend") self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi) elif backend == "Scip": logger.info("Use Scip backend") self.backend = pylp.create_linear_solver(pylp.Preference.Scip) else: raise NotImplementedError("Choose between Gurobi or Scip backend") g1.reindex_edges_save() self.g1 = g1 self.distance_factor = distance_factor self.orientation_factor = orientation_factor self.start_edge_prior = start_edge_prior self.comb_angle_factor = comb_angle_factor self.vertex_selection_cost = vertex_selection_cost self.vertex_cost = g1.get_vertex_cost() self.edge_cost = g1.get_edge_cost(distance_factor, orientation_factor, start_edge_prior) self.edge_combination_cost, self.edges_to_middle =\ g1.get_edge_combination_cost(comb_angle_factor=comb_angle_factor, return_edges_to_middle=True) self.n_vertices = g1.get_number_of_vertices() self.n_dummy = self.n_vertices self.n_edges = g1.get_number_of_edges() + self.n_dummy self.n_comb_edges = len(self.edge_combination_cost) # Variables are vertices, edges and combination of egdes self.n_variables = self.n_vertices + self.n_edges + self.n_comb_edges self.backend.initialize(self.n_variables, pylp.VariableType.Binary) self.objective = pylp.LinearObjective(self.n_variables) """ Set costs """ binary_id = 0 # Add one variable for each vertex (selection cost only for mt's) self.vertex_to_binary = {} self.binary_to_vertex = {} for v in g1.get_vertex_iterator(): self.objective.set_coefficient(binary_id, self.vertex_selection_cost +\ self.vertex_cost[v]) self.vertex_to_binary[v] = binary_id self.binary_to_vertex[binary_id] = v binary_id += 1 assert (binary_id == self.n_vertices) # Add one variable for each edge self.edge_to_binary = {} self.binary_to_edge = {} for e in g1.get_edge_iterator(): self.objective.set_coefficient(binary_id, self.edge_cost[e]) self.edge_to_binary[e] = binary_id self.binary_to_edge[binary_id] = e binary_id += 1 # Add one dummy edge for each vertex self.dummy_to_binary = {} self.binary_to_dummy = {} for v in g1.get_vertex_iterator(): self.objective.set_coefficient(binary_id, self.edge_cost[G1.START_EDGE]) self.dummy_to_binary[v] = binary_id self.binary_to_dummy[binary_id] = v binary_id += 1 assert (binary_id == self.n_vertices + self.n_edges) # Add one variable for each combination of edges: self.comb_to_binary = {} self.binary_to_comb = {} for ee, cost in self.edge_combination_cost.iteritems(): self.objective.set_coefficient(binary_id, cost) self.comb_to_binary[ee] = binary_id self.binary_to_comb[binary_id] = ee binary_id += 1 assert (binary_id == self.n_variables) self.backend.set_objective(self.objective) """ Constraints """ self.constraints = pylp.LinearConstraints() # Edge selection implies vertex selection: for e in g1.get_edge_iterator(): v0 = e.source() v1 = e.target() constraint = pylp.LinearConstraint() constraint.set_coefficient(self.edge_to_binary[e], 2) constraint.set_coefficient(self.vertex_to_binary[v0], -1) constraint.set_coefficient(self.vertex_to_binary[v1], -1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(0) self.constraints.add(constraint) # Vertex selection implies 2 edges: for v in g1.get_vertex_iterator(): incident_edges = g1.get_incident_edges(v) constraint = pylp.LinearConstraint() constraint.set_coefficient(self.vertex_to_binary[v], 2) constraint.set_coefficient(self.dummy_to_binary[v], -1) for e in incident_edges: constraint.set_coefficient(self.edge_to_binary[e], -1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(0) self.constraints.add(constraint) # Combination of 2 edges implies edges and vice versa: for ee in self.edge_combination_cost.keys(): e0 = ee[0] e1 = ee[1] assert (e0 != G1.START_EDGE or e1 != G1.START_EDGE) if e0 == G1.START_EDGE: middle_vertex = self.edges_to_middle[ee] b0 = self.dummy_to_binary[middle_vertex] b1 = self.edge_to_binary[e1] elif e1 == G1.START_EDGE: middle_vertex = self.edges_to_middle[ee] b0 = self.edge_to_binary[e0] b1 = self.dummy_to_binary[middle_vertex] else: b0 = self.edge_to_binary[e0] b1 = self.edge_to_binary[e1] constraint = pylp.LinearConstraint() constraint.set_coefficient(self.comb_to_binary[ee], 2) constraint.set_coefficient(b0, -1) constraint.set_coefficient(b1, -1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(0) self.constraints.add(constraint) # Edges implies combination: constraint = pylp.LinearConstraint() constraint.set_coefficient(b0, 1) constraint.set_coefficient(b1, 1) constraint.set_coefficient(self.comb_to_binary[ee], -1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) self.constraints.add(constraint) # Add partner constraints: for v in g1.get_vertex_iterator(): partner = g1.get_partner(v) if partner != -1: if v < partner: constraint = pylp.LinearConstraint() constraint.set_coefficient(self.vertex_to_binary[v], 1) constraint.set_coefficient(self.vertex_to_binary[partner], 1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) self.constraints.add(constraint) self.backend.set_constraints(self.constraints)
def match(costs, no_match_cost): ''' Arguments: costs (``dict`` from ``tuple`` of ids to ``float``): A dictionary from a pair of edges to the cost of matching those edges together. Assumes edges with no provided cost cannot be matched together. no_match_cost (``float``): The cost of not matching an edge with anything. ''' edge_ids_x = set() edge_ids_y = set() for id_x, id_y in costs.keys(): edge_ids_x.add(id_x) edge_ids_y.add(id_y) edge_ids_x = sorted(edge_ids_x) edge_ids_y = sorted(edge_ids_y) no_match_edge_x = max(edge_ids_x) + 1 no_match_edge_y = max(edge_ids_y) + 1 edge_ids_x.append(no_match_edge_x) edge_ids_y.append(no_match_edge_y) # default cost must be high enough that it will always choose to # not match two edges rather than match them if there # is no cost given for the pair default_cost = 2 * no_match_cost + 1 n = len(edge_ids_x) m = len(edge_ids_y) num_variables = n * m objective = pylp.LinearObjective(num_variables) for i, id_x in enumerate(edge_ids_x): for j, id_y in enumerate(edge_ids_y): key = (id_x, id_y) coeff_index = i * m + j if key in costs: objective.set_coefficient(coeff_index, costs[key]) elif id_x == no_match_edge_x or id_y == no_match_edge_y: if id_x == no_match_edge_x and id_y == no_match_edge_y: continue objective.set_coefficient(coeff_index, no_match_cost) else: objective.set_coefficient(coeff_index, default_cost) constraints = pylp.LinearConstraints() for i in range(n - 1): sum_to_one = pylp.LinearConstraint() for j in range(m): sum_to_one.set_coefficient(i * m + j, 1.0) sum_to_one.set_relation(pylp.Relation.Equal) sum_to_one.set_value(1.0) constraints.add(sum_to_one) for j in range(m - 1): sum_to_one = pylp.LinearConstraint() for i in range(n): sum_to_one.set_coefficient(i * m + j, 1.0) sum_to_one.set_relation(pylp.Relation.Equal) sum_to_one.set_value(1.0) constraints.add(sum_to_one) solver = pylp.LinearSolver(num_variables, pylp.VariableType.Binary, preference=pylp.Preference.Gurobi) solver.set_objective(objective) solver.set_constraints(constraints) solver.set_num_threads(1) solver.set_timeout(240) logger.debug("start solving (num vars %d, num constr. %d, num costs %d)", num_variables, len(constraints), len(costs)) start = time.time() solution, message = solver.solve() end = time.time() logger.debug("solving took %f seconds", end - start) logger.debug("solver message: %s", message) sol_cost = solution.get_value() logger.debug("solution cost: %s", sol_cost) matches = [] for i, id_x in enumerate(edge_ids_x): if id_x == no_match_edge_x: continue for j, id_y in enumerate(edge_ids_y): if id_y == no_match_edge_y: continue if solution[i * m + j] > 0.5: matches.append((id_x, id_y)) return matches, sol_cost
def __init__(self, g2, backend="Gurobi"): self.g2_vertices_N = g2.get_number_of_vertices() if backend == "Gurobi": logger.info("Use Gurobi backend") self.backend = pylp.create_linear_solver(pylp.Preference.Gurobi) elif backend == "Scip": logger.info("Use Scip backend") self.backend = pylp.create_linear_solver(pylp.Preference.Scip) else: raise NotImplementedError("Choose between Gurobi or Scip backend") self.backend.initialize(self.g2_vertices_N, pylp.VariableType.Binary) self.backend.set_num_threads(1) self.objective = pylp.LinearObjective(self.g2_vertices_N) #pylp.set_log_level(pylp.LogLevel.Debug) g2_vertex_index_map = g2.get_vertex_index_map() self.constraints = pylp.LinearConstraints() for v in g2.get_vertex_iterator(): self.objective.set_coefficient(g2_vertex_index_map[v], g2.get_cost(v)) constraint = pylp.LinearConstraint() if g2.get_forced(v): constraint.set_coefficient(v, 1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1) self.constraints.add(constraint) self.backend.set_objective(self.objective) for conflict in g2.get_conflicts(): constraint = pylp.LinearConstraint() all_solved = True for v in conflict: if not g2.get_solved(v): all_solved = False constraint.set_coefficient(v, 1) if not all_solved: constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) self.constraints.add(constraint) for sum_constraint in g2.get_sum_constraints(): vertices_1 = sum_constraint[0] vertices_2 = sum_constraint[1] constraint = pylp.LinearConstraint() for v in vertices_1: constraint.set_coefficient(v, 1) for v in vertices_2: constraint.set_coefficient(v, -1) all_solved = True for v in vertices_1 + vertices_2: if not g2.get_solved(v): all_solved = False break if not all_solved: constraint.set_relation(pylp.Relation.Equal) constraint.set_value(0) self.constraints.add(constraint) for must_pick_one in g2.get_must_pick_one(): constraint = pylp.LinearConstraint() if must_pick_one: for v in must_pick_one: constraint.set_coefficient(v, 1) constraint.set_relation(pylp.Relation.GreaterEqual) constraint.set_value(1) self.constraints.add(constraint) self.backend.set_constraints(self.constraints)
def __init__(self, graph_in, classA, classB, classC): self.graph = graph_in #n_vertices = graph.get_num_vertices() self.backend = pylp.create_linear_solver(pylp.Preference.Scip) #self.graph = graph #vertices = graph.vertices NUM_LEAVES = len(self.graph.nodes[0]['cost']) NUM_GRAPH_NODES = len(self.graph.nodes) NUM_GRAPH_EDGES = len(self.graph.edges) NUM_SOLVER_COEFF = NUM_GRAPH_NODES * NUM_LEAVES + NUM_GRAPH_EDGES GRAPH_NODES = list(self.graph.nodes) GRAPH_EDGES = list(self.graph.edges) self.backend.initialize(NUM_SOLVER_COEFF, pylp.VariableType.Binary) # backend.set_num_threads(1) objective = pylp.LinearObjective(NUM_SOLVER_COEFF) print('adding objective node coefficients') #add variables to objective per vertex id = 0 self.vertex_id = [] self.vertex_class_id = [] for v in GRAPH_NODES: leaf_id = [] class_id = [] for leaf in range(NUM_LEAVES): objective.set_coefficient( id, -1 * self.graph.nodes[v]['cost'][leaf]) leaf_id.append(id) id += 1 # add indicator variables for class A,B,C as well for c in range(3): objective.set_coefficient(id, 1) class_id.append(id) id += 1 self.vertex_id.append(leaf_id) self.vertex_class_id.append(class_id) assert (id == NUM_GRAPH_NODES * NUM_LEAVES) print('adding objective edge coefficients') self.edge_nodes = [] self.obj_edge_id = [] for e in GRAPH_EDGES: objective.set_coefficient(id, 1) self.edge_nodes.append(e) self.obj_edge_id.append(id) id += 1 assert (id == NUM_SOLVER_COEFF) # #add variables to objective per vertex # binary_id = 0 # vertex_to_binary = {} # binary_to_vertex = {} # for v in g1.get_vertex_iterator(): # objective.set_coefficient(binary_id,graph[v]) # # vertex_to_binary[v] = binary_id # binary_to_vertex[binary_id] = v # binary_id += 1 # # assert (binary_id == n_vertices) # #add variables to objective per edge # edge_to_binary = {} # binary_to_edge = {} # for e in g1.get_edge_iterator(): # objective.set_coefficient(binary_id, # edge_cost[e]) # edge_to_binary[e] = binary_id # binary_to_edge[binary_id] = e # binary_id += 1 # assert(binary_id == n_vertices + n_edges) # # self.backend.set_objective(objective) print('adding constraints') self.constraints = pylp.LinearConstraints() #add node constraints: binary, one leaf per node for v in self.vertex_id: constraintV = pylp.LinearConstraint() for l in v: constraint1 = pylp.LinearConstraint() constraint2 = pylp.LinearConstraint() constraint1.set_coefficient(l, 1) constraint1.set_relation(pylp.Relation.GreaterEqual) constraint1.set_value(0) constraint2.set_coefficient(l, 1) constraint2.set_relation(pylp.Relation.LessEqual) constraint2.set_value(1) constraintV.set_coefficient(l, 1) self.constraints.add(constraint1) self.constraints.add(constraint2) constraintV.set_relation(pylp.Relation.Equal) constraintV.set_value(1) self.constraints.add(constraintV) #add node constraints: binary, one class per node (redundant?) for v in self.vertex_class_id: constraintV = pylp.LinearConstraint() for l in v: constraint1 = pylp.LinearConstraint() constraint2 = pylp.LinearConstraint() constraint1.set_coefficient(l, 1) constraint1.set_relation(pylp.Relation.GreaterEqual) constraint1.set_value(0) constraint2.set_coefficient(l, 1) constraint2.set_relation(pylp.Relation.LessEqual) constraint2.set_value(1) constraintV.set_coefficient(l, 1) self.constraints.add(constraint1) self.constraints.add(constraint2) constraintV.set_relation(pylp.Relation.Equal) constraintV.set_value(1) self.constraints.add(constraintV) #add node constraints: node leaf sets node class for v in self.vertex_id: constraintA = pylp.LinearConstraint() for a in classA: constraintA.set_coefficient(v[a], 1) constraintA.set_coefficient(self.vertex_class_id[0], -1) constraintA.set_relation(pylp.Relation.Equal) constraintA.set_value(0) constraintB = pylp.LinearConstraint() for b in classB: constraintB.set_coefficient(v[b], 1) constraintB.set_coefficient(self.vertex_class_id[1], -1) constraintB.set_relation(pylp.Relation.Equal) constraintB.set_value(0) constraintC = pylp.LinearConstraint() for c in classC: constraintC.set_coefficient(v[c], 1) constraintC.set_coefficient(self.vertex_class_id[2], -1) constraintC.set_relation(pylp.Relation.Equal) constraintC.set_value(0) self.constraints.add(constraintA) self.constraints.add(constraintB) self.constraints.add(constraintC) #add edge constraints: edge can't span class A(lumen) to class C(cytosol) #TODO: double check for e in GRAPH_EDGES: constraint_orderF = pylp.LinearConstraint() constraint_orderR = pylp.LinearConstraint() start_idx = GRAPH_NODES.index(e[0]) end_idx = GRAPH_NODES.index(e[1]) constraint_orderF.add_coefficient( self.vertex_class_id[start_idx][0], 1) constraint_orderF.add_coefficient(self.vertex_class_id[end_idx][2], 1) constraint_orderF.add_relation(pylp.Relation.Equal) constraint_orderF.add_value(0) self.constraints.add(constraint_orderF) constraint_orderR.add_coefficient( self.vertex_class_id[start_idx][2], 1) constraint_orderR.add_coefficient(self.vertex_class_id[end_idx][0], 1) constraint_orderR.add_relation(pylp.Relation.Equal) constraint_orderR.add_value(0) self.constraints.add(constraint_orderR) #TODO: add constraint edges with different classes are cut # #Edge selection implies vertex selection # for e in g1.get_edge_iterator(): # v0 = e.source() # v1 = e.target() # # #TODO: XOR logic? # constraint = pylp.LinearConstraint() # constraint.set_coefficient(edge_to_binary[e], 2) # constraint.set_coefficient(vertex_to_binary[v0], -1) # constraint.set_coefficient(vertex_to_binary[v1], -1) # constraint.set_relation(pylp.Relation.LessEqual) # constraint.set_value(0) self.backend.set_objective(objective) self.backend.set_constraints(self.constraints)
def find_optimal_split(synapse_ids, superset_by_synapse_id, nt_by_synapse_id, neurotransmitters, supersets, train_fraction=0.8, ensure_non_empty=True): """Find optimal synapse split per neurotransmitter and synapse superset (e.g. hemi lineage/neuron/brain region) Args: synapse_ids (List): Synapse ids to consider superset_by_synapse_id (dict): Mapping from each synapse_id in synapse_ids to its associated superset. nt_by_synapse_id (dict): Mapping from each synapse_id in synapse_ids to its associated nt. neurotransmitters (List of tuples): List of neurotransmitters to consider. supersets (List of objects): List of supersets to consider. train_fraction (float): Fraction of synapses to assign to training """ # Constuct combined dict: synapses_by_superset_and_nt = {(ss, nt): [] for ss in supersets for nt in neurotransmitters} synapse_ids_by_nt = {nt: [] for nt in neurotransmitters} for synapse_id in synapse_ids: ss = superset_by_synapse_id[synapse_id] nt = nt_by_synapse_id[synapse_id] synapses_by_superset_and_nt[(ss, nt)].append(synapse_id) synapse_ids_by_nt[nt].append(synapse_id) # find optimal 80/20 split num_variables = 0 train_indicators = {} target = {} sum_synapses = {} slack_u = {} slack_l = {} constraints = pylp.LinearConstraints() # for each NT: for nt in neurotransmitters: # compute target value: target_NT target[nt] = int(train_fraction * len(synapse_ids_by_nt[nt])) # let s_NT be sum of synapses in training for NT sum_synapses[nt] = num_variables num_variables += 1 # measure distance to target_NT: d_NT = s_NT - target_NT sum_constraint = pylp.LinearConstraint() # for each HL: for ss in supersets: # add indicator for using (HL, NT) in train: i_HL_NT i = num_variables num_variables += 1 train_indicators[(ss, nt)] = i # i_HL_NT * #_of_synapses... sum_constraint.set_coefficient( i, len(synapses_by_superset_and_nt[(ss, nt)])) # ... - s_NT = 0 sum_constraint.set_coefficient(sum_synapses[nt], -1) sum_constraint.set_relation(pylp.Relation.Equal) sum_constraint.set_value(0) constraints.add(sum_constraint) # add two slack variables for s_NT: slack_u[nt] = num_variables num_variables += 1 slack_l[nt] = num_variables num_variables += 1 # su_NT ≥ d_NT = s_NT - target_NT su_NT ≥ 0 # target_NT ≥ d_NT - su_NT = s_NT - su_NT su_NT ≥ 0 # sl_NT ≥ -d_NT = target_NT - s_NT sl_NT ≥ 0 # -target_NT ≥ -d_NT - sl_NT = -s_NT - sl_NT sl_NT ≥ 0 slack_constraint_u = pylp.LinearConstraint() slack_constraint_l = pylp.LinearConstraint() slack_constraint_u_0 = pylp.LinearConstraint() slack_constraint_l_0 = pylp.LinearConstraint() slack_constraint_u.set_coefficient(sum_synapses[nt], 1) slack_constraint_u.set_coefficient(slack_u[nt], -1) slack_constraint_u.set_relation(pylp.Relation.LessEqual) slack_constraint_u.set_value(target[nt]) slack_constraint_u_0.set_coefficient(slack_u[nt], 1) slack_constraint_u_0.set_relation(pylp.Relation.GreaterEqual) slack_constraint_u_0.set_value(0) slack_constraint_l.set_coefficient(sum_synapses[nt], -1) slack_constraint_l.set_coefficient(slack_l[nt], -1) slack_constraint_l.set_relation(pylp.Relation.LessEqual) slack_constraint_l.set_value(-target[nt]) slack_constraint_l_0.set_coefficient(slack_l[nt], 1) slack_constraint_l_0.set_relation(pylp.Relation.GreaterEqual) slack_constraint_l_0.set_value(0) constraints.add(slack_constraint_u) constraints.add(slack_constraint_l) constraints.add(slack_constraint_u_0) constraints.add(slack_constraint_l_0) # ensure that either all or none of the NTs per hemi-lineages are used for # training for ss in supersets: prev_nt = None for nt in neurotransmitters: if prev_nt is not None: joint_constraint = pylp.LinearConstraint() joint_constraint.set_coefficient( train_indicators[(ss, prev_nt)], 1) joint_constraint.set_coefficient(train_indicators[(ss, nt)], -1) joint_constraint.set_relation(pylp.Relation.Equal) joint_constraint.set_value(0) constraints.add(joint_constraint) prev_nt = nt # Ensure that at least one superset is in test for each nt if ensure_non_empty: for nt in neurotransmitters: non_zero_constraint = pylp.LinearConstraint() non_one_constraint = pylp.LinearConstraint() # Compute number of supersets in nt: non_zero_ss = 0 for ss in supersets: if synapses_by_superset_and_nt[(ss, nt)]: non_zero_ss += 1 # If there are more than one superset # require that the number of supersets # in test and train is at least 1 if non_zero_ss > 1: for ss in supersets: if synapses_by_superset_and_nt[(ss, nt)]: non_zero_constraint.set_coefficient( train_indicators[(ss, nt)], 1) non_one_constraint.set_coefficient( train_indicators[(ss, nt)], 1) non_zero_constraint.set_relation(pylp.Relation.GreaterEqual) non_zero_constraint.set_value(1) non_one_constraint.set_relation(pylp.Relation.LessEqual) non_one_constraint.set_value(non_zero_ss - 1) constraints.add(non_zero_constraint) constraints.add(non_one_constraint) # add sl_NT + su_NT to objective objective = pylp.LinearObjective(num_variables) for nt in neurotransmitters: objective.set_coefficient(slack_u[nt], 1. / target[nt]) objective.set_coefficient(slack_l[nt], 1. / target[nt]) variable_types = pylp.VariableTypeMap() for nt in neurotransmitters: variable_types[slack_u[nt]] = pylp.VariableType.Integer variable_types[slack_l[nt]] = pylp.VariableType.Integer variable_types[sum_synapses[nt]] = pylp.VariableType.Integer solver = pylp.create_linear_solver(pylp.Preference.Gurobi) solver.initialize(num_variables, pylp.VariableType.Binary, variable_types) solver.set_objective(objective) solver.set_constraints(constraints) solution, msg = solver.solve() print(msg) train_synapses_by_ss = {} test_synapses_by_ss = {} for nt in neurotransmitters: print(nt, float(solution[sum_synapses[nt]]) / len(synapse_ids_by_nt[nt]), "% ", solution[sum_synapses[nt]], '/', len(synapse_ids_by_nt[nt]), '(', target[nt], ')') for ss in supersets: if len(synapses_by_superset_and_nt[(ss, nt)]) > 0: if solution[train_indicators[(ss, nt)]] > 0.5: if ss in list(train_synapses_by_ss): pass else: train_synapses_by_ss[ss] = [] train_synapses_by_ss[ss].extend( synapses_by_superset_and_nt[(ss, nt)]) print('+', ss) else: if ss in list(test_synapses_by_ss): pass else: test_synapses_by_ss[ss] = [] test_synapses_by_ss[ss].extend( synapses_by_superset_and_nt[(ss, nt)]) print('-', ss) return train_synapses_by_ss, test_synapses_by_ss
def match_components(nodes_x, nodes_y, edges_xy, node_labels_x, node_labels_y, allow_many_to_many=False, edge_costs=None, no_match_costs=0, edge_conflicts=None): '''Match nodes from X to nodes from Y by selecting candidate edges x <-> y, such that the split/merge error induced from the labels for X and Y is minimized. Example:: X: Y: 1 a | | 2 b | \ 3 h c | | | 4 C i d | | | 5 j e | / 6 f | | 7 g A B 1-7: nodes in X labelled A; a-g: nodes in Y labelled B; h-j: nodes in Y labelled C. Assuming that all nodes in X can be matched to all nodes in Y in the same line (``edges_xy`` would be (1, a), (2, b), (3, h), (3, c), and so on), the solution would be to match: 1 - a 2 - b 3 - c 4 - d 5 - e 6 - f 7 - g h, i, and j would remain unmatched, since matching them would incur a split error of A into B and C. Args: nodes_x, nodes_y (array-like of ``int``): A list of IDs of set X and Y, respectively. edges_xy (array-like of tuple): A list of tuples ``(id_x, id_y)`` of matching edges to chose from. node_labels_x, node_labels_y (``dict``): A dictionary from IDs to labels. allow_many_to_many (``bool``, optional): If ``True``, allow that one node in X can match to multiple nodes in Y and vice versa. Default is ``False``. edge_costs (array-like of ``float``, optional): If given, defines a preference for selecting edges from ``edges_xy`` by contributing costs ``edge_costs[i]`` for edge ``edges_xy[i]``. The edge costs form a secondary objective, i.e., the matching is still performed to minimize the total number of errors (splits, merges, FPs, and FNs). However, for matching problems where several solutions exist with the same number of errors, the edge costs define a preference (e.g., by favouring matches between nodes that are spatially close, if the edge costs represent distances). See also ``no_match_costs``. no_match_costs (``float``, optional): A cost for not matching a node either in X or Y. Complementary to ``edge_costs``. edge_conflicts(``list of lists of tuples (id_x, id_y)`` of edges_xy, optional): Each list in edge conflicts should contain edges_xy that are in conflict with each other. That is for each set of edges edge_conflicts[i] only one edge is picked. Returns: (label_matches, node_matches, num_splits, num_merges, num_fps, num_fns) ``label_matches``: A list of tuples ``(label_x, label_y)`` of labels that got matched. ``node_matches``: A list of tuples ``(id_x, id_y)`` of nodes that got matched. Subset of ``edges_xy``. ``num_splits``, ``num_merges``, ...: The number of label splits, merges, false positives (unmatched in X), and false negatives (unmatched in Y). ''' if edge_costs is None and no_match_costs != 0: edge_costs = [0] * len(edges_xy) num_vars = 0 # add "no match in X" and "no match in Y" dummy nodes no_match_node = max(nodes_x + nodes_y) + 1 no_match_label = max(max(node_labels_x.keys()), max( node_labels_y.keys())) + 1 node_labels_x = dict(node_labels_x) node_labels_y = dict(node_labels_y) node_labels_x.update({no_match_node: no_match_label}) node_labels_y.update({no_match_node: no_match_label}) labels_x = set(node_labels_x.values()) labels_y = set(node_labels_y.values()) # add additional edges to dummy nodes edges_xy += [(n, no_match_node) for n in nodes_x] edges_xy += [(no_match_node, n) for n in nodes_y] # create indicator for each matching edge edge_indicators = {} edges_by_node_x = {} edges_by_node_y = {} for edge in edges_xy: edge_indicators[edge] = num_vars num_vars += 1 u, v = edge if u not in edges_by_node_x: edges_by_node_x[u] = [] if v not in edges_by_node_y: edges_by_node_y[v] = [] edges_by_node_x[u].append(edge) edges_by_node_y[v].append(edge) # Require that each node matches to exactly one (or at least one, depending # on the allow_many_to_many parameter) other node. Dummy nodes can match to # any number. constraints = pylp.LinearConstraints() for nodes, edges_by_node in zip([nodes_x, nodes_y], [edges_by_node_x, edges_by_node_y]): for node in nodes: if node == no_match_node: continue constraint = pylp.LinearConstraint() for edge in edges_by_node[node]: constraint.set_coefficient(edge_indicators[edge], 1) if allow_many_to_many: constraint.set_relation(pylp.Relation.GreaterEqual) else: constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1) constraints.add(constraint) # add indicators for label matches label_indicators = {} edges_by_label_pair = {} for edge in edges_xy: label_pair = node_labels_x[edge[0]], node_labels_y[edge[1]] if label_pair not in label_indicators: label_indicators[label_pair] = num_vars num_vars += 1 if label_pair not in edges_by_label_pair: edges_by_label_pair[label_pair] = [] edges_by_label_pair[label_pair].append(edge) label_indicators[(no_match_label, no_match_label)] = num_vars num_vars += 1 # couple label indicators to edge indicators for label_pair, edges in edges_by_label_pair.items(): # y == 1 <==> sum(x1, ..., xn) > 0 # # y - sum(x1, ..., xn) <= 0 # sum(x1, ..., xn) - n*y <= 0 constraint1 = pylp.LinearConstraint() constraint2 = pylp.LinearConstraint() constraint1.set_coefficient(label_indicators[label_pair], 1) constraint2.set_coefficient(label_indicators[label_pair], -len(edges)) for edge in edges: constraint1.set_coefficient(edge_indicators[edge], -1) constraint2.set_coefficient(edge_indicators[edge], 1) constraint1.set_relation(pylp.Relation.LessEqual) constraint2.set_relation(pylp.Relation.LessEqual) constraint1.set_value(0) constraint2.set_value(0) constraints.add(constraint1) constraints.add(constraint2) if edge_conflicts is not None: for conflict in edge_conflicts: constraint = pylp.LinearConstraint() for edge in conflict: constraint.set_coefficient(edge_indicators[tuple(edge)], 1) constraint.set_relation(pylp.Relation.LessEqual) constraint.set_value(1) constraints.add(constraint) # pin no-match pair indicator to 1 constraint = pylp.LinearConstraint() no_match_indicator = label_indicators[(no_match_label, no_match_label)] constraint.set_coefficient(no_match_indicator, 1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(1) constraints.add(constraint) # add integer for splits # splits = sum of all label pair indicators - n # with n number of labels in x (including no-match) # sum - splits = n splits = num_vars num_vars += 1 constraint = pylp.LinearConstraint() for _, label_indicator in label_indicators.items(): constraint.set_coefficient(label_indicator, 1) constraint.set_coefficient(splits, -1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(len(labels_x)) constraints.add(constraint) # add integer for merges merges = num_vars num_vars += 1 constraint = pylp.LinearConstraint() for _, label_indicator in label_indicators.items(): constraint.set_coefficient(label_indicator, 1) constraint.set_coefficient(merges, -1) constraint.set_relation(pylp.Relation.Equal) constraint.set_value(len(labels_y)) constraints.add(constraint) # set objective objective = pylp.LinearObjective(num_vars) objective.set_coefficient(splits, 1) objective.set_coefficient(merges, 1) min_edge_cost = None if edge_costs is not None: edge_costs, no_match_costs = normalize_matching_costs( len(nodes_x), len(nodes_y), edge_costs, no_match_costs) edge_costs += [no_match_costs] * (len(nodes_x) + len(nodes_y)) min_edge_cost = min(edge_costs) for edge, cost in zip(edges_xy, edge_costs): objective.set_coefficient(edge_indicators[edge], cost) # solve logger.debug("Added %d constraints", len(constraints)) for i in range(len(constraints)): logger.debug(constraints[i]) logger.debug("Creating linear solver") solver = pylp.create_linear_solver(pylp.Preference.Any) variable_types = pylp.VariableTypeMap() variable_types[splits] = pylp.VariableType.Integer variable_types[merges] = pylp.VariableType.Integer if min_edge_cost is None: logger.debug("Set optimality gap to zero") solver.set_optimality_gap(0.0, True) else: logger.debug("Set optimality gap to lowest edge cost") epsilon = 10**(-4) solver.set_optimality_gap(max([min_edge_cost - epsilon, 0.0]), True) logger.debug("Initializing solver with %d variables", num_vars) solver.initialize(num_vars, pylp.VariableType.Binary, variable_types) logger.debug("Setting objective") solver.set_objective(objective) logger.debug("Setting constraints") solver.set_constraints(constraints) logger.debug("Solving...") solution, message = solver.solve() logger.debug("Solver returned: %s", message) if 'NOT' in message: raise RuntimeError("No optimal solution found...") # get label matches label_matches = [] for label_pair, label_indicator in label_indicators.items(): if no_match_node not in label_pair: if solution[label_indicator] > 0.5: label_matches.append(label_pair) # get node matches node_matches = [ e for e in edges_xy if solution[edge_indicators[e]] > 0.5 and no_match_node not in e ] # get error counts num_splits = solution[splits] num_merges = solution[merges] num_fps = 0 num_fns = 0 for label_pair, label_indicator in label_indicators.items(): if label_pair[0] == no_match_label: num_fps += solution[label_indicator] if label_pair[1] == no_match_label: num_fns += solution[label_indicator] num_fps -= 1 num_fns -= 1 num_splits -= num_fps num_merges -= num_fns return (label_matches, node_matches, num_splits, num_merges, num_fps, num_fns)