def testcircular3(self): question, issue, providerservice, answer, provider = "Question", "Issue", "ProviderService", "Answer", "Provider" tuples = [(question, issue), (providerservice, issue), (provider, question), (question, provider), (providerservice, question), (provider, providerservice), (question, answer), (issue, question)] head = topological.sort_as_tree(tuples, [], with_cycles=True) self.assert_sort(tuples, head)
def testcircular3(self): nodes = {} tuples = [('Question', 'Issue'), ('ProviderService', 'Issue'), ('Provider', 'Question'), ('Question', 'Provider'), ('ProviderService', 'Question'), ('Provider', 'ProviderService'), ('Question', 'Answer'), ('Issue', 'Question')] head = topological.sort_as_tree(tuples, [], with_cycles=True) self.assert_sort(tuples, head)
def testsort4(self): node1 = 'keywords' node2 = 'itemkeyowrds' node3 = 'items' node4 = 'hoho' tuples = [(node1, node2), (node4, node1), (node1, node3), (node3, node2)] head = topological.sort_as_tree(tuples, []) self.assert_sort(tuples, head)
def testsort3(self): [ 'Mapper|Keyword|keywords,Mapper|IKAssociation|itemkeywords', 'Mapper|Item|items,Mapper|IKAssociation|itemkeywords' ] node1 = 'keywords' node2 = 'itemkeyowrds' node3 = 'items' tuples = [(node1, node2), (node3, node2), (node1, node3)] head1 = topological.sort_as_tree(tuples, [node1, node2, node3]) head2 = topological.sort_as_tree(tuples, [node3, node1, node2]) head3 = topological.sort_as_tree(tuples, [node3, node2, node1]) # TODO: figure out a "node == node2" function #self.assert_(str(head1) == str(head2) == str(head3)) print "\n" + str(head1) print "\n" + str(head2) print "\n" + str(head3)
def testsort3(self): ['Mapper|Keyword|keywords,Mapper|IKAssociation|itemkeywords', 'Mapper|Item|items,Mapper|IKAssociation|itemkeywords'] node1 = 'keywords' node2 = 'itemkeyowrds' node3 = 'items' tuples = [ (node1, node2), (node3, node2), (node1,node3) ] head1 = topological.sort_as_tree(tuples, [node1, node2, node3]) head2 = topological.sort_as_tree(tuples, [node3, node1, node2]) head3 = topological.sort_as_tree(tuples, [node3, node2, node1]) # TODO: figure out a "node == node2" function #self.assert_(str(head1) == str(head2) == str(head3)) print "\n" + str(head1) print "\n" + str(head2) print "\n" + str(head3)
def testcircular2(self): # this condition was arising from ticket:362 # and was not treated properly by topological sort node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' tuples = [(node1, node2), (node3, node1), (node2, node4), (node3, node2), (node2, node3)] head = topological.sort_as_tree(tuples, [], with_cycles=True) self.assert_sort(tuples, head)
def testcircular(self): node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' node5 = 'node5' tuples = [(node4, node5), (node5, node4), (node1, node2), (node2, node3), (node3, node1), (node4, node1)] allitems = [node1, node2, node3, node4] head = topological.sort_as_tree(tuples, allitems, with_cycles=True) self.assert_sort(tuples, head)
def testsort5(self): # this one, depenending on the weather, node1 = 'node1' #'00B94190' node2 = 'node2' #'00B94990' node3 = 'node3' #'00B9A9B0' node4 = 'node4' #'00B4F210' tuples = [(node4, node1), (node1, node2), (node4, node3), (node2, node3), (node4, node2), (node3, node3)] allitems = [node1, node2, node3, node4] head = topological.sort_as_tree(tuples, allitems, with_cycles=True) self.assert_sort(tuples, head)
def testsort2(self): node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' node5 = 'node5' node6 = 'node6' node7 = 'node7' tuples = [(node1, node2), (node3, node4), (node4, node5), (node5, node6), (node6, node2)] head = topological.sort_as_tree(tuples, [node7]) self.assert_sort(tuples, head, [node7])
def testsort4(self): node1 = 'keywords' node2 = 'itemkeyowrds' node3 = 'items' node4 = 'hoho' tuples = [ (node1, node2), (node4, node1), (node1, node3), (node3, node2) ] head = topological.sort_as_tree(tuples, []) self.assert_sort(tuples, head)
def testsort(self): rootnode = 'root' node2 = 'node2' node3 = 'node3' node4 = 'node4' subnode1 = 'subnode1' subnode2 = 'subnode2' subnode3 = 'subnode3' subnode4 = 'subnode4' subsubnode1 = 'subsubnode1' tuples = [(subnode3, subsubnode1), (node2, subnode1), (node2, subnode2), (rootnode, node2), (rootnode, node3), (rootnode, node4), (node4, subnode3), (node4, subnode4)] head = topological.sort_as_tree(tuples, []) self.assert_sort(tuples, head)
def testcircular2(self): # this condition was arising from ticket:362 # and was not treated properly by topological sort node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' tuples = [ (node1, node2), (node3, node1), (node2, node4), (node3, node2), (node2, node3) ] head = topological.sort_as_tree(tuples, [], with_cycles=True) self.assert_sort(tuples, head)
def testsort2(self): node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' node5 = 'node5' node6 = 'node6' node7 = 'node7' tuples = [ (node1, node2), (node3, node4), (node4, node5), (node5, node6), (node6, node2) ] head = topological.sort_as_tree(tuples, [node7]) self.assert_sort(tuples, head, [node7])
def testcircular(self): node1 = 'node1' node2 = 'node2' node3 = 'node3' node4 = 'node4' node5 = 'node5' tuples = [ (node4, node5), (node5, node4), (node1, node2), (node2, node3), (node3, node1), (node4, node1) ] allitems = [node1, node2, node3, node4] head = topological.sort_as_tree(tuples, allitems, with_cycles=True) self.assert_sort(tuples, head)
def testsort(self): rootnode = 'root' node2 = 'node2' node3 = 'node3' node4 = 'node4' subnode1 = 'subnode1' subnode2 = 'subnode2' subnode3 = 'subnode3' subnode4 = 'subnode4' subsubnode1 = 'subsubnode1' tuples = [ (subnode3, subsubnode1), (node2, subnode1), (node2, subnode2), (rootnode, node2), (rootnode, node3), (rootnode, node4), (node4, subnode3), (node4, subnode4) ] head = topological.sort_as_tree(tuples, []) self.assert_sort(tuples, head)
def testsort5(self): # this one, depenending on the weather, node1 = 'node1' #'00B94190' node2 = 'node2' #'00B94990' node3 = 'node3' #'00B9A9B0' node4 = 'node4' #'00B4F210' tuples = [ (node4, node1), (node1, node2), (node4, node3), (node2, node3), (node4, node2), (node3, node3) ] allitems = [ node1, node2, node3, node4 ] head = topological.sort_as_tree(tuples, allitems, with_cycles=True) self.assert_sort(tuples, head)
def testbigsort(self): tuples = [(i, i + 1) for i in range(0, 1500, 2)] head = topological.sort_as_tree(tuples, [])
def _sort_circular_dependencies(self, trans, cycles): """Topologically sort individual entities with row-level dependencies. Builds a modified UOWTask structure, and is invoked when the per-mapper topological structure is found to have cycles. """ dependencies = {} def set_processor_for_state(state, depprocessor, target_state, isdelete): if state not in dependencies: dependencies[state] = {} tasks = dependencies[state] if depprocessor not in tasks: tasks[depprocessor] = UOWDependencyProcessor( depprocessor.processor, UOWTask(self.uowtransaction, depprocessor.targettask.mapper)) tasks[depprocessor].targettask.append(target_state, isdelete=isdelete) cycles = set(cycles) def dependency_in_cycles(dep): proctask = trans.get_task_by_mapper( dep.processor.mapper.base_mapper, True) targettask = trans.get_task_by_mapper( dep.targettask.mapper.base_mapper, True) return targettask in cycles and (proctask is not None and proctask in cycles) deps_by_targettask = {} extradeplist = [] for task in cycles: for dep in task.polymorphic_dependencies: if not dependency_in_cycles(dep): extradeplist.append(dep) for t in dep.targettask.polymorphic_tasks: l = deps_by_targettask.setdefault(t, []) l.append(dep) object_to_original_task = {} tuples = [] for task in cycles: for subtask in task.polymorphic_tasks: for taskelement in subtask.elements: state = taskelement.state object_to_original_task[state] = subtask if subtask not in deps_by_targettask: continue for dep in deps_by_targettask[subtask]: if dep.processor.no_dependencies or not dependency_in_cycles( dep): continue (processor, targettask) = (dep.processor, dep.targettask) isdelete = taskelement.isdelete # list of dependent objects from this object (added, unchanged, deleted) = dep.get_object_dependencies(state, trans, passive=True) if not added and not unchanged and not deleted: continue # the task corresponding to saving/deleting of those dependent objects childtask = trans.get_task_by_mapper(processor.mapper) childlist = added + unchanged + deleted for o in childlist: if o is None: continue if o not in childtask: childtask.append(o, listonly=True) object_to_original_task[o] = childtask whosdep = dep.whose_dependent_on_who(state, o) if whosdep is not None: tuples.append(whosdep) if whosdep[0] is state: set_processor_for_state(whosdep[0], dep, whosdep[0], isdelete=isdelete) else: set_processor_for_state(whosdep[0], dep, whosdep[1], isdelete=isdelete) else: # TODO: no test coverage here set_processor_for_state(state, dep, state, isdelete=isdelete) t = UOWTask(self.uowtransaction, self.mapper) t.dependencies.update(extradeplist) used_tasks = set() # rationale for "tree" sort as opposed to a straight # dependency - keep non-dependent objects # grouped together, so that insert ordering as determined # by session.add() is maintained. # An alternative might be to represent the "insert order" # as part of the topological sort itself, which would # eliminate the need for this step (but may make the original # topological sort more expensive) head = topological.sort_as_tree(tuples, object_to_original_task.iterkeys()) if head is not None: original_to_tasks = {} stack = [(head, t)] while stack: ((state, cycles, children), parenttask) = stack.pop() originating_task = object_to_original_task[state] used_tasks.add(originating_task) if (parenttask, originating_task) not in original_to_tasks: task = UOWTask(self.uowtransaction, originating_task.mapper) original_to_tasks[(parenttask, originating_task)] = task parenttask.dependent_tasks.append(task) else: task = original_to_tasks[(parenttask, originating_task)] task.append(state, originating_task._objects[state].listonly, isdelete=originating_task._objects[state].isdelete) if state in dependencies: task.cyclical_dependencies.update( dependencies[state].itervalues()) stack += [(n, task) for n in children] ret = [t] # add tasks that were in the cycle, but didnt get assembled # into the cyclical tree, to the start of the list for t2 in cycles: if t2 not in used_tasks and t2 is not self: localtask = UOWTask(self.uowtransaction, t2.mapper) for state in t2.elements: localtask.append(state, t2.listonly, isdelete=t2._objects[state].isdelete) for dep in t2.dependencies: localtask.dependencies.add(dep) ret.insert(0, localtask) return ret
def testbigsort(self): tuples = [] for i in range(0, 1500, 2): tuples.append((i, i + 1)) head = topological.sort_as_tree(tuples, [])
def _sort_circular_dependencies(self, trans, cycles): """Create a hierarchical tree of *subtasks* which associate specific dependency actions with individual objects. This is used for a *cyclical* task, or a task where elements of its object list contain dependencies on each other. This is not the normal case; this logic only kicks in when something like a hierarchical tree is being represented. """ allobjects = [] for task in cycles: allobjects += [e.state for e in task.polymorphic_elements] tuples = [] cycles = util.Set(cycles) extradeplist = [] dependencies = {} def get_dependency_task(state, depprocessor): try: dp = dependencies[state] except KeyError: dp = dependencies.setdefault(state, {}) try: l = dp[depprocessor] except KeyError: l = UOWTask(self.uowtransaction, depprocessor.targettask.mapper) dp[depprocessor] = l return l def dependency_in_cycles(dep): proctask = trans.get_task_by_mapper( dep.processor.mapper.base_mapper, True) targettask = trans.get_task_by_mapper( dep.targettask.mapper.base_mapper, True) return targettask in cycles and (proctask is not None and proctask in cycles) # organize all original UOWDependencyProcessors by their target task deps_by_targettask = {} for task in cycles: for dep in task.polymorphic_dependencies: if not dependency_in_cycles(dep): extradeplist.append(dep) for t in dep.targettask.polymorphic_tasks(): l = deps_by_targettask.setdefault(t, []) l.append(dep) object_to_original_task = {} for task in cycles: for subtask in task.polymorphic_tasks(): for taskelement in subtask.elements: state = taskelement.state object_to_original_task[state] = subtask for dep in deps_by_targettask.get(subtask, []): # is this dependency involved in one of the cycles ? # (don't count the DetectKeySwitch prop) if dep.processor.no_dependencies or not dependency_in_cycles( dep): continue (processor, targettask) = (dep.processor, dep.targettask) isdelete = taskelement.isdelete # list of dependent objects from this object (added, unchanged, deleted) = dep.get_object_dependencies(state, trans, passive=True) if not added and not unchanged and not deleted: continue # the task corresponding to saving/deleting of those dependent objects childtask = trans.get_task_by_mapper(processor.mapper) childlist = added + unchanged + deleted for o in childlist: # other object is None. this can occur if the relationship is many-to-one # or one-to-one, and None was set. the "removed" object will be picked # up in this iteration via the deleted_items() part of the collection. if o is None: continue # the other object is not in the UOWTransaction ! but if we are many-to-one, # we need a task in order to attach dependency operations, so establish a "listonly" # task if o not in childtask: childtask.append(o, listonly=True) object_to_original_task[o] = childtask # create a tuple representing the "parent/child" whosdep = dep.whose_dependent_on_who(state, o) if whosdep is not None: # append the tuple to the partial ordering. tuples.append(whosdep) # create a UOWDependencyProcessor representing this pair of objects. # append it to a UOWTask if whosdep[0] is state: get_dependency_task(whosdep[0], dep).append( whosdep[0], isdelete=isdelete) else: get_dependency_task(whosdep[0], dep).append( whosdep[1], isdelete=isdelete) else: # TODO: no test coverage here get_dependency_task(state, dep).append( state, isdelete=isdelete) head = topological.sort_as_tree(tuples, allobjects) used_tasks = util.Set() def make_task_tree(node, parenttask, nexttasks): (state, cycles, children) = node originating_task = object_to_original_task[state] used_tasks.add(originating_task) t = nexttasks.get(originating_task, None) if t is None: t = UOWTask(self.uowtransaction, originating_task.mapper) nexttasks[originating_task] = t parenttask._append_cyclical_childtask(t) t.append(state, originating_task._objects[state].listonly, isdelete=originating_task._objects[state].isdelete) if state in dependencies: for depprocessor, deptask in dependencies[state].iteritems(): t.cyclical_dependencies.add(depprocessor.branch(deptask)) nd = {} for n in children: t2 = make_task_tree(n, t, nd) return t t = UOWTask(self.uowtransaction, self.mapper) # stick the non-circular dependencies onto the new UOWTask for d in extradeplist: t.dependencies.add(d) if head is not None: make_task_tree(head, t, {}) ret = [t] for t2 in cycles: if t2 not in used_tasks and t2 is not self: # add tasks that were in the cycle, but didnt get assembled # into the cyclical tree, to the start of the list # TODO: no test coverage for this !! localtask = UOWTask(self.uowtransaction, t2.mapper) for state in t2.elements: localtask.append(state, t2.listonly, isdelete=t2._objects[state].isdelete) for dep in t2.dependencies: localtask.dependencies.add(dep) ret.insert(0, localtask) return ret
def testbigsort(self): tuples = [] for i in range(0,1500, 2): tuples.append((i, i+1)) head = topological.sort_as_tree(tuples, [])
def _sort_circular_dependencies(self, trans, cycles): """Topologically sort individual entities with row-level dependencies. Builds a modified UOWTask structure, and is invoked when the per-mapper topological structure is found to have cycles. """ dependencies = {} def set_processor_for_state(state, depprocessor, target_state, isdelete): if state not in dependencies: dependencies[state] = {} tasks = dependencies[state] if depprocessor not in tasks: tasks[depprocessor] = UOWDependencyProcessor( depprocessor.processor, UOWTask(self.uowtransaction, depprocessor.targettask.mapper) ) tasks[depprocessor].targettask.append(target_state, isdelete=isdelete) cycles = set(cycles) def dependency_in_cycles(dep): proctask = trans.get_task_by_mapper(dep.processor.mapper.base_mapper, True) targettask = trans.get_task_by_mapper(dep.targettask.mapper.base_mapper, True) return targettask in cycles and (proctask is not None and proctask in cycles) deps_by_targettask = {} extradeplist = [] for task in cycles: for dep in task.polymorphic_dependencies: if not dependency_in_cycles(dep): extradeplist.append(dep) for t in dep.targettask.polymorphic_tasks: l = deps_by_targettask.setdefault(t, []) l.append(dep) object_to_original_task = {} tuples = [] for task in cycles: for subtask in task.polymorphic_tasks: for taskelement in subtask.elements: state = taskelement.state object_to_original_task[state] = subtask if subtask not in deps_by_targettask: continue for dep in deps_by_targettask[subtask]: if dep.processor.no_dependencies or not dependency_in_cycles(dep): continue (processor, targettask) = (dep.processor, dep.targettask) isdelete = taskelement.isdelete # list of dependent objects from this object (added, unchanged, deleted) = dep.get_object_dependencies(state, trans, passive=True) if not added and not unchanged and not deleted: continue # the task corresponding to saving/deleting of those dependent objects childtask = trans.get_task_by_mapper(processor.mapper) childlist = added + unchanged + deleted for o in childlist: if o is None: continue if o not in childtask: childtask.append(o, listonly=True) object_to_original_task[o] = childtask whosdep = dep.whose_dependent_on_who(state, o) if whosdep is not None: tuples.append(whosdep) if whosdep[0] is state: set_processor_for_state(whosdep[0], dep, whosdep[0], isdelete=isdelete) else: set_processor_for_state(whosdep[0], dep, whosdep[1], isdelete=isdelete) else: # TODO: no test coverage here set_processor_for_state(state, dep, state, isdelete=isdelete) t = UOWTask(self.uowtransaction, self.mapper) t.dependencies.update(extradeplist) used_tasks = set() # rationale for "tree" sort as opposed to a straight # dependency - keep non-dependent objects # grouped together, so that insert ordering as determined # by session.add() is maintained. # An alternative might be to represent the "insert order" # as part of the topological sort itself, which would # eliminate the need for this step (but may make the original # topological sort more expensive) head = topological.sort_as_tree(tuples, object_to_original_task.iterkeys()) if head is not None: original_to_tasks = {} stack = [(head, t)] while stack: ((state, cycles, children), parenttask) = stack.pop() originating_task = object_to_original_task[state] used_tasks.add(originating_task) if (parenttask, originating_task) not in original_to_tasks: task = UOWTask(self.uowtransaction, originating_task.mapper) original_to_tasks[(parenttask, originating_task)] = task parenttask.dependent_tasks.append(task) else: task = original_to_tasks[(parenttask, originating_task)] task.append(state, originating_task._objects[state].listonly, isdelete=originating_task._objects[state].isdelete) if state in dependencies: task.cyclical_dependencies.update(dependencies[state].itervalues()) stack += [(n, task) for n in children] ret = [t] # add tasks that were in the cycle, but didnt get assembled # into the cyclical tree, to the start of the list for t2 in cycles: if t2 not in used_tasks and t2 is not self: localtask = UOWTask(self.uowtransaction, t2.mapper) for state in t2.elements: localtask.append(state, t2.listonly, isdelete=t2._objects[state].isdelete) for dep in t2.dependencies: localtask.dependencies.add(dep) ret.insert(0, localtask) return ret
def _sort_circular_dependencies(self, trans, cycles): """Create a hierarchical tree of *subtasks* which associate specific dependency actions with individual objects. This is used for a *cyclical* task, or a task where elements of its object list contain dependencies on each other. This is not the normal case; this logic only kicks in when something like a hierarchical tree is being represented. """ allobjects = [] for task in cycles: allobjects += [e.state for e in task.polymorphic_elements] tuples = [] cycles = util.Set(cycles) extradeplist = [] dependencies = {} def get_dependency_task(state, depprocessor): try: dp = dependencies[state] except KeyError: dp = dependencies.setdefault(state, {}) try: l = dp[depprocessor] except KeyError: l = UOWTask(self.uowtransaction, depprocessor.targettask.mapper) dp[depprocessor] = l return l def dependency_in_cycles(dep): proctask = trans.get_task_by_mapper(dep.processor.mapper.base_mapper, True) targettask = trans.get_task_by_mapper(dep.targettask.mapper.base_mapper, True) return targettask in cycles and (proctask is not None and proctask in cycles) # organize all original UOWDependencyProcessors by their target task deps_by_targettask = {} for task in cycles: for dep in task.polymorphic_dependencies: if not dependency_in_cycles(dep): extradeplist.append(dep) for t in dep.targettask.polymorphic_tasks(): l = deps_by_targettask.setdefault(t, []) l.append(dep) object_to_original_task = {} for task in cycles: for subtask in task.polymorphic_tasks(): for taskelement in subtask.elements: state = taskelement.state object_to_original_task[state] = subtask for dep in deps_by_targettask.get(subtask, []): # is this dependency involved in one of the cycles ? # (don't count the DetectKeySwitch prop) if dep.processor.no_dependencies or not dependency_in_cycles(dep): continue (processor, targettask) = (dep.processor, dep.targettask) isdelete = taskelement.isdelete # list of dependent objects from this object (added, unchanged, deleted) = dep.get_object_dependencies(state, trans, passive=True) if not added and not unchanged and not deleted: continue # the task corresponding to saving/deleting of those dependent objects childtask = trans.get_task_by_mapper(processor.mapper) childlist = added + unchanged + deleted for o in childlist: # other object is None. this can occur if the relationship is many-to-one # or one-to-one, and None was set. the "removed" object will be picked # up in this iteration via the deleted_items() part of the collection. if o is None: continue # the other object is not in the UOWTransaction ! but if we are many-to-one, # we need a task in order to attach dependency operations, so establish a "listonly" # task if o not in childtask: childtask.append(o, listonly=True) object_to_original_task[o] = childtask # create a tuple representing the "parent/child" whosdep = dep.whose_dependent_on_who(state, o) if whosdep is not None: # append the tuple to the partial ordering. tuples.append(whosdep) # create a UOWDependencyProcessor representing this pair of objects. # append it to a UOWTask if whosdep[0] is state: get_dependency_task(whosdep[0], dep).append(whosdep[0], isdelete=isdelete) else: get_dependency_task(whosdep[0], dep).append(whosdep[1], isdelete=isdelete) else: # TODO: no test coverage here get_dependency_task(state, dep).append(state, isdelete=isdelete) head = topological.sort_as_tree(tuples, allobjects) used_tasks = util.Set() def make_task_tree(node, parenttask, nexttasks): (state, cycles, children) = node originating_task = object_to_original_task[state] used_tasks.add(originating_task) t = nexttasks.get(originating_task, None) if t is None: t = UOWTask(self.uowtransaction, originating_task.mapper) nexttasks[originating_task] = t parenttask._append_cyclical_childtask(t) t.append(state, originating_task._objects[state].listonly, isdelete=originating_task._objects[state].isdelete) if state in dependencies: for depprocessor, deptask in dependencies[state].iteritems(): t.cyclical_dependencies.add(depprocessor.branch(deptask)) nd = {} for n in children: t2 = make_task_tree(n, t, nd) return t t = UOWTask(self.uowtransaction, self.mapper) # stick the non-circular dependencies onto the new UOWTask for d in extradeplist: t.dependencies.add(d) if head is not None: make_task_tree(head, t, {}) ret = [t] # add tasks that were in the cycle, but didnt get assembled # into the cyclical tree, to the start of the list for t2 in cycles: if t2 not in used_tasks and t2 is not self: localtask = UOWTask(self.uowtransaction, t2.mapper) for state in t2.elements: localtask.append(state, t2.listonly, isdelete=t2._objects[state].isdelete) for dep in t2.dependencies: localtask.dependencies.add(dep) ret.insert(0, localtask) return ret