def test_add(): """ add a node to a Graph """ from arbiter.graph import Graph graph = Graph() assert_equals(graph.nodes, frozenset()) assert_equals(graph.roots, frozenset()) assert_false('foo' in graph) graph.add('foo') assert_equals(graph.nodes, frozenset(('foo',))) assert_equals(graph.roots, frozenset(('foo',))) assert_true('foo' in graph) assert_equals(graph.children('foo'), frozenset()) assert_equals(graph.parents('foo'), frozenset()) assert_false(graph.ancestor_of('foo', 'foo')) graph.add('bar', ('foo', 'baz')) assert_equals(graph.nodes, frozenset(('foo', 'bar', 'baz'))) assert_equals(graph.roots, frozenset(('foo',))) assert_true('foo' in graph) assert_true('bar' in graph) assert_true('baz' in graph) assert_equals(graph.children('foo'), frozenset(('bar',))) assert_equals(graph.children('bar'), frozenset()) assert_equals(graph.children('baz'), frozenset(('bar',))) assert_equals(graph.parents('foo'), frozenset()) assert_equals(graph.parents('bar'), frozenset(('foo', 'baz'))) assert_equals(graph.parents('baz'), frozenset()) assert_false(graph.ancestor_of('foo', 'foo')) assert_false(graph.ancestor_of('foo', 'bar')) assert_false(graph.ancestor_of('foo', 'baz')) assert_true(graph.ancestor_of('bar', 'foo')) assert_false(graph.ancestor_of('bar', 'bar')) assert_true(graph.ancestor_of('bar', 'baz')) assert_false(graph.ancestor_of('baz', 'foo')) assert_false(graph.ancestor_of('baz', 'bar')) assert_false(graph.ancestor_of('baz', 'baz')) assert_raises(ValueError, graph.add, 'baz', ('bar',)) assert_raises(ValueError, graph.add, 'ouroboros', ('ouroboros',)) assert_raises(ValueError, graph.add, 'foo') assert_equals(graph.nodes, frozenset(('foo', 'bar', 'baz'))) assert_equals(graph.roots, frozenset(('foo',)))
def test_equality(): """ Graph equality """ from arbiter.graph import Graph graph = Graph() assert_false(graph == 1) assert_true(graph != 0) other = Graph() assert_true(graph == other) assert_false(graph != other) graph.add('foo') assert_false(graph == other) assert_true(graph != other) graph.add('bar', ('foo',)) other.add('bar', ('foo',)) # still shouldn't match graph['foo'] is a stub assert_false(graph == other) assert_true(graph != other) other.add('foo') assert_true(graph == other) assert_false(graph != other)
def test_prune(): """ Prune a Graph """ from arbiter.graph import Graph graph = Graph() graph.add('node') graph.add('bar', ('foo',)) graph.add('baz', ('bar',)) graph.add('beta', ('alpha',)) graph.add('bravo', ('alpha',)) assert_equals( graph.nodes, frozenset( ('node', 'foo', 'bar', 'baz', 'alpha', 'beta', 'bravo') ) ) assert_equals(graph.roots, frozenset(('node',))) assert_equals( graph.prune(), frozenset(('bar', 'baz', 'beta', 'bravo')) ) assert_equals( graph.nodes, frozenset( ('node',) ) ) assert_equals(graph.roots, frozenset(('node',))) assert_equals(graph.prune(), frozenset()) assert_equals( graph.nodes, frozenset( ('node',) ) ) assert_equals(graph.roots, frozenset(('node',)))
def test_remove_orphan(): """ Remove a node from a Graph """ from arbiter.graph import Graph, Strategy graph = Graph() graph.add('node') graph.add('bar', ('foo',)) graph.add('baz', ('bar',)) graph.add('beta', ('alpha',)) graph.add('bravo', ('alpha',)) assert_equals( graph.nodes, frozenset( ('node', 'foo', 'bar', 'baz', 'alpha', 'beta', 'bravo') ) ) assert_equals(graph.roots, frozenset(('node',))) # node with no children/parents assert_equals( graph.remove('node', strategy=Strategy.orphan), frozenset(('node',)) ) assert_equals( graph.nodes, frozenset( ('foo', 'bar', 'baz', 'alpha', 'beta', 'bravo') ) ) assert_equals(graph.roots, frozenset()) # node with child, unique stub parent assert_equals( graph.remove('bar', strategy=Strategy.orphan), frozenset(('bar', 'foo')) ) assert_equals( graph.nodes, frozenset( ('baz', 'alpha', 'beta', 'bravo') ) ) assert_equals(graph.roots, frozenset(('baz',))) # node with non-unique stub parent assert_equals( graph.remove('bravo', strategy=Strategy.orphan), frozenset(('bravo',)) ) assert_equals( graph.nodes, frozenset( ('baz', 'alpha', 'beta',) ) ) assert_equals(graph.roots, frozenset(('baz',))) # stub assert_equals( graph.remove('alpha', strategy=Strategy.orphan), frozenset(('alpha',)) ) assert_equals( graph.nodes, frozenset( ('baz', 'beta') ) ) assert_equals(graph.roots, frozenset(('baz', 'beta'))) assert_raises(KeyError, graph.remove, 'fake')
def test_naming(): """ Node names just need to be hashable. """ from arbiter.graph import Graph, Strategy graph = Graph() for name in (1, float('NaN'), 0, None, '', frozenset(), (), False, sum): graph.add('child1', frozenset((name,))) graph.add(name) assert_true(name in graph) assert_equals(graph.nodes, frozenset((name, 'child1'))) assert_equals(graph.roots, frozenset((name,))) assert_equals(graph.children(name), frozenset(('child1',))) assert_equals(graph.parents(name), frozenset()) assert_false(graph.ancestor_of(name, name)) graph.add('child2', frozenset((name,))) assert_equals( graph.children(name), frozenset(('child1', 'child2')) ) graph.remove(name) graph.remove('child1') graph.remove('child2') assert_equals(graph.nodes, frozenset()) graph.add(None) graph.add('', parents=((),)) graph.add((), parents=(frozenset(),)) graph.add(frozenset()) assert_equals(graph.nodes, frozenset((None, '', (), frozenset()))) assert_equals(graph.roots, frozenset((None, frozenset()))) graph.remove((), strategy=Strategy.remove) assert_equals(graph.nodes, frozenset((None, frozenset()))) assert_equals(graph.roots, frozenset((None, frozenset()))) assert_raises(TypeError, graph.add, []) assert_raises(TypeError, graph.add, 'valid', parents=([],))
def test_remove_promote(): """ Remove a node (transitively making its parents its children's parents) from a Graph. """ from arbiter.graph import Graph graph = Graph() graph.add('aye') graph.add('insect') graph.add('bee', ('aye', 'insect')) graph.add('cee', ('bee',)) graph.add('child', ('stub', 'stub2')) graph.add('grandchild', ('child',)) assert_equals( graph.nodes, frozenset( ( 'aye', 'insect', 'bee', 'cee', 'child', 'stub', 'stub2', 'grandchild', ) ) ) assert_equals(graph.roots, frozenset(('aye', 'insect'))) # two new parents assert_equals(graph.remove('bee'), frozenset(('bee',))) assert_equals( graph.nodes, frozenset( ( 'aye', 'insect', 'cee', 'child', 'stub', 'stub2', 'grandchild', ) ) ) assert_equals( graph.roots, frozenset(('aye', 'insect')) ) assert_equals(graph.children('aye'), frozenset(('cee',))) assert_equals(graph.children('insect'), frozenset(('cee',))) assert_equals(graph.parents('cee'), frozenset(('aye', 'insect'))) # now with stubs assert_equals(graph.remove('child'), frozenset(('child',))) assert_equals( graph.nodes, frozenset( ( 'aye', 'insect', 'cee', 'stub', 'stub2', 'grandchild', ) ) ) assert_equals( graph.roots, frozenset(('aye', 'insect')) ) assert_equals(graph.children('stub'), frozenset(('grandchild',))) assert_equals(graph.children('stub2'), frozenset(('grandchild',))) assert_equals( graph.parents('grandchild'), frozenset(('stub', 'stub2')) ) # delete a stub assert_equals(graph.remove('stub2'), frozenset(('stub2',))) assert_equals( graph.nodes, frozenset( ( 'aye', 'insect', 'cee', 'stub', 'grandchild', ) ) ) assert_equals( graph.roots, frozenset(('aye', 'insect')) ) assert_equals(graph.children('stub'), frozenset(('grandchild',))) assert_equals(graph.parents('grandchild'), frozenset(('stub',)))
class Scheduler(object): """ A dependency scheduler. """ def __init__(self, tasks=None, completed=None, failed=None): if completed is None: completed = set() if failed is None: failed = set() self._graph = Graph() self._tasks = {} self._running = set() self._completed = completed self._failed = failed if tasks is not None: for task in tasks: self.add_task(task) @property def completed(self): """ A copy of the set of successfully completed tasks. """ return frozenset(self._completed) @property def failed(self): """ A copy of the set of failed tasks. """ return frozenset(self._failed) @property def running(self): """ A copy of the set of running tasks. """ return frozenset(self._running) @property def runnable(self): """ Get the set of tasks that are currently runnable. """ return self._graph.roots - self._running def is_finished(self): """ Have all runnable tasks completed? """ return not (self._graph.roots or self._running) def add_task(self, task): """ Add a task to the scheduler. task: The task to add. """ if not self._valid_name(task.name): raise ValueError(task.name) self._tasks[task.name] = task incomplete_dependencies = set() for dependency in task.dependencies: if not self._valid_name(dependency) or dependency in self._failed: # there may already be tasks dependent on this one. self._cascade_failure(task.name) break if dependency not in self._completed: incomplete_dependencies.add(dependency) else: # task hasn't failed try: self._graph.add(task.name, incomplete_dependencies) except ValueError: self._cascade_failure(task.name) def start_task(self, name=None): """ Start a task. Returns the task that was started (or None if no task has been started). name: (optional, None) The task to start. If a name is given, Scheduler will attempt to start the task (and raise an exception if the task doesn't exist or isn't runnable). If no name is given, a task will be chosen arbitrarily """ if name is None: for possibility in self._graph.roots: if possibility not in self._running: name = possibility break else: # all tasks blocked/running/completed/failed return None else: if name not in self._graph.roots or name in self._running: raise ValueError(name) self._running.add(name) return self._tasks[name] def end_task(self, name, success=True): """ End a running task. Raises an exception if the task isn't running. name: The name of the task to complete. success: (optional, True) Whether the task was successful. """ self._running.remove(name) if success: self._completed.add(name) self._graph.remove(name, strategy=Strategy.orphan) else: self._cascade_failure(name) def remove_unrunnable(self): """ Remove any tasks that are dependent on non-existent tasks. """ self._failed.update(self._graph.prune()) def fail_remaining(self): """ Mark all unfinished tasks (including currently running ones) as failed. """ self._failed.update(self._graph.nodes) self._graph = Graph() self._running = set() def _cascade_failure(self, name): """ Mark a task (and anything that depends on it) as failed. name: The name of the offending task """ if name in self._graph: self._failed.update( self._graph.remove(name, strategy=Strategy.remove)) else: self._failed.add(name) def __enter__(self): """ Remove all unrunnable tasks and enter a context manager. When the context manager is exited, all non-complete tasks will be failed. """ self.remove_unrunnable() return self def __exit__(self, exc_type, exc_value, traceback): """ Exit the context manager, stopping (and failing) any non-completed tasks. """ self.fail_remaining() @classmethod def _valid_name(cls, name): """ Check whether a name is valid as a task name. """ return name is not None and isinstance(name, Hashable)
class Scheduler(object): """ A dependency scheduler. """ def __init__(self, tasks=None, completed=None, failed=None): if completed is None: completed = set() if failed is None: failed = set() self._graph = Graph() self._tasks = {} self._running = set() self._completed = completed self._failed = failed if tasks is not None: for task in tasks: self.add_task(task) @property def completed(self): """ A copy of the set of successfully completed tasks. """ return frozenset(self._completed) @property def failed(self): """ A copy of the set of failed tasks. """ return frozenset(self._failed) @property def running(self): """ A copy of the set of running tasks. """ return frozenset(self._running) @property def runnable(self): """ Get the set of tasks that are currently runnable. """ return self._graph.roots - self._running def is_finished(self): """ Have all runnable tasks completed? """ return not (self._graph.roots or self._running) def add_task(self, task): """ Add a task to the scheduler. task: The task to add. """ if not self._valid_name(task.name): raise ValueError(task.name) self._tasks[task.name] = task incomplete_dependencies = set() for dependency in task.dependencies: if not self._valid_name(dependency) or dependency in self._failed: # there may already be tasks dependent on this one. self._cascade_failure(task.name) break if dependency not in self._completed: incomplete_dependencies.add(dependency) else: # task hasn't failed try: self._graph.add(task.name, incomplete_dependencies) except ValueError: self._cascade_failure(task.name) def start_task(self, name=None): """ Start a task. Returns the task that was started (or None if no task has been started). name: (optional, None) The task to start. If a name is given, Scheduler will attempt to start the task (and raise an exception if the task doesn't exist or isn't runnable). If no name is given, a task will be chosen arbitrarily """ if name is None: for possibility in self._graph.roots: if possibility not in self._running: name = possibility break else: # all tasks blocked/running/completed/failed return None else: if name not in self._graph.roots or name in self._running: raise ValueError(name) self._running.add(name) return self._tasks[name] def end_task(self, name, success=True): """ End a running task. Raises an exception if the task isn't running. name: The name of the task to complete. success: (optional, True) Whether the task was successful. """ self._running.remove(name) if success: self._completed.add(name) self._graph.remove(name, strategy=Strategy.orphan) else: self._cascade_failure(name) def remove_unrunnable(self): """ Remove any tasks that are dependent on non-existent tasks. """ self._failed.update(self._graph.prune()) def fail_remaining(self): """ Mark all unfinished tasks (including currently running ones) as failed. """ self._failed.update(self._graph.nodes) self._graph = Graph() self._running = set() def _cascade_failure(self, name): """ Mark a task (and anything that depends on it) as failed. name: The name of the offending task """ if name in self._graph: self._failed.update( self._graph.remove(name, strategy=Strategy.remove) ) else: self._failed.add(name) def __enter__(self): """ Remove all unrunnable tasks and enter a context manager. When the context manager is exited, all non-complete tasks will be failed. """ self.remove_unrunnable() return self def __exit__(self, exc_type, exc_value, traceback): """ Exit the context manager, stopping (and failing) any non-completed tasks. """ self.fail_remaining() @classmethod def _valid_name(cls, name): """ Check whether a name is valid as a task name. """ return name is not None and isinstance(name, Hashable)