def fail_remaining(self):
"""
Mark all unfinished tasks (including currently running ones) as
failed.
"""
self._failed.update(self._graph.nodes)
self._graph = Graph()
self._running = set()
def __init__(self, tasks=None, completed=None, failed=None):
if completed is None:
completed = set()
if failed is None:
failed = set()
self._graph = Graph()
self._tasks = {}
self._running = set()
self._completed = completed
self._failed = failed
if tasks is not None:
for task in tasks:
self.add_task(task)
def fail_remaining(self):
"""
Mark all unfinished tasks (including currently running ones) as
failed.
"""
self._failed.update(self._graph.nodes)
self._graph = Graph()
self._running = set()
def __init__(self, tasks=None, completed=None, failed=None):
if completed is None:
completed = set()
if failed is None:
failed = set()
self._graph = Graph()
self._tasks = {}
self._running = set()
self._completed = completed
self._failed = failed
if tasks is not None:
for task in tasks:
self.add_task(task)
def test_remove_orphan():
"""
Remove a node from a Graph
"""
from arbiter.graph import Graph, Strategy
graph = Graph()
graph.add('node')
graph.add('bar', ('foo',))
graph.add('baz', ('bar',))
graph.add('beta', ('alpha',))
graph.add('bravo', ('alpha',))
assert_equals(
graph.nodes,
frozenset(
('node', 'foo', 'bar', 'baz', 'alpha', 'beta', 'bravo')
)
)
assert_equals(graph.roots, frozenset(('node',)))
# node with no children/parents
assert_equals(
graph.remove('node', strategy=Strategy.orphan),
frozenset(('node',))
)
assert_equals(
graph.nodes,
frozenset(
('foo', 'bar', 'baz', 'alpha', 'beta', 'bravo')
)
)
assert_equals(graph.roots, frozenset())
# node with child, unique stub parent
assert_equals(
graph.remove('bar', strategy=Strategy.orphan),
frozenset(('bar', 'foo'))
)
assert_equals(
graph.nodes,
frozenset(
('baz', 'alpha', 'beta', 'bravo')
)
)
assert_equals(graph.roots, frozenset(('baz',)))
# node with non-unique stub parent
assert_equals(
graph.remove('bravo', strategy=Strategy.orphan),
frozenset(('bravo',))
)
assert_equals(
graph.nodes,
frozenset(
('baz', 'alpha', 'beta',)
)
)
assert_equals(graph.roots, frozenset(('baz',)))
# stub
assert_equals(
graph.remove('alpha', strategy=Strategy.orphan),
frozenset(('alpha',))
)
assert_equals(
graph.nodes,
frozenset(
('baz', 'beta')
)
)
assert_equals(graph.roots, frozenset(('baz', 'beta')))
assert_raises(KeyError, graph.remove, 'fake')
def test_add():
"""
add a node to a Graph
"""
from arbiter.graph import Graph
graph = Graph()
assert_equals(graph.nodes, frozenset())
assert_equals(graph.roots, frozenset())
assert_false('foo' in graph)
graph.add('foo')
assert_equals(graph.nodes, frozenset(('foo',)))
assert_equals(graph.roots, frozenset(('foo',)))
assert_true('foo' in graph)
assert_equals(graph.children('foo'), frozenset())
assert_equals(graph.parents('foo'), frozenset())
assert_false(graph.ancestor_of('foo', 'foo'))
graph.add('bar', ('foo', 'baz'))
assert_equals(graph.nodes, frozenset(('foo', 'bar', 'baz')))
assert_equals(graph.roots, frozenset(('foo',)))
assert_true('foo' in graph)
assert_true('bar' in graph)
assert_true('baz' in graph)
assert_equals(graph.children('foo'), frozenset(('bar',)))
assert_equals(graph.children('bar'), frozenset())
assert_equals(graph.children('baz'), frozenset(('bar',)))
assert_equals(graph.parents('foo'), frozenset())
assert_equals(graph.parents('bar'), frozenset(('foo', 'baz')))
assert_equals(graph.parents('baz'), frozenset())
assert_false(graph.ancestor_of('foo', 'foo'))
assert_false(graph.ancestor_of('foo', 'bar'))
assert_false(graph.ancestor_of('foo', 'baz'))
assert_true(graph.ancestor_of('bar', 'foo'))
assert_false(graph.ancestor_of('bar', 'bar'))
assert_true(graph.ancestor_of('bar', 'baz'))
assert_false(graph.ancestor_of('baz', 'foo'))
assert_false(graph.ancestor_of('baz', 'bar'))
assert_false(graph.ancestor_of('baz', 'baz'))
assert_raises(ValueError, graph.add, 'baz', ('bar',))
assert_raises(ValueError, graph.add, 'ouroboros', ('ouroboros',))
assert_raises(ValueError, graph.add, 'foo')
assert_equals(graph.nodes, frozenset(('foo', 'bar', 'baz')))
assert_equals(graph.roots, frozenset(('foo',)))
def test_naming():
"""
Node names just need to be hashable.
"""
from arbiter.graph import Graph, Strategy
graph = Graph()
for name in (1, float('NaN'), 0, None, '', frozenset(), (), False, sum):
graph.add('child1', frozenset((name,)))
graph.add(name)
assert_true(name in graph)
assert_equals(graph.nodes, frozenset((name, 'child1')))
assert_equals(graph.roots, frozenset((name,)))
assert_equals(graph.children(name), frozenset(('child1',)))
assert_equals(graph.parents(name), frozenset())
assert_false(graph.ancestor_of(name, name))
graph.add('child2', frozenset((name,)))
assert_equals(
graph.children(name),
frozenset(('child1', 'child2'))
)
graph.remove(name)
graph.remove('child1')
graph.remove('child2')
assert_equals(graph.nodes, frozenset())
graph.add(None)
graph.add('', parents=((),))
graph.add((), parents=(frozenset(),))
graph.add(frozenset())
assert_equals(graph.nodes, frozenset((None, '', (), frozenset())))
assert_equals(graph.roots, frozenset((None, frozenset())))
graph.remove((), strategy=Strategy.remove)
assert_equals(graph.nodes, frozenset((None, frozenset())))
assert_equals(graph.roots, frozenset((None, frozenset())))
assert_raises(TypeError, graph.add, [])
assert_raises(TypeError, graph.add, 'valid', parents=([],))
def test_equality():
"""
Graph equality
"""
from arbiter.graph import Graph
graph = Graph()
assert_false(graph == 1)
assert_true(graph != 0)
other = Graph()
assert_true(graph == other)
assert_false(graph != other)
graph.add('foo')
assert_false(graph == other)
assert_true(graph != other)
graph.add('bar', ('foo',))
other.add('bar', ('foo',))
# still shouldn't match graph['foo'] is a stub
assert_false(graph == other)
assert_true(graph != other)
other.add('foo')
assert_true(graph == other)
assert_false(graph != other)
def test_prune():
"""
Prune a Graph
"""
from arbiter.graph import Graph
graph = Graph()
graph.add('node')
graph.add('bar', ('foo',))
graph.add('baz', ('bar',))
graph.add('beta', ('alpha',))
graph.add('bravo', ('alpha',))
assert_equals(
graph.nodes,
frozenset(
('node', 'foo', 'bar', 'baz', 'alpha', 'beta', 'bravo')
)
)
assert_equals(graph.roots, frozenset(('node',)))
assert_equals(
graph.prune(),
frozenset(('bar', 'baz', 'beta', 'bravo'))
)
assert_equals(
graph.nodes,
frozenset(
('node',)
)
)
assert_equals(graph.roots, frozenset(('node',)))
assert_equals(graph.prune(), frozenset())
assert_equals(
graph.nodes,
frozenset(
('node',)
)
)
assert_equals(graph.roots, frozenset(('node',)))
def test_remove_promote():
"""
Remove a node (transitively making its parents its children's
parents) from a Graph.
"""
from arbiter.graph import Graph
graph = Graph()
graph.add('aye')
graph.add('insect')
graph.add('bee', ('aye', 'insect'))
graph.add('cee', ('bee',))
graph.add('child', ('stub', 'stub2'))
graph.add('grandchild', ('child',))
assert_equals(
graph.nodes,
frozenset(
(
'aye', 'insect', 'bee', 'cee', 'child', 'stub', 'stub2',
'grandchild',
)
)
)
assert_equals(graph.roots, frozenset(('aye', 'insect')))
# two new parents
assert_equals(graph.remove('bee'), frozenset(('bee',)))
assert_equals(
graph.nodes,
frozenset(
(
'aye', 'insect', 'cee', 'child', 'stub', 'stub2', 'grandchild',
)
)
)
assert_equals(
graph.roots,
frozenset(('aye', 'insect'))
)
assert_equals(graph.children('aye'), frozenset(('cee',)))
assert_equals(graph.children('insect'), frozenset(('cee',)))
assert_equals(graph.parents('cee'), frozenset(('aye', 'insect')))
# now with stubs
assert_equals(graph.remove('child'), frozenset(('child',)))
assert_equals(
graph.nodes,
frozenset(
(
'aye', 'insect', 'cee', 'stub', 'stub2', 'grandchild',
)
)
)
assert_equals(
graph.roots,
frozenset(('aye', 'insect'))
)
assert_equals(graph.children('stub'), frozenset(('grandchild',)))
assert_equals(graph.children('stub2'), frozenset(('grandchild',)))
assert_equals(
graph.parents('grandchild'),
frozenset(('stub', 'stub2'))
)
# delete a stub
assert_equals(graph.remove('stub2'), frozenset(('stub2',)))
assert_equals(
graph.nodes,
frozenset(
(
'aye', 'insect', 'cee', 'stub', 'grandchild',
)
)
)
assert_equals(
graph.roots,
frozenset(('aye', 'insect'))
)
assert_equals(graph.children('stub'), frozenset(('grandchild',)))
assert_equals(graph.parents('grandchild'), frozenset(('stub',)))
class Scheduler(object):
"""
A dependency scheduler.
"""
def __init__(self, tasks=None, completed=None, failed=None):
if completed is None:
completed = set()
if failed is None:
failed = set()
self._graph = Graph()
self._tasks = {}
self._running = set()
self._completed = completed
self._failed = failed
if tasks is not None:
for task in tasks:
self.add_task(task)
@property
def completed(self):
"""
A copy of the set of successfully completed tasks.
"""
return frozenset(self._completed)
@property
def failed(self):
"""
A copy of the set of failed tasks.
"""
return frozenset(self._failed)
@property
def running(self):
"""
A copy of the set of running tasks.
"""
return frozenset(self._running)
@property
def runnable(self):
"""
Get the set of tasks that are currently runnable.
"""
return self._graph.roots - self._running
def is_finished(self):
"""
Have all runnable tasks completed?
"""
return not (self._graph.roots or self._running)
def add_task(self, task):
"""
Add a task to the scheduler.
task: The task to add.
"""
if not self._valid_name(task.name):
raise ValueError(task.name)
self._tasks[task.name] = task
incomplete_dependencies = set()
for dependency in task.dependencies:
if not self._valid_name(dependency) or dependency in self._failed:
# there may already be tasks dependent on this one.
self._cascade_failure(task.name)
break
if dependency not in self._completed:
incomplete_dependencies.add(dependency)
else: # task hasn't failed
try:
self._graph.add(task.name, incomplete_dependencies)
except ValueError:
self._cascade_failure(task.name)
def start_task(self, name=None):
"""
Start a task.
Returns the task that was started (or None if no task has been
started).
name: (optional, None) The task to start. If a name is given,
Scheduler will attempt to start the task (and raise an
exception if the task doesn't exist or isn't runnable). If
no name is given, a task will be chosen arbitrarily
"""
if name is None:
for possibility in self._graph.roots:
if possibility not in self._running:
name = possibility
break
else: # all tasks blocked/running/completed/failed
return None
else:
if name not in self._graph.roots or name in self._running:
raise ValueError(name)
self._running.add(name)
return self._tasks[name]
def end_task(self, name, success=True):
"""
End a running task. Raises an exception if the task isn't
running.
name: The name of the task to complete.
success: (optional, True) Whether the task was successful.
"""
self._running.remove(name)
if success:
self._completed.add(name)
self._graph.remove(name, strategy=Strategy.orphan)
else:
self._cascade_failure(name)
def remove_unrunnable(self):
"""
Remove any tasks that are dependent on non-existent tasks.
"""
self._failed.update(self._graph.prune())
def fail_remaining(self):
"""
Mark all unfinished tasks (including currently running ones) as
failed.
"""
self._failed.update(self._graph.nodes)
self._graph = Graph()
self._running = set()
def _cascade_failure(self, name):
"""
Mark a task (and anything that depends on it) as failed.
name: The name of the offending task
"""
if name in self._graph:
self._failed.update(
self._graph.remove(name, strategy=Strategy.remove))
else:
self._failed.add(name)
def __enter__(self):
"""
Remove all unrunnable tasks and enter a context manager. When
the context manager is exited, all non-complete tasks will be
failed.
"""
self.remove_unrunnable()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""
Exit the context manager, stopping (and failing) any
non-completed tasks.
"""
self.fail_remaining()
@classmethod
def _valid_name(cls, name):
"""
Check whether a name is valid as a task name.
"""
return name is not None and isinstance(name, Hashable)
class Scheduler(object):
"""
A dependency scheduler.
"""
def __init__(self, tasks=None, completed=None, failed=None):
if completed is None:
completed = set()
if failed is None:
failed = set()
self._graph = Graph()
self._tasks = {}
self._running = set()
self._completed = completed
self._failed = failed
if tasks is not None:
for task in tasks:
self.add_task(task)
@property
def completed(self):
"""
A copy of the set of successfully completed tasks.
"""
return frozenset(self._completed)
@property
def failed(self):
"""
A copy of the set of failed tasks.
"""
return frozenset(self._failed)
@property
def running(self):
"""
A copy of the set of running tasks.
"""
return frozenset(self._running)
@property
def runnable(self):
"""
Get the set of tasks that are currently runnable.
"""
return self._graph.roots - self._running
def is_finished(self):
"""
Have all runnable tasks completed?
"""
return not (self._graph.roots or self._running)
def add_task(self, task):
"""
Add a task to the scheduler.
task: The task to add.
"""
if not self._valid_name(task.name):
raise ValueError(task.name)
self._tasks[task.name] = task
incomplete_dependencies = set()
for dependency in task.dependencies:
if not self._valid_name(dependency) or dependency in self._failed:
# there may already be tasks dependent on this one.
self._cascade_failure(task.name)
break
if dependency not in self._completed:
incomplete_dependencies.add(dependency)
else: # task hasn't failed
try:
self._graph.add(task.name, incomplete_dependencies)
except ValueError:
self._cascade_failure(task.name)
def start_task(self, name=None):
"""
Start a task.
Returns the task that was started (or None if no task has been
started).
name: (optional, None) The task to start. If a name is given,
Scheduler will attempt to start the task (and raise an
exception if the task doesn't exist or isn't runnable). If
no name is given, a task will be chosen arbitrarily
"""
if name is None:
for possibility in self._graph.roots:
if possibility not in self._running:
name = possibility
break
else: # all tasks blocked/running/completed/failed
return None
else:
if name not in self._graph.roots or name in self._running:
raise ValueError(name)
self._running.add(name)
return self._tasks[name]
def end_task(self, name, success=True):
"""
End a running task. Raises an exception if the task isn't
running.
name: The name of the task to complete.
success: (optional, True) Whether the task was successful.
"""
self._running.remove(name)
if success:
self._completed.add(name)
self._graph.remove(name, strategy=Strategy.orphan)
else:
self._cascade_failure(name)
def remove_unrunnable(self):
"""
Remove any tasks that are dependent on non-existent tasks.
"""
self._failed.update(self._graph.prune())
def fail_remaining(self):
"""
Mark all unfinished tasks (including currently running ones) as
failed.
"""
self._failed.update(self._graph.nodes)
self._graph = Graph()
self._running = set()
def _cascade_failure(self, name):
"""
Mark a task (and anything that depends on it) as failed.
name: The name of the offending task
"""
if name in self._graph:
self._failed.update(
self._graph.remove(name, strategy=Strategy.remove)
)
else:
self._failed.add(name)
def __enter__(self):
"""
Remove all unrunnable tasks and enter a context manager. When
the context manager is exited, all non-complete tasks will be
failed.
"""
self.remove_unrunnable()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""
Exit the context manager, stopping (and failing) any
non-completed tasks.
"""
self.fail_remaining()
@classmethod
def _valid_name(cls, name):
"""
Check whether a name is valid as a task name.
"""
return name is not None and isinstance(name, Hashable)