def add(self, *items):
"""Adds a given task/tasks/flow/flows to this flow."""
if not items:
return self
# NOTE(imelnikov): we add item to the end of flow, so it should
# not provide anything previous items of the flow require.
requires = self.requires
provides = self.provides
for item in items:
requires |= item.requires
out_of_order = requires & item.provides
if out_of_order:
raise exceptions.InvariantViolation(
"%(item)s provides %(oo)s that are required "
"by previous item(s) of linear flow %(flow)s"
% dict(item=item.name, flow=self.name,
oo=sorted(out_of_order)))
same_provides = provides & item.provides
if same_provides:
raise exceptions.DependencyFailure(
"%(item)s provides %(value)s but is already being"
" provided by %(flow)s and duplicate producers"
" are disallowed"
% dict(item=item.name, flow=self.name,
value=sorted(same_provides)))
provides |= item.provides
self._children.extend(items)
return self
def _build_arg_mapping(self,
executor,
requires=None,
rebind=None,
auto_extract=True,
ignore_list=None):
required, optional = _build_arg_mapping(self.name,
requires,
rebind,
executor,
auto_extract,
ignore_list=ignore_list)
rebind = OrderedDict()
for (arg_name, bound_name) in itertools.chain(six.iteritems(required),
six.iteritems(optional)):
rebind.setdefault(arg_name, bound_name)
self.rebind = rebind
self.requires = sets.OrderedSet(six.itervalues(required))
self.optional = sets.OrderedSet(six.itervalues(optional))
if self.inject:
inject_keys = frozenset(six.iterkeys(self.inject))
self.requires -= inject_keys
self.optional -= inject_keys
out_of_order = self.provides.intersection(self.requires)
if out_of_order:
raise exceptions.DependencyFailure(
"Atom %(item)s provides %(oo)s that are required "
"by this atom" % dict(item=self.name, oo=sorted(out_of_order)))
def _validate(self, graph=None):
if graph is None:
graph = self._graph
# Ensure that there is a valid topological ordering.
if not nx.is_directed_acyclic_graph(graph):
raise exc.DependencyFailure("No path through the items in the"
" graph produces an ordering that"
" will allow for correct dependency"
" resolution")
def _build_arg_mapping(self, executor, requires=None, rebind=None,
auto_extract=True, ignore_list=None):
self.rebind = _build_arg_mapping(self.name, requires, rebind,
executor, auto_extract, ignore_list)
out_of_order = self.provides.intersection(self.requires)
if out_of_order:
raise exceptions.DependencyFailure(
"Atom %(item)s provides %(oo)s that are required "
"by this atom"
% dict(item=self.name, oo=sorted(out_of_order)))
def _swap(self, graph):
"""Validates the replacement graph and then swaps the underlying graph.
After swapping occurs the underlying graph will be frozen so that the
immutability invariant is maintained (we may be able to relax this
constraint in the future since our exposed public api does not allow
direct access to the underlying graph).
"""
if not graph.is_directed_acyclic():
raise exc.DependencyFailure("No path through the node(s) in the"
" graph produces an ordering that"
" will allow for logical"
" edge traversal")
self._graph = graph.freeze()
def add(self, *items):
"""Adds a given task/tasks/flow/flows to this flow."""
items = [i for i in items if not self._graph.has_node(i)]
if not items:
return self
requirements = collections.defaultdict(list)
provided = {}
def update_requirements(node):
for value in node.requires:
requirements[value].append(node)
for node in self:
update_requirements(node)
for value in node.provides:
provided[value] = node
# NOTE(harlowja): Add items and edges to a temporary copy of the
# underlying graph and only if that is successful added to do we then
# swap with the underlying graph.
tmp_graph = nx.DiGraph(self._graph)
for item in items:
tmp_graph.add_node(item)
update_requirements(item)
for value in item.provides:
if value in provided:
raise exc.DependencyFailure(
"%(item)s provides %(value)s but is already being"
" provided by %(flow)s and duplicate producers"
" are disallowed" % dict(item=item.name,
flow=provided[value].name,
value=value))
provided[value] = item
for value in item.requires:
if value in provided:
self._link(provided[value],
item,
graph=tmp_graph,
reason=value)
for value in item.provides:
if value in requirements:
for node in requirements[value]:
self._link(item, node, graph=tmp_graph, reason=value)
self._swap(tmp_graph)
return self
def add(self, *items):
"""Adds a given task/tasks/flow/flows to this flow."""
if not items:
return self
# NOTE(harlowja): check that items to be added are actually
# independent.
provides = self.provides
old_requires = self.requires
for item in items:
item_provides = item.provides
bad_provs = item_provides & old_requires
if bad_provs:
raise exceptions.InvariantViolation(
"%(item)s provides %(oo)s that are required "
"by other item(s) of unordered flow %(flow)s"
% dict(item=item.name, flow=self.name,
oo=sorted(bad_provs)))
same_provides = provides & item.provides
if same_provides:
raise exceptions.DependencyFailure(
"%(item)s provides %(value)s but is already being"
" provided by %(flow)s and duplicate producers"
" are disallowed"
% dict(item=item.name, flow=self.name,
value=sorted(same_provides)))
provides |= item.provides
for item in items:
bad_reqs = provides & item.requires
if bad_reqs:
raise exceptions.InvariantViolation(
"%(item)s requires %(oo)s that are provided "
"by other item(s) of unordered flow %(flow)s"
% dict(item=item.name, flow=self.name,
oo=sorted(bad_reqs)))
self._children.update(items)
return self