def add(self, *items): """Adds a given task/tasks/flow/flows to this flow.""" if not items: return self # NOTE(imelnikov): we add item to the end of flow, so it should # not provide anything previous items of the flow require. requires = self.requires provides = self.provides for item in items: requires |= item.requires out_of_order = requires & item.provides if out_of_order: raise exceptions.InvariantViolation( "%(item)s provides %(oo)s that are required " "by previous item(s) of linear flow %(flow)s" % dict(item=item.name, flow=self.name, oo=sorted(out_of_order))) same_provides = provides & item.provides if same_provides: raise exceptions.DependencyFailure( "%(item)s provides %(value)s but is already being" " provided by %(flow)s and duplicate producers" " are disallowed" % dict(item=item.name, flow=self.name, value=sorted(same_provides))) provides |= item.provides self._children.extend(items) return self
def _build_arg_mapping(self, executor, requires=None, rebind=None, auto_extract=True, ignore_list=None): required, optional = _build_arg_mapping(self.name, requires, rebind, executor, auto_extract, ignore_list=ignore_list) rebind = OrderedDict() for (arg_name, bound_name) in itertools.chain(six.iteritems(required), six.iteritems(optional)): rebind.setdefault(arg_name, bound_name) self.rebind = rebind self.requires = sets.OrderedSet(six.itervalues(required)) self.optional = sets.OrderedSet(six.itervalues(optional)) if self.inject: inject_keys = frozenset(six.iterkeys(self.inject)) self.requires -= inject_keys self.optional -= inject_keys out_of_order = self.provides.intersection(self.requires) if out_of_order: raise exceptions.DependencyFailure( "Atom %(item)s provides %(oo)s that are required " "by this atom" % dict(item=self.name, oo=sorted(out_of_order)))
def _validate(self, graph=None): if graph is None: graph = self._graph # Ensure that there is a valid topological ordering. if not nx.is_directed_acyclic_graph(graph): raise exc.DependencyFailure("No path through the items in the" " graph produces an ordering that" " will allow for correct dependency" " resolution")
def _build_arg_mapping(self, executor, requires=None, rebind=None, auto_extract=True, ignore_list=None): self.rebind = _build_arg_mapping(self.name, requires, rebind, executor, auto_extract, ignore_list) out_of_order = self.provides.intersection(self.requires) if out_of_order: raise exceptions.DependencyFailure( "Atom %(item)s provides %(oo)s that are required " "by this atom" % dict(item=self.name, oo=sorted(out_of_order)))
def _swap(self, graph): """Validates the replacement graph and then swaps the underlying graph. After swapping occurs the underlying graph will be frozen so that the immutability invariant is maintained (we may be able to relax this constraint in the future since our exposed public api does not allow direct access to the underlying graph). """ if not graph.is_directed_acyclic(): raise exc.DependencyFailure("No path through the node(s) in the" " graph produces an ordering that" " will allow for logical" " edge traversal") self._graph = graph.freeze()
def add(self, *items): """Adds a given task/tasks/flow/flows to this flow.""" items = [i for i in items if not self._graph.has_node(i)] if not items: return self requirements = collections.defaultdict(list) provided = {} def update_requirements(node): for value in node.requires: requirements[value].append(node) for node in self: update_requirements(node) for value in node.provides: provided[value] = node # NOTE(harlowja): Add items and edges to a temporary copy of the # underlying graph and only if that is successful added to do we then # swap with the underlying graph. tmp_graph = nx.DiGraph(self._graph) for item in items: tmp_graph.add_node(item) update_requirements(item) for value in item.provides: if value in provided: raise exc.DependencyFailure( "%(item)s provides %(value)s but is already being" " provided by %(flow)s and duplicate producers" " are disallowed" % dict(item=item.name, flow=provided[value].name, value=value)) provided[value] = item for value in item.requires: if value in provided: self._link(provided[value], item, graph=tmp_graph, reason=value) for value in item.provides: if value in requirements: for node in requirements[value]: self._link(item, node, graph=tmp_graph, reason=value) self._swap(tmp_graph) return self
def add(self, *items): """Adds a given task/tasks/flow/flows to this flow.""" if not items: return self # NOTE(harlowja): check that items to be added are actually # independent. provides = self.provides old_requires = self.requires for item in items: item_provides = item.provides bad_provs = item_provides & old_requires if bad_provs: raise exceptions.InvariantViolation( "%(item)s provides %(oo)s that are required " "by other item(s) of unordered flow %(flow)s" % dict(item=item.name, flow=self.name, oo=sorted(bad_provs))) same_provides = provides & item.provides if same_provides: raise exceptions.DependencyFailure( "%(item)s provides %(value)s but is already being" " provided by %(flow)s and duplicate producers" " are disallowed" % dict(item=item.name, flow=self.name, value=sorted(same_provides))) provides |= item.provides for item in items: bad_reqs = provides & item.requires if bad_reqs: raise exceptions.InvariantViolation( "%(item)s requires %(oo)s that are provided " "by other item(s) of unordered flow %(flow)s" % dict(item=item.name, flow=self.name, oo=sorted(bad_reqs))) self._children.update(items) return self