def _build_arg_mapping(self,
executor,
requires=None,
rebind=None,
auto_extract=True,
ignore_list=None):
required, optional = _build_arg_mapping(self.name,
requires,
rebind,
executor,
auto_extract,
ignore_list=ignore_list)
# Form the real rebind mapping, if a key name is the same as the
# key value, then well there is no rebinding happening, otherwise
# there will be.
rebind = collections.OrderedDict()
for (arg_name, bound_name) in itertools.chain(six.iteritems(required),
six.iteritems(optional)):
rebind.setdefault(arg_name, bound_name)
requires = sets.OrderedSet(six.itervalues(required))
optional = sets.OrderedSet(six.itervalues(optional))
if self.inject:
inject_keys = frozenset(six.iterkeys(self.inject))
requires -= inject_keys
optional -= inject_keys
return rebind, requires, optional
def _build_arg_mapping(self,
executor,
requires=None,
rebind=None,
auto_extract=True,
ignore_list=None):
required, optional = _build_arg_mapping(self.name,
requires,
rebind,
executor,
auto_extract,
ignore_list=ignore_list)
rebind = OrderedDict()
for (arg_name, bound_name) in itertools.chain(six.iteritems(required),
six.iteritems(optional)):
rebind.setdefault(arg_name, bound_name)
self.rebind = rebind
self.requires = sets.OrderedSet(six.itervalues(required))
self.optional = sets.OrderedSet(six.itervalues(optional))
if self.inject:
inject_keys = frozenset(six.iterkeys(self.inject))
self.requires -= inject_keys
self.optional -= inject_keys
out_of_order = self.provides.intersection(self.requires)
if out_of_order:
raise exceptions.DependencyFailure(
"Atom %(item)s provides %(oo)s that are required "
"by this atom" % dict(item=self.name, oo=sorted(out_of_order)))
def test_difference(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3])
es = set(s)
es2 = set(s2)
self.assertEqual(es.difference(es2), s.difference(s2))
self.assertEqual(es2.difference(es), s2.difference(s))
def test_intersection(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3, 4, 5])
es = set(s)
es2 = set(s2)
self.assertEqual(es.intersection(es2), s.intersection(s2))
self.assertEqual(es2.intersection(s), s2.intersection(s))
def test_union(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3, 4])
es = set(s)
es2 = set(s2)
self.assertEqual(es.union(es2), s.union(s2))
self.assertEqual(es2.union(es), s2.union(s))
def __init__(self, name=None, provides=None, inject=None):
self.name = name
self.version = (1, 0)
self.inject = inject
self.save_as = _save_as_to_mapping(provides)
self.requires = sets.OrderedSet()
self.optional = sets.OrderedSet()
self.provides = sets.OrderedSet(self.save_as)
self.rebind = OrderedDict()
def test_multi_union(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3, 4])
s3 = sets.OrderedSet([4, 5, 6])
es = set(s)
es2 = set(s2)
es3 = set(s3)
self.assertEqual(es.union(es2, es3), s.union(s2, s3))
def test_multi_intersection(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3, 4, 5])
s3 = sets.OrderedSet([1, 2])
es = set(s)
es2 = set(s2)
es3 = set(s3)
self.assertEqual(es.intersection(s2, s3), s.intersection(s2, s3))
self.assertEqual(es2.intersection(es3), s2.intersection(s3))
def test_multi_difference(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3])
s3 = sets.OrderedSet([3, 4, 5])
es = set(s)
es2 = set(s2)
es3 = set(s3)
self.assertEqual(es3.difference(es), s3.difference(s))
self.assertEqual(es.difference(es3), s.difference(s3))
self.assertEqual(es2.difference(es, es3), s2.difference(s, s3))
def _build_arg_mapping(self, executor, requires=None, rebind=None,
auto_extract=True, ignore_list=None):
required, optional = _build_arg_mapping(self.name, requires, rebind,
executor, auto_extract,
ignore_list=ignore_list)
rebind = collections.OrderedDict()
for (arg_name, bound_name) in itertools.chain(six.iteritems(required),
six.iteritems(optional)):
rebind.setdefault(arg_name, bound_name)
self.rebind = rebind
self.requires = sets.OrderedSet(six.itervalues(required))
self.optional = sets.OrderedSet(six.itervalues(optional))
if self.inject:
inject_keys = frozenset(six.iterkeys(self.inject))
self.requires -= inject_keys
self.optional -= inject_keys
def __init__(self, name=None, provides=None, requires=None,
auto_extract=True, rebind=None, inject=None,
ignore_list=None, revert_rebind=None, revert_requires=None):
if provides is None:
provides = self.default_provides
self.name = name
self.version = (1, 0)
self.inject = inject
self.save_as = _save_as_to_mapping(provides)
self.provides = sets.OrderedSet(self.save_as)
if ignore_list is None:
ignore_list = []
self.rebind, exec_requires, self.optional = self._build_arg_mapping(
self.execute,
requires=requires,
rebind=rebind, auto_extract=auto_extract,
ignore_list=ignore_list
)
revert_ignore = ignore_list + list(_default_revert_args)
revert_mapping = self._build_arg_mapping(
self.revert,
requires=revert_requires or requires,
rebind=revert_rebind or rebind,
auto_extract=auto_extract,
ignore_list=revert_ignore
)
(self.revert_rebind, addl_requires,
self.revert_optional) = revert_mapping
self.requires = exec_requires.union(addl_requires)
def test_pickleable(self):
items = [10, 9, 8, 7]
s = sets.OrderedSet(items)
self.assertEqual(items, list(s))
s_bin = pickle.dumps(s)
s2 = pickle.loads(s_bin)
self.assertEqual(s, s2)
self.assertEqual(items, list(s2))
def test_deps(self):
data = {
"id": "i",
"action": "a",
"driver": "print",
"provides": ["foo"],
"requires": ["bar"],
"needed-by": ["baz"],
}
obj = base.BaseTask("test", data, [])
self.assertEqual(obj.driver, "print")
self.assertEqual(obj.task_provides, ["foo"])
self.assertEqual(obj.task_requires, ["bar"])
self.assertEqual(obj.task_needed_by, ["baz"])
self.assertEqual(obj.requires, sets.OrderedSet(["bar"]))
obj.update_requires(["buzz"])
self.assertEqual(obj.requires, sets.OrderedSet(["bar", "buzz"]))
def __init__(self,
name=None,
provides=None,
requires=None,
auto_extract=True,
rebind=None,
inject=None,
ignore_list=None,
revert_rebind=None,
revert_requires=None):
if provides is None:
provides = self.default_provides
self.name = name
self.version = (1, 0)
self.inject = inject
self.save_as = _save_as_to_mapping(provides)
self.provides = sets.OrderedSet(self.save_as)
if ignore_list is None:
ignore_list = []
self.rebind, exec_requires, self.optional = self._build_arg_mapping(
self.execute,
requires=requires,
rebind=rebind,
auto_extract=auto_extract,
ignore_list=ignore_list)
revert_ignore = ignore_list + list(_default_revert_args)
revert_mapping = self._build_arg_mapping(self.revert,
requires=revert_requires
or requires,
rebind=revert_rebind
or rebind,
auto_extract=auto_extract,
ignore_list=revert_ignore)
(self.revert_rebind, addl_requires,
self.revert_optional) = revert_mapping
# TODO(bnemec): This should be documented as an ivar, but can't be due
# to https://github.com/sphinx-doc/sphinx/issues/2549
#: A :py:class:`~taskflow.types.sets.OrderedSet` of inputs this atom
#: requires to function.
self.requires = exec_requires.union(addl_requires)
def test_empty_difference(self):
s = sets.OrderedSet([1, 2, 3])
es = set(s)
self.assertEqual(es.difference(), s.difference())
def test_retain_ordering(self):
items = [10, 9, 8, 7]
s = sets.OrderedSet(iter(items))
self.assertEqual(items, list(s))
def _build_arg_mapping(atom_name,
reqs,
rebind_args,
function,
do_infer,
ignore_list=None):
"""Builds an input argument mapping for a given function.
Given a function, its requirements and a rebind mapping this helper
function will build the correct argument mapping for the given function as
well as verify that the final argument mapping does not have missing or
extra arguments (where applicable).
"""
# Build a list of required arguments based on function signature.
req_args = reflection.get_callable_args(function, required_only=True)
all_args = reflection.get_callable_args(function, required_only=False)
# Remove arguments that are part of ignore list.
if ignore_list:
for arg in ignore_list:
if arg in req_args:
req_args.remove(arg)
else:
ignore_list = []
# Build the required names.
required = OrderedDict()
# Add required arguments to required mappings if inference is enabled.
if do_infer:
required.update((a, a) for a in req_args)
# Add additional manually provided requirements to required mappings.
if reqs:
if isinstance(reqs, six.string_types):
required.update({reqs: reqs})
else:
required.update((a, a) for a in reqs)
# Update required mappings values based on rebinding of arguments names.
required.update(_build_rebind_dict(req_args, rebind_args))
# Determine if there are optional arguments that we may or may not take.
if do_infer:
opt_args = sets.OrderedSet(all_args)
opt_args = opt_args - set(
itertools.chain(six.iterkeys(required), iter(ignore_list)))
optional = OrderedDict((a, a) for a in opt_args)
else:
optional = OrderedDict()
# Check if we are given some extra arguments that we aren't able to accept.
if not reflection.accepts_kwargs(function):
extra_args = sets.OrderedSet(six.iterkeys(required))
extra_args -= all_args
if extra_args:
raise ValueError('Extra arguments given to atom %s: %s' %
(atom_name, list(extra_args)))
# NOTE(imelnikov): don't use set to preserve order in error message
missing_args = [arg for arg in req_args if arg not in required]
if missing_args:
raise ValueError('Missing arguments for atom %s: %s' %
(atom_name, missing_args))
return required, optional
def update_requires(self, vals: list):
LOG.debug("Updating %s requires to include %s", self.name, vals)
self.requires = self.requires.union(sets.OrderedSet(vals))
def test_retain_duplicate_ordering(self):
items = [10, 9, 10, 8, 9, 7, 8]
s = sets.OrderedSet(iter(items))
self.assertEqual([10, 9, 8, 7], list(s))
def test_duplicate_length(self):
items = [10, 9, 10, 8, 9, 7, 8]
s = sets.OrderedSet(iter(items))
self.assertEqual(4, len(s))
def test_length(self):
items = [10, 9, 8, 7]
s = sets.OrderedSet(iter(items))
self.assertEqual(4, len(s))
def test_contains(self):
items = [10, 9, 8, 7]
s = sets.OrderedSet(iter(items))
for i in items:
self.assertIn(i, s)
def test_copy(self):
items = [10, 9, 8, 7]
s = sets.OrderedSet(iter(items))
s2 = s.copy()
self.assertEqual(s, s2)
self.assertEqual(items, list(s2))
def test_empty_union(self):
s = sets.OrderedSet([1, 2, 3])
es = set(s)
self.assertEqual(es.union(), s.union())
def test_empty_intersection(self):
s = sets.OrderedSet([1, 2, 3])
es = set(s)
self.assertEqual(es.intersection(), s.intersection())
def test_subset(self):
s = sets.OrderedSet([1, 2, 3])
s2 = sets.OrderedSet([2, 3])
self.assertTrue(s2.issubset(s))
self.assertFalse(s2.issuperset(s))
