def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
ignore_list=None):
"""Builds an input argument mapping for a given function.
Given a function, its requirements and a rebind mapping this helper
function will build the correct argument mapping for the given function as
well as verify that the final argument mapping does not have missing or
extra arguments (where applicable).
"""
# build a list of required arguments based on function signature
req_args = reflection.get_callable_args(function, required_only=True)
all_args = reflection.get_callable_args(function, required_only=False)
# remove arguments that are part of ignore_list
if ignore_list:
for arg in ignore_list:
if arg in req_args:
req_args.remove(arg)
else:
ignore_list = []
required = {}
# add reqs to required mappings
if reqs:
if isinstance(reqs, six.string_types):
required.update({reqs: reqs})
else:
required.update((a, a) for a in reqs)
# add req_args to required mappings if do_infer is set
if do_infer:
required.update((a, a) for a in req_args)
# update required mappings based on rebind_args
required.update(_build_rebind_dict(req_args, rebind_args))
if do_infer:
opt_args = set(all_args) - set(required) - set(ignore_list)
optional = dict((a, a) for a in opt_args)
else:
optional = {}
if not reflection.accepts_kwargs(function):
extra_args = set(required) - set(all_args)
if extra_args:
extra_args_str = ', '.join(sorted(extra_args))
raise ValueError('Extra arguments given to atom %s: %s'
% (atom_name, extra_args_str))
# NOTE(imelnikov): don't use set to preserve order in error message
missing_args = [arg for arg in req_args if arg not in required]
if missing_args:
raise ValueError('Missing arguments for atom %s: %s'
% (atom_name, ' ,'.join(missing_args)))
return required, optional
def __init__(self, functor, requires, name=None, provides=None,
auto_extract=True, rebind=None, inject=None):
if not six.callable(functor):
raise ValueError("Function to use for reduce must be callable")
f_args = reflection.get_callable_args(functor)
if len(f_args) != 2:
raise ValueError("%s arguments were provided. Reduce functor "
"must take exactly 2 arguments." % len(f_args))
if not misc.is_iterable(requires):
raise TypeError("%s type was provided for requires. Requires "
"must be an iterable." % type(requires))
if len(requires) < 2:
raise ValueError("%s elements were provided. Requires must have "
"at least 2 elements." % len(requires))
if name is None:
name = reflection.get_callable_name(functor)
super(ReduceFunctorTask, self).__init__(name=name,
provides=provides,
inject=inject,
requires=requires,
rebind=rebind,
auto_extract=auto_extract)
self._functor = functor
def __init__(self, obj, config, migration, **kwargs):
super(MigrationTask, self).__init__(
name='{0}_{1}'.format(utils.qualname(self.__class__),
taskflow_utils.object_name(obj)),
requires=reflection.get_callable_args(self.migrate),
**kwargs)
self.src_object = obj
self.config = config
self.migration = migration
self.created_object = None
def execute(self, wrapper, *_args, **_kwargs):
"""Invoke saved callable with saved args."""
if not ('provided' in reflection.get_callable_args(self._func)
or reflection.accepts_kwargs(self._func)):
_kwargs.pop('provided', None)
if self._logspec:
# Execute the log method (the first element in the list) with its
# arguments (the remaining elements in the list).
self._logspec[0](*self._logspec[1:])
return self._func(wrapper, *_args, **_kwargs)
def cleanup(self, on_expired_callback=None):
"""Delete out-dated keys & values from the cache."""
with self._lock:
expired_values = [(k, v) for k, v in six.iteritems(self._data)
if v.expired]
for (k, _v) in expired_values:
del self._data[k]
if on_expired_callback is not None:
arg_c = len(reflection.get_callable_args(on_expired_callback))
for (k, v) in expired_values:
if arg_c == 2:
on_expired_callback(k, v)
else:
on_expired_callback(v)
def clear(self, on_cleared_callback=None):
"""Removes all keys & values from the cache."""
cleared_items = []
with self._lock:
if on_cleared_callback is not None:
cleared_items.extend(six.iteritems(self._data))
self._data.clear()
if on_cleared_callback is not None:
arg_c = len(reflection.get_callable_args(on_cleared_callback))
for (k, v) in cleared_items:
if arg_c == 2:
on_cleared_callback(k, v)
else:
on_cleared_callback(v)
def _execute(wrapper):
update_needed = False
for task in self._tasks:
kwargs = task.save_kwargs
if ('provided' in reflection.get_callable_args(task.execute)
or reflection.accepts_kwargs(task.execute)):
kwargs['provided'] = self.provided
ret = task.execute(wrapper, *task.save_args, **kwargs)
if task.flag_update and ret:
update_needed = True
if task.provides is not None:
self.provided[task.provides] = ret
if update_needed:
wrapper = wrapper.update(timeout=self.update_timeout)
return wrapper
def __init__(self, functor, requires, name=None, provides=None,
auto_extract=True, rebind=None, inject=None):
if not six.callable(functor):
raise ValueError("Function to use for map must be callable")
f_args = reflection.get_callable_args(functor)
if len(f_args) != 1:
raise ValueError("%s arguments were provided. Map functor must "
"take exactly 1 argument." % len(f_args))
if not misc.is_iterable(requires):
raise TypeError("%s type was provided for requires. Requires "
"must be an iterable." % type(requires))
if name is None:
name = reflection.get_callable_name(functor)
super(MapFunctorTask, self).__init__(name=name, provides=provides,
inject=inject)
self._functor = functor
self._build_arg_mapping(executor=self.execute, requires=requires,
rebind=rebind, auto_extract=auto_extract)
def test_class_constructor(self):
result = reflection.get_callable_args(ClassWithInit)
self.assertEqual(['k', 'l'], result)
def test_class_method(self):
result = reflection.get_callable_args(Class.class_method)
self.assertEqual(['g', 'h'], result)
def test_instance_method(self):
result = reflection.get_callable_args(Class().method)
self.assertEqual(['c', 'd'], result)
def test_method(self):
result = reflection.get_callable_args(Class.method)
self.assertEqual(['self', 'c', 'd'], result)
def test_required_only(self):
result = reflection.get_callable_args(function_with_defs,
required_only=True)
self.assertEqual(['a', 'b'], result)
def test_function_with_defaults(self):
result = reflection.get_callable_args(function_with_defs)
self.assertEqual(['a', 'b', 'optional'], result)
def test_mere_function(self):
result = reflection.get_callable_args(mere_function)
self.assertEqual(['a', 'b'], result)
def findall(self, **kwargs):
"""Find all items with attributes matching ``**kwargs``."""
found = ListWithMeta([], None)
searches = kwargs.items()
detailed = True
list_kwargs = {}
list_argspec = reflection.get_callable_args(self.list)
if 'detailed' in list_argspec:
detailed = ("human_id" not in kwargs and
"name" not in kwargs and
"display_name" not in kwargs)
list_kwargs['detailed'] = detailed
if 'is_public' in list_argspec and 'is_public' in kwargs:
is_public = kwargs['is_public']
list_kwargs['is_public'] = is_public
if is_public is None:
tmp_kwargs = kwargs.copy()
del tmp_kwargs['is_public']
searches = tmp_kwargs.items()
if 'search_opts' in list_argspec:
# pass search_opts in to do server side based filtering.
# TODO(jogo) not all search_opts support regex, find way to
# identify when to use regex and when to use string matching.
# volumes does not support regex while servers does. So when
# doing findall on servers some client side filtering is still
# needed.
if "human_id" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["human_id"]}
elif "name" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["name"]}
elif "display_name" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["display_name"]}
if "all_tenants" in kwargs:
all_tenants = kwargs['all_tenants']
list_kwargs['search_opts']['all_tenants'] = all_tenants
searches = [(k, v) for k, v in searches if k != 'all_tenants']
if "deleted" in kwargs:
deleted = kwargs['deleted']
list_kwargs['search_opts']['deleted'] = deleted
searches = [(k, v) for k, v in searches if k != 'deleted']
listing = self.list(**list_kwargs)
found.append_request_ids(listing.request_ids)
for obj in listing:
try:
if all(getattr(obj, attr) == value
for (attr, value) in searches):
if detailed:
found.append(obj)
else:
detail = self.get(obj.id)
found.append(detail)
found.append_request_ids(detail.request_ids)
except AttributeError:
continue
return found
def test_function_with_defaults(self):
result = reflection.get_callable_args(function_with_defs)
self.assertEqual(['a', 'b', 'optional'], result)
def _build_arg_mapping(atom_name, reqs, rebind_args, function, do_infer,
ignore_list=None):
"""Builds an input argument mapping for a given function.
Given a function, its requirements and a rebind mapping this helper
function will build the correct argument mapping for the given function as
well as verify that the final argument mapping does not have missing or
extra arguments (where applicable).
"""
# Build a list of required arguments based on function signature.
req_args = reflection.get_callable_args(function, required_only=True)
all_args = reflection.get_callable_args(function, required_only=False)
# Remove arguments that are part of ignore list.
if ignore_list:
for arg in ignore_list:
if arg in req_args:
req_args.remove(arg)
else:
ignore_list = []
# Build the required names.
required = collections.OrderedDict()
# Add required arguments to required mappings if inference is enabled.
if do_infer:
required.update((a, a) for a in req_args)
# Add additional manually provided requirements to required mappings.
if reqs:
if isinstance(reqs, six.string_types):
required.update({reqs: reqs})
else:
required.update((a, a) for a in reqs)
# Update required mappings values based on rebinding of arguments names.
required.update(_build_rebind_dict(req_args, rebind_args))
# Determine if there are optional arguments that we may or may not take.
if do_infer:
opt_args = sets.OrderedSet(all_args)
opt_args = opt_args - set(itertools.chain(six.iterkeys(required),
iter(ignore_list)))
optional = collections.OrderedDict((a, a) for a in opt_args)
else:
optional = collections.OrderedDict()
# Check if we are given some extra arguments that we aren't able to accept.
if not reflection.accepts_kwargs(function):
extra_args = sets.OrderedSet(six.iterkeys(required))
extra_args -= all_args
if extra_args:
raise ValueError('Extra arguments given to atom %s: %s'
% (atom_name, list(extra_args)))
# NOTE(imelnikov): don't use set to preserve order in error message
missing_args = [arg for arg in req_args if arg not in required]
if missing_args:
raise ValueError('Missing arguments for atom %s: %s'
% (atom_name, missing_args))
return required, optional
def test_mere_function(self):
result = reflection.get_callable_args(mere_function)
self.assertEqual(['a', 'b'], result)
def test_class_with_call(self):
result = reflection.get_callable_args(CallableClass())
self.assertEqual(['i', 'j'], result)
def test_required_only(self):
result = reflection.get_callable_args(function_with_defs,
required_only=True)
self.assertEqual(['a', 'b'], result)
def test_method(self):
result = reflection.get_callable_args(Class.method)
self.assertEqual(['self', 'c', 'd'], result)
def test_instance_method(self):
result = reflection.get_callable_args(Class().method)
self.assertEqual(['c', 'd'], result)
def test_decorators_work(self):
@dummy_decorator
def special_fun(x, y):
pass
result = reflection.get_callable_args(special_fun)
self.assertEqual(['x', 'y'], result)
def findall(self, **kwargs):
"""Find all items with attributes matching ``**kwargs``."""
found = ListWithMeta([], None)
searches = kwargs.items()
detailed = True
list_kwargs = {}
list_argspec = reflection.get_callable_args(self.list)
if 'detailed' in list_argspec:
detailed = ("human_id" not in kwargs and
"name" not in kwargs and
"display_name" not in kwargs)
list_kwargs['detailed'] = detailed
if 'is_public' in list_argspec and 'is_public' in kwargs:
is_public = kwargs['is_public']
list_kwargs['is_public'] = is_public
if is_public is None:
tmp_kwargs = kwargs.copy()
del tmp_kwargs['is_public']
searches = tmp_kwargs.items()
if 'search_opts' in list_argspec:
# pass search_opts in to do server side based filtering.
# TODO(jogo) not all search_opts support regex, find way to
# identify when to use regex and when to use string matching.
# volumes does not support regex while servers does. So when
# doing findall on servers some client side filtering is still
# needed.
if "human_id" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["human_id"]}
elif "name" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["name"]}
elif "display_name" in kwargs:
list_kwargs['search_opts'] = {"name": kwargs["display_name"]}
if "all_tenants" in kwargs:
all_tenants = kwargs['all_tenants']
list_kwargs['search_opts']['all_tenants'] = all_tenants
searches = [(k, v) for k, v in searches if k != 'all_tenants']
if "deleted" in kwargs:
deleted = kwargs['deleted']
list_kwargs['search_opts']['deleted'] = deleted
searches = [(k, v) for k, v in searches if k != 'deleted']
listing = self.list(**list_kwargs)
found.append_request_ids(listing.request_ids)
for obj in listing:
try:
if all(getattr(obj, attr) == value
for (attr, value) in searches):
if detailed:
found.append(obj)
else:
detail = self.get(obj.id)
found.append(detail)
found.append_request_ids(detail.request_ids)
except AttributeError:
continue
return found
