def _normalize_parameters(self, thing, action=None, additional_args=None):
'''
arguments can be fuzzy. Deal with all the forms.
'''
additional_args = {} if additional_args is None else additional_args
# final args are the ones we'll eventually return, so first update
# them with any additional args specified, which have lower priority
# than those which may be parsed/normalized next
final_args = dict()
if additional_args:
if isinstance(additional_args, string_types):
templar = Templar(loader=None)
if templar.is_template(additional_args):
final_args['_variable_params'] = additional_args
else:
raise AssibleParserError("Complex args containing variables cannot use bare variables (without Jinja2 delimiters), "
"and must use the full variable style ('{{var_name}}')")
elif isinstance(additional_args, dict):
final_args.update(additional_args)
else:
raise AssibleParserError('Complex args must be a dictionary or variable string ("{{var}}").')
# how we normalize depends if we figured out what the module name is
# yet. If we have already figured it out, it's a 'new style' invocation.
# otherwise, it's not
if action is not None:
args = self._normalize_new_style_args(thing, action)
else:
(action, args) = self._normalize_old_style_args(thing)
# this can occasionally happen, simplify
if args and 'args' in args:
tmp_args = args.pop('args')
if isinstance(tmp_args, string_types):
tmp_args = parse_kv(tmp_args)
args.update(tmp_args)
# only internal variables can start with an underscore, so
# we don't allow users to set them directly in arguments
if args and action not in FREEFORM_ACTIONS:
for arg in args:
arg = to_text(arg)
if arg.startswith('_assible_'):
raise AssibleError("invalid parameter specified for action '%s': '%s'" % (action, arg))
# finally, update the args we're going to return with the ones
# which were normalized above
if args:
final_args.update(args)
return (action, final_args)
def _get_delegated_vars(self, play, task, existing_variables):
# This method has a lot of code copied from ``TaskExecutor._get_loop_items``
# if this is failing, and ``TaskExecutor._get_loop_items`` is not
# then more will have to be copied here.
# TODO: dedupe code here and with ``TaskExecutor._get_loop_items``
# this may be possible once we move pre-processing pre fork
if not hasattr(task, 'loop'):
# This "task" is not a Task, so we need to skip it
return {}, None
# we unfortunately need to template the delegate_to field here,
# as we're fetching vars before post_validate has been called on
# the task that has been passed in
vars_copy = existing_variables.copy()
# get search path for this task to pass to lookup plugins
vars_copy['assible_search_path'] = task.get_search_path()
# ensure basedir is always in (dwim already searches here but we need to display it)
if self._loader.get_basedir() not in vars_copy['assible_search_path']:
vars_copy['assible_search_path'].append(self._loader.get_basedir())
templar = Templar(loader=self._loader, variables=vars_copy)
items = []
has_loop = True
if task.loop_with is not None:
if task.loop_with in lookup_loader:
fail = True
if task.loop_with == 'first_found':
# first_found loops are special. If the item is undefined then we want to fall through to the next
fail = False
try:
loop_terms = listify_lookup_plugin_terms(
terms=task.loop,
templar=templar,
loader=self._loader,
fail_on_undefined=fail,
convert_bare=False)
if not fail:
loop_terms = [
t for t in loop_terms if not templar.is_template(t)
]
mylookup = lookup_loader.get(task.loop_with,
loader=self._loader,
templar=templar)
# give lookup task 'context' for subdir (mostly needed for first_found)
for subdir in ['template', 'var',
'file']: # TODO: move this to constants?
if subdir in task.action:
break
setattr(mylookup, '_subdir', subdir + 's')
items = wrap_var(
mylookup.run(terms=loop_terms, variables=vars_copy))
except AssibleTemplateError:
# This task will be skipped later due to this, so we just setup
# a dummy array for the later code so it doesn't fail
items = [None]
else:
raise AssibleError(
"Failed to find the lookup named '%s' in the available lookup plugins"
% task.loop_with)
elif task.loop is not None:
try:
items = templar.template(task.loop)
except AssibleTemplateError:
# This task will be skipped later due to this, so we just setup
# a dummy array for the later code so it doesn't fail
items = [None]
else:
has_loop = False
items = [None]
# since host can change per loop, we keep dict per host name resolved
delegated_host_vars = dict()
item_var = getattr(task.loop_control, 'loop_var', 'item')
cache_items = False
for item in items:
# update the variables with the item value for templating, in case we need it
if item is not None:
vars_copy[item_var] = item
templar.available_variables = vars_copy
delegated_host_name = templar.template(task.delegate_to,
fail_on_undefined=False)
if delegated_host_name != task.delegate_to:
cache_items = True
if delegated_host_name is None:
raise AssibleError(
message="Undefined delegate_to host for task:",
obj=task._ds)
if not isinstance(delegated_host_name, string_types):
raise AssibleError(
message=
"the field 'delegate_to' has an invalid type (%s), and could not be"
" converted to a string type." % type(delegated_host_name),
obj=task._ds)
if delegated_host_name in delegated_host_vars:
# no need to repeat ourselves, as the delegate_to value
# does not appear to be tied to the loop item variable
continue
# now try to find the delegated-to host in inventory, or failing that,
# create a new host on the fly so we can fetch variables for it
delegated_host = None
if self._inventory is not None:
delegated_host = self._inventory.get_host(delegated_host_name)
# try looking it up based on the address field, and finally
# fall back to creating a host on the fly to use for the var lookup
if delegated_host is None:
for h in self._inventory.get_hosts(
ignore_limits=True, ignore_restrictions=True):
# check if the address matches, or if both the delegated_to host
# and the current host are in the list of localhost aliases
if h.address == delegated_host_name:
delegated_host = h
break
else:
delegated_host = Host(name=delegated_host_name)
else:
delegated_host = Host(name=delegated_host_name)
# now we go fetch the vars for the delegated-to host and save them in our
# master dictionary of variables to be used later in the TaskExecutor/PlayContext
delegated_host_vars[delegated_host_name] = self.get_vars(
play=play,
host=delegated_host,
task=task,
include_delegate_to=False,
include_hostvars=True,
)
delegated_host_vars[delegated_host_name][
'inventory_hostname'] = vars_copy.get('inventory_hostname')
_assible_loop_cache = None
if has_loop and cache_items:
# delegate_to templating produced a change, so we will cache the templated items
# in a special private hostvar
# this ensures that delegate_to+loop doesn't produce different results than TaskExecutor
# which may reprocess the loop
_assible_loop_cache = items
return delegated_host_vars, _assible_loop_cache
def _get_magic_variables(self,
play,
host,
task,
include_hostvars,
include_delegate_to,
_hosts=None,
_hosts_all=None):
'''
Returns a dictionary of so-called "magic" variables in Assible,
which are special variables we set internally for use.
'''
variables = {}
variables['playbook_dir'] = os.path.abspath(self._loader.get_basedir())
variables['assible_playbook_python'] = sys.executable
variables['assible_config_file'] = C.CONFIG_FILE
if play:
# This is a list of all role names of all dependencies for all roles for this play
dependency_role_names = list(
set([
d.get_name() for r in play.roles
for d in r.get_all_dependencies()
]))
# This is a list of all role names of all roles for this play
play_role_names = [r.get_name() for r in play.roles]
# assible_role_names includes all role names, dependent or directly referenced by the play
variables['assible_role_names'] = list(
set(dependency_role_names + play_role_names))
# assible_play_role_names includes the names of all roles directly referenced by this play
# roles that are implicitly referenced via dependencies are not listed.
variables['assible_play_role_names'] = play_role_names
# assible_dependent_role_names includes the names of all roles that are referenced via dependencies
# dependencies that are also explicitly named as roles are included in this list
variables['assible_dependent_role_names'] = dependency_role_names
# DEPRECATED: role_names should be deprecated in favor of assible_role_names or assible_play_role_names
variables['role_names'] = variables['assible_play_role_names']
variables['assible_play_name'] = play.get_name()
if task:
if task._role:
variables['role_name'] = task._role.get_name(
include_role_fqcn=False)
variables['role_path'] = task._role._role_path
variables['role_uuid'] = text_type(task._role._uuid)
variables[
'assible_collection_name'] = task._role._role_collection
variables['assible_role_name'] = task._role.get_name()
if self._inventory is not None:
variables['groups'] = self._inventory.get_groups_dict()
if play:
templar = Templar(loader=self._loader)
if templar.is_template(play.hosts):
pattern = 'all'
else:
pattern = play.hosts or 'all'
# add the list of hosts in the play, as adjusted for limit/filters
if not _hosts_all:
_hosts_all = [
h.name for h in self._inventory.get_hosts(
pattern=pattern, ignore_restrictions=True)
]
if not _hosts:
_hosts = [h.name for h in self._inventory.get_hosts()]
variables['assible_play_hosts_all'] = _hosts_all[:]
variables['assible_play_hosts'] = [
x for x in variables['assible_play_hosts_all']
if x not in play._removed_hosts
]
variables['assible_play_batch'] = [
x for x in _hosts if x not in play._removed_hosts
]
# DEPRECATED: play_hosts should be deprecated in favor of assible_play_batch,
# however this would take work in the templating engine, so for now we'll add both
variables['play_hosts'] = variables['assible_play_batch']
# the 'omit' value allows params to be left out if the variable they are based on is undefined
variables['omit'] = self._omit_token
# Set options vars
for option, option_value in iteritems(self._options_vars):
variables[option] = option_value
if self._hostvars is not None and include_hostvars:
variables['hostvars'] = self._hostvars
return variables
def load_list_of_tasks(ds,
play,
block=None,
role=None,
task_include=None,
use_handlers=False,
variable_manager=None,
loader=None):
'''
Given a list of task datastructures (parsed from YAML),
return a list of Task() or TaskInclude() objects.
'''
# we import here to prevent a circular dependency with imports
from assible.playbook.block import Block
from assible.playbook.handler import Handler
from assible.playbook.task import Task
from assible.playbook.task_include import TaskInclude
from assible.playbook.role_include import IncludeRole
from assible.playbook.handler_task_include import HandlerTaskInclude
from assible.template import Templar
if not isinstance(ds, list):
raise AssibleAssertionError(
'The ds (%s) should be a list but was a %s' % (ds, type(ds)))
task_list = []
for task_ds in ds:
if not isinstance(task_ds, dict):
raise AssibleAssertionError(
'The ds (%s) should be a dict but was a %s' % (ds, type(ds)))
if 'block' in task_ds:
t = Block.load(
task_ds,
play=play,
parent_block=block,
role=role,
task_include=task_include,
use_handlers=use_handlers,
variable_manager=variable_manager,
loader=loader,
)
task_list.append(t)
else:
args_parser = ModuleArgsParser(task_ds)
try:
(action, args,
delegate_to) = args_parser.parse(skip_action_validation=True)
except AssibleParserError as e:
# if the raises exception was created with obj=ds args, then it includes the detail
# so we dont need to add it so we can just re raise.
if e._obj:
raise
# But if it wasn't, we can add the yaml object now to get more detail
raise AssibleParserError(to_native(e), obj=task_ds, orig_exc=e)
if action in ('include', 'import_tasks', 'include_tasks'):
if use_handlers:
include_class = HandlerTaskInclude
else:
include_class = TaskInclude
t = include_class.load(task_ds,
block=block,
role=role,
task_include=None,
variable_manager=variable_manager,
loader=loader)
all_vars = variable_manager.get_vars(play=play, task=t)
templar = Templar(loader=loader, variables=all_vars)
# check to see if this include is dynamic or static:
# 1. the user has set the 'static' option to false or true
# 2. one of the appropriate config options was set
if action == 'include_tasks':
is_static = False
elif action == 'import_tasks':
is_static = True
elif t.static is not None:
display.deprecated(
"The use of 'static' has been deprecated. "
"Use 'import_tasks' for static inclusion, or 'include_tasks' for dynamic inclusion",
version='2.12',
collection_name='assible.builtin')
is_static = t.static
else:
is_static = C.DEFAULT_TASK_INCLUDES_STATIC or \
(use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC) or \
(not templar.is_template(t.args['_raw_params']) and t.all_parents_static() and not t.loop)
if is_static:
if t.loop is not None:
if action == 'import_tasks':
raise AssibleParserError(
"You cannot use loops on 'import_tasks' statements. You should use 'include_tasks' instead.",
obj=task_ds)
else:
raise AssibleParserError(
"You cannot use 'static' on an include with a loop",
obj=task_ds)
# we set a flag to indicate this include was static
t.statically_loaded = True
# handle relative includes by walking up the list of parent include
# tasks and checking the relative result to see if it exists
parent_include = block
cumulative_path = None
found = False
subdir = 'tasks'
if use_handlers:
subdir = 'handlers'
while parent_include is not None:
if not isinstance(parent_include, TaskInclude):
parent_include = parent_include._parent
continue
try:
parent_include_dir = os.path.dirname(
templar.template(
parent_include.args.get('_raw_params')))
except AssibleUndefinedVariable as e:
if not parent_include.statically_loaded:
raise AssibleParserError(
"Error when evaluating variable in dynamic parent include path: %s. "
"When using static imports, the parent dynamic include cannot utilize host facts "
"or variables from inventory" %
parent_include.args.get('_raw_params'),
obj=task_ds,
suppress_extended_error=True,
orig_exc=e)
raise
if cumulative_path is None:
cumulative_path = parent_include_dir
elif not os.path.isabs(cumulative_path):
cumulative_path = os.path.join(
parent_include_dir, cumulative_path)
include_target = templar.template(
t.args['_raw_params'])
if t._role:
new_basedir = os.path.join(t._role._role_path,
subdir, cumulative_path)
include_file = loader.path_dwim_relative(
new_basedir, subdir, include_target)
else:
include_file = loader.path_dwim_relative(
loader.get_basedir(), cumulative_path,
include_target)
if os.path.exists(include_file):
found = True
break
else:
parent_include = parent_include._parent
if not found:
try:
include_target = templar.template(
t.args['_raw_params'])
except AssibleUndefinedVariable as e:
raise AssibleParserError(
"Error when evaluating variable in import path: %s.\n\n"
"When using static imports, ensure that any variables used in their names are defined in vars/vars_files\n"
"or extra-vars passed in from the command line. Static imports cannot use variables from facts or inventory\n"
"sources like group or host vars." %
t.args['_raw_params'],
obj=task_ds,
suppress_extended_error=True,
orig_exc=e)
if t._role:
include_file = loader.path_dwim_relative(
t._role._role_path, subdir, include_target)
else:
include_file = loader.path_dwim(include_target)
try:
data = loader.load_from_file(include_file)
if data is None:
display.warning(
'file %s is empty and had no tasks to include'
% include_file)
continue
elif not isinstance(data, list):
raise AssibleParserError(
"included task files must contain a list of tasks",
obj=data)
# since we can't send callbacks here, we display a message directly in
# the same fashion used by the on_include callback. We also do it here,
# because the recursive nature of helper methods means we may be loading
# nested includes, and we want the include order printed correctly
display.vv("statically imported: %s" % include_file)
except AssibleFileNotFound:
if action != 'include' or t.static or \
C.DEFAULT_TASK_INCLUDES_STATIC or \
C.DEFAULT_HANDLER_INCLUDES_STATIC and use_handlers:
raise
display.deprecated(
"Included file '%s' not found, however since this include is not "
"explicitly marked as 'static: yes', we will try and include it dynamically "
"later. In the future, this will be an error unless 'static: no' is used "
"on the include task. If you do not want missing includes to be considered "
"dynamic, use 'static: yes' on the include or set the global assible.cfg "
"options to make all includes static for tasks and/or handlers"
% include_file,
version="2.12",
collection_name='assible.builtin')
task_list.append(t)
continue
ti_copy = t.copy(exclude_parent=True)
ti_copy._parent = block
included_blocks = load_list_of_blocks(
data,
play=play,
parent_block=None,
task_include=ti_copy,
role=role,
use_handlers=use_handlers,
loader=loader,
variable_manager=variable_manager,
)
# FIXME: remove once 'include' is removed
# pop tags out of the include args, if they were specified there, and assign
# them to the include. If the include already had tags specified, we raise an
# error so that users know not to specify them both ways
tags = ti_copy.vars.pop('tags', [])
if isinstance(tags, string_types):
tags = tags.split(',')
if len(tags) > 0:
if action in ('include_tasks', 'import_tasks'):
raise AssibleParserError(
'You cannot specify "tags" inline to the task, it is a task keyword'
)
if len(ti_copy.tags) > 0:
raise AssibleParserError(
"Include tasks should not specify tags in more than one way (both via args and directly on the task). "
"Mixing styles in which tags are specified is prohibited for whole import hierarchy, not only for single import statement",
obj=task_ds,
suppress_extended_error=True,
)
display.deprecated(
"You should not specify tags in the include parameters. All tags should be specified using the task-level option",
version="2.12",
collection_name='assible.builtin')
else:
tags = ti_copy.tags[:]
# now we extend the tags on each of the included blocks
for b in included_blocks:
b.tags = list(set(b.tags).union(tags))
# END FIXME
# FIXME: handlers shouldn't need this special handling, but do
# right now because they don't iterate blocks correctly
if use_handlers:
for b in included_blocks:
task_list.extend(b.block)
else:
task_list.extend(included_blocks)
else:
t.is_static = False
task_list.append(t)
elif action in ('include_role', 'import_role'):
ir = IncludeRole.load(
task_ds,
block=block,
role=role,
task_include=None,
variable_manager=variable_manager,
loader=loader,
)
# 1. the user has set the 'static' option to false or true
# 2. one of the appropriate config options was set
is_static = False
if action == 'import_role':
is_static = True
elif ir.static is not None:
display.deprecated(
"The use of 'static' for 'include_role' has been deprecated. "
"Use 'import_role' for static inclusion, or 'include_role' for dynamic inclusion",
version='2.12',
collection_name='assible.builtin')
is_static = ir.static
if is_static:
if ir.loop is not None:
if action == 'import_role':
raise AssibleParserError(
"You cannot use loops on 'import_role' statements. You should use 'include_role' instead.",
obj=task_ds)
else:
raise AssibleParserError(
"You cannot use 'static' on an include_role with a loop",
obj=task_ds)
# we set a flag to indicate this include was static
ir.statically_loaded = True
# template the role name now, if needed
all_vars = variable_manager.get_vars(play=play, task=ir)
templar = Templar(loader=loader, variables=all_vars)
ir._role_name = templar.template(ir._role_name)
# uses compiled list from object
blocks, _ = ir.get_block_list(
variable_manager=variable_manager, loader=loader)
task_list.extend(blocks)
else:
# passes task object itself for latter generation of list
task_list.append(ir)
else:
if use_handlers:
t = Handler.load(task_ds,
block=block,
role=role,
task_include=task_include,
variable_manager=variable_manager,
loader=loader)
else:
t = Task.load(task_ds,
block=block,
role=role,
task_include=task_include,
variable_manager=variable_manager,
loader=loader)
task_list.append(t)
return task_list
def parse(self, skip_action_validation=False):
'''
Given a task in one of the supported forms, parses and returns
returns the action, arguments, and delegate_to values for the
task, dealing with all sorts of levels of fuzziness.
'''
thing = None
action = None
delegate_to = self._task_ds.get('delegate_to', Sentinel)
args = dict()
self.internal_redirect_list = []
# This is the standard YAML form for command-type modules. We grab
# the args and pass them in as additional arguments, which can/will
# be overwritten via dict updates from the other arg sources below
additional_args = self._task_ds.get('args', dict())
# We can have one of action, local_action, or module specified
# action
if 'action' in self._task_ds:
# an old school 'action' statement
thing = self._task_ds['action']
action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
# local_action
if 'local_action' in self._task_ds:
# local_action is similar but also implies a delegate_to
if action is not None:
raise AssibleParserError("action and local_action are mutually exclusive", obj=self._task_ds)
thing = self._task_ds.get('local_action', '')
delegate_to = 'localhost'
action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
# module: <stuff> is the more new-style invocation
# filter out task attributes so we're only querying unrecognized keys as actions/modules
non_task_ds = dict((k, v) for k, v in iteritems(self._task_ds) if (k not in self._task_attrs) and (not k.startswith('with_')))
# walk the filtered input dictionary to see if we recognize a module name
for item, value in iteritems(non_task_ds):
is_action_candidate = False
if item in BUILTIN_TASKS:
is_action_candidate = True
elif skip_action_validation:
is_action_candidate = True
else:
# If the plugin is resolved and redirected smuggle the list of candidate names via the task attribute 'internal_redirect_list'
context = action_loader.find_plugin_with_context(item, collection_list=self._collection_list)
if not context.resolved:
context = module_loader.find_plugin_with_context(item, collection_list=self._collection_list)
if context.resolved and context.redirect_list:
self.internal_redirect_list = context.redirect_list
elif context.redirect_list:
self.internal_redirect_list = context.redirect_list
is_action_candidate = bool(self.internal_redirect_list)
if is_action_candidate:
# finding more than one module name is a problem
if action is not None:
raise AssibleParserError("conflicting action statements: %s, %s" % (action, item), obj=self._task_ds)
action = item
thing = value
action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
# if we didn't see any module in the task at all, it's not a task really
if action is None:
if non_task_ds: # there was one non-task action, but we couldn't find it
bad_action = list(non_task_ds.keys())[0]
raise AssibleParserError("couldn't resolve module/action '{0}'. This often indicates a "
"misspelling, missing collection, or incorrect module path.".format(bad_action),
obj=self._task_ds)
else:
raise AssibleParserError("no module/action detected in task.",
obj=self._task_ds)
elif args.get('_raw_params', '') != '' and action not in RAW_PARAM_MODULES:
templar = Templar(loader=None)
raw_params = args.pop('_raw_params')
if templar.is_template(raw_params):
args['_variable_params'] = raw_params
else:
raise AssibleParserError("this task '%s' has extra params, which is only allowed in the following modules: %s" % (action,
", ".join(RAW_PARAM_MODULES)),
obj=self._task_ds)
return (action, args, delegate_to)