def _file_transport_command(self, in_path, out_path, sftp_action):
# scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too.
host = '[%s]' % self.host
# Transfer methods to try
methods = []
# Use the transfer_method option if set, otherwise use scp_if_ssh
ssh_transfer_method = self._play_context.ssh_transfer_method
if ssh_transfer_method is not None:
if not (ssh_transfer_method in ('smart', 'sftp', 'scp', 'piped')):
raise AnsibleOptionsError(
'transfer_method needs to be one of [smart|sftp|scp|piped]'
)
if ssh_transfer_method == 'smart':
methods = ['sftp', 'scp', 'piped']
else:
methods = [ssh_transfer_method]
else:
# since this can be a non-bool now, we need to handle it correctly
scp_if_ssh = C.DEFAULT_SCP_IF_SSH
if not isinstance(scp_if_ssh, bool):
scp_if_ssh = scp_if_ssh.lower()
if scp_if_ssh in BOOLEANS:
scp_if_ssh = boolean(scp_if_ssh, strict=False)
elif scp_if_ssh != 'smart':
raise AnsibleOptionsError(
'scp_if_ssh needs to be one of [smart|True|False]')
if scp_if_ssh == 'smart':
methods = ['sftp', 'scp', 'piped']
elif scp_if_ssh is True:
methods = ['scp']
else:
methods = ['sftp']
for method in methods:
returncode = stdout = stderr = None
if method == 'sftp':
cmd = self._build_command('sftp', to_bytes(host))
in_data = u"{0} {1} {2}\n".format(sftp_action,
shlex_quote(in_path),
shlex_quote(out_path))
in_data = to_bytes(in_data, nonstring='passthru')
(returncode, stdout, stderr) = self._bare_run(cmd,
in_data,
checkrc=False)
elif method == 'scp':
if sftp_action == 'get':
cmd = self._build_command(
'scp', u'{0}:{1}'.format(host, shlex_quote(in_path)),
out_path)
else:
cmd = self._build_command(
'scp', in_path,
u'{0}:{1}'.format(host, shlex_quote(out_path)))
in_data = None
(returncode, stdout, stderr) = self._bare_run(cmd,
in_data,
checkrc=False)
elif method == 'piped':
if sftp_action == 'get':
# we pass sudoable=False to disable pty allocation, which
# would end up mixing stdout/stderr and screwing with newlines
(returncode, stdout, stderr) = self.exec_command(
'dd if=%s bs=%s' % (in_path, BUFSIZE), sudoable=False)
out_file = open(
to_bytes(out_path, errors='surrogate_or_strict'),
'wb+')
out_file.write(stdout)
out_file.close()
else:
in_data = open(
to_bytes(in_path, errors='surrogate_or_strict'),
'rb').read()
in_data = to_bytes(in_data, nonstring='passthru')
(returncode, stdout, stderr) = self.exec_command(
'dd of=%s bs=%s' % (out_path, BUFSIZE),
in_data=in_data)
# Check the return code and rollover to next method if failed
if returncode == 0:
return (returncode, stdout, stderr)
else:
# If not in smart mode, the data will be printed by the raise below
if len(methods) > 1:
display.warning(
msg=
'%s transfer mechanism failed on %s. Use ANSIBLE_DEBUG=1 to see detailed information'
% (method, host))
display.debug(msg='%s' % to_native(stdout))
display.debug(msg='%s' % to_native(stderr))
if returncode == 255:
raise AnsibleConnectionFailure(
"Failed to connect to the host via %s: %s" %
(method, to_native(stderr)))
else:
raise AnsibleError("failed to transfer file to %s %s:\n%s\n%s" %
(to_native(in_path), to_native(out_path),
to_native(stdout), to_native(stderr)))
def _get_connection(self, variables, templar):
'''
Reads the connection property for the host, and returns the
correct connection object from the list of connection plugins
'''
if self._task.delegate_to is not None:
# since we're delegating, we don't want to use interpreter values
# which would have been set for the original target host
for i in variables.keys():
if isinstance(i, string_types) and i.startswith(
'ansible_') and i.endswith('_interpreter'):
del variables[i]
# now replace the interpreter values with those that may have come
# from the delegated-to host
delegated_vars = variables.get('ansible_delegated_vars',
dict()).get(self._task.delegate_to,
dict())
if isinstance(delegated_vars, dict):
for i in delegated_vars:
if isinstance(i, string_types) and i.startswith(
"ansible_") and i.endswith("_interpreter"):
variables[i] = delegated_vars[i]
conn_type = self._play_context.connection
if conn_type == 'smart':
conn_type = 'ssh'
if sys.platform.startswith(
'darwin') and self._play_context.password:
# due to a current bug in sshpass on OSX, which can trigger
# a kernel panic even for non-privileged users, we revert to
# paramiko on that OS when a SSH password is specified
conn_type = "paramiko"
else:
# see if SSH can support ControlPersist if not use paramiko
try:
cmd = subprocess.Popen(['ssh', '-o', 'ControlPersist'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = cmd.communicate()
err = to_unicode(err)
if u"Bad configuration option" in err or u"Usage:" in err:
conn_type = "paramiko"
except OSError:
conn_type = "paramiko"
connection = self._shared_loader_obj.connection_loader.get(
conn_type, self._play_context, self._new_stdin)
if not connection:
raise AnsibleError("the connection plugin '%s' was not found" %
conn_type)
if self._play_context.accelerate:
# accelerate is deprecated as of 2.1...
display.deprecated(
'Accelerated mode is deprecated. Consider using SSH with ControlPersist and pipelining enabled instead'
)
# launch the accelerated daemon here
ssh_connection = connection
handler = self._shared_loader_obj.action_loader.get(
'normal',
task=self._task,
connection=ssh_connection,
play_context=self._play_context,
loader=self._loader,
templar=templar,
shared_loader_obj=self._shared_loader_obj,
)
key = key_for_hostname(self._play_context.remote_addr)
accelerate_args = dict(
password=base64.b64encode(key.__str__()),
port=self._play_context.accelerate_port,
minutes=C.ACCELERATE_DAEMON_TIMEOUT,
ipv6=self._play_context.accelerate_ipv6,
debug=self._play_context.verbosity,
)
connection = self._shared_loader_obj.connection_loader.get(
'accelerate', self._play_context, self._new_stdin)
if not connection:
raise AnsibleError("the connection plugin '%s' was not found" %
conn_type)
try:
connection._connect()
except AnsibleConnectionFailure:
display.debug('connection failed, fallback to accelerate')
res = handler._execute_module(module_name='accelerate',
module_args=accelerate_args,
task_vars=variables,
delete_remote_tmp=False)
display.debug(res)
connection._connect()
return connection
def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None):
''' Given a config key figure out the actual value and report on the origin of the settings '''
if cfile is None:
# use default config
cfile = self._config_file
# Note: sources that are lists listed in low to high precedence (last one wins)
value = None
origin = None
defs = self.get_configuration_definitions(plugin_type, plugin_name)
if config in defs:
# direct setting via plugin arguments, can set to None so we bypass rest of processing/defaults
direct_aliases = []
if direct:
direct_aliases = [direct[alias] for alias in defs[config].get('aliases', []) if alias in direct]
if direct and config in direct:
value = direct[config]
origin = 'Direct'
elif direct and direct_aliases:
value = direct_aliases[0]
origin = 'Direct'
else:
# Use 'variable overrides' if present, highest precedence, but only present when querying running play
if variables and defs[config].get('vars'):
value, origin = self._loop_entries(variables, defs[config]['vars'])
origin = 'var: %s' % origin
# use playbook keywords if you have em
if value is None and keys and config in keys:
value, origin = keys[config], 'keyword'
origin = 'keyword: %s' % origin
# env vars are next precedence
if value is None and defs[config].get('env'):
value, origin = self._loop_entries(py3compat.environ, defs[config]['env'])
origin = 'env: %s' % origin
# try config file entries next, if we have one
if self._parsers.get(cfile, None) is None:
self._parse_config_file(cfile)
if value is None and cfile is not None:
ftype = get_config_type(cfile)
if ftype and defs[config].get(ftype):
if ftype == 'ini':
# load from ini config
try: # FIXME: generalize _loop_entries to allow for files also, most of this code is dupe
for ini_entry in defs[config]['ini']:
temp_value = get_ini_config_value(self._parsers[cfile], ini_entry)
if temp_value is not None:
value = temp_value
origin = cfile
if 'deprecated' in ini_entry:
self.DEPRECATED.append(('[%s]%s' % (ini_entry['section'], ini_entry['key']), ini_entry['deprecated']))
except Exception as e:
sys.stderr.write("Error while loading ini config %s: %s" % (cfile, to_native(e)))
elif ftype == 'yaml':
# FIXME: implement, also , break down key from defs (. notation???)
origin = cfile
# set default if we got here w/o a value
if value is None:
if defs[config].get('required', False):
if not plugin_type or config not in INTERNAL_DEFS.get(plugin_type, {}):
raise AnsibleError("No setting was provided for required configuration %s" %
to_native(_get_entry(plugin_type, plugin_name, config)))
else:
value = defs[config].get('default')
origin = 'default'
# skip typing as this is a templated default that will be resolved later in constants, which has needed vars
if plugin_type is None and isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')):
return value, origin
# ensure correct type, can raise exceptions on mismatched types
try:
value = ensure_type(value, defs[config].get('type'), origin=origin)
except ValueError as e:
if origin.startswith('env:') and value == '':
# this is empty env var for non string so we can set to default
origin = 'default'
value = ensure_type(defs[config].get('default'), defs[config].get('type'), origin=origin)
else:
raise AnsibleOptionsError('Invalid type for configuration option %s: %s' %
(to_native(_get_entry(plugin_type, plugin_name, config)), to_native(e)))
# deal with deprecation of the setting
if 'deprecated' in defs[config] and origin != 'default':
self.DEPRECATED.append((config, defs[config].get('deprecated')))
else:
raise AnsibleError('Requested entry (%s) was not defined in configuration.' % to_native(_get_entry(plugin_type, plugin_name, config)))
return value, origin
def do_template(self,
data,
preserve_trailing_newlines=True,
escape_backslashes=True,
fail_on_undefined=None,
overrides=None,
disable_lookups=False):
if self.jinja2_native and not isinstance(data, string_types):
return data
# For preserving the number of input newlines in the output (used
# later in this method)
data_newlines = _count_newlines_from_end(data)
if fail_on_undefined is None:
fail_on_undefined = self._fail_on_undefined_errors
try:
# allows template header overrides to change jinja2 options.
if overrides is None:
myenv = self.environment
else:
myenv = self.environment.overlay(overrides)
# Get jinja env overrides from template
if hasattr(data,
'startswith') and data.startswith(JINJA2_OVERRIDE):
eol = data.find('\n')
line = data[len(JINJA2_OVERRIDE):eol]
data = data[eol + 1:]
for pair in line.split(','):
(key, val) = pair.split(':')
key = key.strip()
setattr(myenv, key, ast.literal_eval(val.strip()))
if escape_backslashes:
# Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\".
data = _escape_backslashes(data, myenv)
try:
t = myenv.from_string(data)
except TemplateSyntaxError as e:
raise AnsibleError(
"template error while templating string: %s. String: %s" %
(to_native(e), to_native(data)))
except Exception as e:
if 'recursion' in to_native(e):
raise AnsibleError(
"recursive loop detected in template string: %s" %
to_native(data))
else:
return data
if disable_lookups:
t.globals['query'] = t.globals['q'] = t.globals[
'lookup'] = self._fail_lookup
jvars = AnsibleJ2Vars(self, t.globals)
self.cur_context = new_context = t.new_context(jvars, shared=True)
rf = t.root_render_func(new_context)
try:
if self.jinja2_native:
res = ansible_native_concat(rf)
else:
res = j2_concat(rf)
if getattr(new_context, 'unsafe', False):
res = wrap_var(res)
except TypeError as te:
if 'AnsibleUndefined' in to_native(te):
errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(
data)
errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(
te)
raise AnsibleUndefinedVariable(errmsg)
else:
display.debug(
"failing because of a type error, template data is: %s"
% to_text(data))
raise AnsibleError(
"Unexpected templating type error occurred on (%s): %s"
% (to_native(data), to_native(te)))
if self.jinja2_native and not isinstance(res, string_types):
return res
if preserve_trailing_newlines:
# The low level calls above do not preserve the newline
# characters at the end of the input data, so we use the
# calculate the difference in newlines and append them
# to the resulting output for parity
#
# jinja2 added a keep_trailing_newline option in 2.7 when
# creating an Environment. That would let us make this code
# better (remove a single newline if
# preserve_trailing_newlines is False). Once we can depend on
# that version being present, modify our code to set that when
# initializing self.environment and remove a single trailing
# newline here if preserve_newlines is False.
res_newlines = _count_newlines_from_end(res)
if data_newlines > res_newlines:
res += self.environment.newline_sequence * (data_newlines -
res_newlines)
return res
except (UndefinedError, AnsibleUndefinedVariable) as e:
if fail_on_undefined:
raise AnsibleUndefinedVariable(e)
else:
display.debug("Ignoring undefined failure: %s" % to_text(e))
return data
def _get_action_handler(self, connection, templar):
'''
Returns the correct action plugin to handle the requestion task action
'''
if self._task.action in self._shared_loader_obj.action_loader:
if self._task. async != 0:
raise AnsibleError(
"async mode is not supported with the %s module" %
self._task.action)
handler_name = self._task.action
elif self._task. async == 0:
handler_name = 'normal'
else:
handler_name = 'async'
handler = self._shared_loader_obj.action_loader.get(
handler_name,
task=self._task,
connection=connection,
play_context=self._play_context,
loader=self._loader,
templar=templar,
shared_loader_obj=self._shared_loader_obj,
)
if not handler:
raise AnsibleError("the handler '%s' was not found" % handler_name)
return handler
def _execute_module(self,
module_name=None,
module_args=None,
tmp=None,
task_vars=None,
persist_files=False,
delete_remote_tmp=True):
'''
Transfer and run a module along with its arguments.
'''
if task_vars is None:
task_vars = dict()
# if a module name was not specified for this execution, use
# the action from the task
if module_name is None:
module_name = self._task.action
if module_args is None:
module_args = self._task.args
# set check mode in the module arguments, if required
if self._play_context.check_mode:
if not self._supports_check_mode:
raise AnsibleError(
"check mode is not supported for this operation")
module_args['_ansible_check_mode'] = True
else:
module_args['_ansible_check_mode'] = False
# Get the connection user for permission checks
remote_user = task_vars.get(
'ansible_ssh_user') or self._play_context.remote_user
# set no log in the module arguments, if required
module_args[
'_ansible_no_log'] = self._play_context.no_log or C.DEFAULT_NO_TARGET_SYSLOG
# set debug in the module arguments, if required
module_args['_ansible_debug'] = C.DEFAULT_DEBUG
# let module know we are in diff mode
module_args['_ansible_diff'] = self._play_context.diff
# let module know our verbosity
module_args['_ansible_verbosity'] = display.verbosity
# give the module information about the ansible version
module_args['_ansible_version'] = __version__
# give the module information about its name
module_args['_ansible_module_name'] = module_name
# set the syslog facility to be used in the module
module_args['_ansible_syslog_facility'] = task_vars.get(
'ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY)
# let module know about filesystems that selinux treats specially
module_args[
'_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS
(module_style, shebang, module_data,
module_path) = self._configure_module(module_name=module_name,
module_args=module_args,
task_vars=task_vars)
display.vvv("Using module file %s" % module_path)
if not shebang and module_style != 'binary':
raise AnsibleError("module (%s) is missing interpreter line" %
module_name)
# a remote tmp path may be necessary and not already created
remote_module_path = None
args_file_path = None
if not tmp and self._late_needs_tmp_path(tmp, module_style):
tmp = self._make_tmp_path(remote_user)
if tmp and \
(module_style != 'new' or \
not self._connection.has_pipelining or \
not self._play_context.pipelining or \
C.DEFAULT_KEEP_REMOTE_FILES or \
self._play_context.become_method == 'su'):
remote_module_filename = self._connection._shell.get_remote_filename(
module_path)
remote_module_path = self._connection._shell.join_path(
tmp, remote_module_filename)
if module_style in ('old', 'non_native_want_json', 'binary'):
# we'll also need a temp file to hold our module arguments
args_file_path = self._connection._shell.join_path(tmp, 'args')
if remote_module_path or module_style != 'new':
display.debug("transferring module to remote %s" %
remote_module_path)
if module_style == 'binary':
self._transfer_file(module_path, remote_module_path)
else:
self._transfer_data(remote_module_path, module_data)
if module_style == 'old':
# we need to dump the module args to a k=v string in a file on
# the remote system, which can be read and parsed by the module
args_data = ""
for k, v in iteritems(module_args):
args_data += '%s=%s ' % (k, pipes.quote(text_type(v)))
self._transfer_data(args_file_path, args_data)
elif module_style in ('non_native_want_json', 'binary'):
self._transfer_data(args_file_path, json.dumps(module_args))
display.debug("done transferring module to remote")
environment_string = self._compute_environment_string()
remote_files = None
if args_file_path:
remote_files = tmp, remote_module_path, args_file_path
elif remote_module_path:
remote_files = tmp, remote_module_path
# Fix permissions of the tmp path and tmp files. This should be
# called after all files have been transferred.
if remote_files:
self._fixup_perms2(remote_files, remote_user)
cmd = ""
in_data = None
if self._connection.has_pipelining and self._play_context.pipelining and not C.DEFAULT_KEEP_REMOTE_FILES and module_style == 'new':
in_data = module_data
else:
if remote_module_path:
cmd = remote_module_path
rm_tmp = None
if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
if not self._play_context.become or self._play_context.become_user == 'root':
# not sudoing or sudoing to root, so can cleanup files in the same step
rm_tmp = tmp
cmd = self._connection._shell.build_module_command(
environment_string,
shebang,
cmd,
arg_path=args_file_path,
rm_tmp=rm_tmp)
cmd = cmd.strip()
sudoable = True
if module_name == "accelerate":
# always run the accelerate module as the user
# specified in the play, not the sudo_user
sudoable = False
res = self._low_level_execute_command(cmd,
sudoable=sudoable,
in_data=in_data)
if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
if self._play_context.become and self._play_context.become_user != 'root':
# not sudoing to root, so maybe can't delete files as that other user
# have to clean up temp files as original user in a second step
tmp_rm_cmd = self._connection._shell.remove(tmp, recurse=True)
tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd,
sudoable=False)
tmp_rm_data = self._parse_returned_data(tmp_rm_res)
if tmp_rm_data.get('rc', 0) != 0:
display.warning(
'Error deleting remote temporary files (rc: {0}, stderr: {1})'
.format(
tmp_rm_res.get('rc'),
tmp_rm_res.get('stderr',
'No error string available.')))
# parse the main result
data = self._parse_returned_data(res)
# pre-split stdout into lines, if stdout is in the data and there
# isn't already a stdout_lines value there
if 'stdout' in data and 'stdout_lines' not in data:
data['stdout_lines'] = data.get('stdout', u'').splitlines()
display.debug("done with _execute_module (%s, %s)" %
(module_name, module_args))
return data
def _fail_lookup(self, name, *args, **kwargs):
raise AnsibleError(
"The lookup `%s` was found, however lookups were disabled from templating"
% name)
import json
import re
from ansible import constants as C
from ansible.module_utils._text import to_text, to_bytes
from ansible.errors import AnsibleConnectionFailure, AnsibleError
from ansible.plugins.netconf import NetconfBase
from ansible.plugins.netconf import ensure_connected
try:
from ncclient import manager
from ncclient.operations import RPCError
from ncclient.transport.errors import SSHUnknownHostError
from ncclient.xml_ import to_ele, to_xml, new_ele
except ImportError:
raise AnsibleError("ncclient is not installed")
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
class Netconf(NetconfBase):
def get_text(self, ele, tag):
try:
return to_text(ele.find(tag).text,
errors='surrogate_then_replace').strip()
except AttributeError:
pass
def _parse(self, err):
all_hosts = {}
# not passing from_remote because data from CMDB is trusted
try:
self.raw = self._loader.load(self.data)
except Exception as e:
sys.stderr.write(to_native(err) + "\n")
raise AnsibleError(
"failed to parse executable inventory script results from {0}: {1}"
.format(to_native(self.filename), to_native(e)))
if not isinstance(self.raw, Mapping):
sys.stderr.write(to_native(err) + "\n")
raise AnsibleError(
"failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict"
.format(to_native(self.filename)))
group = None
for (group_name, data) in self.raw.items():
# in Ansible 1.3 and later, a "_meta" subelement may contain
# a variable "hostvars" which contains a hash for each host
# if this "hostvars" exists at all then do not call --host for each
# host. This is for efficiency and scripts should still return data
# if called with --host for backwards compat with 1.2 and earlier.
if group_name == '_meta':
if 'hostvars' in data:
self.host_vars_from_top = data['hostvars']
continue
if group_name not in self.groups:
group = self.groups[group_name] = Group(group_name)
group = self.groups[group_name]
host = None
if not isinstance(data, dict):
data = {'hosts': data}
# is not those subkeys, then simplified syntax, host with vars
elif not any(k in data for k in ('hosts', 'vars', 'children')):
data = {'hosts': [group_name], 'vars': data}
if 'hosts' in data:
if not isinstance(data['hosts'], list):
raise AnsibleError("You defined a group \"%s\" with bad "
"data for the host list:\n %s" %
(group_name, data))
for hostname in data['hosts']:
if hostname not in all_hosts:
all_hosts[hostname] = Host(hostname)
host = all_hosts[hostname]
group.add_host(host)
if 'vars' in data:
if not isinstance(data['vars'], dict):
raise AnsibleError("You defined a group \"%s\" with bad "
"data for variables:\n %s" %
(group_name, data))
for k, v in iteritems(data['vars']):
group.set_variable(k, v)
# Separate loop to ensure all groups are defined
for (group_name, data) in self.raw.items():
if group_name == '_meta':
continue
if isinstance(data, dict) and 'children' in data:
for child_name in data['children']:
if child_name in self.groups:
self.groups[group_name].add_child_group(
self.groups[child_name])
# Finally, add all top-level groups as children of 'all'.
# We exclude ungrouped here because it was already added as a child of
# 'all' at the time it was created.
for group in self.groups.values():
if group.depth == 0 and group.name not in ('all', 'ungrouped'):
self.groups['all'].add_child_group(group)
def _run_loop(self, items):
'''
Runs the task with the loop items specified and collates the result
into an array named 'results' which is inserted into the final result
along with the item for which the loop ran.
'''
results = []
# make copies of the job vars and task so we can add the item to
# the variables and re-validate the task with the item variable
# task_vars = self._job_vars.copy()
task_vars = self._job_vars
loop_var = 'item'
index_var = None
label = None
loop_pause = 0
templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
if self._task.loop_control:
# FIXME: move this to the object itself to allow post_validate to take care of templating
loop_var = templar.template(self._task.loop_control.loop_var)
index_var = templar.template(self._task.loop_control.index_var)
loop_pause = templar.template(self._task.loop_control.pause)
# the these may be 'None', so we still need to default to something useful
# this is tempalted below after an item is assigned
label = (self._task.loop_control.label or ('{{' + loop_var + '}}'))
if loop_var in task_vars:
display.warning(u"The loop variable '%s' is already in use. "
u"You should set the `loop_var` value in the `loop_control` option for the task"
u" to something else to avoid variable collisions and unexpected behavior." % loop_var)
ran_once = False
if self._task.loop_with:
# Only squash with 'with_:' not with the 'loop:', 'magic' squashing can be removed once with_ loops are
items = self._squash_items(items, loop_var, task_vars)
for item_index, item in enumerate(items):
task_vars[loop_var] = item
if index_var:
task_vars[index_var] = item_index
# pause between loop iterations
if loop_pause and ran_once:
try:
time.sleep(float(loop_pause))
except ValueError as e:
raise AnsibleError('Invalid pause value: %s, produced error: %s' % (loop_pause, to_native(e)))
else:
ran_once = True
try:
tmp_task = self._task.copy(exclude_parent=True, exclude_tasks=True)
tmp_task._parent = self._task._parent
tmp_play_context = self._play_context.copy()
except AnsibleParserError as e:
results.append(dict(failed=True, msg=to_text(e)))
continue
# now we swap the internal task and play context with their copies,
# execute, and swap them back so we can do the next iteration cleanly
(self._task, tmp_task) = (tmp_task, self._task)
(self._play_context, tmp_play_context) = (tmp_play_context, self._play_context)
res = self._execute(variables=task_vars)
task_fields = self._task.dump_attrs()
(self._task, tmp_task) = (tmp_task, self._task)
(self._play_context, tmp_play_context) = (tmp_play_context, self._play_context)
# now update the result with the item info, and append the result
# to the list of results
res[loop_var] = item
if index_var:
res[index_var] = item_index
res['_ansible_item_result'] = True
res['_ansible_ignore_errors'] = task_fields.get('ignore_errors')
if label is not None:
res['_ansible_item_label'] = templar.template(label, cache=False)
self._rslt_q.put(
TaskResult(
self._host.name,
self._task._uuid,
res,
task_fields=task_fields,
),
block=False,
)
results.append(res)
del task_vars[loop_var]
return results
def check_module_args(self):
if self.module_name in C.MODULE_REQUIRE_ARGS and not self.module_args:
err = "No argument passed to '%s' module." % self.module_name
raise AnsibleError(err)
def _get_loop_items(self):
'''
Loads a lookup plugin to handle the with_* portion of a task (if specified),
and returns the items result.
'''
# save the play context variables to a temporary dictionary,
# so that we can modify the job vars without doing a full copy
# and later restore them to avoid modifying things too early
play_context_vars = dict()
self._play_context.update_vars(play_context_vars)
old_vars = dict()
for k in play_context_vars:
if k in self._job_vars:
old_vars[k] = self._job_vars[k]
self._job_vars[k] = play_context_vars[k]
# get search path for this task to pass to lookup plugins
self._job_vars['ansible_search_path'] = self._task.get_search_path()
templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
items = None
if self._task.loop_with:
if self._task.loop_with in self._shared_loader_obj.lookup_loader:
fail = True
if self._task.loop_with == 'first_found':
# first_found loops are special. If the item is undefined then we want to fall through to the next value rather than failing.
fail = False
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop, templar=templar, loader=self._loader, fail_on_undefined=fail,
convert_bare=False)
if not fail:
loop_terms = [t for t in loop_terms if not templar._contains_vars(t)]
# get lookup
mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop_with, loader=self._loader, templar=templar)
# give lookup task 'context' for subdir (mostly needed for first_found)
for subdir in ['template', 'var', 'file']: # TODO: move this to constants?
if subdir in self._task.action:
break
setattr(mylookup, '_subdir', subdir + 's')
# run lookup
items = mylookup.run(terms=loop_terms, variables=self._job_vars, wantlist=True)
else:
raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop_with)
elif self._task.loop:
items = templar.template(self._task.loop)
if not isinstance(items, list):
raise AnsibleError(
"Invalid data passed to 'loop', it requires a list, got this instead: %s."
" Hint: If you passed a list/dict of just one element,"
" try adding wantlist=True to your lookup invocation or use q/query instead of lookup." % items
)
# now we restore any old job variables that may have been modified,
# and delete them if they were in the play context vars but not in
# the old variables dictionary
for k in play_context_vars:
if k in old_vars:
self._job_vars[k] = old_vars[k]
else:
del self._job_vars[k]
if items:
for idx, item in enumerate(items):
if item is not None and not isinstance(item, UnsafeProxy):
items[idx] = UnsafeProxy(item)
# ensure basedir is always in (dwim already searches here but we need to display it)
if self._loader.get_basedir() not in self._job_vars['ansible_search_path']:
self._job_vars['ansible_search_path'].append(self._loader.get_basedir())
return items
def _search_executable(self, executable):
cmd = distutils.spawn.find_executable(executable)
if not cmd:
raise AnsibleError("%s command not found in PATH") % executable
return cmd
#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.errors import AnsibleError
try:
import luna
except ImportError:
raise AnsibleError("luna is not installed")
from luna_ansible.helpers import StreamStringLogger
import logging
if luna.__version__ != '1.2':
raise AnsibleError("Only luna-1.2 is supported")
def luna_group_present(data):
data.pop('state')
name = data.pop('name')
# perform interface data checking, make sure networks are exist,
# interface names are unique, etc.
#
interfaces = data.pop('interfaces')
tmp_interfaces = {}
for elem in interfaces:
if not 'name' in elem:
return True, False, "No name for interface defined"
def _make_tmp_path(self, remote_user):
'''
Create and return a temporary path on a remote box.
'''
basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(
0, 2**48))
use_system_tmp = False
if self._play_context.become and self._play_context.become_user not in (
'root', remote_user):
use_system_tmp = True
tmp_mode = 0o700
cmd = self._connection._shell.mkdtemp(basefile, use_system_tmp,
tmp_mode)
result = self._low_level_execute_command(cmd, sudoable=False)
# error handling on this seems a little aggressive?
if result['rc'] != 0:
if result['rc'] == 5:
output = 'Authentication failure.'
elif result['rc'] == 255 and self._connection.transport in (
'ssh', ):
if self._play_context.verbosity > 3:
output = u'SSH encountered an unknown error. The output was:\n%s%s' % (
result['stdout'], result['stderr'])
else:
output = (
u'SSH encountered an unknown error during the connection.'
' We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue'
)
elif u'No space left on device' in result['stderr']:
output = result['stderr']
else:
output = (
'Authentication or permission failure.'
' In some cases, you may have been able to authenticate and did not have permissions on the remote directory.'
' Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp".'
' Failed command was: %s, exited with result %d' %
(cmd, result['rc']))
if 'stdout' in result and result['stdout'] != u'':
output = output + u": %s" % result['stdout']
raise AnsibleConnectionFailure(output)
try:
stdout_parts = result['stdout'].strip().split('%s=' % basefile, 1)
rc = self._connection._shell.join_path(stdout_parts[-1],
u'').splitlines()[-1]
except IndexError:
# stdout was empty or just space, set to / to trigger error in next if
rc = '/'
# Catch failure conditions, files should never be
# written to locations in /.
if rc == '/':
raise AnsibleError(
'failed to resolve remote temporary directory from %s: `%s` returned empty string'
% (basefile, cmd))
return rc
"""
RETURN = """
_raw:
description:
Returns a boolean when I(attribute) is check_ec2_classic. Otherwise returns the value(s) of the attribute
(or all attributes if one is not specified).
"""
from ansible.errors import AnsibleError
try:
import boto3
import botocore
except ImportError:
raise AnsibleError(
"The lookup aws_account_attribute requires boto3 and botocore.")
from ansible.plugins import AnsiblePlugin
from ansible.plugins.lookup import LookupBase
from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import boto3_conn, get_aws_connection_info
from ansible.module_utils._text import to_native
from ansible.module_utils.six import string_types
import os
def _boto3_conn(region, credentials):
boto_profile = credentials.pop('aws_profile', None)
try:
connection = boto3.session.Session(profile_name=boto_profile).client(
'ec2', region, **credentials)
def _fixup_perms2(self, remote_paths, remote_user, execute=True):
"""
We need the files we upload to be readable (and sometimes executable)
by the user being sudo'd to but we want to limit other people's access
(because the files could contain passwords or other private
information. We achieve this in one of these ways:
* If no sudo is performed or the remote_user is sudo'ing to
themselves, we don't have to change permissions.
* If the remote_user sudo's to a privileged user (for instance, root),
we don't have to change permissions
* If the remote_user sudo's to an unprivileged user then we attempt to
grant the unprivileged user access via file system acls.
* If granting file system acls fails we try to change the owner of the
file with chown which only works in case the remote_user is
privileged or the remote systems allows chown calls by unprivileged
users (e.g. HP-UX)
* If the chown fails we can set the file to be world readable so that
the second unprivileged user can read the file.
Since this could allow other users to get access to private
information we only do this ansible is configured with
"allow_world_readable_tmpfiles" in the ansible.cfg
"""
if self._connection._shell.SHELL_FAMILY == 'powershell':
# This won't work on Powershell as-is, so we'll just completely skip until
# we have a need for it, at which point we'll have to do something different.
return remote_paths
if self._play_context.become and self._play_context.become_user not in (
'root', remote_user):
# Unprivileged user that's different than the ssh user. Let's get
# to work!
# Try to use file system acls to make the files readable for sudo'd
# user
if execute:
mode = 'rx'
else:
mode = 'rX'
res = self._remote_set_user_facl(remote_paths,
self._play_context.become_user,
mode)
if res['rc'] != 0:
# File system acls failed; let's try to use chown next
# Set executable bit first as on some systems an
# unprivileged user can use chown
if execute:
res = self._remote_chmod(remote_paths, 'u+x')
if res['rc'] != 0:
raise AnsibleError(
'Failed to set file mode on remote temporary files (rc: {0}, err: {1})'
.format(res['rc'], res['stderr']))
res = self._remote_chown(remote_paths,
self._play_context.become_user)
if res['rc'] != 0 and remote_user == 'root':
# chown failed even if remove_user is root
raise AnsibleError(
'Failed to change ownership of the temporary files Ansible needs to create despite connecting as root.'
' Unprivileged become user would be unable to read the file.'
)
elif res['rc'] != 0:
if C.ALLOW_WORLD_READABLE_TMPFILES:
# chown and fs acls failed -- do things this insecure
# way only if the user opted in in the config file
display.warning(
'Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user.'
' This may be insecure. For information on securing this, see'
' https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'
)
res = self._remote_chmod(remote_paths, 'a+%s' % mode)
if res['rc'] != 0:
raise AnsibleError(
'Failed to set file mode on remote files (rc: {0}, err: {1})'
.format(res['rc'], res['stderr']))
else:
raise AnsibleError(
'Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user'
' (rc: {0}, err: {1}). For information on working around this,'
' see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'
.format(res['rc'], res['stderr']))
elif execute:
# Can't depend on the file being transferred with execute
# permissions. Only need user perms because no become was
# used here
res = self._remote_chmod(remote_paths, 'u+x')
if res['rc'] != 0:
raise AnsibleError(
'Failed to set file mode on remote files (rc: {0}, err: {1})'
.format(res['rc'], res['stderr']))
return remote_paths
def wrapped(self, *args, **kwargs):
prompt = self._connection.get_prompt()
if not to_text(prompt,
errors='surrogate_or_strict').strip().endswith('#'):
raise AnsibleError('operation requires privilege escalation')
return func(self, *args, **kwargs)
def _get_diff_data(self, destination, source, task_vars, source_file=True):
diff = {}
display.debug("Going to peek to see if file has changed permissions")
peek_result = self._execute_module(module_name='file',
module_args=dict(path=destination,
diff_peek=True),
task_vars=task_vars,
persist_files=True)
if not ('failed' in peek_result
and peek_result['failed']) or peek_result.get('rc', 0) == 0:
if peek_result['state'] == 'absent':
diff['before'] = ''
elif peek_result['appears_binary']:
diff['dst_binary'] = 1
elif C.MAX_FILE_SIZE_FOR_DIFF > 0 and peek_result[
'size'] > C.MAX_FILE_SIZE_FOR_DIFF:
diff['dst_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
else:
display.debug("Slurping the file %s" % source)
dest_result = self._execute_module(
module_name='slurp',
module_args=dict(path=destination),
task_vars=task_vars,
persist_files=True)
if 'content' in dest_result:
dest_contents = dest_result['content']
if dest_result['encoding'] == 'base64':
dest_contents = base64.b64decode(dest_contents)
else:
raise AnsibleError(
"unknown encoding in content option, failed: %s" %
dest_result)
diff['before_header'] = destination
diff['before'] = dest_contents
if source_file:
st = os.stat(source)
if C.MAX_FILE_SIZE_FOR_DIFF > 0 and st[
stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
else:
display.debug("Reading local copy of the file %s" % source)
try:
src = open(source)
src_contents = src.read()
except Exception as e:
raise AnsibleError(
"Unexpected error while reading source (%s) for diff: %s "
% (source, str(e)))
if "\x00" in src_contents:
diff['src_binary'] = 1
else:
diff['after_header'] = source
diff['after'] = src_contents
else:
display.debug("source of file passed in")
diff['after_header'] = 'dynamically generated'
diff['after'] = source
if self._play_context.no_log:
if 'before' in diff:
diff["before"] = ""
if 'after' in diff:
diff[
"after"] = " [[ Diff output has been hidden because 'no_log: true' was specified for this result ]]"
return diff
def run(self):
super(DocCLI, self).run()
plugin_type = self.options.type
# choose plugin type
if plugin_type == 'cache':
loader = cache_loader
elif plugin_type == 'callback':
loader = callback_loader
elif plugin_type == 'connection':
loader = connection_loader
elif plugin_type == 'lookup':
loader = lookup_loader
elif plugin_type == 'strategy':
loader = strategy_loader
else:
loader = module_loader
# add to plugin path from command line
if self.options.module_path is not None:
for i in self.options.module_path.split(os.pathsep):
loader.add_directory(i)
# list plugins for type
if self.options.list_dir:
paths = loader._get_paths()
for path in paths:
self.find_plugins(path, plugin_type)
self.pager(self.get_plugin_list_text(loader))
return 0
# process all plugins of type
if self.options.all_plugins:
paths = loader._get_paths()
for path in paths:
self.find_plugins(path, plugin_type)
if len(self.args) == 0:
raise AnsibleOptionsError("Incorrect options passed")
# process command line list
text = ''
for plugin in self.args:
try:
# if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
filename = loader.find_plugin(plugin,
mod_type='.py',
ignore_deprecated=True)
if filename is None:
display.warning(
"%s %s not found in %s\n" %
(plugin_type, plugin, DocCLI.print_paths(loader)))
continue
if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
continue
try:
doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(
filename, verbose=(self.options.verbosity > 0))
except:
display.vvv(traceback.format_exc())
display.error(
"%s %s has a documentation error formatting or is missing documentation."
% (plugin_type, plugin))
continue
if doc is not None:
# assign from other sections
doc['plainexamples'] = plainexamples
doc['returndocs'] = returndocs
doc['metadata'] = metadata
# generate extra data
if plugin_type == 'module':
# is there corresponding action plugin?
if plugin in action_loader:
doc['action'] = True
else:
doc['action'] = False
doc['filename'] = filename
doc['now_date'] = datetime.date.today().strftime(
'%Y-%m-%d')
doc['docuri'] = doc[plugin_type].replace('_', '-')
if self.options.show_snippet and plugin_type == 'module':
text += self.get_snippet_text(doc)
else:
text += self.get_man_text(doc)
else:
# this typically means we couldn't even parse the docstring, not just that the YAML is busted,
# probably a quoting issue.
raise AnsibleError("Parsing produced an empty object.")
except Exception as e:
display.vvv(traceback.format_exc())
raise AnsibleError(
"%s %s missing documentation (or could not parse documentation): %s\n"
% (plugin_type, plugin, str(e)))
if text:
self.pager(text)
return 0
def _lookup(self, name, *args, **kwargs):
instance = lookup_loader.get(name, loader=self._loader, templar=self)
if instance is None:
raise AnsibleError("lookup plugin (%s) not found" % name)
wantlist = kwargs.pop('wantlist', False)
allow_unsafe = kwargs.pop('allow_unsafe',
C.DEFAULT_ALLOW_UNSAFE_LOOKUPS)
errors = kwargs.pop('errors', 'strict')
loop_terms = listify_lookup_plugin_terms(terms=args,
templar=self,
loader=self._loader,
fail_on_undefined=True,
convert_bare=False)
# safely catch run failures per #5059
try:
ran = instance.run(loop_terms,
variables=self._available_variables,
**kwargs)
except (AnsibleUndefinedVariable, UndefinedError) as e:
raise AnsibleUndefinedVariable(e)
except AnsibleOptionsError as e:
# invalid options given to lookup, just reraise
raise e
except AnsibleLookupError as e:
# lookup handled error but still decided to bail
msg = 'Lookup failed but the error is being ignored: %s' % to_native(
e)
if errors == 'warn':
display.warning(msg)
elif errors == 'ignore':
display.display(msg, log_only=True)
else:
raise e
return [] if wantlist else None
except Exception as e:
# errors not handled by lookup
msg = u"An unhandled exception occurred while running the lookup plugin '%s'. Error was a %s, original message: %s" % \
(name, type(e), to_text(e))
if errors == 'warn':
display.warning(msg)
elif errors == 'ignore':
display.display(msg, log_only=True)
else:
display.vvv('exception during Jinja2 execution: {0}'.format(
format_exc()))
raise AnsibleError(to_native(msg), orig_exc=e)
return [] if wantlist else None
if ran and allow_unsafe is False:
if self.cur_context:
self.cur_context.unsafe = True
if wantlist:
return wrap_var(ran)
try:
if self.jinja2_native and isinstance(ran[0], NativeJinjaText):
ran = wrap_var(NativeJinjaText(",".join(ran)))
else:
ran = wrap_var(",".join(ran))
except TypeError:
# Lookup Plugins should always return lists. Throw an error if that's not
# the case:
if not isinstance(ran, Sequence):
raise AnsibleError(
"The lookup plugin '%s' did not return a list." % name)
# The TypeError we can recover from is when the value *inside* of the list
# is not a string
if len(ran) == 1:
ran = wrap_var(ran[0])
else:
ran = wrap_var(ran)
return ran
def make_become_cmd(self, cmd, executable=None):
""" helper function to create privilege escalation commands """
prompt = None
success_key = None
self.prompt = None
if self.become:
if not executable:
executable = self.executable
becomecmd = None
randbits = ''.join(random.choice(string.ascii_lowercase) for x in range(32))
success_key = 'BECOME-SUCCESS-%s' % randbits
success_cmd = shlex_quote('echo %s; %s' % (success_key, cmd))
if executable:
command = '%s -c %s' % (executable, success_cmd)
else:
command = success_cmd
# set executable to use for the privilege escalation method, with various overrides
exe = self.become_exe or \
getattr(self, '%s_exe' % self.become_method, None) or \
C.DEFAULT_BECOME_EXE or \
getattr(C, 'DEFAULT_%s_EXE' % self.become_method.upper(), None) or \
self.become_method
# set flags to use for the privilege escalation method, with various overrides
flags = self.become_flags or \
getattr(self, '%s_flags' % self.become_method, None) or \
C.DEFAULT_BECOME_FLAGS or \
getattr(C, 'DEFAULT_%s_FLAGS' % self.become_method.upper(), None) or \
''
if self.become_method == 'sudo':
# If we have a password, we run sudo with a randomly-generated
# prompt set using -p. Otherwise we run it with default -n, which makes
# it fail if it would have prompted for a password.
# Cannot rely on -n as it can be removed from defaults, which should be
# done for older versions of sudo that do not support the option.
#
# Passing a quoted compound command to sudo (or sudo -s)
# directly doesn't work, so we shellquote it with shlex_quote()
# and pass the quoted string to the user's shell.
# force quick error if password is required but not supplied, should prevent sudo hangs.
if self.become_pass:
prompt = '[sudo via ansible, key=%s] password: '******'%s %s -p "%s" -u %s %s' % (exe, flags.replace('-n',''), prompt, self.become_user, command)
else:
becomecmd = '%s %s -u %s %s' % (exe, flags, self.become_user, command)
elif self.become_method == 'su':
# passing code ref to examine prompt as simple string comparisson isn't good enough with su
def detect_su_prompt(b_data):
b_password_string = b"|".join([b'(\w+\'s )?' + x for x in b_SU_PROMPT_LOCALIZATIONS])
# Colon or unicode fullwidth colon
b_password_string = b_password_string + to_bytes(u' ?(:|：) ?')
b_SU_PROMPT_LOCALIZATIONS_RE = re.compile(b_password_string, flags=re.IGNORECASE)
return bool(b_SU_PROMPT_LOCALIZATIONS_RE.match(b_data))
prompt = detect_su_prompt
becomecmd = '%s %s %s -c %s' % (exe, flags, self.become_user, shlex_quote(command))
elif self.become_method == 'pbrun':
prompt='Password:'******'%s %s -u %s %s' % (exe, flags, self.become_user, success_cmd)
elif self.become_method == 'ksu':
def detect_ksu_prompt(b_data):
return re.match(b"Kerberos password for .*@.*:", b_data)
prompt = detect_ksu_prompt
becomecmd = '%s %s %s -e %s' % (exe, self.become_user, flags, command)
elif self.become_method == 'pfexec':
# No user as it uses it's own exec_attr to figure it out
becomecmd = '%s %s "%s"' % (exe, flags, success_cmd)
elif self.become_method == 'runas':
raise AnsibleError("'runas' is not yet implemented")
#FIXME: figure out prompt
# this is not for use with winrm plugin but if they ever get ssh native on windoez
becomecmd = '%s %s /user:%s "%s"' % (exe, flags, self.become_user, success_cmd)
elif self.become_method == 'doas':
prompt = 'doas (%s@' % self.remote_user
exe = self.become_exe or 'doas'
if not self.become_pass:
flags += ' -n '
if self.become_user:
flags += ' -u %s ' % self.become_user
#FIXME: make shell independent
becomecmd = '%s %s echo %s && %s %s env ANSIBLE=true %s' % (exe, flags, success_key, exe, flags, cmd)
elif self.become_method == 'dzdo':
exe = self.become_exe or 'dzdo'
if self.become_pass:
prompt = '[dzdo via ansible, key=%s] password: '******'%s -p %s -u %s %s' % (exe, shlex_quote(prompt), self.become_user, command)
else:
becomecmd = '%s -u %s %s' % (exe, self.become_user, command)
else:
raise AnsibleError("Privilege escalation method not found: %s" % self.become_method)
if self.become_pass:
self.prompt = prompt
self.success_key = success_key
return becomecmd
return cmd
def fetch_file(self, in_path, out_path):
super(Connection, self).fetch_file(in_path, out_path)
display.vvv("FETCH %s TO %s" % (in_path, out_path),
host=self._psrp_host)
in_path = self._shell._unquote(in_path)
out_path = out_path.replace('\\', '/')
# because we are dealing with base64 data we need to get the max size
# of the bytes that the base64 size would equal
max_b64_size = int(self.runspace.connection.max_payload_size -
(self.runspace.connection.max_payload_size / 4 * 3))
buffer_size = max_b64_size - (max_b64_size % 1024)
# setup the file stream with read only mode
setup_script = '''$ErrorActionPreference = "Stop"
$path = '%s'
if (Test-Path -Path $path -PathType Leaf) {
$fs = New-Object -TypeName System.IO.FileStream -ArgumentList @(
$path,
[System.IO.FileMode]::Open,
[System.IO.FileAccess]::Read,
[System.IO.FileShare]::Read
)
$buffer_size = %d
} elseif (Test-Path -Path $path -PathType Container) {
Write-Output -InputObject "[DIR]"
} else {
Write-Error -Message "$path does not exist"
$host.SetShouldExit(1)
}''' % (self._shell._escape(in_path), buffer_size)
# read the file stream at the offset and return the b64 string
read_script = '''$ErrorActionPreference = "Stop"
$fs.Seek(%d, [System.IO.SeekOrigin]::Begin) > $null
$buffer = New-Object -TypeName byte[] -ArgumentList $buffer_size
$bytes_read = $fs.Read($buffer, 0, $buffer_size)
if ($bytes_read -gt 0) {
$bytes = $buffer[0..($bytes_read - 1)]
Write-Output -InputObject ([System.Convert]::ToBase64String($bytes))
}'''
# need to run the setup script outside of the local scope so the
# file stream stays active between fetch operations
rc, stdout, stderr = self._exec_psrp_script(setup_script,
use_local_scope=False,
force_stop=True)
if rc != 0:
raise AnsibleError("failed to setup file stream for fetch '%s': %s"
% (out_path, to_native(stderr)))
elif stdout.strip() == '[DIR]':
# to be consistent with other connection plugins, we assume the caller has created the target dir
return
b_out_path = to_bytes(out_path, errors='surrogate_or_strict')
# to be consistent with other connection plugins, we assume the caller has created the target dir
offset = 0
with open(b_out_path, 'wb') as out_file:
while True:
display.vvvvv("PSRP FETCH %s to %s (offset=%d" %
(in_path, out_path, offset), host=self._psrp_host)
rc, stdout, stderr = self._exec_psrp_script(read_script % offset, force_stop=True)
if rc != 0:
raise AnsibleError("failed to transfer file to '%s': %s"
% (out_path, to_native(stderr)))
data = base64.b64decode(stdout.strip())
out_file.write(data)
if len(data) < buffer_size:
break
offset += len(data)
rc, stdout, stderr = self._exec_psrp_script("$fs.Close()", force_stop=True)
if rc != 0:
display.warning("failed to close remote file stream of file "
"'%s': %s" % (in_path, to_native(stderr)))
def _do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None):
# For preserving the number of input newlines in the output (used
# later in this method)
data_newlines = _count_newlines_from_end(data)
if fail_on_undefined is None:
fail_on_undefined = self._fail_on_undefined_errors
try:
# allows template header overrides to change jinja2 options.
if overrides is None:
myenv = self.environment.overlay()
else:
myenv = self.environment.overlay(overrides)
# Get jinja env overrides from template
if data.startswith(JINJA2_OVERRIDE):
eol = data.find('\n')
line = data[len(JINJA2_OVERRIDE):eol]
data = data[eol+1:]
for pair in line.split(','):
(key,val) = pair.split(':')
key = key.strip()
setattr(myenv, key, ast.literal_eval(val.strip()))
#FIXME: add tests
myenv.filters.update(self._get_filters())
myenv.tests.update(self._get_tests())
if escape_backslashes:
# Allow users to specify backslashes in playbooks as "\\"
# instead of as "\\\\".
data = _escape_backslashes(data, myenv)
try:
t = myenv.from_string(data)
except TemplateSyntaxError as e:
raise AnsibleError("template error while templating string: %s. String: %s" % (str(e), data))
except Exception as e:
if 'recursion' in str(e):
raise AnsibleError("recursive loop detected in template string: %s" % data)
else:
return data
t.globals['lookup'] = self._lookup
t.globals['finalize'] = self._finalize
jvars = AnsibleJ2Vars(self, t.globals)
new_context = t.new_context(jvars, shared=True)
rf = t.root_render_func(new_context)
try:
res = j2_concat(rf)
except TypeError as te:
if 'StrictUndefined' in str(te):
raise AnsibleUndefinedVariable(
"Unable to look up a name or access an attribute in template string. " + \
"Make sure your variable name does not contain invalid characters like '-'."
)
else:
debug("failing because of a type error, template data is: %s" % data)
raise AnsibleError("an unexpected type error occurred. Error was %s" % te)
if preserve_trailing_newlines:
# The low level calls above do not preserve the newline
# characters at the end of the input data, so we use the
# calculate the difference in newlines and append them
# to the resulting output for parity
#
# jinja2 added a keep_trailing_newline option in 2.7 when
# creating an Environment. That would let us make this code
# better (remove a single newline if
# preserve_trailing_newlines is False). Once we can depend on
# that version being present, modify our code to set that when
# initializing self.environment and remove a single trailing
# newline here if preserve_newlines is False.
res_newlines = _count_newlines_from_end(res)
if data_newlines > res_newlines:
res += '\n' * (data_newlines - res_newlines)
return res
except (UndefinedError, AnsibleUndefinedVariable) as e:
if fail_on_undefined:
raise AnsibleUndefinedVariable(e)
else:
#TODO: return warning about undefined var
return data
def _get_loop_items(self):
'''
Loads a lookup plugin to handle the with_* portion of a task (if specified),
and returns the items result.
'''
# save the play context variables to a temporary dictionary,
# so that we can modify the job vars without doing a full copy
# and later restore them to avoid modifying things too early
play_context_vars = dict()
self._play_context.update_vars(play_context_vars)
old_vars = dict()
for k in play_context_vars.keys():
if k in self._job_vars:
old_vars[k] = self._job_vars[k]
self._job_vars[k] = play_context_vars[k]
templar = Templar(loader=self._loader,
shared_loader_obj=self._shared_loader_obj,
variables=self._job_vars)
items = None
if self._task.loop:
if self._task.loop in self._shared_loader_obj.lookup_loader:
#TODO: remove convert_bare true and deprecate this in with_
if self._task.loop == 'first_found':
# first_found loops are special. If the item is undefined
# then we want to fall through to the next value rather
# than failing.
loop_terms = listify_lookup_plugin_terms(
terms=self._task.loop_args,
templar=templar,
loader=self._loader,
fail_on_undefined=False,
convert_bare=True)
loop_terms = [
t for t in loop_terms if not templar._contains_vars(t)
]
else:
try:
loop_terms = listify_lookup_plugin_terms(
terms=self._task.loop_args,
templar=templar,
loader=self._loader,
fail_on_undefined=True,
convert_bare=True)
except AnsibleUndefinedVariable as e:
display.deprecated(
"Skipping task due to undefined Error, in the future this will be a fatal error.: %s"
% to_bytes(e))
return None
items = self._shared_loader_obj.lookup_loader.get(
self._task.loop, loader=self._loader,
templar=templar).run(terms=loop_terms,
variables=self._job_vars,
wantlist=True)
else:
raise AnsibleError(
"Unexpected failure in finding the lookup named '%s' in the available lookup plugins"
% self._task.loop)
# now we restore any old job variables that may have been modified,
# and delete them if they were in the play context vars but not in
# the old variables dictionary
for k in play_context_vars.keys():
if k in old_vars:
self._job_vars[k] = old_vars[k]
else:
del self._job_vars[k]
if items:
from ansible.vars.unsafe_proxy import UnsafeProxy
for idx, item in enumerate(items):
if item is not None and not isinstance(item, UnsafeProxy):
items[idx] = UnsafeProxy(item)
return items
def _clean_data(self, orig_data):
''' remove jinja2 template tags from data '''
if hasattr(orig_data, '__ENCRYPTED__'):
ret = orig_data
elif isinstance(orig_data, list):
clean_list = []
for list_item in orig_data:
clean_list.append(self._clean_data(list_item))
ret = clean_list
elif isinstance(orig_data, dict):
clean_dict = {}
for k in orig_data:
clean_dict[self._clean_data(k)] = self._clean_data(
orig_data[k])
ret = clean_dict
elif isinstance(orig_data, string_types):
# This will error with str data (needs unicode), but all strings should already be converted already.
# If you get exception, the problem is at the data origin, do not add to_text here.
with contextlib.closing(StringIO(orig_data)) as data:
# these variables keep track of opening block locations, as we only
# want to replace matched pairs of print/block tags
print_openings = []
block_openings = []
for mo in self._clean_regex.finditer(orig_data):
token = mo.group(0)
token_start = mo.start(0)
if token[0] == self.environment.variable_start_string[0]:
if token == self.environment.block_start_string:
block_openings.append(token_start)
elif token == self.environment.variable_start_string:
print_openings.append(token_start)
elif token[1] == self.environment.variable_end_string[1]:
prev_idx = None
if token == self.environment.block_end_string and block_openings:
prev_idx = block_openings.pop()
elif token == self.environment.variable_end_string and print_openings:
prev_idx = print_openings.pop()
if prev_idx is not None:
# replace the opening
data.seek(prev_idx, os.SEEK_SET)
data.write(
to_text(self.environment.comment_start_string))
# replace the closing
data.seek(token_start, os.SEEK_SET)
data.write(
to_text(self.environment.comment_end_string))
else:
raise AnsibleError(
"Error while cleaning data for safety: unhandled regex match"
)
ret = data.getvalue()
else:
ret = orig_data
return ret
'''
import re
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict
from ansible.module_utils.ec2 import camel_dict_to_snake_dict
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
from ansible.utils.display import Display
try:
import boto3
import botocore
except ImportError:
raise AnsibleError('The ec2 dynamic inventory plugin requires boto3 and botocore.')
display = Display()
# The mappings give an array of keys to get from the filter name to the value
# returned by boto3's EC2 describe_instances method.
instance_meta_filter_to_boto_attr = {
'group-id': ('Groups', 'GroupId'),
'group-name': ('Groups', 'GroupName'),
'network-interface.attachment.instance-owner-id': ('OwnerId',),
'owner-id': ('OwnerId',),
'requester-id': ('RequesterId',),
'reservation-id': ('ReservationId',),
}
def _configure_module(self, module_name, module_args, task_vars=None):
'''
Handles the loading and templating of the module code through the
modify_module() function.
'''
if task_vars is None:
task_vars = dict()
# Search module path(s) for named module.
for mod_type in self._connection.module_implementation_preferences:
# Check to determine if PowerShell modules are supported, and apply
# some fixes (hacks) to module name + args.
if mod_type == '.ps1':
# win_stat, win_file, and win_copy are not just like their
# python counterparts but they are compatible enough for our
# internal usage
if module_name in ('stat', 'file', 'copy'
) and self._task.action != module_name:
module_name = 'win_%s' % module_name
# Remove extra quotes surrounding path parameters before sending to module.
if module_name in ('win_stat', 'win_file', 'win_copy',
'slurp') and module_args and hasattr(
self._connection._shell, '_unquote'):
for key in ('src', 'dest', 'path'):
if key in module_args:
module_args[
key] = self._connection._shell._unquote(
module_args[key])
module_path = self._shared_loader_obj.module_loader.find_plugin(
module_name, mod_type)
if module_path:
break
else: # This is a for-else: http://bit.ly/1ElPkyg
# Use Windows version of ping module to check module paths when
# using a connection that supports .ps1 suffixes. We check specifically
# for win_ping here, otherwise the code would look for ping.ps1
if '.ps1' in self._connection.module_implementation_preferences:
ping_module = 'win_ping'
else:
ping_module = 'ping'
module_path2 = self._shared_loader_obj.module_loader.find_plugin(
ping_module,
self._connection.module_implementation_preferences)
if module_path2 is not None:
raise AnsibleError(
"The module %s was not found in configured module paths" %
(module_name))
else:
raise AnsibleError(
"The module %s was not found in configured module paths. "
"Additionally, core modules are missing. If this is a checkout, "
"run 'git submodule update --init --recursive' to correct this problem."
% (module_name))
# insert shared code and arguments into the module
(module_data, module_style, module_shebang) = modify_module(
module_name,
module_path,
module_args,
task_vars=task_vars,
module_compression=self._play_context.module_compression)
return (module_style, module_shebang, module_data, module_path)
def _get_galaxy_yml(b_galaxy_yml_path):
meta_info = get_collections_galaxy_meta_info()
mandatory_keys = set()
string_keys = set()
list_keys = set()
dict_keys = set()
for info in meta_info:
if info.get('required', False):
mandatory_keys.add(info['key'])
key_list_type = {
'str': string_keys,
'list': list_keys,
'dict': dict_keys,
}[info.get('type', 'str')]
key_list_type.add(info['key'])
all_keys = frozenset(list(mandatory_keys) + list(string_keys) + list(list_keys) + list(dict_keys))
try:
with open(b_galaxy_yml_path, 'rb') as g_yaml:
galaxy_yml = yaml.safe_load(g_yaml)
except YAMLError as err:
raise AnsibleError("Failed to parse the galaxy.yml at '%s' with the following error:\n%s"
% (to_native(b_galaxy_yml_path), to_native(err)))
set_keys = set(galaxy_yml.keys())
missing_keys = mandatory_keys.difference(set_keys)
if missing_keys:
raise AnsibleError("The collection galaxy.yml at '%s' is missing the following mandatory keys: %s"
% (to_native(b_galaxy_yml_path), ", ".join(sorted(missing_keys))))
extra_keys = set_keys.difference(all_keys)
if len(extra_keys) > 0:
display.warning("Found unknown keys in collection galaxy.yml at '%s': %s"
% (to_text(b_galaxy_yml_path), ", ".join(extra_keys)))
# Add the defaults if they have not been set
for optional_string in string_keys:
if optional_string not in galaxy_yml:
galaxy_yml[optional_string] = None
for optional_list in list_keys:
list_val = galaxy_yml.get(optional_list, None)
if list_val is None:
galaxy_yml[optional_list] = []
elif not isinstance(list_val, list):
galaxy_yml[optional_list] = [list_val]
for optional_dict in dict_keys:
if optional_dict not in galaxy_yml:
galaxy_yml[optional_dict] = {}
# license is a builtin var in Python, to avoid confusion we just rename it to license_ids
galaxy_yml['license_ids'] = galaxy_yml['license']
del galaxy_yml['license']
return galaxy_yml
def _bare_run(self, cmd, in_data, sudoable=True, checkrc=True):
'''
Starts the command and communicates with it until it ends.
'''
display_cmd = list(map(shlex_quote, map(to_text, cmd)))
display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)),
host=self.host)
# Start the given command. If we don't need to pipeline data, we can try
# to use a pseudo-tty (ssh will have been invoked with -tt). If we are
# pipelining data, or can't create a pty, we fall back to using plain
# old pipes.
p = None
if isinstance(cmd, (text_type, binary_type)):
cmd = to_bytes(cmd)
else:
cmd = list(map(to_bytes, cmd))
if not in_data:
try:
# Make sure stdin is a proper pty to avoid tcgetattr errors
master, slave = pty.openpty()
if PY3 and self._play_context.password:
p = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
pass_fds=self.sshpass_pipe)
else:
p = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdin = os.fdopen(master, 'wb', 0)
os.close(slave)
except (OSError, IOError):
p = None
if not p:
if PY3 and self._play_context.password:
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
pass_fds=self.sshpass_pipe)
else:
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdin = p.stdin
# If we are using SSH password authentication, write the password into
# the pipe we opened in _build_command.
if self._play_context.password:
os.close(self.sshpass_pipe[0])
try:
os.write(self.sshpass_pipe[1],
to_bytes(self._play_context.password) + b'\n')
except OSError as e:
# Ignore broken pipe errors if the sshpass process has exited.
if e.errno != errno.EPIPE or p.poll() is None:
raise
os.close(self.sshpass_pipe[1])
#
# SSH state machine
#
# Now we read and accumulate output from the running process until it
# exits. Depending on the circumstances, we may also need to write an
# escalation password and/or pipelined input to the process.
states = [
'awaiting_prompt', 'awaiting_escalation', 'ready_to_send',
'awaiting_exit'
]
# Are we requesting privilege escalation? Right now, we may be invoked
# to execute sftp/scp with sudoable=True, but we can request escalation
# only when using ssh. Otherwise we can send initial data straightaway.
state = states.index('ready_to_send')
if b'ssh' in cmd:
if self._play_context.prompt:
# We're requesting escalation with a password, so we have to
# wait for a password prompt.
state = states.index('awaiting_prompt')
display.debug(u'Initial state: %s: %s' %
(states[state], self._play_context.prompt))
elif self._play_context.become and self._play_context.success_key:
# We're requesting escalation without a password, so we have to
# detect success/failure before sending any initial data.
state = states.index('awaiting_escalation')
display.debug(u'Initial state: %s: %s' %
(states[state], self._play_context.success_key))
# We store accumulated stdout and stderr output from the process here,
# but strip any privilege escalation prompt/confirmation lines first.
# Output is accumulated into tmp_*, complete lines are extracted into
# an array, then checked and removed or copied to stdout or stderr. We
# set any flags based on examining the output in self._flags.
b_stdout = b_stderr = b''
b_tmp_stdout = b_tmp_stderr = b''
self._flags = dict(become_prompt=False,
become_success=False,
become_error=False,
become_nopasswd_error=False)
# select timeout should be longer than the connect timeout, otherwise
# they will race each other when we can't connect, and the connect
# timeout usually fails
timeout = 2 + self._play_context.timeout
for fd in (p.stdout, p.stderr):
fcntl.fcntl(fd, fcntl.F_SETFL,
fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# TODO: bcoca would like to use SelectSelector() when open
# filehandles is low, then switch to more efficient ones when higher.
# select is faster when filehandles is low.
selector = selectors.DefaultSelector()
selector.register(p.stdout, selectors.EVENT_READ)
selector.register(p.stderr, selectors.EVENT_READ)
# If we can send initial data without waiting for anything, we do so
# before we start polling
if states[state] == 'ready_to_send' and in_data:
self._send_initial_data(stdin, in_data)
state += 1
try:
while True:
poll = p.poll()
events = selector.select(timeout)
# We pay attention to timeouts only while negotiating a prompt.
if not events:
# We timed out
if state <= states.index('awaiting_escalation'):
# If the process has already exited, then it's not really a
# timeout; we'll let the normal error handling deal with it.
if poll is not None:
break
self._terminate_process(p)
raise AnsibleError(
'Timeout (%ds) waiting for privilege escalation prompt: %s'
% (timeout, to_native(b_stdout)))
# Read whatever output is available on stdout and stderr, and stop
# listening to the pipe if it's been closed.
for key, event in events:
if key.fileobj == p.stdout:
b_chunk = p.stdout.read()
if b_chunk == b'':
# stdout has been closed, stop watching it
selector.unregister(p.stdout)
# When ssh has ControlMaster (+ControlPath/Persist) enabled, the
# first connection goes into the background and we never see EOF
# on stderr. If we see EOF on stdout, lower the select timeout
# to reduce the time wasted selecting on stderr if we observe
# that the process has not yet existed after this EOF. Otherwise
# we may spend a long timeout period waiting for an EOF that is
# not going to arrive until the persisted connection closes.
timeout = 1
b_tmp_stdout += b_chunk
display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" %
(state, to_text(b_chunk)))
elif key.fileobj == p.stderr:
b_chunk = p.stderr.read()
if b_chunk == b'':
# stderr has been closed, stop watching it
selector.unregister(p.stderr)
b_tmp_stderr += b_chunk
display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" %
(state, to_text(b_chunk)))
# We examine the output line-by-line until we have negotiated any
# privilege escalation prompt and subsequent success/error message.
# Afterwards, we can accumulate output without looking at it.
if state < states.index('ready_to_send'):
if b_tmp_stdout:
b_output, b_unprocessed = self._examine_output(
'stdout', states[state], b_tmp_stdout, sudoable)
b_stdout += b_output
b_tmp_stdout = b_unprocessed
if b_tmp_stderr:
b_output, b_unprocessed = self._examine_output(
'stderr', states[state], b_tmp_stderr, sudoable)
b_stderr += b_output
b_tmp_stderr = b_unprocessed
else:
b_stdout += b_tmp_stdout
b_stderr += b_tmp_stderr
b_tmp_stdout = b_tmp_stderr = b''
# If we see a privilege escalation prompt, we send the password.
# (If we're expecting a prompt but the escalation succeeds, we
# didn't need the password and can carry on regardless.)
if states[state] == 'awaiting_prompt':
if self._flags['become_prompt']:
display.debug(
'Sending become_pass in response to prompt')
stdin.write(
to_bytes(self._play_context.become_pass) + b'\n')
self._flags['become_prompt'] = False
state += 1
elif self._flags['become_success']:
state += 1
# We've requested escalation (with or without a password), now we
# wait for an error message or a successful escalation.
if states[state] == 'awaiting_escalation':
if self._flags['become_success']:
display.debug('Escalation succeeded')
self._flags['become_success'] = False
state += 1
elif self._flags['become_error']:
display.debug('Escalation failed')
self._terminate_process(p)
self._flags['become_error'] = False
raise AnsibleError('Incorrect %s password' %
self._play_context.become_method)
elif self._flags['become_nopasswd_error']:
display.debug('Escalation requires password')
self._terminate_process(p)
self._flags['become_nopasswd_error'] = False
raise AnsibleError('Missing %s password' %
self._play_context.become_method)
elif self._flags['become_prompt']:
# This shouldn't happen, because we should see the "Sorry,
# try again" message first.
display.debug('Escalation prompt repeated')
self._terminate_process(p)
self._flags['become_prompt'] = False
raise AnsibleError('Incorrect %s password' %
self._play_context.become_method)
# Once we're sure that the privilege escalation prompt, if any, has
# been dealt with, we can send any initial data and start waiting
# for output.
if states[state] == 'ready_to_send':
if in_data:
self._send_initial_data(stdin, in_data)
state += 1
# Now we're awaiting_exit: has the child process exited? If it has,
# and we've read all available output from it, we're done.
if poll is not None:
if not selector.get_map() or not events:
break
# We should not see further writes to the stdout/stderr file
# descriptors after the process has closed, set the select
# timeout to gather any last writes we may have missed.
timeout = 0
continue
# If the process has not yet exited, but we've already read EOF from
# its stdout and stderr (and thus no longer watching any file
# descriptors), we can just wait for it to exit.
elif not selector.get_map():
p.wait()
break
# Otherwise there may still be outstanding data to read.
finally:
selector.close()
# close stdin after process is terminated and stdout/stderr are read
# completely (see also issue #848)
stdin.close()
if C.HOST_KEY_CHECKING:
if cmd[0] == b"sshpass" and p.returncode == 6:
raise AnsibleError(
'Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support '
'this. Please add this host\'s fingerprint to your known_hosts file to manage this host.'
)
controlpersisterror = b'Bad configuration option: ControlPersist' in b_stderr or b'unknown configuration option: ControlPersist' in b_stderr
if p.returncode != 0 and controlpersisterror:
raise AnsibleError(
'using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" '
'(or ssh_args in [ssh_connection] section of the config file) before running again'
)
# If we find a broken pipe because of ControlPersist timeout expiring (see #16731),
# we raise a special exception so that we can retry a connection.
controlpersist_broken_pipe = b'mux_client_hello_exchange: write packet: Broken pipe' in b_stderr
if p.returncode == 255 and controlpersist_broken_pipe:
raise AnsibleControlPersistBrokenPipeError(
'SSH Error: data could not be sent because of ControlPersist broken pipe.'
)
if p.returncode == 255 and in_data and checkrc:
raise AnsibleConnectionFailure(
'SSH Error: data could not be sent to remote host "%s". Make sure this host can be reached over ssh'
% self.host)
return (p.returncode, b_stdout, b_stderr)
