def fetch_file(self, in_path, out_path): ''' fetch a file from lxc to local ''' super(Connection, self).fetch_file(in_path, out_path) self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name) in_path = to_bytes(in_path, errors='strict') out_path = to_bytes(out_path, errors='strict') try: dst_file = open(out_path, "wb") except IOError: traceback.print_exc() msg = "failed to open output file %s" % out_path raise errors.AnsibleError(msg) try: def write_file(args): try: with open(in_path, 'rb') as src_file: shutil.copyfileobj(src_file, dst_file) finally: # this is needed in the lxc child process # to flush internal python buffers dst_file.close() try: self.container.attach_wait(write_file, None) except IOError: traceback.print_exc() msg = "failed to transfer file from %s to %s" % (in_path, out_path) raise errors.AnsibleError(msg) finally: dst_file.close()
def read_vault_password_file(self, vault_password_file): """ Read a vault password from a file or if executable, execute the script and retrieve password from STDOUT """ this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict')) if not os.path.exists(to_bytes(this_path, errors='strict')): raise AnsibleFileNotFound("The vault password file %s was not found" % this_path) if self.is_executable(this_path): try: # STDERR not captured to make it easier for users to prompt for input in their scripts p = subprocess.Popen(this_path, stdout=subprocess.PIPE) except OSError as e: raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e)) stdout, stderr = p.communicate() self.set_vault_password(stdout.strip('\r\n')) else: try: f = open(this_path, "rb") self.set_vault_password(f.read().strip()) f.close() except (OSError, IOError) as e: raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))
def ask_vault_passwords(ask_new_vault_pass=False, rekey=False): ''' prompt for vault password and/or password change ''' vault_pass = None new_vault_pass = None try: if rekey or not ask_new_vault_pass: vault_pass = getpass.getpass(prompt="Vault password: "******"New Vault password: "******"Confirm New Vault password: "******"Passwords do not match") except EOFError: pass # enforce no newline chars at the end of passwords if vault_pass: vault_pass = to_bytes(vault_pass, errors='strict', nonstring='simplerepr').strip() if new_vault_pass: new_vault_pass = to_bytes(new_vault_pass, errors='strict', nonstring='simplerepr').strip() if ask_new_vault_pass and not rekey: vault_pass = new_vault_pass return vault_pass, new_vault_pass
def ask_vault_passwords(ask_vault_pass=False, ask_new_vault_pass=False, confirm_vault=False, confirm_new=False): ''' prompt for vault password and/or password change ''' vault_pass = None new_vault_pass = None if ask_vault_pass: vault_pass = getpass.getpass(prompt="Vault password: "******"Confirm Vault password: "******"Passwords do not match") if ask_new_vault_pass: new_vault_pass = getpass.getpass(prompt="New Vault password: "******"Confirm New Vault password: "******"Passwords do not match") # enforce no newline chars at the end of passwords if vault_pass: vault_pass = to_bytes(vault_pass, errors='strict', nonstring='simplerepr').strip() if new_vault_pass: new_vault_pass = to_bytes(new_vault_pass, errors='strict', nonstring='simplerepr').strip() return vault_pass, new_vault_pass
def ask_passwords(self): ''' prompt for connection and become passwords if needed ''' op = self.options sshpass = None becomepass = None become_prompt = '' try: if op.ask_pass: sshpass = getpass.getpass(prompt="SSH password: "******"%s password[defaults to SSH password]: " % op.become_method.upper() if sshpass: sshpass = to_bytes(sshpass, errors='strict', nonstring='simplerepr') else: become_prompt = "%s password: " % op.become_method.upper() if op.become_ask_pass: becomepass = getpass.getpass(prompt=become_prompt) if op.ask_pass and becomepass == '': becomepass = sshpass if becomepass: becomepass = to_bytes(becomepass) except EOFError: pass return (sshpass, becomepass)
def put_file(self, in_path, out_path): ''' transfer a file from local to lxc ''' super(Connection, self).put_file(in_path, out_path) self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name) in_path = to_bytes(in_path, errors='strict') out_path = to_bytes(out_path, errors='strict') if not os.path.exists(in_path): msg = "file or module does not exist: %s" % in_path raise errors.AnsibleFileNotFound(msg) try: src_file = open(in_path, "rb") except IOError: traceback.print_exc() raise errors.AnsibleError("failed to open input file to %s" % in_path) try: def write_file(args): with open(out_path, 'wb+') as dst_file: shutil.copyfileobj(src_file, dst_file) try: self.container.attach_wait(write_file, None) except IOError: traceback.print_exc() msg = "failed to transfer file to %s" % out_path raise errors.AnsibleError(msg) finally: src_file.close()
def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False): # FIXME: this needs to be implemented #msg = utils.sanitize_output(msg) msg2 = self._safe_output(msg, stderr=stderr) if color: msg2 = stringc(msg, color) if not log_only: b_msg2 = to_bytes(msg2) if not stderr: print(b_msg2) sys.stdout.flush() else: print(b_msg2, file=sys.stderr) sys.stderr.flush() if logger and not screen_only: while msg.startswith("\n"): msg = msg.replace("\n","") b_msg = to_bytes(msg) if color == 'red': logger.error(b_msg) else: logger.info(b_msg)
def exec_command(self, cmd, in_data=None, sudoable=True): super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) cmd_parts = shlex.split(to_bytes(cmd), posix=False) cmd_parts = map(to_unicode, cmd_parts) script = None cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or '' # Support running .ps1 files (via script/raw). if cmd_ext == '.ps1': script = '& %s' % cmd # Support running .bat/.cmd files; change back to the default system encoding instead of UTF-8. elif cmd_ext in ('.bat', '.cmd'): script = '[System.Console]::OutputEncoding = [System.Text.Encoding]::Default; & %s' % cmd # Encode the command if not already encoded; supports running simple PowerShell commands via raw. elif '-EncodedCommand' not in cmd_parts: script = cmd if script: cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False) if '-EncodedCommand' in cmd_parts: encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1] decoded_cmd = to_unicode(base64.b64decode(encoded_cmd).decode('utf-16-le')) display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host) else: display.vvv("EXEC %s" % cmd, host=self._winrm_host) try: result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True) except Exception: traceback.print_exc() raise AnsibleError("failed to exec cmd %s" % cmd) result.std_out = to_bytes(result.std_out) result.std_err = to_bytes(result.std_err) return (result.status_code, result.std_out, result.std_err)
def put_file(self, in_path, out_path): """ Transfer a file from local to docker container """ super(Connection, self).put_file(in_path, out_path) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) out_path = self._prefix_login_path(out_path) if not os.path.exists(to_bytes(in_path, errors='strict')): raise AnsibleFileNotFound( "file or module does not exist: %s" % in_path) out_path = pipes.quote(out_path) # Older docker doesn't have native support for copying files into # running containers, so we use docker exec to implement this # Although docker version 1.8 and later provide support, the # owner and group of the files are always set to root args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)]) args = [to_bytes(i, errors='strict') for i in args] with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: try: p = subprocess.Popen(args, stdin=in_file, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError: raise AnsibleError("docker connection requires dd command in the container to put files") stdout, stderr = p.communicate() if p.returncode != 0: raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
def put_file(self, in_path, out_path): """ transfer a file from local to remote """ super(Connection, self).put_file(in_path, out_path) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host) if not os.path.exists(to_bytes(in_path, errors="strict")): raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) # scp and sftp require square brackets for IPv6 addresses, but # accept them for hostnames and IPv4 addresses too. host = "[%s]" % self.host if C.DEFAULT_SCP_IF_SSH: cmd = self._build_command("scp", in_path, u"{0}:{1}".format(host, pipes.quote(out_path))) in_data = None else: cmd = self._build_command("sftp", to_bytes(host)) in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) in_data = to_bytes(in_data, nonstring="passthru") (returncode, stdout, stderr) = self._run(cmd, in_data) if returncode != 0: raise AnsibleError( "failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr)) )
def encrypt(self, data, password): salt = os.urandom(32) key1, key2, iv = self.gen_key_initctr(password, salt) # PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3 bs = AES.block_size padding_length = (bs - len(data) % bs) or bs data += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict') # COUNTER.new PARAMETERS # 1) nbits (integer) - Length of the counter, in bits. # 2) initial_value (integer) - initial value of the counter. "iv" from gen_key_initctr ctr = Counter.new(128, initial_value=int(iv, 16)) # AES.new PARAMETERS # 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from gen_key_initctr # 2) MODE_CTR, is the recommended mode # 3) counter=<CounterObject> cipher = AES.new(key1, AES.MODE_CTR, counter=ctr) # ENCRYPT PADDED DATA cryptedData = cipher.encrypt(data) # COMBINE SALT, DIGEST AND DATA hmac = HMAC.new(key2, cryptedData, SHA256) message = b'%s\n%s\n%s' % (hexlify(salt), to_bytes(hmac.hexdigest()), hexlify(cryptedData)) message = hexlify(message) return message
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): """ Used to insert chunks of code into modules before transfer rather than doing regular python imports. This allows for more efficient transfer in a non-bootstrapping scenario by not moving extra files over the wire and also takes care of embedding arguments in the transferred modules. This version is done in such a way that local imports can still be used in the module code, so IDEs don't have to be aware of what is going on. Example: from ansible.module_utils.basic import * ... will result in the insertion of basic.py into the module from the module_utils/ directory in the source tree. All modules are required to import at least basic, though there will also be other snippets. For powershell, there's equivalent conventions like this: # POWERSHELL_COMMON which results in the inclusion of the common code from powershell.ps1 """ with open(module_path, 'rb') as f: # read in the module source module_data = f.read() (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): shebang = lines[0].strip() args = shlex.split(str(shebang[2:])) interpreter = args[0] interpreter = to_bytes(interpreter) new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru') if new_shebang: lines[0] = shebang = new_shebang if os.path.basename(interpreter).startswith(b'python'): lines.insert(1, to_bytes(ENCODING_STRING)) else: # No shebang, assume a binary module? pass module_data = b"\n".join(lines) else: shebang = to_bytes(shebang, errors='strict') return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))
def decrypt_file(self, filename, output_file=None): check_prereqs() ciphertext = self.read_data(filename) try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) self.write_data(plaintext, output_file or filename, shred=False)
def _safe_output(self, msg, stderr=False): if not stderr and sys.stdout.encoding: msg = to_bytes(msg, sys.stdout.encoding) elif stderr and sys.stderr.encoding: msg = to_bytes(msg, sys.stderr.encoding) else: msg = to_bytes(msg) return msg
def plaintext(self, filename): check_prereqs() ciphertext = self.read_data(filename) try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) return plaintext
def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases): modstring = module if modstring.startswith('_'): modstring = module[1:] modname = modstring if module in deprecated: modstring = modstring + DEPRECATED elif module not in core: modstring = modstring + NOTCORE category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False): """ Display a message to the user Note: msg *must* be a unicode string to prevent UnicodeError tracebacks. """ # FIXME: this needs to be implemented #msg = utils.sanitize_output(msg) nocolor = msg if color: msg = stringc(msg, color) if not log_only: if not msg.endswith(u'\n'): msg2 = msg + u'\n' else: msg2 = msg msg2 = to_bytes(msg2, encoding=self._output_encoding(stderr=stderr)) if sys.version_info >= (3,): # Convert back to text string on python3 # We first convert to a byte string so that we get rid of # characters that are invalid in the user's locale msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) if not stderr: fileobj = sys.stdout else: fileobj = sys.stderr fileobj.write(msg2) try: fileobj.flush() except IOError as e: # Ignore EPIPE in case fileobj has been prematurely closed, eg. # when piping to "head -n1" if e.errno != errno.EPIPE: raise if logger and not screen_only: msg2 = nocolor.lstrip(u'\n') msg2 = to_bytes(msg2) if sys.version_info >= (3,): # Convert back to text string on python3 # We first convert to a byte string so that we get rid of # characters that are invalid in the user's locale msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) if color == C.COLOR_ERROR: logger.error(msg2) else: logger.info(msg2)
def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases): modstring = module modname = module if module in deprecated: modstring = modstring + DEPRECATED modname = "_" + module elif module not in core: modstring = modstring + NOTCORE result = process_module(modname, options, env, template, outputname, module_map, aliases) if result != "SKIPPED": category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(result)), to_bytes(module)))
def _add_header(self, data): # combine header and encrypted data in 80 char columns #tmpdata = hexlify(data) tmpdata = [to_bytes(data[i:i+80]) for i in range(0, len(data), 80)] if not self.cipher_name: raise errors.AnsibleError("the cipher must be set before adding a header") dirty_data = to_bytes(HEADER + ";" + self.version + ";" + self.cipher_name + "\n") for l in tmpdata: dirty_data += l + b'\n' return dirty_data
def __init__(self, play_context, new_stdin, *args, **kwargs): super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs) self.zone = self._play_context.remote_addr if os.geteuid() != 0: raise AnsibleError("zone connection requires running as root") self.zoneadm_cmd = to_bytes(self._search_executable('zoneadm')) self.zlogin_cmd = to_bytes(self._search_executable('zlogin')) if self.zone not in self.list_zones(): raise AnsibleError("incorrect zone name %s" % self.zone)
def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False): """ Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel to the inventory base directory or in the same directory as the playbook. Variables in the playbook dir will win over the inventory dir if files are in both. """ results = {} scan_pass = 0 _basedir = self._basedir _playbook_basedir = self._playbook_basedir # look in both the inventory base directory and the playbook base directory # unless we do an update for a new playbook base dir if not new_pb_basedir and _playbook_basedir: basedirs = [_basedir, _playbook_basedir] else: basedirs = [_basedir] for basedir in basedirs: # this can happen from particular API usages, particularly if not run # from /usr/bin/ansible-playbook if basedir in ('', None): basedir = './' scan_pass = scan_pass + 1 # it's not an eror if the directory does not exist, keep moving if not os.path.exists(basedir): continue # save work of second scan if the directories are the same if _basedir == _playbook_basedir and scan_pass != 1: continue # Before trying to load vars from file, check that the directory contains relvant file names if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)): # load vars in dir/group_vars/name_of_group base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict') host_results = self._variable_manager.add_group_vars_file(base_path, self._loader) if return_results: results = combine_vars(results, host_results) elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)): # same for hostvars in dir/host_vars/name_of_host base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict') group_results = self._variable_manager.add_host_vars_file(base_path, self._loader) if return_results: results = combine_vars(results, group_results) # all done, results is a dictionary of variables for this particular host. return results
def exec_command(self, cmd, in_data=None, sudoable=False): ''' run a command on the chroot ''' super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) executable = to_bytes(self._play_context.executable, errors='strict') local_cmd = [executable, '-c', to_bytes(cmd, errors='strict')] read_stdout, write_stdout = None, None read_stderr, write_stderr = None, None read_stdin, write_stdin = None, None try: read_stdout, write_stdout = os.pipe() read_stderr, write_stderr = os.pipe() kwargs = { 'stdout': self._set_nonblocking(write_stdout), 'stderr': self._set_nonblocking(write_stderr), 'env_policy': _lxc.LXC_ATTACH_CLEAR_ENV } if in_data: read_stdin, write_stdin = os.pipe() kwargs['stdin'] = self._set_nonblocking(read_stdin) self._display.vvv("EXEC %s" % (local_cmd), host=self.container_name) pid = self.container.attach(_lxc.attach_run_command, local_cmd, **kwargs) if pid == -1: msg = "failed to attach to container %s" % self.container_name raise errors.AnsibleError(msg) write_stdout = os.close(write_stdout) write_stderr = os.close(write_stderr) if read_stdin: read_stdin = os.close(read_stdin) return self._communicate(pid, in_data, write_stdin, read_stdout, read_stderr) finally: fds = [read_stdout, write_stdout, read_stderr, write_stderr, read_stdin, write_stdin] for fd in fds: if fd: os.close(fd)
def put_file(self, in_path, out_path): ''' transfer a file from local to local ''' super(Connection, self).put_file(in_path, out_path) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr) if not os.path.exists(to_bytes(in_path, errors='strict')): raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) try: shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) except shutil.Error: raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path))) except IOError as e: raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e)))
def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000): in_size = os.path.getsize(to_bytes(in_path, errors='strict')) offset = 0 with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: for out_data in iter((lambda:in_file.read(buffer_size)), ''): offset += len(out_data) self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host) # yes, we're double-encoding over the wire in this case- we want to ensure that the data shipped to the end PS pipeline is still b64-encoded b64_data = base64.b64encode(out_data) + '\r\n' # cough up the data, as well as an indicator if this is the last chunk so winrm_send knows to set the End signal yield b64_data, (in_file.tell() == in_size) if offset == 0: # empty file, return an empty buffer + eof to close it yield "", True
def path_dwim_relative(self, path, dirname, source): ''' find one file in either a role or playbook dir with or without explicitly named dirname subdirs Used in action plugins and lookups to find supplemental files that could be in either place. ''' search = [] isrole = False # I have full path, nothing else needs to be looked at if source.startswith('~') or source.startswith('/'): search.append(self.path_dwim(source)) else: # base role/play path + templates/files/vars + relative filename search.append(os.path.join(path, dirname, source)) basedir = unfrackpath(path) # is it a role and if so make sure you get correct base path if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \ or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')): isrole = True if path.endswith('tasks'): basedir = unfrackpath(os.path.dirname(path)) cur_basedir = self._basedir self.set_basedir(basedir) # resolved base role/play path + templates/files/vars + relative filename search.append(self.path_dwim(os.path.join(basedir, dirname, source))) self.set_basedir(cur_basedir) if isrole and not source.endswith(dirname): # look in role's tasks dir w/o dirname search.append(self.path_dwim(os.path.join(basedir, 'tasks', source))) # try to create absolute path for loader basedir + templates/files/vars + filename search.append(self.path_dwim(os.path.join(dirname,source))) search.append(self.path_dwim(os.path.join(basedir, source))) # try to create absolute path for loader basedir + filename search.append(self.path_dwim(source)) for candidate in search: if os.path.exists(to_bytes(candidate, errors='strict')): break return candidate
def put_file(self, in_path, out_path): """ put a file from local to lxd """ super(Connection, self).put_file(in_path, out_path) self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host) if not os.path.isfile(to_bytes(in_path, errors='strict')): raise AnsibleFileNotFound("input path is not a file: %s" % in_path) local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path] local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] call(local_cmd)
def edit_file(self, filename): check_prereqs() ciphertext = self.read_data(filename) try: plaintext = self.vault.decrypt(ciphertext) except AnsibleError as e: raise AnsibleError("%s for %s" % (to_bytes(e),to_bytes(filename))) if self.vault.cipher_name not in CIPHER_WRITE_WHITELIST: # we want to get rid of files encrypted with the AES cipher self._edit_file_helper(filename, existing_data=plaintext, force_save=True) else: self._edit_file_helper(filename, existing_data=plaintext, force_save=False)
def decrypt(self, data): data = to_bytes(data) if self.password is None: raise errors.AnsibleError("A vault password must be specified to decrypt data") if not self.is_encrypted(data): raise errors.AnsibleError("data is not encrypted") # clean out header data = self._split_header(data) # create the cipher object ciphername = to_unicode(self.cipher_name) if 'Vault' + ciphername in globals() and ciphername in CIPHER_WHITELIST: cipher = globals()['Vault' + ciphername] this_cipher = cipher() else: raise errors.AnsibleError("{} cipher could not be found".format(ciphername)) # try to unencrypt data data = this_cipher.decrypt(data, self.password) if data is None: raise errors.AnsibleError("Decryption failed") return data
def decrypt(self, data, password): # SPLIT SALT, DIGEST, AND DATA data = b''.join(data.split(b"\n")) data = unhexlify(data) salt, cryptedHmac, cryptedData = data.split(b"\n", 2) salt = unhexlify(salt) cryptedData = unhexlify(cryptedData) key1, key2, iv = self.gen_key_initctr(password, salt) # EXIT EARLY IF DIGEST DOESN'T MATCH hmacDecrypt = HMAC.new(key2, cryptedData, SHA256) if not self.is_equal(cryptedHmac, to_bytes(hmacDecrypt.hexdigest())): return None # SET THE COUNTER AND THE CIPHER ctr = Counter.new(128, initial_value=int(iv, 16)) cipher = AES.new(key1, AES.MODE_CTR, counter=ctr) # DECRYPT PADDED DATA decryptedData = cipher.decrypt(cryptedData) # UNPAD DATA try: padding_length = ord(decryptedData[-1]) except TypeError: padding_length = decryptedData[-1] decryptedData = decryptedData[:-padding_length] return to_unicode(decryptedData)
def _transfer_data(self, remote_path, data): ''' Copies the module data out to the temporary module path. ''' if isinstance(data, dict): data = jsonify(data) afd, afile = tempfile.mkstemp() afo = os.fdopen(afd, 'w') try: data = to_bytes(data, errors='strict') afo.write(data) except Exception as e: #raise AnsibleError("failure encoding into utf-8: %s" % str(e)) raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e)) afo.flush() afo.close() try: self._connection.put_file(afile, remote_path) finally: os.unlink(afile) return remote_path
def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE): ''' run a command on the chroot. This is only needed for implementing put_file() get_file() so that we don't have to read the whole file into memory. compared to exec_command() it looses some niceties like being able to return the process's exit code immediately. ''' executable = C.DEFAULT_EXECUTABLE.split( )[0] if C.DEFAULT_EXECUTABLE else '/bin/sh' local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd] display.vvv("EXEC %s" % (local_cmd), host=self.chroot) local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p
def _format_output(self, b_data): """ Add header and format to 80 columns :arg b_data: the encrypted and hexlified data as a byte string :returns: a byte str that should be dumped into a file. It's formatted to 80 char columns and has the header prepended """ if not self.cipher_name: raise AnsibleError("the cipher must be set before adding a header") header = b';'.join([ b_HEADER, self.b_version, to_bytes(self.cipher_name, errors='strict', encoding='utf-8') ]) tmpdata = [header] tmpdata += [b_data[i:i + 80] for i in range(0, len(b_data), 80)] tmpdata += [b''] tmpdata = b'\n'.join(tmpdata) return tmpdata
def put_file(self, in_path, out_path): ''' transfer a file from local to jail ''' super(Connection, self).put_file(in_path, out_path) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.jail) out_path = pipes.quote(self._prefix_login_path(out_path)) try: with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: try: p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) except OSError: raise AnsibleError("jail connection requires dd command in the jail") try: stdout, stderr = p.communicate() except: traceback.print_exc() raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path)) if p.returncode != 0: raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr)) except IOError: raise AnsibleError("file or module does not exist at: %s" % in_path)
def write_data(self, data, filename, shred=True): """write data to given path :arg data: the encrypted and hexlified data as a utf-8 byte string :arg filename: filename to save 'data' to. :arg shred: if shred==True, make sure that the original data is first shredded so that is cannot be recovered. """ # FIXME: do we need this now? data_bytes should always be a utf-8 byte string b_file_data = to_bytes(data, errors='strict') if filename == '-': sys.stdout.write(b_file_data) else: if os.path.isfile(filename): if shred: self._shred_file(filename) else: os.remove(filename) with open(filename, "wb") as fh: fh.write(b_file_data)
def _winrm_exec(self, command, args=(), from_exec=False, stdin_iterator=None): if from_exec: display.vvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) else: display.vvvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) if not self.protocol: self.protocol = self._winrm_connect() if not self.shell_id: self.shell_id = self.protocol.open_shell(codepage=65001) # UTF-8 command_id = None try: stdin_push_failed = False command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args), console_mode_stdin=(stdin_iterator == None)) # TODO: try/except around this, so we can get/return the command result on a broken pipe or other failure (probably more useful than the 500 that comes from this) try: if stdin_iterator: for (data, is_last) in stdin_iterator: self._winrm_send_input(self.protocol, self.shell_id, command_id, data, eof=is_last) except: stdin_push_failed = True # NB: this could hang if the receiver is still running (eg, network failed a Send request but the server's still happy). # FUTURE: Consider adding pywinrm status check/abort operations to see if the target is still running after a failure. response = Response(self.protocol.get_command_output(self.shell_id, command_id)) if from_exec: display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) else: display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host) display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host) if stdin_push_failed: raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err)) return response finally: if command_id: self.protocol.cleanup_command(self.shell_id, command_id)
def get(self, key): if self.has_expired(key) or key == "": raise KeyError if key in self._cache: return self._cache.get(key) cachefile = "%s/%s" % (self._cache_dir, key) try: with codecs.open(cachefile, 'r', encoding='utf-8') as f: try: value = json.load(f) self._cache[key] = value return value except ValueError as e: self._display.warning("error while trying to read %s : %s. Most likely a corrupt file, so erasing and failing." % (cachefile, to_bytes(e))) self.delete(key) raise AnsibleError("The JSON cache file %s was corrupt, or did not otherwise contain valid JSON data. It has been removed, so you can re-run your command now." % cachefile) except (OSError,IOError) as e: self._display.warning("error while trying to read %s : %s" % (cachefile, to_bytes(e))) raise KeyError
def test_encrypt_decrypt_aes256_bad_hmac(self): # FIXME This test isn't working quite yet. raise SkipTest if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2: raise SkipTest v = VaultLib('test-vault-password') v.cipher_name = 'AES256' # plaintext = "Setec Astronomy" enc_data = '''$ANSIBLE_VAULT;1.1;AES256 33363965326261303234626463623963633531343539616138316433353830356566396130353436 3562643163366231316662386565383735653432386435610a306664636137376132643732393835 63383038383730306639353234326630666539346233376330303938323639306661313032396437 6233623062366136310a633866373936313238333730653739323461656662303864663666653563 3138''' b_data = to_bytes(enc_data, errors='strict', encoding='utf-8') b_data = v._split_header(b_data) foo = binascii.unhexlify(b_data) lines = foo.splitlines() # line 0 is salt, line 1 is hmac, line 2+ is ciphertext b_salt = lines[0] b_hmac = lines[1] b_ciphertext_data = b'\n'.join(lines[2:]) b_ciphertext = binascii.unhexlify(b_ciphertext_data) # b_orig_ciphertext = b_ciphertext[:] # now muck with the text # b_munged_ciphertext = b_ciphertext[:10] + b'\x00' + b_ciphertext[11:] # b_munged_ciphertext = b_ciphertext # assert b_orig_ciphertext != b_munged_ciphertext b_ciphertext_data = binascii.hexlify(b_ciphertext) b_payload = b'\n'.join([b_salt, b_hmac, b_ciphertext_data]) # reformat b_invalid_ciphertext = v._format_output(b_payload) # assert we throw an error v.decrypt(b_invalid_ciphertext)
def exec_command(self, cmd, in_data=None, sudoable=True): super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) cmd_parts = shlex.split(to_bytes(cmd), posix=False) cmd_parts = map(to_unicode, cmd_parts) script = None cmd_ext = cmd_parts and self._shell._unquote( cmd_parts[0]).lower()[-4:] or '' # Support running .ps1 files (via script/raw). if cmd_ext == '.ps1': script = '& %s' % cmd # Support running .bat/.cmd files; change back to the default system encoding instead of UTF-8. elif cmd_ext in ('.bat', '.cmd'): script = '[System.Console]::OutputEncoding = [System.Text.Encoding]::Default; & %s' % cmd # Encode the command if not already encoded; supports running simple PowerShell commands via raw. elif '-EncodedCommand' not in cmd_parts: script = cmd if script: cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False) if '-EncodedCommand' in cmd_parts: encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1] decoded_cmd = to_unicode( base64.b64decode(encoded_cmd).decode('utf-16-le')) self._display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host) else: self._display.vvv("EXEC %s" % cmd, host=self._winrm_host) try: result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True) except Exception as e: traceback.print_exc() raise AnsibleError("failed to exec cmd %s" % cmd) result.std_out = to_unicode(result.std_out) result.std_err = to_unicode(result.std_err) return (result.status_code, result.std_out, result.std_err)
def _split_ssh_args(argstring): """ Takes a string like '-o Foo=1 -o Bar="foo bar"' and returns a list ['-o', 'Foo=1', '-o', 'Bar=foo bar'] that can be added to the argument list. The list will not contain any empty elements. """ try: # Python 2.6.x shlex doesn't handle unicode type so we have to # convert args to byte string for that case. More efficient to # try without conversion first but python2.6 doesn't throw an # exception, it merely mangles the output: # >>> shlex.split(u't e') # ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00'] return [ to_unicode(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip() ] except AttributeError: return [ to_unicode(x.strip()) for x in shlex.split(argstring) if x.strip() ]
def fetch_file(self, in_path, out_path): ''' fetch a file from remote to local ''' super(Connection, self).fetch_file(in_path, out_path) display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host) # scp and sftp require square brackets for IPv6 addresses, but # accept them for hostnames and IPv4 addresses too. host = '[%s]' % self.host if C.DEFAULT_SCP_IF_SSH: cmd = self._build_command('scp', u'{0}:{1}'.format(host, pipes.quote(in_path)), out_path) in_data = None else: cmd = self._build_command('sftp', host) in_data = u"get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) in_data = to_bytes(in_data, nonstring='passthru') (returncode, stdout, stderr) = self._run(cmd, in_data) if returncode != 0: raise AnsibleError("failed to transfer file from {0}:\n{1}\n{2}".format(in_path, stdout, stderr))
def _winrm_exec(self, command, args=(), from_exec=False): if from_exec: self._display.vvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) else: self._display.vvvvvv("WINRM EXEC %r %r" % (command, args), host=self._winrm_host) if not self.protocol: self.protocol = self._winrm_connect() if not self.shell_id: self.shell_id = self.protocol.open_shell(codepage=65001) # UTF-8 command_id = None try: command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args)) response = Response(self.protocol.get_command_output(self.shell_id, command_id)) if from_exec: self._display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) else: self._display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) self._display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host) self._display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host) return response finally: if command_id: self.protocol.cleanup_command(self.shell_id, command_id)
def _winrm_send_input(self, protocol, shell_id, command_id, stdin, eof=False): rq = { 'env:Envelope': protocol._get_soap_header( resource_uri= 'http://schemas.microsoft.com/wbem/wsman/1/windows/shell/cmd', action= 'http://schemas.microsoft.com/wbem/wsman/1/windows/shell/Send', shell_id=shell_id) } stream = rq['env:Envelope'].setdefault('env:Body', {}).setdefault('rsp:Send', {})\ .setdefault('rsp:Stream', {}) stream['@Name'] = 'stdin' stream['@CommandId'] = command_id stream['#text'] = base64.b64encode(to_bytes(stdin)) if eof: stream['@End'] = 'true' rs = protocol.send_message(xmltodict.unparse(rq))
def fetch_file(self, in_path, out_path): ''' fetch a file from jail to local ''' super(Connection, self).fetch_file(in_path, out_path) display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.jail) in_path = pipes.quote(self._prefix_login_path(in_path)) try: p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE)) except OSError: raise AnsibleError("jail connection requires dd command in the jail") with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: try: chunk = p.stdout.read(BUFSIZE) while chunk: out_file.write(chunk) chunk = p.stdout.read(BUFSIZE) except: traceback.print_exc() raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path)) stdout, stderr = p.communicate() if p.returncode != 0: raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
def get_real_file(self, file_path): """ If the file is vault encrypted return a path to a temporary decrypted file If the file is not encrypted then the path is returned Temporary files are cleanup in the destructor """ if not file_path or not isinstance(file_path, string_types): raise AnsibleParserError("Invalid filename: '%s'" % str(file_path)) if not self.path_exists(file_path) or not self.is_file(file_path): raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_path) if not self._vault: self._vault = VaultLib(password="") real_path = self.path_dwim(file_path) try: with open(to_bytes(real_path), 'rb') as f: data = f.read() if self._vault.is_encrypted(data): # if the file is encrypted and no password was specified, # the decrypt call would throw an error, but we check first # since the decrypt function doesn't know the file name if not self._vault_password: raise AnsibleParserError("A vault password must be specified to decrypt %s" % file_path) data = self._vault.decrypt(data) # Make a temp file real_path = self._create_content_tempfile(data) self._tempfiles.add(real_path) return real_path except (IOError, OSError) as e: raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (real_path, str(e)))
def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None): if sender is None: sender = '<root>' if to is None: to = 'root' if smtphost is None: smtphost = os.getenv('SMTPHOST', 'localhost') if body is None: body = subject smtp = smtplib.SMTP(smtphost) b_sender = to_bytes(sender) b_to = to_bytes(to) b_cc = to_bytes(cc) b_bcc = to_bytes(bcc) b_subject = to_bytes(subject) b_body = to_bytes(body) b_content = b'From: %s\n' % b_sender b_content += b'To: %s\n' % b_to if cc: b_content += b'Cc: %s\n' % b_cc b_content += b'Subject: %s\n\n' % b_subject b_content += b_body b_addresses = b_to.split(b',') if b_cc: b_addresses += b_cc.split(b',') if b_bcc: b_addresses += b_bcc.split(b',') for b_address in b_addresses: smtp.sendmail(b_sender, b_address, b_content) smtp.quit()
def log(self, host, code, data): ''' if type(data) == dict: if '_ansible_verbose_override' in data: data = 'omitted' else: data = data.copy() invocation = data.pop('invocation', None) data = json.dumps(data) if invocation is not None: data = json.dumps(invocation) + " => %s " % data ''' newdata = {} if data.has_key('cmd'): newdata['cmd'] = data['cmd'] if data.has_key('start'): newdata['start'] = data['start'] if data.has_key('end'): newdata['end'] = data['end'] if data.has_key('delta'): newdata['delta'] = data['delta'] if data.has_key('stderr'): newdata['stderr'] = data['stderr'] if data.has_key('stdout'): newdata['stdout'] = data['stdout'] if data.has_key('stdout_lines'): newdata['stdout_lines'] = data['stdout_lines'] self._step_callback(ip=host, code=code, data=newdata) if self._debug: now = time.strftime(self.TIME_FORMAT, time.localtime()) msg = to_bytes(self.MSG_FORMAT % dict(now=now, category=code, data=data)) print(msg)
def test_action_base__configure_module(self): fake_loader = DictDataLoader({}) # create our fake task mock_task = MagicMock() mock_task.action = "copy" # create a mock connection, so we don't actually try and connect to things mock_connection = MagicMock() # create a mock shared loader object def mock_find_plugin(name, options): if name == 'badmodule': return None elif '.ps1' in options: return '/fake/path/to/%s.ps1' % name else: return '/fake/path/to/%s' % name mock_module_loader = MagicMock() mock_module_loader.find_plugin.side_effect = mock_find_plugin mock_shared_obj_loader = MagicMock() mock_shared_obj_loader.module_loader = mock_module_loader # we're using a real play context here play_context = PlayContext() # our test class action_base = DerivedActionBase( task=mock_task, connection=mock_connection, play_context=play_context, loader=fake_loader, templar=None, shared_loader_obj=mock_shared_obj_loader, ) # test python module formatting with patch.object( builtins, 'open', mock_open(read_data=to_bytes(python_module_replacers.strip(), encoding='utf-8'))) as m: with patch.object(os, 'rename') as m: mock_task.args = dict(a=1, foo='fö〩') mock_connection.module_implementation_preferences = ('', ) (style, shebang, data, path) = action_base._configure_module(mock_task.action, mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, u"#!/usr/bin/python") # test module not found self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args) # test powershell module formatting with patch.object( builtins, 'open', mock_open( read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))) as m: mock_task.action = 'win_copy' mock_task.args = dict(b=2) mock_connection.module_implementation_preferences = ('.ps1', ) (style, shebang, data, path) = action_base._configure_module('stat', mock_task.args) self.assertEqual(style, "new") self.assertEqual(shebang, u'#!powershell') # test module not found self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args)
def run(self, tmp=None, task_vars=None): ''' handler for fetch operations ''' if task_vars is None: task_vars = dict() result = super(ActionModule, self).run(tmp, task_vars) if self._play_context.check_mode: result['skipped'] = True result['msg'] = 'check mode not (yet) supported for this module' return result source = self._task.args.get('src', None) dest = self._task.args.get('dest', None) flat = boolean(self._task.args.get('flat')) fail_on_missing = boolean(self._task.args.get('fail_on_missing')) validate_checksum = boolean(self._task.args.get('validate_checksum', self._task.args.get('validate_md5'))) if 'validate_md5' in self._task.args and 'validate_checksum' in self._task.args: result['failed'] = True result['msg'] = "validate_checksum and validate_md5 cannot both be specified" return result if source is None or dest is None: result['failed'] = True result['msg'] = "src and dest are required" return result source = self._connection._shell.join_path(source) source = self._remote_expand_user(source) remote_checksum = None if not self._play_context.become: # calculate checksum for the remote file, don't bother if using become as slurp will be used remote_checksum = self._remote_checksum(source, all_vars=task_vars) # use slurp if permissions are lacking or privilege escalation is needed remote_data = None if remote_checksum in ('1', '2', None): slurpres = self._execute_module(module_name='slurp', module_args=dict(src=source), task_vars=task_vars, tmp=tmp) if slurpres.get('failed'): if not fail_on_missing and (slurpres.get('msg').startswith('file not found') or remote_checksum == '1'): result['msg'] = "the remote file does not exist, not transferring, ignored" result['file'] = source result['changed'] = False else: result.update(slurpres) return result else: if slurpres['encoding'] == 'base64': remote_data = base64.b64decode(slurpres['content']) if remote_data is not None: remote_checksum = checksum_s(remote_data) # the source path may have been expanded on the # target system, so we compare it here and use the # expanded version if it's different remote_source = slurpres.get('source') if remote_source and remote_source != source: source = remote_source # calculate the destination name if os.path.sep not in self._connection._shell.join_path('a', ''): source = self._connection._shell._unquote(source) source_local = source.replace('\\', '/') else: source_local = source dest = os.path.expanduser(dest) if flat: if dest.endswith(os.sep): # if the path ends with "/", we'll use the source filename as the # destination filename base = os.path.basename(source_local) dest = os.path.join(dest, base) if not dest.startswith("/"): # if dest does not start with "/", we'll assume a relative path dest = self._loader.path_dwim(dest) else: # files are saved in dest dir, with a subdir for each host, then the filename if 'inventory_hostname' in task_vars: target_name = task_vars['inventory_hostname'] else: target_name = self._play_context.remote_addr dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local) dest = dest.replace("//","/") if remote_checksum in ('0', '1', '2', '3', '4'): # these don't fail because you may want to transfer a log file that # possibly MAY exist but keep going to fetch other log files if remote_checksum == '0': result['msg'] = "unable to calculate the checksum of the remote file" result['file'] = source result['changed'] = False elif remote_checksum == '1': if fail_on_missing: result['failed'] = True result['msg'] = "the remote file does not exist" result['file'] = source else: result['msg'] = "the remote file does not exist, not transferring, ignored" result['file'] = source result['changed'] = False elif remote_checksum == '2': result['msg'] = "no read permission on remote file, not transferring, ignored" result['file'] = source result['changed'] = False elif remote_checksum == '3': result['msg'] = "remote file is a directory, fetch cannot work on directories" result['file'] = source result['changed'] = False elif remote_checksum == '4': result['msg'] = "python isn't present on the system. Unable to compute checksum" result['file'] = source result['changed'] = False return result # calculate checksum for the local file local_checksum = checksum(dest) if remote_checksum != local_checksum: # create the containing directories, if needed makedirs_safe(os.path.dirname(dest)) # fetch the file and check for changes if remote_data is None: self._connection.fetch_file(source, dest) else: try: f = open(to_bytes(dest, errors='strict'), 'w') f.write(remote_data) f.close() except (IOError, OSError) as e: raise AnsibleError("Failed to fetch the file: %s" % e) new_checksum = secure_hash(dest) # For backwards compatibility. We'll return None on FIPS enabled systems try: new_md5 = md5(dest) except ValueError: new_md5 = None if validate_checksum and new_checksum != remote_checksum: result.update(dict(failed=True, md5sum=new_md5, msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum)) else: result.update(dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum)) else: # For backwards compatibility. We'll return None on FIPS enabled systems try: local_md5 = md5(dest) except ValueError: local_md5 = None result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)) return result
def exec_command(self, cmd, in_data=None, sudoable=True): ''' run a command on the local host ''' super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) display.debug("in local.exec_command()") executable = C.DEFAULT_EXECUTABLE.split( )[0] if C.DEFAULT_EXECUTABLE else None display.vvv(u"{0} EXEC {1}".format(self._play_context.remote_addr, cmd)) # FIXME: cwd= needs to be set to the basedir of the playbook display.debug("opening command with Popen()") if isinstance(cmd, (text_type, binary_type)): cmd = to_bytes(cmd) else: cmd = map(to_bytes, cmd) p = subprocess.Popen( cmd, shell=isinstance(cmd, basestring), executable=executable, #cwd=... stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) display.debug("done running command with Popen()") if self._play_context.prompt and sudoable: fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK) become_output = '' while not self.check_become_success( become_output) and not self.check_password_prompt( become_output): rfd, wfd, efd = select.select([p.stdout, p.stderr], [], [p.stdout, p.stderr], self._play_context.timeout) if p.stdout in rfd: chunk = p.stdout.read() elif p.stderr in rfd: chunk = p.stderr.read() else: stdout, stderr = p.communicate() raise AnsibleError( 'timeout waiting for privilege escalation password prompt:\n' + become_output) if not chunk: stdout, stderr = p.communicate() raise AnsibleError( 'privilege output closed while waiting for password prompt:\n' + become_output) become_output += chunk if not self.check_become_success(become_output): p.stdin.write(self._play_context.become_pass + '\n') fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK) display.debug("getting output with communicate()") stdout, stderr = p.communicate(in_data) display.debug("done communicating") display.debug("done with local.exec_command()") return (p.returncode, stdout, stderr)
def _run(self, cmd, in_data, sudoable=True): ''' Starts the command and communicates with it until it ends. ''' display_cmd = map(to_unicode, map(pipes.quote, cmd)) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) # Start the given command. If we don't need to pipeline data, we can try # to use a pseudo-tty (ssh will have been invoked with -tt). If we are # pipelining data, or can't create a pty, we fall back to using plain # old pipes. p = None if isinstance(cmd, (text_type, binary_type)): cmd = to_bytes(cmd) else: cmd = list(map(to_bytes, cmd)) if not in_data: try: # Make sure stdin is a proper pty to avoid tcgetattr errors master, slave = pty.openpty() p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = os.fdopen(master, 'w', 0) os.close(slave) except (OSError, IOError): p = None if not p: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = p.stdin # If we are using SSH password authentication, write the password into # the pipe we opened in _build_command. if self._play_context.password: os.close(self.sshpass_pipe[0]) os.write(self.sshpass_pipe[1], "{0}\n".format(to_bytes(self._play_context.password))) os.close(self.sshpass_pipe[1]) ## SSH state machine # # Now we read and accumulate output from the running process until it # exits. Depending on the circumstances, we may also need to write an # escalation password and/or pipelined input to the process. states = [ 'awaiting_prompt', 'awaiting_escalation', 'ready_to_send', 'awaiting_exit' ] # Are we requesting privilege escalation? Right now, we may be invoked # to execute sftp/scp with sudoable=True, but we can request escalation # only when using ssh. Otherwise we can send initial data straightaway. state = states.index('ready_to_send') if b'ssh' in cmd: if self._play_context.prompt: # We're requesting escalation with a password, so we have to # wait for a password prompt. state = states.index('awaiting_prompt') display.debug('Initial state: %s: %s' % (states[state], self._play_context.prompt)) elif self._play_context.become and self._play_context.success_key: # We're requesting escalation without a password, so we have to # detect success/failure before sending any initial data. state = states.index('awaiting_escalation') display.debug('Initial state: %s: %s' % (states[state], self._play_context.success_key)) # We store accumulated stdout and stderr output from the process here, # but strip any privilege escalation prompt/confirmation lines first. # Output is accumulated into tmp_*, complete lines are extracted into # an array, then checked and removed or copied to stdout or stderr. We # set any flags based on examining the output in self._flags. stdout = stderr = '' tmp_stdout = tmp_stderr = '' self._flags = dict(become_prompt=False, become_success=False, become_error=False, become_nopasswd_error=False) # select timeout should be longer than the connect timeout, otherwise # they will race each other when we can't connect, and the connect # timeout usually fails timeout = 2 + self._play_context.timeout rpipes = [p.stdout, p.stderr] for fd in rpipes: fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) # If we can send initial data without waiting for anything, we do so # before we call select. if states[state] == 'ready_to_send' and in_data: self._send_initial_data(stdin, in_data) state += 1 while True: rfd, wfd, efd = select.select(rpipes, [], [], timeout) # We pay attention to timeouts only while negotiating a prompt. if not rfd: if state <= states.index('awaiting_escalation'): # If the process has already exited, then it's not really a # timeout; we'll let the normal error handling deal with it. if p.poll() is not None: break self._terminate_process(p) raise AnsibleError( 'Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, stdout)) # Read whatever output is available on stdout and stderr, and stop # listening to the pipe if it's been closed. if p.stdout in rfd: chunk = p.stdout.read() if chunk == '': rpipes.remove(p.stdout) tmp_stdout += chunk display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, chunk)) if p.stderr in rfd: chunk = p.stderr.read() if chunk == '': rpipes.remove(p.stderr) tmp_stderr += chunk display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, chunk)) # We examine the output line-by-line until we have negotiated any # privilege escalation prompt and subsequent success/error message. # Afterwards, we can accumulate output without looking at it. if state < states.index('ready_to_send'): if tmp_stdout: output, unprocessed = self._examine_output( 'stdout', states[state], tmp_stdout, sudoable) stdout += output tmp_stdout = unprocessed if tmp_stderr: output, unprocessed = self._examine_output( 'stderr', states[state], tmp_stderr, sudoable) stderr += output tmp_stderr = unprocessed else: stdout += tmp_stdout stderr += tmp_stderr tmp_stdout = tmp_stderr = '' # If we see a privilege escalation prompt, we send the password. # (If we're expecting a prompt but the escalation succeeds, we # didn't need the password and can carry on regardless.) if states[state] == 'awaiting_prompt': if self._flags['become_prompt']: display.debug('Sending become_pass in response to prompt') stdin.write('{0}\n'.format( to_bytes(self._play_context.become_pass))) self._flags['become_prompt'] = False state += 1 elif self._flags['become_success']: state += 1 # We've requested escalation (with or without a password), now we # wait for an error message or a successful escalation. if states[state] == 'awaiting_escalation': if self._flags['become_success']: display.debug('Escalation succeeded') self._flags['become_success'] = False state += 1 elif self._flags['become_error']: display.debug('Escalation failed') self._terminate_process(p) self._flags['become_error'] = False raise AnsibleError('Incorrect %s password' % self._play_context.become_method) elif self._flags['become_nopasswd_error']: display.debug('Escalation requires password') self._terminate_process(p) self._flags['become_nopasswd_error'] = False raise AnsibleError('Missing %s password' % self._play_context.become_method) elif self._flags['become_prompt']: # This shouldn't happen, because we should see the "Sorry, # try again" message first. display.debug('Escalation prompt repeated') self._terminate_process(p) self._flags['become_prompt'] = False raise AnsibleError('Incorrect %s password' % self._play_context.become_method) # Once we're sure that the privilege escalation prompt, if any, has # been dealt with, we can send any initial data and start waiting # for output. if states[state] == 'ready_to_send': if in_data: self._send_initial_data(stdin, in_data) state += 1 # Now we're awaiting_exit: has the child process exited? If it has, # and we've read all available output from it, we're done. if p.poll() is not None: if not rpipes or not rfd: break # When ssh has ControlMaster (+ControlPath/Persist) enabled, the # first connection goes into the background and we never see EOF # on stderr. If we see EOF on stdout and the process has exited, # we're probably done. We call select again with a zero timeout, # just to make certain we don't miss anything that may have been # written to stderr between the time we called select() and when # we learned that the process had finished. if p.stdout not in rpipes: timeout = 0 continue # If the process has not yet exited, but we've already read EOF from # its stdout and stderr (and thus removed both from rpipes), we can # just wait for it to exit. elif not rpipes: p.wait() break # Otherwise there may still be outstanding data to read. # close stdin after process is terminated and stdout/stderr are read # completely (see also issue #848) stdin.close() if C.HOST_KEY_CHECKING: if cmd[0] == b"sshpass" and p.returncode == 6: raise AnsibleError( 'Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this. Please add this host\'s fingerprint to your known_hosts file to manage this host.' ) controlpersisterror = 'Bad configuration option: ControlPersist' in stderr or 'unknown configuration option: ControlPersist' in stderr if p.returncode != 0 and controlpersisterror: raise AnsibleError( 'using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again' ) if p.returncode == 255 and in_data: raise AnsibleConnectionFailure( 'SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh' ) return (p.returncode, stdout, stderr)
def _build_command(self, binary, *other_args): ''' Takes a binary (ssh, scp, sftp) and optional extra arguments and returns a command line as an array that can be passed to subprocess.Popen. ''' self._command = [] ## First, the command name. # If we want to use password authentication, we have to set up a pipe to # write the password to sshpass. if self._play_context.password: if not self._sshpass_available(): raise AnsibleError( "to use the 'ssh' connection type with passwords, you must install the sshpass program" ) self.sshpass_pipe = os.pipe() self._command += ['sshpass', '-d{0}'.format(self.sshpass_pipe[0])] self._command += [binary] ## Next, additional arguments based on the configuration. # sftp batch mode allows us to correctly catch failed transfers, but can # be disabled if the client side doesn't support the option. if binary == 'sftp' and C.DEFAULT_SFTP_BATCH_MODE: self._command += ['-b', '-'] self._command += ['-C'] if self._play_context.verbosity > 3: self._command += ['-vvv'] elif binary == 'ssh': # Older versions of ssh (e.g. in RHEL 6) don't accept sftp -q. self._command += ['-q'] # Next, we add [ssh_connection]ssh_args from ansible.cfg. if self._play_context.ssh_args: args = self._split_ssh_args(self._play_context.ssh_args) self._add_args("ansible.cfg set ssh_args", args) # Now we add various arguments controlled by configuration file settings # (e.g. host_key_checking) or inventory variables (ansible_ssh_port) or # a combination thereof. # BB Mod: Rely on ssh_config's strict host key checking; IOW don't add an explict SSH arg # if not C.HOST_KEY_CHECKING: # self._add_args( # "ANSIBLE_HOST_KEY_CHECKING/host_key_checking disabled", # ("-o", "StrictHostKeyChecking=no") # ) if self._play_context.port is not None: self._add_args("ANSIBLE_REMOTE_PORT/remote_port/ansible_port set", ("-o", "Port={0}".format(self._play_context.port))) key = self._play_context.private_key_file if key: self._add_args( "ANSIBLE_PRIVATE_KEY_FILE/private_key_file/ansible_ssh_private_key_file set", ("-o", "IdentityFile=\"{0}\"".format(os.path.expanduser(key)))) # BB Mod: Stick to authmethods: hostbased,publickey if not self._play_context.password: self._add_args("ansible_password/ansible_ssh_pass not set", ("-o", "KbdInteractiveAuthentication=no", "-o", "PreferredAuthentications=hostbased,publickey", "-o", "PasswordAuthentication=no")) user = self._play_context.remote_user if user: self._add_args( "ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set", ("-o", "User={0}".format( to_bytes(self._play_context.remote_user)))) self._add_args( "ANSIBLE_TIMEOUT/timeout set", ("-o", "ConnectTimeout={0}".format(self._play_context.timeout))) # Add in any common or binary-specific arguments from the PlayContext # (i.e. inventory or task settings or overrides on the command line). for opt in ['ssh_common_args', binary + '_extra_args']: attr = getattr(self._play_context, opt, None) if attr is not None: args = self._split_ssh_args(attr) self._add_args("PlayContext set %s" % opt, args) # Check if ControlPersist is enabled and add a ControlPath if one hasn't # already been set. controlpersist, controlpath = self._persistence_controls(self._command) if controlpersist: self._persistent = True if not controlpath: cpdir = unfrackpath('$HOME/.ansible/cp') # The directory must exist and be writable. makedirs_safe(cpdir, 0o700) if not os.access(cpdir, os.W_OK): raise AnsibleError("Cannot write to ControlPath %s" % cpdir) args = ("-o", "ControlPath={0}".format( to_bytes(C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)))) self._add_args("found only ControlPersist; added ControlPath", args) ## Finally, we add any caller-supplied extras. if other_args: self._command += other_args return self._command
def exec_command(self, cmd, in_data=None, sudoable=True): ''' run a command on the remote host ''' super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) if in_data: raise AnsibleError( "Internal Error: this module does not support optimized module pipelining" ) bufsize = 4096 try: self.ssh.get_transport().set_keepalive(5) chan = self.ssh.get_transport().open_session() except Exception as e: msg = "Failed to open session" if len(str(e)) > 0: msg += ": %s" % str(e) raise AnsibleConnectionFailure(msg) # sudo usually requires a PTY (cf. requiretty option), therefore # we give it one by default (pty=True in ansble.cfg), and we try # to initialise from the calling environment when sudoable is enabled if C.PARAMIKO_PTY and sudoable: chan.get_pty(term=os.getenv('TERM', 'vt100'), width=int(os.getenv('COLUMNS', 0)), height=int(os.getenv('LINES', 0))) display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr) cmd = to_bytes(cmd, errors='strict') no_prompt_out = '' no_prompt_err = '' become_output = '' try: chan.exec_command(cmd) if self._play_context.prompt: passprompt = False become_sucess = False while not (become_sucess or passprompt): display.debug('Waiting for Privilege Escalation input') chunk = chan.recv(bufsize) display.debug("chunk is: %s" % chunk) if not chunk: if 'unknown user' in become_output: raise AnsibleError('user %s does not exist' % self._play_context.become_user) else: break #raise AnsibleError('ssh connection closed waiting for password prompt') become_output += chunk # need to check every line because we might get lectured # and we might get the middle of a line in a chunk for l in become_output.splitlines(True): if self.check_become_success(l): become_sucess = True break elif self.check_password_prompt(l): passprompt = True break if passprompt: if self._play_context.become and self._play_context.become_pass: chan.sendall(self._play_context.become_pass + '\n') else: raise AnsibleError( "A password is reqired but none was supplied") else: no_prompt_out += become_output no_prompt_err += become_output except socket.timeout: raise AnsibleError( 'ssh timed out waiting for privilege escalation.\n' + become_output) stdout = ''.join(chan.makefile('rb', bufsize)) stderr = ''.join(chan.makefile_stderr('rb', bufsize)) return (chan.recv_exit_status(), no_prompt_out + stdout, no_prompt_out + stderr)
def modify_module(module_path, module_args, task_vars=dict(), strip_comments=False): """ Used to insert chunks of code into modules before transfer rather than doing regular python imports. This allows for more efficient transfer in a non-bootstrapping scenario by not moving extra files over the wire and also takes care of embedding arguments in the transferred modules. This version is done in such a way that local imports can still be used in the module code, so IDEs don't have to be aware of what is going on. Example: from ansible.module_utils.basic import * ... will result in the insertion of basic.py into the module from the module_utils/ directory in the source tree. All modules are required to import at least basic, though there will also be other snippets. For powershell, there's equivalent conventions like this: # POWERSHELL_COMMON which results in the inclusion of the common code from powershell.ps1 """ ### TODO: Optimization ideas if this code is actually a source of slowness: # * Fix comment stripping: Currently doesn't preserve shebangs and encoding info (but we unconditionally add encoding info) # * Use pyminifier if installed # * comment stripping/pyminifier needs to have config setting to turn it # off for debugging purposes (goes along with keep remote but should be # separate otherwise users wouldn't be able to get info on what the # minifier output) # * Only split into lines and recombine into strings once # * Cache the modified module? If only the args are different and we do # that as the last step we could cache all the work up to that point. with open(module_path) as f: # read in the module source module_data = f.read() (module_data, module_style) = _find_snippet_imports(module_data, module_path, strip_comments) module_args_json = json.dumps(module_args).encode('utf-8') python_repred_args = repr(module_args_json) # these strings should be part of the 'basic' snippet which is required to be included module_data = module_data.replace(REPLACER_VERSION, repr(__version__)) module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args) module_data = module_data.replace(REPLACER_WINARGS, module_args_json) module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) module_data = module_data.replace(REPLACER_SELINUX, ','.join(C.DEFAULT_SELINUX_SPECIAL_FS)) if module_style == 'new': facility = C.DEFAULT_SYSLOG_FACILITY if 'ansible_syslog_facility' in task_vars: facility = task_vars['ansible_syslog_facility'] module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility) lines = module_data.split(b"\n", 1) shebang = None if lines[0].startswith(b"#!"): shebang = lines[0].strip() args = shlex.split(str(shebang[2:])) interpreter = args[0] interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter) if interpreter_config in task_vars: interpreter = to_bytes(task_vars[interpreter_config], errors='strict') lines[0] = shebang = b"#!{0} {1}".format(interpreter, b" ".join(args[1:])) if os.path.basename(interpreter).startswith('python'): lines.insert(1, ENCODING_STRING) else: # No shebang, assume a binary module? pass module_data = b"\n".join(lines) return (module_data, module_style, shebang)
def __init__(self, password): self.b_password = to_bytes(password, errors='strict', encoding='utf-8') self.cipher_name = None self.b_version = b'1.1'
def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False): """ Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel to the inventory base directory or in the same directory as the playbook. Variables in the playbook dir will win over the inventory dir if files are in both. """ results = {} scan_pass = 0 _basedir = self._basedir _playbook_basedir = self._playbook_basedir # look in both the inventory base directory and the playbook base directory # unless we do an update for a new playbook base dir if not new_pb_basedir and _playbook_basedir: basedirs = [_basedir, _playbook_basedir] else: basedirs = [_basedir] for basedir in basedirs: # this can happen from particular API usages, particularly if not run # from /usr/bin/ansible-playbook if basedir in ('', None): basedir = './' scan_pass = scan_pass + 1 # it's not an eror if the directory does not exist, keep moving if not os.path.exists(basedir): continue # save work of second scan if the directories are the same if _basedir == _playbook_basedir and scan_pass != 1: continue # Before trying to load vars from file, check that the directory contains relvant file names if host is None and any( map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)): # load vars in dir/group_vars/name_of_group base_path = to_unicode(os.path.abspath( os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict') host_results = self._variable_manager.add_group_vars_file( base_path, self._loader) if return_results: results = combine_vars(results, host_results) elif group is None and any( map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)): # same for hostvars in dir/host_vars/name_of_host base_path = to_unicode(os.path.abspath( os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict') group_results = self._variable_manager.add_host_vars_file( base_path, self._loader) if return_results: results = combine_vars(results, group_results) # all done, results is a dictionary of variables for this particular host. return results
def write_data(self, data, filename): if os.path.isfile(filename): os.remove(filename) f = open(filename, "wb") f.write(to_bytes(data)) f.close()
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): """ Given the source of the module, convert it to a Jinja2 template to insert module code and return whether it's a new or old style module. """ module_substyle = module_style = 'old' # module_style is something important to calling code (ActionBase). It # determines how arguments are formatted (json vs k=v) and whether # a separate arguments file needs to be sent over the wire. # module_substyle is extra information that's useful internally. It tells # us what we have to look to substitute in the module files and whether # we're using module replacer or ziploader to format the module itself. if _is_binary(module_data): module_substyle = module_style = 'binary' elif REPLACER in module_data: # Do REPLACER before from ansible.module_utils because we need make sure # we substitute "from ansible.module_utils basic" for REPLACER module_style = 'new' module_substyle = 'python' module_data = module_data.replace( REPLACER, b'from ansible.module_utils.basic import *') elif b'from ansible.module_utils.' in module_data: module_style = 'new' module_substyle = 'python' elif REPLACER_WINDOWS in module_data: module_style = 'new' module_substyle = 'powershell' elif REPLACER_JSONARGS in module_data: module_style = 'new' module_substyle = 'jsonargs' elif b'WANT_JSON' in module_data: module_substyle = module_style = 'non_native_want_json' shebang = None # Neither old-style, non_native_want_json nor binary modules should be modified # except for the shebang line (Done by modify_module) if module_style in ('old', 'non_native_want_json', 'binary'): return module_data, module_style, shebang output = BytesIO() py_module_names = set() if module_substyle == 'python': params = dict(ANSIBLE_MODULE_ARGS=module_args, ) python_repred_params = repr(json.dumps(params)) try: compression_method = getattr(zipfile, module_compression) except AttributeError: display.warning( u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression) compression_method = zipfile.ZIP_STORED lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ziploader_cache') cached_module_filename = os.path.join( lookup_path, "%s-%s" % (module_name, module_compression)) zipdata = None # Optimization -- don't lock if the module has already been cached if os.path.exists(cached_module_filename): display.debug('ZIPLOADER: using cached module: %s' % cached_module_filename) zipdata = open(cached_module_filename, 'rb').read() # Fool the check later... I think we should just remove the check py_module_names.add(('basic', )) else: if module_name in strategy.action_write_locks: display.debug('ZIPLOADER: Using lock for %s' % module_name) lock = strategy.action_write_locks[module_name] else: # If the action plugin directly invokes the module (instead of # going through a strategy) then we don't have a cross-process # Lock specifically for this module. Use the "unexpected # module" lock instead display.debug('ZIPLOADER: Using generic lock for %s' % module_name) lock = strategy.action_write_locks[None] display.debug('ZIPLOADER: Acquiring lock') with lock: display.debug('ZIPLOADER: Lock acquired: %s' % id(lock)) # Check that no other process has created this while we were # waiting for the lock if not os.path.exists(cached_module_filename): display.debug('ZIPLOADER: Creating module') # Create the module zip data zipoutput = BytesIO() zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) zf.writestr( 'ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\ntry:\n from ansible.release import __version__,__author__\nexcept ImportError:\n __version__="' + to_bytes(__version__) + b'"\n __author__="' + to_bytes(__author__) + b'"\n') zf.writestr( 'ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n' ) zf.writestr('ansible_module_%s.py' % module_name, module_data) py_module_cache = {('__init__', ): b''} recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf) zf.close() zipdata = base64.b64encode(zipoutput.getvalue()) # Write the assembled module to a temp file (write to temp # so that no one looking for the file reads a partially # written file) if not os.path.exists(lookup_path): # Note -- if we have a global function to setup, that would # be a better place to run this os.mkdir(lookup_path) display.debug('ZIPLOADER: Writing module') with open(cached_module_filename + '-part', 'wb') as f: f.write(zipdata) # Rename the file into its final position in the cache so # future users of this module can read it off the # filesystem instead of constructing from scratch. display.debug('ZIPLOADER: Renaming module') os.rename(cached_module_filename + '-part', cached_module_filename) display.debug('ZIPLOADER: Done creating module') if zipdata is None: display.debug('ZIPLOADER: Reading module after lock') # Another process wrote the file while we were waiting for # the write lock. Go ahead and read the data from disk # instead of re-creating it. try: zipdata = open(cached_module_filename, 'rb').read() except IOError: raise AnsibleError( 'A different worker process failed to create module file. Look at traceback for that process for debugging information.' ) # Fool the check later... I think we should just remove the check py_module_names.add(('basic', )) zipdata = to_unicode(zipdata, errors='strict') shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars) if shebang is None: shebang = u'#!/usr/bin/python' executable = interpreter.split(u' ', 1) if len(executable) == 2 and executable[0].endswith(u'env'): # Handle /usr/bin/env python style interpreter settings interpreter = u"'{0}', '{1}'".format(*executable) else: # Still have to enclose the parts of the interpreter in quotes # because we're substituting it into the template as a python # string interpreter = u"'{0}'".format(interpreter) output.write( to_bytes(ACTIVE_ZIPLOADER_TEMPLATE % dict( zipdata=zipdata, ansible_module=module_name, params=python_repred_params, shebang=shebang, interpreter=interpreter, coding=ENCODING_STRING, ))) module_data = output.getvalue() # Sanity check from 1.x days. Maybe too strict. Some custom python # modules that use ziploader may implement their own helpers and not # need basic.py. All the constants that we substituted into basic.py # for module_replacer are now available in other, better ways. if ('basic', ) not in py_module_names: raise AnsibleError( "missing required import in %s: Did not import ansible.module_utils.basic for boilerplate helper code" % module_path) elif module_substyle == 'powershell': # Module replacer for jsonargs and windows lines = module_data.split(b'\n') for line in lines: if REPLACER_WINDOWS in line: ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1")) output.write(ps_data) py_module_names.add((b'powershell', )) continue output.write(line + b'\n') module_data = output.getvalue() module_args_json = to_bytes(json.dumps(module_args)) module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) # Sanity check from 1.x days. This is currently useless as we only # get here if we are going to substitute powershell.ps1 into the # module anyway. Leaving it for when/if we add other powershell # module_utils files. if (b'powershell', ) not in py_module_names: raise AnsibleError( "missing required import in %s: # POWERSHELL_COMMON" % module_path) elif module_substyle == 'jsonargs': module_args_json = to_bytes(json.dumps(module_args)) # these strings could be included in a third-party module but # officially they were included in the 'basic' snippet for new-style # python modules (which has been replaced with something else in # ziploader) If we remove them from jsonargs-style module replacer # then we can remove them everywhere. python_repred_args = to_bytes(repr(module_args_json)) module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args) module_data = module_data.replace( REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) # The main event -- substitute the JSON args string into the module module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) facility = b'syslog.' + to_bytes(task_vars.get( 'ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict') module_data = module_data.replace(b'syslog.LOG_USER', facility) return (module_data, module_style, shebang)
def run(self, tmp=None, task_vars=dict()): ''' handler for template operations ''' source = self._task.args.get('src', None) dest = self._task.args.get('dest', None) faf = self._task.first_available_file if (source is None and faf is not None) or dest is None: return dict(failed=True, msg="src and dest are required") if tmp is None: tmp = self._make_tmp_path() if faf: #FIXME: issue deprecation warning for first_available_file, use with_first_found or lookup('first_found',...) instead found = False for fn in faf: fn_orig = fn fnt = self._templar.template(fn) fnd = self._loader.path_dwim(self._task._role_._role_path, 'templates', fnt) if not os.path.exists(fnd): of = task_vars.get('_original_file', None) if of is not None: fnd = self._loader.path_dwim( self._task._role_._role_path, 'templates', of) if os.path.exists(fnd): source = fnd found = True break if not found: return dict( failed=True, msg="could not find src in first_available_file list") else: if self._task._role is not None: source = self._loader.path_dwim_relative( self._task._role._role_path, 'templates', source) else: source = self._loader.path_dwim(source) # Expand any user home dir specification dest = self._remote_expand_user(dest, tmp) directory_prepended = False if dest.endswith(os.sep): directory_prepended = True base = os.path.basename(source) dest = os.path.join(dest, base) # template the source data locally & get ready to transfer try: with open(source, 'r') as f: template_data = to_unicode(f.read()) try: template_uid = pwd.getpwuid(os.stat(source).st_uid).pw_name except: template_uid = os.stat(source).st_uid temp_vars = task_vars.copy() temp_vars['template_host'] = os.uname()[1] temp_vars['template_path'] = source temp_vars['template_mtime'] = datetime.datetime.fromtimestamp( os.path.getmtime(source)) temp_vars['template_uid'] = template_uid temp_vars['template_fullpath'] = os.path.abspath(source) temp_vars['template_run_date'] = datetime.datetime.now() managed_default = C.DEFAULT_MANAGED_STR managed_str = managed_default.format( host=temp_vars['template_host'], uid=temp_vars['template_uid'], file=to_bytes(temp_vars['template_path'])) temp_vars['ansible_managed'] = time.strftime( managed_str, time.localtime(os.path.getmtime(source))) old_vars = self._templar._available_variables self._templar.set_available_variables(temp_vars) resultant = self._templar.template(template_data, preserve_trailing_newlines=True) self._templar.set_available_variables(old_vars) except Exception as e: return dict(failed=True, msg=type(e).__name__ + ": " + str(e)) local_checksum = checksum_s(resultant) remote_checksum = self.get_checksum(tmp, dest, not directory_prepended, source=source) if isinstance(remote_checksum, dict): # Error from remote_checksum is a dict. Valid return is a str return remote_checksum if local_checksum != remote_checksum: # if showing diffs, we need to get the remote value dest_contents = '' # FIXME: still need to implement diff mechanism #if self.runner.diff: # # using persist_files to keep the temp directory around to avoid needing to grab another # dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % dest, task_vars=task_vars, persist_files=True) # if 'content' in dest_result.result: # dest_contents = dest_result.result['content'] # if dest_result.result['encoding'] == 'base64': # dest_contents = base64.b64decode(dest_contents) # else: # raise Exception("unknown encoding, failed: %s" % dest_result.result) xfered = self._transfer_data( self._connection._shell.join_path(tmp, 'source'), resultant) # fix file permissions when the copy is done as a different user if self._connection_info.become and self._connection_info.become_user != 'root': self._remote_chmod('a+r', xfered, tmp) # run the copy module new_module_args = self._task.args.copy() new_module_args.update( dict( src=xfered, dest=dest, original_basename=os.path.basename(source), follow=True, ), ) result = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars) if result.get('changed', False): result['diff'] = dict(before=dest_contents, after=resultant) return result else: # when running the file module based on the template data, we do # not want the source filename (the name of the template) to be used, # since this would mess up links, so we clear the src param and tell # the module to follow links. When doing that, we have to set # original_basename to the template just in case the dest is # a directory. new_module_args = self._task.args.copy() new_module_args.update( dict( src=None, original_basename=os.path.basename(source), follow=True, ), ) return self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars)
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): """ Used to insert chunks of code into modules before transfer rather than doing regular python imports. This allows for more efficient transfer in a non-bootstrapping scenario by not moving extra files over the wire and also takes care of embedding arguments in the transferred modules. This version is done in such a way that local imports can still be used in the module code, so IDEs don't have to be aware of what is going on. Example: from ansible.module_utils.basic import * ... will result in the insertion of basic.py into the module from the module_utils/ directory in the source tree. All modules are required to import at least basic, though there will also be other snippets. For powershell, there's equivalent conventions like this: # POWERSHELL_COMMON which results in the inclusion of the common code from powershell.ps1 """ with open(module_path, 'rb') as f: # read in the module source module_data = f.read() (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): shebang = lines[0].strip() args = shlex.split(str(shebang[2:])) interpreter = args[0] interpreter = to_bytes(interpreter) new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru') if new_shebang: lines[0] = shebang = new_shebang if os.path.basename(interpreter).startswith(b'python'): lines.insert(1, to_bytes(ENCODING_STRING)) else: # No shebang, assume a binary module? pass module_data = b"\n".join(lines) else: shebang = to_bytes(shebang, errors='strict') return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))
def run(self, tmp=None, task_vars=None): ''' handler for file transfer operations ''' if task_vars is None: task_vars = dict() result = super(ActionModule, self).run(tmp, task_vars) source = self._task.args.get('src', None) content = self._task.args.get('content', None) dest = self._task.args.get('dest', None) raw = boolean(self._task.args.get('raw', 'no')) force = boolean(self._task.args.get('force', 'yes')) faf = self._task.first_available_file remote_src = boolean(self._task.args.get('remote_src', False)) follow = boolean(self._task.args.get('follow', False)) if (source is None and content is None and faf is None) or dest is None: result['failed'] = True result['msg'] = "src (or content) and dest are required" return result elif (source is not None or faf is not None) and content is not None: result['failed'] = True result['msg'] = "src and content are mutually exclusive" return result elif content is not None and dest is not None and dest.endswith("/"): result['failed'] = True result['msg'] = "dest must be a file if content is defined" return result # Check if the source ends with a "/" source_trailing_slash = False if source: source_trailing_slash = self._connection._shell.path_has_trailing_slash( source) # Define content_tempfile in case we set it after finding content populated. content_tempfile = None # If content is defined make a temp file and write the content into it. if content is not None: try: # If content comes to us as a dict it should be decoded json. # We need to encode it back into a string to write it out. if isinstance(content, dict) or isinstance(content, list): content_tempfile = self._create_content_tempfile( json.dumps(content)) else: content_tempfile = self._create_content_tempfile(content) source = content_tempfile except Exception as err: result['failed'] = True result['msg'] = "could not write content temp file: %s" % err return result # if we have first_available_file in our vars # look up the files and use the first one we find as src elif faf: source = self._get_first_available_file( faf, task_vars.get('_original_file', None)) if source is None: result['failed'] = True result[ 'msg'] = "could not find src in first_available_file list" return result elif remote_src: result.update( self._execute_module(module_name='copy', module_args=self._task.args, task_vars=task_vars, delete_remote_tmp=False)) return result else: if self._task._role is not None: source = self._loader.path_dwim_relative( self._task._role._role_path, 'files', source) else: source = self._loader.path_dwim_relative( self._loader.get_basedir(), 'files', source) # A list of source file tuples (full_path, relative_path) which will try to copy to the destination source_files = [] # If source is a directory populate our list else source is a file and translate it to a tuple. if os.path.isdir(to_bytes(source, errors='strict')): # Get the amount of spaces to remove to get the relative path. if source_trailing_slash: sz = len(source) else: sz = len(source.rsplit('/', 1)[0]) + 1 # Walk the directory and append the file tuples to source_files. for base_path, sub_folders, files in os.walk(source): for file in files: full_path = os.path.join(base_path, file) rel_path = full_path[sz:] if rel_path.startswith('/'): rel_path = rel_path[1:] source_files.append((full_path, rel_path)) # If it's recursive copy, destination is always a dir, # explicitly mark it so (note - copy module relies on this). if not self._connection._shell.path_has_trailing_slash(dest): dest = self._connection._shell.join_path(dest, '') else: source_files.append((source, os.path.basename(source))) changed = False module_return = dict(changed=False) # A register for if we executed a module. # Used to cut down on command calls when not recursive. module_executed = False # Tell _execute_module to delete the file if there is one file. delete_remote_tmp = (len(source_files) == 1) # If this is a recursive action create a tmp path that we can share as the _exec_module create is too late. remote_user = task_vars.get( 'ansible_ssh_user') or self._play_context.remote_user if not delete_remote_tmp: if tmp is None or "-tmp-" not in tmp: tmp = self._make_tmp_path(remote_user) self._cleanup_remote_tmp = True # expand any user home dir specifier dest = self._remote_expand_user(dest) diffs = [] for source_full, source_rel in source_files: source_full = self._loader.get_real_file(source_full) # Generate a hash of the local file. local_checksum = checksum(source_full) # If local_checksum is not defined we can't find the file so we should fail out. if local_checksum is None: result['failed'] = True result['msg'] = "could not find src=%s" % source_full self._remove_tmp_path(tmp) return result # This is kind of optimization - if user told us destination is # dir, do path manipulation right away, otherwise we still check # for dest being a dir via remote call below. if self._connection._shell.path_has_trailing_slash(dest): dest_file = self._connection._shell.join_path(dest, source_rel) else: dest_file = self._connection._shell.join_path(dest) # Attempt to get remote file info dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp) if dest_status['exists'] and dest_status['isdir']: # The dest is a directory. if content is not None: # If source was defined as content remove the temporary file and fail out. self._remove_tempfile_if_content_defined( content, content_tempfile) self._remove_tmp_path(tmp) result['failed'] = True result['msg'] = "can not use content with a dir as dest" return result else: # Append the relative source location to the destination and get remote stats again dest_file = self._connection._shell.join_path( dest, source_rel) dest_status = self._execute_remote_stat(dest_file, all_vars=task_vars, follow=follow, tmp=tmp) if dest_status['exists'] and not force: # remote_file does not exist so continue to next iteration. continue if local_checksum != dest_status['checksum']: # The checksums don't match and we will change or error out. changed = True # Create a tmp path if missing only if this is not recursive. # If this is recursive we already have a tmp path. if delete_remote_tmp: if tmp is None or "-tmp-" not in tmp: tmp = self._make_tmp_path(remote_user) self._cleanup_remote_tmp = True if self._play_context.diff and not raw: diffs.append( self._get_diff_data(dest_file, source_full, task_vars)) if self._play_context.check_mode: self._remove_tempfile_if_content_defined( content, content_tempfile) changed = True module_return = dict(changed=True) continue # Define a remote directory that we will copy the file to. tmp_src = self._connection._shell.join_path(tmp, 'source') remote_path = None if not raw: remote_path = self._transfer_file(source_full, tmp_src) else: self._transfer_file(source_full, dest_file) # We have copied the file remotely and no longer require our content_tempfile self._remove_tempfile_if_content_defined( content, content_tempfile) self._loader.cleanup_tmp_file(source_full) # fix file permissions when the copy is done as a different user if remote_path: self._fixup_perms2((tmp, remote_path), remote_user) if raw: # Continue to next iteration if raw is defined. continue # Run the copy module # src and dest here come after original and override them # we pass dest only to make sure it includes trailing slash in case of recursive copy new_module_args = self._task.args.copy() new_module_args.update( dict( src=tmp_src, dest=dest, original_basename=source_rel, )) if 'content' in new_module_args: del new_module_args['content'] module_return = self._execute_module( module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_executed = True else: # no need to transfer the file, already correct hash, but still need to call # the file module in case we want to change attributes self._remove_tempfile_if_content_defined( content, content_tempfile) self._loader.cleanup_tmp_file(source_full) if raw: # Continue to next iteration if raw is defined. self._remove_tmp_path(tmp) continue # Build temporary module_args. new_module_args = self._task.args.copy() new_module_args.update( dict(src=source_rel, dest=dest, original_basename=source_rel)) # Execute the file module. module_return = self._execute_module( module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_executed = True if not module_return.get('checksum'): module_return['checksum'] = local_checksum if module_return.get('failed'): result.update(module_return) if not delete_remote_tmp: self._remove_tmp_path(tmp) return result if module_return.get('changed'): changed = True # the file module returns the file path as 'path', but # the copy module uses 'dest', so add it if it's not there if 'path' in module_return and 'dest' not in module_return: module_return['dest'] = module_return['path'] # Delete tmp path if we were recursive or if we did not execute a module. if not delete_remote_tmp or (delete_remote_tmp and not module_executed): self._remove_tmp_path(tmp) if module_executed and len(source_files) == 1: result.update(module_return) else: result.update(dict(dest=dest, src=source, changed=changed)) if diffs: result['diff'] = diffs return result