def test_plugins_connection_ssh_put_file(self, mock_ospe, mock_sleep): pc = PlayContext() new_stdin = StringIO() conn = connection_loader.get('ssh', pc, new_stdin) conn._build_command = MagicMock() conn._bare_run = MagicMock() mock_ospe.return_value = True conn._build_command.return_value = 'some command to run' conn._bare_run.return_value = (0, '', '') conn.host = "some_host" C.ASSIBLE_SSH_RETRIES = 9 # Test with C.DEFAULT_SCP_IF_SSH set to smart # Test when SFTP works C.DEFAULT_SCP_IF_SSH = 'smart' expected_in_data = b' '.join((b'put', to_bytes(shlex_quote('/path/to/in/file')), to_bytes(shlex_quote('/path/to/dest/file')))) + b'\n' conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False) # Test when SFTP doesn't work but SCP does conn._bare_run.side_effect = [(1, 'stdout', 'some errors'), (0, '', '')] conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._bare_run.assert_called_with('some command to run', None, checkrc=False) conn._bare_run.side_effect = None # test with C.DEFAULT_SCP_IF_SSH enabled C.DEFAULT_SCP_IF_SSH = True conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._bare_run.assert_called_with('some command to run', None, checkrc=False) conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._bare_run.assert_called_with('some command to run', None, checkrc=False) # test with C.DEFAULT_SCP_IF_SSH disabled C.DEFAULT_SCP_IF_SSH = False expected_in_data = b' '.join((b'put', to_bytes(shlex_quote('/path/to/in/file')), to_bytes(shlex_quote('/path/to/dest/file')))) + b'\n' conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False) expected_in_data = b' '.join((b'put', to_bytes(shlex_quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(shlex_quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False) # test that a non-zero rc raises an error conn._bare_run.return_value = (1, 'stdout', 'some errors') self.assertRaises(AssibleError, conn.put_file, '/path/to/bad/file', '/remote/path/to/file') # test that a not-found path raises an error mock_ospe.return_value = False conn._bare_run.return_value = (0, 'stdout', '') self.assertRaises(AssibleFileNotFound, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')
def checksum(self, path, python_interp): # In the following test, each condition is a check and logical # comparison (|| or &&) that sets the rc value. Every check is run so # the last check in the series to fail will be the rc that is returned. # # If a check fails we error before invoking the hash functions because # hash functions may successfully take the hash of a directory on BSDs # (UFS filesystem?) which is not what the rest of the assible code expects # # If all of the available hashing methods fail we fail with an rc of 0. # This logic is added to the end of the cmd at the bottom of this function. # Return codes: # checksum: success! # 0: Unknown error # 1: Remote file does not exist # 2: No read permissions on the file # 3: File is a directory # 4: No python interpreter # Quoting gets complex here. We're writing a python string that's # used by a variety of shells on the remote host to invoke a python # "one-liner". shell_escaped_path = shlex_quote(path) test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc} \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR) # NOQA csums = [ u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python > 2.4 (including python3) u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python == 2.4 ] cmd = (" %s " % self._SHELL_OR).join(csums) cmd = "%s; %s %s (echo \'0 \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path) return cmd
def set_user_facl(self, paths, user, mode): """Only sets acls for users as that's really all we need""" cmd = ['setfacl', '-m', 'u:%s:%s' % (user, mode)] cmd.extend(paths) cmd = [shlex_quote(c) for c in cmd] return ' '.join(cmd)
def db_dump(module, target, target_opts="", db=None, dump_extra_args=None, user=None, password=None, host=None, port=None, **kw): flags = login_flags(db, host, port, user, db_prefix=False) cmd = module.get_bin_path('pg_dump', True) comp_prog_path = None if os.path.splitext(target)[-1] == '.tar': flags.append(' --format=t') elif os.path.splitext(target)[-1] == '.pgc': flags.append(' --format=c') if os.path.splitext(target)[-1] == '.gz': if module.get_bin_path('pigz'): comp_prog_path = module.get_bin_path('pigz', True) else: comp_prog_path = module.get_bin_path('gzip', True) elif os.path.splitext(target)[-1] == '.bz2': comp_prog_path = module.get_bin_path('bzip2', True) elif os.path.splitext(target)[-1] == '.xz': comp_prog_path = module.get_bin_path('xz', True) cmd += "".join(flags) if dump_extra_args: cmd += " {0} ".format(dump_extra_args) if target_opts: cmd += " {0} ".format(target_opts) if comp_prog_path: # Use a fifo to be notified of an error in pg_dump # Using shell pipe has no way to return the code of the first command # in a portable way. fifo = os.path.join(module.tmpdir, 'pg_fifo') os.mkfifo(fifo) cmd = '{1} <{3} > {2} & {0} >{3}'.format(cmd, comp_prog_path, shlex_quote(target), fifo) else: cmd = '{0} > {1}'.format(cmd, shlex_quote(target)) return do_with_password(module, cmd, password)
def db_restore(module, target, target_opts="", db=None, user=None, password=None, host=None, port=None, **kw): flags = login_flags(db, host, port, user) comp_prog_path = None cmd = module.get_bin_path('psql', True) if os.path.splitext(target)[-1] == '.sql': flags.append(' --file={0}'.format(target)) elif os.path.splitext(target)[-1] == '.tar': flags.append(' --format=Tar') cmd = module.get_bin_path('pg_restore', True) elif os.path.splitext(target)[-1] == '.pgc': flags.append(' --format=Custom') cmd = module.get_bin_path('pg_restore', True) elif os.path.splitext(target)[-1] == '.gz': comp_prog_path = module.get_bin_path('zcat', True) elif os.path.splitext(target)[-1] == '.bz2': comp_prog_path = module.get_bin_path('bzcat', True) elif os.path.splitext(target)[-1] == '.xz': comp_prog_path = module.get_bin_path('xzcat', True) cmd += "".join(flags) if target_opts: cmd += " {0} ".format(target_opts) if comp_prog_path: env = os.environ.copy() if password: env = {"PGPASSWORD": password} p1 = subprocess.Popen([comp_prog_path, target], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p2 = subprocess.Popen(cmd, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=env) (stdout2, stderr2) = p2.communicate() p1.stdout.close() p1.wait() if p1.returncode != 0: stderr1 = p1.stderr.read() return p1.returncode, '', stderr1, 'cmd: ****' else: return p2.returncode, '', stderr2, 'cmd: ****' else: cmd = '{0} < {1}'.format(cmd, shlex_quote(target)) return do_with_password(module, cmd, password)
def login_flags(db, host, port, user, db_prefix=True): """ returns a list of connection argument strings each prefixed with a space and quoted where necessary to later be combined in a single shell string with `"".join(rv)` db_prefix determines if "--dbname" is prefixed to the db argument, since the argument was introduced in 9.3. """ flags = [] if db: if db_prefix: flags.append(' --dbname={0}'.format(shlex_quote(db))) else: flags.append(' {0}'.format(shlex_quote(db))) if host: flags.append(' --host={0}'.format(host)) if port: flags.append(' --port={0}'.format(port)) if user: flags.append(' --username={0}'.format(user)) return flags
def build_module_command(self, env_string, shebang, cmd, arg_path=None): # don't quote the cmd if it's an empty string, because this will break pipelining mode if cmd.strip() != '': cmd = shlex_quote(cmd) cmd_parts = [] if shebang: shebang = shebang.replace("#!", "").strip() else: shebang = "" cmd_parts.extend([env_string.strip(), shebang, cmd]) if arg_path is not None: cmd_parts.append(arg_path) new_cmd = " ".join(cmd_parts) return new_cmd
def build_become_command(self, cmd, shell): super(BecomeModule, self).build_become_command(cmd, shell) # Prompt handling for ``su`` is more complicated, this # is used to satisfy the connection plugin self.prompt = True if not cmd: return cmd exe = self.get_option('become_exe') or self.name flags = self.get_option('become_flags') or '' user = self.get_option('become_user') or '' success_cmd = self._build_success_command(cmd, shell) return "%s %s %s -c %s" % (exe, flags, user, shlex_quote(success_cmd))
def _build_success_command(self, cmd, shell, noexe=False): if not all((cmd, shell, self.success)): return cmd try: cmd = shlex_quote( '%s %s %s %s' % (shell.ECHO, self.success, shell.COMMAND_SEP, cmd)) except AttributeError: # TODO: This should probably become some more robust functionlity used to detect incompat raise AssibleError( 'The %s shell family is incompatible with the %s become plugin' % (shell.SHELL_FAMILY, self.name)) exe = getattr(shell, 'executable', None) if exe and not noexe: cmd = '%s -c %s' % (exe, cmd) return cmd
def _write_execute(self, path): """ Return the command line for writing a crontab """ user = '' if self.user: if platform.system() in ['SunOS', 'HP-UX', 'AIX']: return "chown %s %s ; su '%s' -c '%s %s'" % (shlex_quote( self.user), shlex_quote(path), shlex_quote( self.user), self.cron_cmd, shlex_quote(path)) elif pwd.getpwuid(os.getuid())[0] != self.user: user = '******' % shlex_quote(self.user) return "%s %s %s" % (self.cron_cmd, user, shlex_quote(path))
def expand_user(self, user_home_path, username=''): ''' Return a command to expand tildes in a path It can be either "~" or "~username". We just ignore $HOME We use the POSIX definition of a username: http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426 http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276 Falls back to 'current working directory' as we assume 'home is where the remote user ends up' ''' # Check that the user_path to expand is safe if user_home_path != '~': if not _USER_HOME_PATH_RE.match(user_home_path): # shlex_quote will make the shell return the string verbatim user_home_path = shlex_quote(user_home_path) elif username: # if present the user name is appended to resolve "that user's home" user_home_path += username return 'echo %s' % user_home_path
def _read_user_execute(self): """ Returns the command line for reading a crontab """ user = '' if self.user: if platform.system() == 'SunOS': return "su %s -c '%s -l'" % (shlex_quote( self.user), shlex_quote(self.cron_cmd)) elif platform.system() == 'AIX': return "%s -l %s" % (shlex_quote( self.cron_cmd), shlex_quote(self.user)) elif platform.system() == 'HP-UX': return "%s %s %s" % (self.cron_cmd, '-l', shlex_quote( self.user)) elif pwd.getpwuid(os.getuid())[0] != self.user: user = '******' % shlex_quote(self.user) return "%s %s %s" % (self.cron_cmd, user, '-l')
def remove(self, path, recurse=False): path = shlex_quote(path) cmd = 'rm -f ' if recurse: cmd += '-r ' return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL)
def quote(a): ''' return its argument quoted for shell usage ''' if a is None: a = u'' return shlex_quote(to_text(a))
def env_prefix(**args): return ' '.join([ '%s=%s' % (k, shlex_quote(text_type(v))) for k, v in args.items() ])
def quote(self, cmd): """Returns a shell-escaped string that can be safely used as one token in a shell command line""" return shlex_quote(cmd)
def chmod(self, paths, mode): cmd = ['chmod', mode] cmd.extend(paths) cmd = [shlex_quote(c) for c in cmd] return ' '.join(cmd)
def chown(self, paths, user): cmd = ['chown', user] cmd.extend(paths) cmd = [shlex_quote(c) for c in cmd] return ' '.join(cmd)
def chgrp(self, paths, group): cmd = ['chgrp', group] cmd.extend(paths) cmd = [shlex_quote(c) for c in cmd] return ' '.join(cmd)
def run(self): ''' use Runner lib to do SSH things ''' super(PullCLI, self).run() # log command line now = datetime.datetime.now() display.display(now.strftime("Starting Assible Pull at %F %T")) display.display(' '.join(sys.argv)) # Build Checkout command # Now construct the assible command node = platform.node() host = socket.getfqdn() limit_opts = 'localhost,%s,127.0.0.1' % ','.join( set([host, node, host.split('.')[0], node.split('.')[0]])) base_opts = '-c local ' if context.CLIARGS['verbosity'] > 0: base_opts += ' -%s' % ''.join( ["v" for x in range(0, context.CLIARGS['verbosity'])]) # Attempt to use the inventory passed in as an argument # It might not yet have been downloaded so use localhost as default inv_opts = self._get_inv_cli() if not inv_opts: inv_opts = " -i localhost, " # avoid interpreter discovery since we already know which interpreter to use on localhost inv_opts += '-e %s ' % shlex_quote( 'assible_python_interpreter=%s' % sys.executable) # SCM specific options if context.CLIARGS['module_name'] == 'git': repo_opts = "name=%s dest=%s" % (context.CLIARGS['url'], context.CLIARGS['dest']) if context.CLIARGS['checkout']: repo_opts += ' version=%s' % context.CLIARGS['checkout'] if context.CLIARGS['accept_host_key']: repo_opts += ' accept_hostkey=yes' if context.CLIARGS['private_key_file']: repo_opts += ' key_file=%s' % context.CLIARGS[ 'private_key_file'] if context.CLIARGS['verify']: repo_opts += ' verify_commit=yes' if context.CLIARGS['tracksubs']: repo_opts += ' track_submodules=yes' if not context.CLIARGS['fullclone']: repo_opts += ' depth=1' elif context.CLIARGS['module_name'] == 'subversion': repo_opts = "repo=%s dest=%s" % (context.CLIARGS['url'], context.CLIARGS['dest']) if context.CLIARGS['checkout']: repo_opts += ' revision=%s' % context.CLIARGS['checkout'] if not context.CLIARGS['fullclone']: repo_opts += ' export=yes' elif context.CLIARGS['module_name'] == 'hg': repo_opts = "repo=%s dest=%s" % (context.CLIARGS['url'], context.CLIARGS['dest']) if context.CLIARGS['checkout']: repo_opts += ' revision=%s' % context.CLIARGS['checkout'] elif context.CLIARGS['module_name'] == 'bzr': repo_opts = "name=%s dest=%s" % (context.CLIARGS['url'], context.CLIARGS['dest']) if context.CLIARGS['checkout']: repo_opts += ' version=%s' % context.CLIARGS['checkout'] else: raise AssibleOptionsError( 'Unsupported (%s) SCM module for pull, choices are: %s' % (context.CLIARGS['module_name'], ','.join(self.REPO_CHOICES))) # options common to all supported SCMS if context.CLIARGS['clean']: repo_opts += ' force=yes' path = module_loader.find_plugin(context.CLIARGS['module_name']) if path is None: raise AssibleOptionsError( ("module '%s' not found.\n" % context.CLIARGS['module_name'])) bin_path = os.path.dirname(os.path.abspath(sys.argv[0])) # hardcode local and inventory/host as this is just meant to fetch the repo cmd = '%s/assible %s %s -m %s -a "%s" all -l "%s"' % ( bin_path, inv_opts, base_opts, context.CLIARGS['module_name'], repo_opts, limit_opts) for ev in context.CLIARGS['extra_vars']: cmd += ' -e %s' % shlex_quote(ev) # Nap? if context.CLIARGS['sleep']: display.display("Sleeping for %d seconds..." % context.CLIARGS['sleep']) time.sleep(context.CLIARGS['sleep']) # RUN the Checkout command display.debug("running assible with VCS module to checkout repo") display.vvvv('EXEC: %s' % cmd) rc, b_out, b_err = run_cmd(cmd, live=True) if rc != 0: if context.CLIARGS['force']: display.warning( "Unable to update repository. Continuing with (forced) run of playbook." ) else: return rc elif context.CLIARGS['ifchanged'] and b'"changed": true' not in b_out: display.display("Repository has not changed, quitting.") return 0 playbook = self.select_playbook(context.CLIARGS['dest']) if playbook is None: raise AssibleOptionsError("Could not find a playbook to run.") # Build playbook command cmd = '%s/assible-playbook %s %s' % (bin_path, base_opts, playbook) if context.CLIARGS['vault_password_files']: for vault_password_file in context.CLIARGS['vault_password_files']: cmd += " --vault-password-file=%s" % vault_password_file if context.CLIARGS['vault_ids']: for vault_id in context.CLIARGS['vault_ids']: cmd += " --vault-id=%s" % vault_id for ev in context.CLIARGS['extra_vars']: cmd += ' -e %s' % shlex_quote(ev) if context.CLIARGS['become_ask_pass']: cmd += ' --ask-become-pass' if context.CLIARGS['skip_tags']: cmd += ' --skip-tags "%s"' % to_native(u','.join( context.CLIARGS['skip_tags'])) if context.CLIARGS['tags']: cmd += ' -t "%s"' % to_native(u','.join(context.CLIARGS['tags'])) if context.CLIARGS['subset']: cmd += ' -l "%s"' % context.CLIARGS['subset'] else: cmd += ' -l "%s"' % limit_opts if context.CLIARGS['check']: cmd += ' -C' if context.CLIARGS['diff']: cmd += ' -D' os.chdir(context.CLIARGS['dest']) # redo inventory options as new files might exist now inv_opts = self._get_inv_cli() if inv_opts: cmd += inv_opts # RUN THE PLAYBOOK COMMAND display.debug("running assible-playbook to do actual work") display.debug('EXEC: %s' % cmd) rc, b_out, b_err = run_cmd(cmd, live=True) if context.CLIARGS['purge']: os.chdir('/') try: shutil.rmtree(context.CLIARGS['dest']) except Exception as e: display.error(u"Failed to remove %s: %s" % (context.CLIARGS['dest'], to_text(e))) return rc
def exists(self, path): cmd = ['test', '-e', shlex_quote(path)] return ' '.join(cmd)