def test_plugins_connection_ssh_put_file(self, mock_ospe, mock_sleep):
pc = PlayContext()
new_stdin = StringIO()
conn = connection_loader.get('ssh', pc, new_stdin)
conn._build_command = MagicMock()
conn._bare_run = MagicMock()
mock_ospe.return_value = True
conn._build_command.return_value = 'some command to run'
conn._bare_run.return_value = (0, '', '')
conn.host = "some_host"
C.ASSIBLE_SSH_RETRIES = 9
# Test with C.DEFAULT_SCP_IF_SSH set to smart
# Test when SFTP works
C.DEFAULT_SCP_IF_SSH = 'smart'
expected_in_data = b' '.join((b'put', to_bytes(shlex_quote('/path/to/in/file')), to_bytes(shlex_quote('/path/to/dest/file')))) + b'\n'
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False)
# Test when SFTP doesn't work but SCP does
conn._bare_run.side_effect = [(1, 'stdout', 'some errors'), (0, '', '')]
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._bare_run.assert_called_with('some command to run', None, checkrc=False)
conn._bare_run.side_effect = None
# test with C.DEFAULT_SCP_IF_SSH enabled
C.DEFAULT_SCP_IF_SSH = True
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._bare_run.assert_called_with('some command to run', None, checkrc=False)
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._bare_run.assert_called_with('some command to run', None, checkrc=False)
# test with C.DEFAULT_SCP_IF_SSH disabled
C.DEFAULT_SCP_IF_SSH = False
expected_in_data = b' '.join((b'put', to_bytes(shlex_quote('/path/to/in/file')), to_bytes(shlex_quote('/path/to/dest/file')))) + b'\n'
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False)
expected_in_data = b' '.join((b'put',
to_bytes(shlex_quote('/path/to/in/file/with/unicode-fö〩')),
to_bytes(shlex_quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._bare_run.assert_called_with('some command to run', expected_in_data, checkrc=False)
# test that a non-zero rc raises an error
conn._bare_run.return_value = (1, 'stdout', 'some errors')
self.assertRaises(AssibleError, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')
# test that a not-found path raises an error
mock_ospe.return_value = False
conn._bare_run.return_value = (0, 'stdout', '')
self.assertRaises(AssibleFileNotFound, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')
def checksum(self, path, python_interp):
# In the following test, each condition is a check and logical
# comparison (|| or &&) that sets the rc value. Every check is run so
# the last check in the series to fail will be the rc that is returned.
#
# If a check fails we error before invoking the hash functions because
# hash functions may successfully take the hash of a directory on BSDs
# (UFS filesystem?) which is not what the rest of the assible code expects
#
# If all of the available hashing methods fail we fail with an rc of 0.
# This logic is added to the end of the cmd at the bottom of this function.
# Return codes:
# checksum: success!
# 0: Unknown error
# 1: Remote file does not exist
# 2: No read permissions on the file
# 3: File is a directory
# 4: No python interpreter
# Quoting gets complex here. We're writing a python string that's
# used by a variety of shells on the remote host to invoke a python
# "one-liner".
shell_escaped_path = shlex_quote(path)
test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc} \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR) # NOQA
csums = [
u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python > 2.4 (including python3)
u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # NOQA Python == 2.4
]
cmd = (" %s " % self._SHELL_OR).join(csums)
cmd = "%s; %s %s (echo \'0 \'%s)" % (test, cmd, self._SHELL_OR, shell_escaped_path)
return cmd
def set_user_facl(self, paths, user, mode):
"""Only sets acls for users as that's really all we need"""
cmd = ['setfacl', '-m', 'u:%s:%s' % (user, mode)]
cmd.extend(paths)
cmd = [shlex_quote(c) for c in cmd]
return ' '.join(cmd)
def db_dump(module, target, target_opts="",
db=None,
dump_extra_args=None,
user=None,
password=None,
host=None,
port=None,
**kw):
flags = login_flags(db, host, port, user, db_prefix=False)
cmd = module.get_bin_path('pg_dump', True)
comp_prog_path = None
if os.path.splitext(target)[-1] == '.tar':
flags.append(' --format=t')
elif os.path.splitext(target)[-1] == '.pgc':
flags.append(' --format=c')
if os.path.splitext(target)[-1] == '.gz':
if module.get_bin_path('pigz'):
comp_prog_path = module.get_bin_path('pigz', True)
else:
comp_prog_path = module.get_bin_path('gzip', True)
elif os.path.splitext(target)[-1] == '.bz2':
comp_prog_path = module.get_bin_path('bzip2', True)
elif os.path.splitext(target)[-1] == '.xz':
comp_prog_path = module.get_bin_path('xz', True)
cmd += "".join(flags)
if dump_extra_args:
cmd += " {0} ".format(dump_extra_args)
if target_opts:
cmd += " {0} ".format(target_opts)
if comp_prog_path:
# Use a fifo to be notified of an error in pg_dump
# Using shell pipe has no way to return the code of the first command
# in a portable way.
fifo = os.path.join(module.tmpdir, 'pg_fifo')
os.mkfifo(fifo)
cmd = '{1} <{3} > {2} & {0} >{3}'.format(cmd, comp_prog_path, shlex_quote(target), fifo)
else:
cmd = '{0} > {1}'.format(cmd, shlex_quote(target))
return do_with_password(module, cmd, password)
def db_restore(module, target, target_opts="",
db=None,
user=None,
password=None,
host=None,
port=None,
**kw):
flags = login_flags(db, host, port, user)
comp_prog_path = None
cmd = module.get_bin_path('psql', True)
if os.path.splitext(target)[-1] == '.sql':
flags.append(' --file={0}'.format(target))
elif os.path.splitext(target)[-1] == '.tar':
flags.append(' --format=Tar')
cmd = module.get_bin_path('pg_restore', True)
elif os.path.splitext(target)[-1] == '.pgc':
flags.append(' --format=Custom')
cmd = module.get_bin_path('pg_restore', True)
elif os.path.splitext(target)[-1] == '.gz':
comp_prog_path = module.get_bin_path('zcat', True)
elif os.path.splitext(target)[-1] == '.bz2':
comp_prog_path = module.get_bin_path('bzcat', True)
elif os.path.splitext(target)[-1] == '.xz':
comp_prog_path = module.get_bin_path('xzcat', True)
cmd += "".join(flags)
if target_opts:
cmd += " {0} ".format(target_opts)
if comp_prog_path:
env = os.environ.copy()
if password:
env = {"PGPASSWORD": password}
p1 = subprocess.Popen([comp_prog_path, target], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p2 = subprocess.Popen(cmd, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=env)
(stdout2, stderr2) = p2.communicate()
p1.stdout.close()
p1.wait()
if p1.returncode != 0:
stderr1 = p1.stderr.read()
return p1.returncode, '', stderr1, 'cmd: ****'
else:
return p2.returncode, '', stderr2, 'cmd: ****'
else:
cmd = '{0} < {1}'.format(cmd, shlex_quote(target))
return do_with_password(module, cmd, password)
def login_flags(db, host, port, user, db_prefix=True):
"""
returns a list of connection argument strings each prefixed
with a space and quoted where necessary to later be combined
in a single shell string with `"".join(rv)`
db_prefix determines if "--dbname" is prefixed to the db argument,
since the argument was introduced in 9.3.
"""
flags = []
if db:
if db_prefix:
flags.append(' --dbname={0}'.format(shlex_quote(db)))
else:
flags.append(' {0}'.format(shlex_quote(db)))
if host:
flags.append(' --host={0}'.format(host))
if port:
flags.append(' --port={0}'.format(port))
if user:
flags.append(' --username={0}'.format(user))
return flags
def build_module_command(self, env_string, shebang, cmd, arg_path=None):
# don't quote the cmd if it's an empty string, because this will break pipelining mode
if cmd.strip() != '':
cmd = shlex_quote(cmd)
cmd_parts = []
if shebang:
shebang = shebang.replace("#!", "").strip()
else:
shebang = ""
cmd_parts.extend([env_string.strip(), shebang, cmd])
if arg_path is not None:
cmd_parts.append(arg_path)
new_cmd = " ".join(cmd_parts)
return new_cmd
def build_become_command(self, cmd, shell):
super(BecomeModule, self).build_become_command(cmd, shell)
# Prompt handling for ``su`` is more complicated, this
# is used to satisfy the connection plugin
self.prompt = True
if not cmd:
return cmd
exe = self.get_option('become_exe') or self.name
flags = self.get_option('become_flags') or ''
user = self.get_option('become_user') or ''
success_cmd = self._build_success_command(cmd, shell)
return "%s %s %s -c %s" % (exe, flags, user, shlex_quote(success_cmd))
def _build_success_command(self, cmd, shell, noexe=False):
if not all((cmd, shell, self.success)):
return cmd
try:
cmd = shlex_quote(
'%s %s %s %s' %
(shell.ECHO, self.success, shell.COMMAND_SEP, cmd))
except AttributeError:
# TODO: This should probably become some more robust functionlity used to detect incompat
raise AssibleError(
'The %s shell family is incompatible with the %s become plugin'
% (shell.SHELL_FAMILY, self.name))
exe = getattr(shell, 'executable', None)
if exe and not noexe:
cmd = '%s -c %s' % (exe, cmd)
return cmd
def _write_execute(self, path):
"""
Return the command line for writing a crontab
"""
user = ''
if self.user:
if platform.system() in ['SunOS', 'HP-UX', 'AIX']:
return "chown %s %s ; su '%s' -c '%s %s'" % (shlex_quote(
self.user), shlex_quote(path), shlex_quote(
self.user), self.cron_cmd, shlex_quote(path))
elif pwd.getpwuid(os.getuid())[0] != self.user:
user = '******' % shlex_quote(self.user)
return "%s %s %s" % (self.cron_cmd, user, shlex_quote(path))
def expand_user(self, user_home_path, username=''):
''' Return a command to expand tildes in a path
It can be either "~" or "~username". We just ignore $HOME
We use the POSIX definition of a username:
http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276
Falls back to 'current working directory' as we assume 'home is where the remote user ends up'
'''
# Check that the user_path to expand is safe
if user_home_path != '~':
if not _USER_HOME_PATH_RE.match(user_home_path):
# shlex_quote will make the shell return the string verbatim
user_home_path = shlex_quote(user_home_path)
elif username:
# if present the user name is appended to resolve "that user's home"
user_home_path += username
return 'echo %s' % user_home_path
def _read_user_execute(self):
"""
Returns the command line for reading a crontab
"""
user = ''
if self.user:
if platform.system() == 'SunOS':
return "su %s -c '%s -l'" % (shlex_quote(
self.user), shlex_quote(self.cron_cmd))
elif platform.system() == 'AIX':
return "%s -l %s" % (shlex_quote(
self.cron_cmd), shlex_quote(self.user))
elif platform.system() == 'HP-UX':
return "%s %s %s" % (self.cron_cmd, '-l', shlex_quote(
self.user))
elif pwd.getpwuid(os.getuid())[0] != self.user:
user = '******' % shlex_quote(self.user)
return "%s %s %s" % (self.cron_cmd, user, '-l')
def remove(self, path, recurse=False):
path = shlex_quote(path)
cmd = 'rm -f '
if recurse:
cmd += '-r '
return cmd + "%s %s" % (path, self._SHELL_REDIRECT_ALLNULL)
def quote(a):
''' return its argument quoted for shell usage '''
if a is None:
a = u''
return shlex_quote(to_text(a))
def env_prefix(**args):
return ' '.join([
'%s=%s' % (k, shlex_quote(text_type(v)))
for k, v in args.items()
])
def quote(self, cmd):
"""Returns a shell-escaped string that can be safely used as one token in a shell command line"""
return shlex_quote(cmd)
def chmod(self, paths, mode):
cmd = ['chmod', mode]
cmd.extend(paths)
cmd = [shlex_quote(c) for c in cmd]
return ' '.join(cmd)
def chown(self, paths, user):
cmd = ['chown', user]
cmd.extend(paths)
cmd = [shlex_quote(c) for c in cmd]
return ' '.join(cmd)
def chgrp(self, paths, group):
cmd = ['chgrp', group]
cmd.extend(paths)
cmd = [shlex_quote(c) for c in cmd]
return ' '.join(cmd)
def run(self):
''' use Runner lib to do SSH things '''
super(PullCLI, self).run()
# log command line
now = datetime.datetime.now()
display.display(now.strftime("Starting Assible Pull at %F %T"))
display.display(' '.join(sys.argv))
# Build Checkout command
# Now construct the assible command
node = platform.node()
host = socket.getfqdn()
limit_opts = 'localhost,%s,127.0.0.1' % ','.join(
set([host, node,
host.split('.')[0],
node.split('.')[0]]))
base_opts = '-c local '
if context.CLIARGS['verbosity'] > 0:
base_opts += ' -%s' % ''.join(
["v" for x in range(0, context.CLIARGS['verbosity'])])
# Attempt to use the inventory passed in as an argument
# It might not yet have been downloaded so use localhost as default
inv_opts = self._get_inv_cli()
if not inv_opts:
inv_opts = " -i localhost, "
# avoid interpreter discovery since we already know which interpreter to use on localhost
inv_opts += '-e %s ' % shlex_quote(
'assible_python_interpreter=%s' % sys.executable)
# SCM specific options
if context.CLIARGS['module_name'] == 'git':
repo_opts = "name=%s dest=%s" % (context.CLIARGS['url'],
context.CLIARGS['dest'])
if context.CLIARGS['checkout']:
repo_opts += ' version=%s' % context.CLIARGS['checkout']
if context.CLIARGS['accept_host_key']:
repo_opts += ' accept_hostkey=yes'
if context.CLIARGS['private_key_file']:
repo_opts += ' key_file=%s' % context.CLIARGS[
'private_key_file']
if context.CLIARGS['verify']:
repo_opts += ' verify_commit=yes'
if context.CLIARGS['tracksubs']:
repo_opts += ' track_submodules=yes'
if not context.CLIARGS['fullclone']:
repo_opts += ' depth=1'
elif context.CLIARGS['module_name'] == 'subversion':
repo_opts = "repo=%s dest=%s" % (context.CLIARGS['url'],
context.CLIARGS['dest'])
if context.CLIARGS['checkout']:
repo_opts += ' revision=%s' % context.CLIARGS['checkout']
if not context.CLIARGS['fullclone']:
repo_opts += ' export=yes'
elif context.CLIARGS['module_name'] == 'hg':
repo_opts = "repo=%s dest=%s" % (context.CLIARGS['url'],
context.CLIARGS['dest'])
if context.CLIARGS['checkout']:
repo_opts += ' revision=%s' % context.CLIARGS['checkout']
elif context.CLIARGS['module_name'] == 'bzr':
repo_opts = "name=%s dest=%s" % (context.CLIARGS['url'],
context.CLIARGS['dest'])
if context.CLIARGS['checkout']:
repo_opts += ' version=%s' % context.CLIARGS['checkout']
else:
raise AssibleOptionsError(
'Unsupported (%s) SCM module for pull, choices are: %s' %
(context.CLIARGS['module_name'], ','.join(self.REPO_CHOICES)))
# options common to all supported SCMS
if context.CLIARGS['clean']:
repo_opts += ' force=yes'
path = module_loader.find_plugin(context.CLIARGS['module_name'])
if path is None:
raise AssibleOptionsError(
("module '%s' not found.\n" % context.CLIARGS['module_name']))
bin_path = os.path.dirname(os.path.abspath(sys.argv[0]))
# hardcode local and inventory/host as this is just meant to fetch the repo
cmd = '%s/assible %s %s -m %s -a "%s" all -l "%s"' % (
bin_path, inv_opts, base_opts, context.CLIARGS['module_name'],
repo_opts, limit_opts)
for ev in context.CLIARGS['extra_vars']:
cmd += ' -e %s' % shlex_quote(ev)
# Nap?
if context.CLIARGS['sleep']:
display.display("Sleeping for %d seconds..." %
context.CLIARGS['sleep'])
time.sleep(context.CLIARGS['sleep'])
# RUN the Checkout command
display.debug("running assible with VCS module to checkout repo")
display.vvvv('EXEC: %s' % cmd)
rc, b_out, b_err = run_cmd(cmd, live=True)
if rc != 0:
if context.CLIARGS['force']:
display.warning(
"Unable to update repository. Continuing with (forced) run of playbook."
)
else:
return rc
elif context.CLIARGS['ifchanged'] and b'"changed": true' not in b_out:
display.display("Repository has not changed, quitting.")
return 0
playbook = self.select_playbook(context.CLIARGS['dest'])
if playbook is None:
raise AssibleOptionsError("Could not find a playbook to run.")
# Build playbook command
cmd = '%s/assible-playbook %s %s' % (bin_path, base_opts, playbook)
if context.CLIARGS['vault_password_files']:
for vault_password_file in context.CLIARGS['vault_password_files']:
cmd += " --vault-password-file=%s" % vault_password_file
if context.CLIARGS['vault_ids']:
for vault_id in context.CLIARGS['vault_ids']:
cmd += " --vault-id=%s" % vault_id
for ev in context.CLIARGS['extra_vars']:
cmd += ' -e %s' % shlex_quote(ev)
if context.CLIARGS['become_ask_pass']:
cmd += ' --ask-become-pass'
if context.CLIARGS['skip_tags']:
cmd += ' --skip-tags "%s"' % to_native(u','.join(
context.CLIARGS['skip_tags']))
if context.CLIARGS['tags']:
cmd += ' -t "%s"' % to_native(u','.join(context.CLIARGS['tags']))
if context.CLIARGS['subset']:
cmd += ' -l "%s"' % context.CLIARGS['subset']
else:
cmd += ' -l "%s"' % limit_opts
if context.CLIARGS['check']:
cmd += ' -C'
if context.CLIARGS['diff']:
cmd += ' -D'
os.chdir(context.CLIARGS['dest'])
# redo inventory options as new files might exist now
inv_opts = self._get_inv_cli()
if inv_opts:
cmd += inv_opts
# RUN THE PLAYBOOK COMMAND
display.debug("running assible-playbook to do actual work")
display.debug('EXEC: %s' % cmd)
rc, b_out, b_err = run_cmd(cmd, live=True)
if context.CLIARGS['purge']:
os.chdir('/')
try:
shutil.rmtree(context.CLIARGS['dest'])
except Exception as e:
display.error(u"Failed to remove %s: %s" %
(context.CLIARGS['dest'], to_text(e)))
return rc
def exists(self, path):
cmd = ['test', '-e', shlex_quote(path)]
return ' '.join(cmd)
