def pair(address, key): ''' Pair the bluetooth adapter with a device CLI Example: .. code-block:: bash salt '*' bluetooth.pair DE:AD:BE:EF:CA:FE 1234 Where DE:AD:BE:EF:CA:FE is the address of the device to pair with, and 1234 is the passphrase. TODO: This function is currently broken, as the bluez-simple-agent program no longer ships with BlueZ >= 5.0. It needs to be refactored. ''' if not salt.utils.validate.net.mac(address): raise CommandExecutionError( 'Invalid BD address passed to bluetooth.pair' ) try: int(key) except Exception: raise CommandExecutionError( 'bluetooth.pair requires a numerical key to be used' ) addy = address_() cmd = 'echo {0} | bluez-simple-agent {1} {2}'.format( _cmd_quote(addy['device']), _cmd_quote(address), _cmd_quote(key) ) out = __salt__['cmd.run'](cmd, python_shell=True).splitlines() return out
def pair(address, key): ''' Pair the bluetooth adapter with a device CLI Example: .. code-block:: bash salt '*' bluetooth.pair DE:AD:BE:EF:CA:FE 1234 Where DE:AD:BE:EF:CA:FE is the address of the device to pair with, and 1234 is the passphrase. TODO: This function is currently broken, as the bluez-simple-agent program no longer ships with BlueZ >= 5.0. It needs to be refactored. ''' if not salt.utils.validate.net.mac(address): raise CommandExecutionError( 'Invalid BD address passed to bluetooth.pair') try: int(key) except Exception: raise CommandExecutionError( 'bluetooth.pair requires a numerical key to be used') addy = address_() cmd = 'echo {0} | bluez-simple-agent {1} {2}'.format( _cmd_quote(addy['device']), _cmd_quote(address), _cmd_quote(key)) out = __salt__['cmd.run'](cmd, python_shell=True).splitlines() return out
def _rbenv_exec(command, args='', env=None, runas=None, ret=None): if not is_installed(runas): return False binary = _rbenv_bin(runas) path = _rbenv_path(runas) environ = _parse_env(env) environ['RBENV_ROOT'] = path args = ' '.join([_cmd_quote(arg) for arg in _shlex_split(args)]) result = __salt__['cmd.run_all']( '{0} {1} {2}'.format(binary, _cmd_quote(command), args), runas=runas, env=environ ) if isinstance(ret, dict): ret.update(result) return ret if result['retcode'] == 0: return result['stdout'] else: return False
def execute(opts, data, func, args, kwargs): """ Allow for the calling of execution modules via sudo. This module is invoked by the minion if the ``sudo_user`` minion config is present. Example minion config: .. code-block:: yaml sudo_user: saltdev Once this setting is made, any execution module call done by the minion will be run under ``sudo -u <sudo_user> salt-call``. For example, with the above minion config, .. code-block:: bash salt sudo_minion cmd.run 'cat /etc/sudoers' is equivalent to .. code-block:: bash sudo -u saltdev salt-call cmd.run 'cat /etc/sudoers' being run on ``sudo_minion``. """ cmd = [ "sudo", "-u", opts.get("sudo_user"), "salt-call", "--out", "json", "--metadata", "-c", opts.get("config_dir"), "--", data.get("fun"), ] if data["fun"] in ("state.sls", "state.highstate", "state.apply"): kwargs["concurrent"] = True for arg in args: cmd.append(_cmd_quote(six.text_type(arg))) for key in kwargs: cmd.append(_cmd_quote("{0}={1}".format(key, kwargs[key]))) cmd_ret = __salt__["cmd.run_all"](cmd, use_vt=True, python_shell=False) if cmd_ret["retcode"] == 0: cmd_meta = salt.utils.json.loads(cmd_ret["stdout"])["local"] ret = cmd_meta["return"] __context__["retcode"] = cmd_meta.get("retcode", 0) else: ret = cmd_ret["stderr"] __context__["retcode"] = cmd_ret["retcode"] return ret
def _rbenv_exec(command, args='', env=None, runas=None, ret=None): if not is_installed(runas): return False binary = _rbenv_bin(runas) path = _rbenv_path(runas) environ = _parse_env(env) environ['RBENV_ROOT'] = path args = ' '.join([_cmd_quote(arg) for arg in _shlex_split(args)]) result = __salt__['cmd.run_all']('{0} {1} {2}'.format( binary, _cmd_quote(command), args), runas=runas, env=environ) if isinstance(ret, dict): ret.update(result) return ret if result['retcode'] == 0: return result['stdout'] else: return False
def set_(name, value, user=None, **kwargs): cmd = 'xdg-mime default {} {}'.format(_cmd_quote(value), _cmd_quote(name)) result = __salt__['cmd.run_all'](cmd, runas=user) log.debug(result) return result['retcode'] == 0
def set_(name, value, subprop=None, user=None, **kwargs): subpropParam = '' if subprop is None else _cmd_quote(subprop) cmd = 'xdg-settings set {} {} {}'.format(_cmd_quote(name), subpropParam, _cmd_quote(value)) result = __salt__['cmd.run_all'](cmd, runas=user) log.debug(result) return result['retcode'] == 0
def get(name, subprop=None, user=None, **kwargs): subpropParam = '' if subprop is None else _cmd_quote(subprop) cmd = 'xdg-settings get {} {}'.format(_cmd_quote(name), subpropParam) output = __salt__['cmd.run'](cmd, runas=user).splitlines() if len(output) != 1: log.debug(output) return None return output[0]
def execute(opts, data, func, args, kwargs): ''' Allow for the calling of execution modules via sudo. This module is invoked by the minion if the ``sudo_user`` minion config is present. Example minion config: .. code-block:: yaml sudo_user: saltdev Once this setting is made, any execution module call done by the minion will be run under ``sudo -u <sudo_user> salt-call``. For example, with the above minion config, .. code-block:: bash salt sudo_minion cmd.run 'cat /etc/sudoers' is equivalent to .. code-block:: bash sudo -u saltdev salt-call cmd.run 'cat /etc/sudoers' being run on ``sudo_minion``. ''' cmd = ['sudo', '-u', opts.get('sudo_user'), 'salt-call', '--out', 'json', '--metadata', '-c', salt.syspaths.CONFIG_DIR, '--', data.get('fun')] if data['fun'] in ('state.sls', 'state.highstate', 'state.apply'): kwargs['concurrent'] = True for arg in args: cmd.append(_cmd_quote(six.text_type(arg))) for key in kwargs: cmd.append(_cmd_quote('{0}={1}'.format(key, kwargs[key]))) cmd_ret = __salt__['cmd.run_all'](cmd, use_vt=True, python_shell=False) if cmd_ret['retcode'] == 0: cmd_meta = salt.utils.json.loads(cmd_ret['stdout'])['local'] ret = cmd_meta['return'] __context__['retcode'] = cmd_meta.get('retcode', 0) else: ret = cmd_ret['stderr'] __context__['retcode'] = cmd_ret['retcode'] return ret
def version_cmp( pkg1, pkg2, ignore_epoch=False, **kwargs ): # pylint: disable=unused-argument """ Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem making the comparison. ignore_epoch : False Set to ``True`` to ignore the epoch when comparing versions .. versionadded:: 2016.3.4 CLI Example: .. code-block:: bash salt '*' pkg.version_cmp '0.2.4-0' '0.2.4.1-0' """ normalize = ( lambda x: six.text_type(x).split(":", 1)[-1] if ignore_epoch else six.text_type(x) ) pkg1 = normalize(pkg1) pkg2 = normalize(pkg2) output = __salt__["cmd.run_stdout"]( ["opkg", "--version"], output_loglevel="trace", python_shell=False ) opkg_version = output.split(" ")[2].strip() if salt.utils.versions.LooseVersion( opkg_version ) >= salt.utils.versions.LooseVersion("0.3.4"): cmd_compare = ["opkg", "compare-versions"] elif salt.utils.path.which("opkg-compare-versions"): cmd_compare = ["opkg-compare-versions"] else: log.warning( "Unable to find a compare-versions utility installed. Either upgrade opkg to " "version > 0.3.4 (preferred) or install the older opkg-compare-versions script." ) return None for oper, ret in (("<<", -1), ("=", 0), (">>", 1)): cmd = cmd_compare[:] cmd.append(_cmd_quote(pkg1)) cmd.append(oper) cmd.append(_cmd_quote(pkg2)) retcode = __salt__["cmd.retcode"]( cmd, output_loglevel="trace", ignore_retcode=True, python_shell=False ) if retcode == 0: return ret return None
def set_(name, capabilities, **kwargs): capabilities_str = sanitize(capabilities, toString=True) realpath = os.path.realpath(name) cmd = 'setcap {} {}'.format(_cmd_quote(capabilities_str), _cmd_quote(realpath)) result = __salt__['cmd.run_all'](cmd) log.debug(result) return result['retcode'] == 0
def version_cmp(pkg1, pkg2, ignore_epoch=False): ''' Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem making the comparison. ignore_epoch : False Set to ``True`` to ignore the epoch when comparing versions .. versionadded:: 2016.3.4 CLI Example: .. code-block:: bash salt '*' pkg.version_cmp '0.2.4-0' '0.2.4.1-0' ''' normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x) pkg1 = normalize(pkg1) pkg2 = normalize(pkg2) output = __salt__['cmd.run_stdout'](['opkg', '--version'], output_loglevel='trace', python_shell=False) opkg_version = output.split(' ')[2].strip() if salt.utils.versions.LooseVersion(opkg_version) >= \ salt.utils.versions.LooseVersion('0.3.4'): cmd_compare = ['opkg', 'compare-versions'] elif salt.utils.path.which('opkg-compare-versions'): cmd_compare = ['opkg-compare-versions'] else: log.warning( 'Unable to find a compare-versions utility installed. Either upgrade opkg to ' 'version > 0.3.4 (preferred) or install the older opkg-compare-versions script.' ) return None for oper, ret in (("<<", -1), ("=", 0), (">>", 1)): cmd = cmd_compare[:] cmd.append(_cmd_quote(pkg1)) cmd.append(oper) cmd.append(_cmd_quote(pkg2)) retcode = __salt__['cmd.retcode'](cmd, output_loglevel='trace', ignore_retcode=True, python_shell=False) if retcode == 0: return ret return None
def delete(target, stop=True): ''' Deletes a gluster volume target Volume to delete stop Stop volume before delete if it is started, True by default ''' if target not in list_volumes(): return 'Volume does not exist' cmd = 'yes | gluster volume delete {0}'.format(_cmd_quote(target)) # Stop volume if requested to and it is running if stop is True and isinstance(status(target), dict): stop_volume(target) stopped = True else: stopped = False # Warn volume is running if stop not requested if isinstance(status(target), dict): return 'Error: Volume must be stopped before deletion' result = __salt__['cmd.run'](cmd, python_shell=True) if result.splitlines()[0].endswith('success'): if stopped: return 'Volume {0} stopped and deleted'.format(target) else: return 'Volume {0} deleted'.format(target) else: return result
def _install_rbenv(path, runas=None): if os.path.isdir(path): return True return 0 == __salt__['cmd.retcode']( 'git clone https://github.com/sstephenson/rbenv.git {0}' .format(_cmd_quote(path)), runas=runas)
def do(cmdline=None, runas=None): ''' Execute a ruby command with rbenv's shims from the user or the system. CLI Example: .. code-block:: bash salt '*' rbenv.do 'gem list bundler' salt '*' rbenv.do 'gem list bundler' deploy ''' path = _rbenv_path(runas) environ = {'PATH': '{0}/shims:{1}'.format(path, os.environ['PATH'])} cmdline = ' '.join([_cmd_quote(cmd) for cmd in _shlex_split(cmdline)]) result = __salt__['cmd.run_all']( cmdline, runas=runas, env=environ ) if result['retcode'] == 0: rehash(runas=runas) return result['stdout'] else: return False
def generate_cert(key, config=None, subj='/CN=localhost', expire=30): r''' Generate a self-signed certificate. key The PEM-encoded RSA private key. config The location of an OpenSSL configuration file on the minion, useful for specifying X.509 extensions in generated certificates. subj The subject name of the certificate, using ``/`` to separate relative distinguished names. Default is ``/CN=localhost``. expire Number of days until the certificate expires. Default is ``30``. ''' ret = '' if subj is None: subj = '/CN=localhost' cmd = 'openssl' opts = ['req -new -x509 -sha256 -batch'] if salt.utils.platform.is_windows(): opts.append('-key -') else: opts.append('-key /dev/stdin') if config: if not os.path.exists(config): raise CommandExecutionError( '\'{0}\' file not found on minion'.format(config)) opts.append(' '.join(('-config', _cmd_quote(config)))) opts.append(' '.join(('-subj', _cmd_quote(subj)))) try: expire = int(expire) except TypeError: raise SaltInvocationError('\'{0}\' is not an integer'.format(expire)) opts.append(' '.join(('-days', str(expire)))) if not salt.utils.path.which(cmd): raise CommandExecutionError( '\'{0}\' command not found on minion'.format(cmd)) opts = ' '.join(opts) res = __salt__['cmd.run_all'](' '.join((cmd, opts)), stdin=key) if res['retcode'] != 0: raise CommandExecutionError(res['stdout'] + res['stderr']) ret = res['stdout'] return ret
def bin_pkg_info(path, saltenv='base'): ''' .. versionadded:: 2015.8.0 Parses RPM metadata and returns a dictionary of information about the package (name, version, etc.). path Path to the file. Can either be an absolute path to a file on the minion, or a salt fileserver URL (e.g. ``salt://path/to/file.rpm``). If a salt fileserver URL is passed, the file will be cached to the minion so that it can be examined. saltenv : base Salt fileserver envrionment from which to retrieve the package. Ignored if ``path`` is a local file path on the minion. CLI Example: .. code-block:: bash salt '*' lowpkg.bin_pkg_info /root/salt-2015.5.1-2.el7.noarch.rpm salt '*' lowpkg.bin_pkg_info salt://salt-2015.5.1-2.el7.noarch.rpm ''' # If the path is a valid protocol, pull it down using cp.cache_file if __salt__['config.valid_fileproto'](path): newpath = __salt__['cp.cache_file'](path, saltenv) if not newpath: raise CommandExecutionError( 'Unable to retrieve {0} from saltenv \'{1}\'' .format(path, saltenv) ) path = newpath else: if not os.path.exists(path): raise CommandExecutionError( '{0} does not exist on minion'.format(path) ) elif not os.path.isabs(path): raise SaltInvocationError( '{0} does not exist on minion'.format(path) ) # REPOID is not a valid tag for the rpm command. Remove it and replace it # with 'none' queryformat = salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', 'none') output = __salt__['cmd.run_stdout']( 'rpm -qp --queryformat {0} {1}'.format(_cmd_quote(queryformat), path), output_loglevel='trace', ignore_retcode=True ) ret = {} pkginfo = salt.utils.pkg.rpm.parse_pkginfo( output, osarch=__grains__['osarch'] ) for field in pkginfo._fields: ret[field] = getattr(pkginfo, field) return ret
def _get_osx(**kwargs): type_ = kwargs.get('type', 'HostName') if type_ not in OSX_HOSTNAME_TYPES: raise salt.exceptions.SaltInvocationError( 'Invalid hostname type given: {0}'.format(type_)) cmd = 'scutil --get {0}'.format(_cmd_quote(type_)) return __salt__['cmd.run'](cmd)
def _install_ruby_build(path, runas=None): path = '{0}/plugins/ruby-build'.format(path) if os.path.isdir(path): return True return 0 == __salt__['cmd.retcode']( 'git clone https://github.com/sstephenson/ruby-build.git {0}' .format(_cmd_quote(path)), runas=runas)
def _install_rbenv(path, runas=None): if os.path.isdir(path): return True return 0 == __salt__['cmd.retcode']( 'git clone https://github.com/sstephenson/rbenv.git {0}'.format( _cmd_quote(path)), runas=runas)
def get(name, user=None, **kwargs): cmd = 'xdg-mime query default {}'.format(_cmd_quote(name)) output = __salt__['cmd.run'](cmd, runas=user).splitlines() if len(output) != 1: log.debug(output) return None return output[0]
def _rbenv_path(runas=None): path = None if runas in (None, 'root'): path = __salt__['config.option']('rbenv.root') or '/usr/local/rbenv' else: path = (__salt__['config.option']('rbenv.root') or '~{0}/.rbenv'.format(runas)) return _cmd_quote(os.path.expanduser(path))
def _install_ruby_build(path, runas=None): path = '{0}/plugins/ruby-build'.format(path) if os.path.isdir(path): return True return 0 == __salt__['cmd.retcode']( 'git clone https://github.com/sstephenson/ruby-build.git {0}'.format( _cmd_quote(path)), runas=runas)
def get(name, **kwargs): realpath = os.path.realpath(name) cmd = 'getcap {}'.format(_cmd_quote(realpath)) output = __salt__['cmd.run'](cmd).splitlines() if len(output) != 1: log.debug(output) return [] prefix = '{} = '.format(realpath) return _removePrefix(output[0], prefix).split(',')
def enable(name, **kwargs): ''' Enable the named service to start at boot CLI Example: .. code-block:: bash salt '*' service.enable <service name> ''' osmajor = _osrel()[0] if osmajor < '6': cmd = 'update-rc.d -f {0} defaults 99'.format(_cmd_quote(name)) else: cmd = 'update-rc.d {0} enable'.format(_cmd_quote(name)) try: if int(osmajor) >= 6: cmd = 'insserv {0} && '.format(_cmd_quote(name)) + cmd except ValueError: if osmajor == 'testing/unstable' or osmajor == 'unstable': cmd = 'insserv {0} && '.format(_cmd_quote(name)) + cmd return not __salt__['cmd.retcode'](cmd, python_shell=True)
def enable(name, **kwargs): """ Enable the named service to start at boot CLI Example: .. code-block:: bash salt '*' service.enable <service name> """ osmajor = _osrel()[0] if osmajor < "6": cmd = "update-rc.d -f {0} defaults 99".format(_cmd_quote(name)) else: cmd = "update-rc.d {0} enable".format(_cmd_quote(name)) try: if int(osmajor) >= 6: cmd = "insserv {0} && ".format(_cmd_quote(name)) + cmd except ValueError: osrel = _osrel() if osrel == "testing/unstable" or osrel == "unstable" or osrel.endswith( "/sid"): cmd = "insserv {0} && ".format(_cmd_quote(name)) + cmd return not __salt__["cmd.retcode"](cmd, python_shell=True)
def do(cmdline=None, runas=None): ''' Execute a ruby command with rbenv's shims from the user or the system. CLI Example: .. code-block:: bash salt '*' rbenv.do 'gem list bundler' salt '*' rbenv.do 'gem list bundler' deploy ''' path = _rbenv_path(runas) environ = {'PATH': '{0}/shims:{1}'.format(path, os.environ['PATH'])} cmdline = ' '.join([_cmd_quote(cmd) for cmd in _shlex_split(cmdline)]) result = __salt__['cmd.run_all'](cmdline, runas=runas, env=environ) if result['retcode'] == 0: rehash(runas=runas) return result['stdout'] else: return False
def stop_volume(name): ''' Stop a gluster volume. name Volume name CLI Example: .. code-block:: bash salt '*' glusterfs.stop_volume mycluster ''' vol_status = status(name) if isinstance(vol_status, dict): cmd = 'yes | gluster volume stop {0}'.format(_cmd_quote(name)) result = __salt__['cmd.run'](cmd, python_shell=True) if result.splitlines()[0].endswith('success'): return 'Volume {0} stopped'.format(name) else: return result return vol_status
def _set_osx_of_type(type_, name): cmd = 'scutil --set {0} {1}'.format(type_, _cmd_quote(name)) return __salt__['cmd.run_all'](cmd)
def at(*args, **kwargs): # pylint: disable=C0103 ''' Add a job to the queue. The 'timespec' follows the format documented in the at(1) manpage. CLI Example: .. code-block:: bash salt '*' at.at <timespec> <cmd> [tag=<tag>] [runas=<user>] salt '*' at.at 12:05am '/sbin/reboot' tag=reboot salt '*' at.at '3:05am +3 days' 'bin/myscript' tag=nightly runas=jim ''' if len(args) < 2: return {'jobs': []} # Shim to produce output similar to what __virtual__() should do # but __salt__ isn't available in __virtual__() binary = salt.utils.which('at') if not binary: return '\'at.at\' is not available.' if __grains__['os_family'] == 'RedHat': echo_cmd = 'echo -e' else: echo_cmd = 'echo' if 'tag' in kwargs: cmd = '{4} "### SALT: {0}\n{1}" | {2} {3}'.format(_cmd_quote(kwargs['tag']), _cmd_quote(' '.join(args[1:])), binary, _cmd_quote(args[0]), echo_cmd) else: cmd = '{3} "{1}" | {2} {0}'.format(args[0], ' '.join(args[1:]), binary, echo_cmd) # Can't use _cmd here since we need to prepend 'echo_cmd' if 'runas' in kwargs: output = __salt__['cmd.run']('{0}'.format(cmd), runas=kwargs['runas']) else: output = __salt__['cmd.run']('{0}'.format(cmd)) if output is None: return '\'at.at\' is not available.' if output.endswith('Garbled time'): return {'jobs': [], 'error': 'invalid timespec'} if output.startswith('warning: commands'): output = output.splitlines()[1] if output.startswith('commands will be executed'): output = output.splitlines()[1] output = output.split()[1] if __grains__['os'] in BSD: return atq(str(output)) else: return atq(int(output))
def _extract(cwd, sfn, archive_format=None, archive_compression=None, archive_files=None): ret = '' if archive_format is None: ext = sfn.split('.')[-1].lower() if ext == 'zip': archive_format = 'zip' elif ext in ('tar', 'tgz', 'tb2', 'tbz', 'tbz2', 'txz'): archive_format = 'tar' else: ext = sfn.split('.')[-2] if ext == 'tar': archive_format = 'tar' if archive_format not in ('tar', 'zip'): raise SaltInvocationError( 'Invalid archive format \'{0}\'.'.format(archive_format)) if archive_format == 'zip': cmd = 'unzip' if not salt.utils.path.which(cmd): raise CommandExecutionError( '\'{0}\' command not found on minion.'.format(cmd)) opts = ['{0}'] if archive_files: for f in archive_files: opts.append(_cmd_quote(f)) elif archive_format == 'tar': if archive_compression is None: ext = sfn.split('.')[-1].lower() if ext in ('gz', 'tgz'): archive_compression = 'gz' elif ext in ('bz2', 'tb2', 'tbz', 'tbz2'): archive_compression = 'bz2' elif ext in ('xz', 'txz'): archive_compression = 'xz' elif ext == 'tar': archive_compression = None else: archive_compression = ext if archive_compression not in (None, 'gz', 'bz2', 'xz'): raise SaltInvocationError( 'Invalid compression method \'{0}\'.'.format( archive_compression)) if archive_compression == 'gz': cmd = 'zcat' elif archive_compression == 'bz2': cmd = 'bzcat' elif archive_compression == 'xz': cmd = 'xzcat' else: cmd = 'cat' for i in (cmd, 'tar'): if not salt.utils.path.which(i): raise CommandExecutionError( '\'{0}\' command not found on minion.'.format(i)) opts = ['{0} | tar -x -v -f -'] if __grains__['kernel'] == 'Linux': # assume GNU tar opts.append('--no-same-owner') if archive_files: for f in archive_files: opts.append(_cmd_quote(f)) res = __salt__['cmd.run_all']( ' '.join((cmd, ' '.join(opts).format(_cmd_quote(sfn)))), cwd=cwd, python_shell=True, ) if res['retcode'] != 0: raise CommandExecutionError(res['stdout'] + res['stderr']) ret = res['stdout'] return ret
def extracted(name, source, archive_format, archive_user=None, password=None, user=None, group=None, tar_options=None, source_hash=None, if_missing=None, keep=False, trim_output=False, skip_verify=False, source_hash_update=None): ''' .. versionadded:: 2014.1.0 State that make sure an archive is extracted in a directory. The downloaded archive is erased if successfully extracted. The archive is downloaded only if necessary. .. note:: If ``if_missing`` is not defined, this state will check for ``name`` instead. If ``name`` exists, it will assume the archive was previously extracted successfully and will not extract it again. Example, tar with flag for lmza compression: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - tar_options: J - archive_format: tar - if_missing: /opt/graylog2-server-0.9.6p1/ Example, tar with flag for verbose output: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - archive_format: tar - tar_options: v - user: root - group: root - if_missing: /opt/graylog2-server-0.9.6p1/ Example, tar with flag for lmza compression and update based if source_hash differs from what was previously extracted: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - source_hash_update: true - tar_options: J - archive_format: tar - if_missing: /opt/graylog2-server-0.9.6p1/ name Location where archive should be extracted password Password to use with password protected zip files. Currently only zip files with passwords are supported. .. versionadded:: 2016.3.0 source Archive source, same syntax as file.managed source argument. source_hash Hash of source file, or file with list of hash-to-file mappings. It uses the same syntax as the file.managed source_hash argument. source_hash_update Set this to ``True`` if archive should be extracted if source_hash has changed. This would extract regardless of the ``if_missing`` parameter. .. versionadded:: 2016.3.0 skip_verify:False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. .. versionadded:: 2016.3.4 archive_format ``tar``, ``zip`` or ``rar`` archive_user The user to own each extracted file. .. deprecated:: 2014.7.2 Replaced by ``user`` parameter user The user to own each extracted file. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. group The group to own each extracted file. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. if_missing If specified, this path will be checked, and if it exists then the archive will not be extracted. This can be helpful if the archive extracts all files into a subfolder. This path can be either a directory or a file, so this option can also be used to check for a semaphore file and conditionally skip extraction. .. versionchanged:: 2016.3.0 When used in combination with either ``user`` or ``group``, ownership will only be enforced when ``if_missing`` is a directory. tar_options If ``archive_format`` is set to ``tar``, this option can be used to specify a string of additional arguments to pass to the tar command. If ``archive_format`` is set to ``tar`` and this option is *not* used, then the minion will attempt to use Python's native tarfile_ support to extract it. Python's native tarfile_ support can only handle gzip and bzip2 compression, however. .. versionchanged:: 2015.8.11,2016.3.2 XZ-compressed archives no longer require ``J`` to manually be set in the ``tar_options``, they are now detected automatically and Salt will extract them using ``xz-utils``. This is a more platform-independent solution, as not all tar implementations support the ``J`` argument for extracting archives. .. note:: Main operators like -x, --extract, --get, -c and -f/--file **should not be used** here. Using this option means that the ``tar`` command will be used, which is less platform-independent, so keep this in mind when using this option; the options must be valid options for the ``tar`` implementation on the minion's OS. .. _tarfile: https://docs.python.org/2/library/tarfile.html keep Keep the archive in the minion's cache trim_output The number of files we should output on success before the rest are trimmed, if this is set to True then it will default to 100 .. versionadded:: 2016.3.0 ''' ret = {'name': name, 'result': None, 'changes': {}, 'comment': ''} valid_archives = ('tar', 'rar', 'zip') if archive_format not in valid_archives: ret['result'] = False ret['comment'] = '{0} is not supported, valid formats are: {1}'.format( archive_format, ','.join(valid_archives)) return ret # remove this whole block after formal deprecation. if archive_user is not None: warn_until( 'Carbon', 'Passing \'archive_user\' is deprecated.' 'Pass \'user\' instead.' ) if user is None: user = archive_user if not name.endswith('/'): name += '/' if __opts__['test']: source_match = source else: try: source_match = __salt__['file.source_list'](source, source_hash, __env__)[0] except CommandExecutionError as exc: ret['result'] = False ret['comment'] = exc.strerror return ret urlparsed_source = _urlparse(source_match) source_hash_name = urlparsed_source.path or urlparsed_source.netloc source_is_local = urlparsed_source.scheme in ('', 'file') if source_is_local: # Get rid of "file://" from start of source_match source_match = urlparsed_source.path if not os.path.isfile(source_match): ret['comment'] = 'Source file \'{0}\' does not exist'.format(source_match) return ret if if_missing is None: if_missing = name if source_hash and source_hash_update: if urlparsed_source.scheme != '': ret['result'] = False ret['comment'] = ( '\'source_hash_update\' is not yet implemented for a remote ' 'source_hash' ) return ret else: try: hash_type, hsum = source_hash.split('=') except ValueError: ret['result'] = False ret['comment'] = 'Invalid source_hash format' return ret source_file = '{0}.{1}'.format(os.path.basename(source), hash_type) hash_fname = os.path.join(__opts__['cachedir'], 'files', __env__, source_file) if _compare_checksum(hash_fname, name, hsum): ret['result'] = True ret['comment'] = 'Hash {0} has not changed'.format(hsum) return ret elif ( __salt__['file.directory_exists'](if_missing) or __salt__['file.file_exists'](if_missing) ): ret['result'] = True ret['comment'] = '{0} already exists'.format(if_missing) return ret log.debug('Input seem valid so far') if source_is_local: filename = source_match else: filename = os.path.join( __opts__['cachedir'], 'files', __env__, '{0}.{1}'.format(re.sub('[:/\\\\]', '_', if_missing), archive_format)) if not source_is_local and not os.path.isfile(filename): if __opts__['test']: ret['result'] = None ret['comment'] = \ '{0} {1} would be downloaded to cache'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match ) return ret log.debug('%s is not in cache, downloading it', source_match) file_result = __states__['file.managed'](filename, source=source_match, source_hash=source_hash, makedirs=True, skip_verify=skip_verify, source_hash_name=source_hash_name) log.debug('file.managed: {0}'.format(file_result)) # get value of first key try: file_result = file_result[next(six.iterkeys(file_result))] except AttributeError: pass try: if not file_result['result']: log.debug('failed to download {0}'.format(source)) return file_result except TypeError: if not file_result: log.debug('failed to download {0}'.format(source)) return file_result else: log.debug('Archive %s is already in cache', source) if __opts__['test']: ret['result'] = None ret['comment'] = '{0} {1} would be extracted to {2}'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match, name ) return ret created_destdir = False if __salt__['file.file_exists'](name.rstrip('/')): ret['result'] = False ret['comment'] = ('{0} exists and is not a directory' .format(name.rstrip('/'))) return ret elif not __salt__['file.directory_exists'](name): __salt__['file.makedirs'](name, user=archive_user) created_destdir = True log.debug('Extracting {0} to {1}'.format(filename, name)) if archive_format == 'zip': if password is None and salt.utils.which('unzip'): files = __salt__['archive.cmd_unzip'](filename, name, trim_output=trim_output) else: # https://bugs.python.org/issue15795 if password is not None: log.warning('Password supplied: using archive.unzip') if not salt.utils.which('unzip'): log.warning('Cannot find unzip command for archive.cmd_unzip:' ' using archive.unzip instead') files = __salt__['archive.unzip'](filename, name, trim_output=trim_output, password=password) elif archive_format == 'rar': files = __salt__['archive.unrar'](filename, name, trim_output=trim_output) else: if tar_options is None: try: with closing(tarfile.open(filename, 'r')) as tar: files = tar.getnames() tar.extractall(name) except tarfile.ReadError: if salt.utils.which('xz'): if __salt__['cmd.retcode'](['xz', '-l', filename], python_shell=False, ignore_retcode=True) == 0: # XZ-compressed data log.debug( 'Tar file is XZ-compressed, attempting ' 'decompression and extraction using xz-utils ' 'and the tar command' ) # Must use python_shell=True here because not all tar # implementations support the -J flag for decompressing # XZ-compressed data. We need to dump the decompressed # data to stdout and pipe it to tar for extraction. cmd = 'xz --decompress --stdout {0} | tar xvf -' results = __salt__['cmd.run_all']( cmd.format(_cmd_quote(filename)), cwd=name, python_shell=True) if results['retcode'] != 0: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] else: # Failed to open tar archive and it is not # XZ-compressed, gracefully fail the state if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive using Python\'s ' 'native tar file support. If archive is ' 'compressed using something other than gzip or ' 'bzip2, the \'tar_options\' parameter may be ' 'required to pass the correct options to the tar ' 'command in order to extract the archive.' ) return ret else: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive. If it is ' 'XZ-compressed, install xz-utils to attempt ' 'extraction.' ) return ret else: try: tar_opts = tar_options.split(' ') except AttributeError: tar_opts = str(tar_options).split(' ') tar_cmd = ['tar'] tar_shortopts = 'x' tar_longopts = [] for position, opt in enumerate(tar_opts): if opt.startswith('-'): tar_longopts.append(opt) else: if position > 0: tar_longopts.append(opt) else: append_opt = opt append_opt = append_opt.replace('x', '').replace('f', '') tar_shortopts = tar_shortopts + append_opt tar_cmd.append(tar_shortopts) tar_cmd.extend(tar_longopts) tar_cmd.extend(['-f', filename]) results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False) if results['retcode'] != 0: ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] if not files: files = 'no tar output so far' # Recursively set user and group ownership of files after extraction. # Note: We do this here because we might not have access to the cachedir. if user or group: if os.path.isdir(if_missing): recurse = [] if user: recurse.append('user') if group: recurse.append('group') dir_result = __states__['file.directory'](if_missing, user=user, group=group, recurse=recurse) log.debug('file.directory: %s', dir_result) elif os.path.isfile(if_missing): log.debug('if_missing (%s) is a file, not enforcing user/group ' 'permissions', if_missing) if len(files) > 0: ret['result'] = True ret['changes']['directories_created'] = [name] ret['changes']['extracted_files'] = files ret['comment'] = '{0} extracted to {1}'.format(source_match, name) if not source_is_local and not keep: os.unlink(filename) if source_hash and source_hash_update: _update_checksum(hash_fname, name, hash[1]) else: __salt__['file.remove'](if_missing) ret['result'] = False ret['comment'] = 'Can\'t extract content of {0}'.format(source_match) return ret
def extracted(name, source, archive_format, password=None, user=None, group=None, tar_options=None, zip_options=None, source_hash=None, if_missing=None, keep=False, trim_output=False, skip_verify=False, source_hash_update=None, use_cmd_unzip=False, **kwargs): ''' .. versionadded:: 2014.1.0 State that make sure an archive is extracted in a directory. The downloaded archive is erased if successfully extracted. The archive is downloaded only if necessary. .. note:: If ``if_missing`` is not defined, this state will check for ``name`` instead. If ``name`` exists, it will assume the archive was previously extracted successfully and will not extract it again. Example, tar with flag for lmza compression: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - tar_options: J - archive_format: tar - if_missing: /opt/graylog2-server-0.9.6p1/ Example, tar with flag for verbose output: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - archive_format: tar - tar_options: v - user: root - group: root - if_missing: /opt/graylog2-server-0.9.6p1/ Example, tar with flag for lmza compression and update based if source_hash differs from what was previously extracted: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - source_hash_update: true - tar_options: J - archive_format: tar - if_missing: /opt/graylog2-server-0.9.6p1/ name Location where archive should be extracted password Password to use with password protected zip files. Currently only zip files with passwords are supported. .. versionadded:: 2016.3.0 source Archive source, same syntax as file.managed source argument. source_hash Hash of source file, or file with list of hash-to-file mappings. It uses the same syntax as the file.managed source_hash argument. source_hash_update Set this to ``True`` if archive should be extracted if source_hash has changed. This would extract regardless of the ``if_missing`` parameter. .. versionadded:: 2016.3.0 skip_verify:False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. .. versionadded:: 2016.3.4 archive_format ``tar``, ``zip`` or ``rar`` user The user to own each extracted file. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. group The group to own each extracted file. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. if_missing If specified, this path will be checked, and if it exists then the archive will not be extracted. This can be helpful if the archive extracts all files into a subfolder. This path can be either a directory or a file, so this option can also be used to check for a semaphore file and conditionally skip extraction. .. versionchanged:: 2016.3.0 When used in combination with either ``user`` or ``group``, ownership will only be enforced when ``if_missing`` is a directory. tar_options If ``archive_format`` is set to ``tar``, this option can be used to specify a string of additional arguments to pass to the tar command. If ``archive_format`` is set to ``tar`` and this option is *not* used, then the minion will attempt to use Python's native tarfile_ support to extract it. Python's native tarfile_ support can only handle gzip and bzip2 compression, however. .. versionchanged:: 2015.8.11,2016.3.2 XZ-compressed archives no longer require ``J`` to manually be set in the ``tar_options``, they are now detected automatically and Salt will extract them using ``xz-utils``. This is a more platform-independent solution, as not all tar implementations support the ``J`` argument for extracting archives. .. note:: Main operators like -x, --extract, --get, -c and -f/--file **should not be used** here. Using this option means that the ``tar`` command will be used, which is less platform-independent, so keep this in mind when using this option; the options must be valid options for the ``tar`` implementation on the minion's OS. .. _tarfile: https://docs.python.org/2/library/tarfile.html zip_options Optional when using ``zip`` archives, ignored when usign other archives files. This is mostly used to overwrite exsiting files with ``o``. This options are only used when ``unzip`` binary is used. .. versionadded:: 2016.3.1 keep Keep the archive in the minion's cache trim_output The number of files we should output on success before the rest are trimmed, if this is set to True then it will default to 100 .. versionadded:: 2016.3.0 use_cmd_unzip When archive_format is zip, setting this flag to True will use the archive.cmd_unzip module function .. versionadded:: Carbon kwargs kwargs to pass to the archive.unzip or archive.unrar function .. versionadded:: Carbon ''' ret = {'name': name, 'result': None, 'changes': {}, 'comment': ''} valid_archives = ('tar', 'rar', 'zip') if archive_format not in valid_archives: ret['result'] = False ret['comment'] = '{0} is not supported, valid formats are: {1}'.format( archive_format, ','.join(valid_archives)) return ret if not name.endswith('/'): name += '/' if __opts__['test']: source_match = source else: try: source_match = __salt__['file.source_list'](source, source_hash, __env__)[0] except CommandExecutionError as exc: ret['result'] = False ret['comment'] = exc.strerror return ret urlparsed_source = _urlparse(source_match) source_hash_name = urlparsed_source.path or urlparsed_source.netloc if if_missing is None: if_missing = name if source_hash and source_hash_update: if urlparsed_source.scheme != '': ret['result'] = False ret['comment'] = ( '\'source_hash_update\' is not yet implemented for a remote ' 'source_hash' ) return ret else: try: hash_type, hsum = source_hash.split('=') except ValueError: ret['result'] = False ret['comment'] = 'Invalid source_hash format' return ret source_file = '{0}.{1}'.format(os.path.basename(source), hash_type) hash_fname = os.path.join(__opts__['cachedir'], 'files', __env__, source_file) if _compare_checksum(hash_fname, name, hsum): ret['result'] = True ret['comment'] = 'Hash {0} has not changed'.format(hsum) return ret elif ( __salt__['file.directory_exists'](if_missing) or __salt__['file.file_exists'](if_missing) ): ret['result'] = True ret['comment'] = '{0} already exists'.format(if_missing) return ret log.debug('Input seem valid so far') filename = os.path.join(__opts__['cachedir'], 'files', __env__, '{0}.{1}'.format(re.sub('[:/\\\\]', '_', if_missing), archive_format)) if not os.path.exists(filename): if __opts__['test']: ret['result'] = None ret['comment'] = \ '{0} {1} would be downloaded to cache'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match ) return ret log.debug('%s is not in cache, downloading it', source_match) file_result = __salt__['state.single']('file.managed', filename, source=source_match, source_hash=source_hash, makedirs=True, skip_verify=skip_verify, saltenv=__env__, source_hash_name=source_hash_name) log.debug('file.managed: {0}'.format(file_result)) # get value of first key try: file_result = file_result[next(six.iterkeys(file_result))] except AttributeError: pass try: if not file_result['result']: log.debug('failed to download {0}'.format(source)) return file_result except TypeError: if not file_result: log.debug('failed to download {0}'.format(source)) return file_result else: log.debug('Archive %s is already in cache', source) if __opts__['test']: ret['result'] = None ret['comment'] = '{0} {1} would be extracted to {2}'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match, name ) return ret created_destdir = False if __salt__['file.file_exists'](name.rstrip('/')): ret['result'] = False ret['comment'] = ('{0} exists and is not a directory' .format(name.rstrip('/'))) return ret elif not __salt__['file.directory_exists'](name): __salt__['file.makedirs'](name, user=user, group=group) created_destdir = True log.debug('Extracting {0} to {1}'.format(filename, name)) if archive_format == 'zip': if use_cmd_unzip: files = __salt__['archive.cmd_unzip'](filename, name, options=zip_options, trim_output=trim_output, **kwargs) else: files = __salt__['archive.unzip'](filename, name, options=zip_options, trim_output=trim_output, password=password, **kwargs) elif archive_format == 'rar': files = __salt__['archive.unrar'](filename, name, trim_output=trim_output, **kwargs) else: if tar_options is None: try: with closing(tarfile.open(filename, 'r')) as tar: files = tar.getnames() tar.extractall(name) except tarfile.ReadError: if salt.utils.which('xz'): if __salt__['cmd.retcode'](['xz', '-l', filename], python_shell=False, ignore_retcode=True) == 0: # XZ-compressed data log.debug( 'Tar file is XZ-compressed, attempting ' 'decompression and extraction using xz-utils ' 'and the tar command' ) # Must use python_shell=True here because not all tar # implementations support the -J flag for decompressing # XZ-compressed data. We need to dump the decompressed # data to stdout and pipe it to tar for extraction. cmd = 'xz --decompress --stdout {0} | tar xvf -' results = __salt__['cmd.run_all']( cmd.format(_cmd_quote(filename)), cwd=name, python_shell=True) if results['retcode'] != 0: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] else: # Failed to open tar archive and it is not # XZ-compressed, gracefully fail the state if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive using Python\'s ' 'native tar file support. If archive is ' 'compressed using something other than gzip or ' 'bzip2, the \'tar_options\' parameter may be ' 'required to pass the correct options to the tar ' 'command in order to extract the archive.' ) return ret else: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive. If it is ' 'XZ-compressed, install xz-utils to attempt ' 'extraction.' ) return ret else: try: tar_opts = tar_options.split(' ') except AttributeError: tar_opts = str(tar_options).split(' ') tar_cmd = ['tar'] tar_shortopts = 'x' tar_longopts = [] for position, opt in enumerate(tar_opts): if opt.startswith('-'): tar_longopts.append(opt) else: if position > 0: tar_longopts.append(opt) else: append_opt = opt append_opt = append_opt.replace('x', '').replace('f', '') tar_shortopts = tar_shortopts + append_opt if __grains__['os'] == 'OpenBSD': tar_shortopts = '-' + tar_shortopts tar_cmd.append(tar_shortopts) tar_cmd.extend(tar_longopts) tar_cmd.extend(['-f', filename]) results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False) if results['retcode'] != 0: ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] if not files: files = 'no tar output so far' # Recursively set user and group ownership of files after extraction. # Note: We do this here because we might not have access to the cachedir. if user or group: if os.path.isdir(if_missing): recurse = [] if user: recurse.append('user') if group: recurse.append('group') dir_result = __salt__['state.single']('file.directory', if_missing, user=user, group=group, recurse=recurse) log.debug('file.directory: %s', dir_result) elif os.path.isfile(if_missing): log.debug('if_missing (%s) is a file, not enforcing user/group ' 'permissions', if_missing) if len(files) > 0: ret['result'] = True ret['changes']['directories_created'] = [name] ret['changes']['extracted_files'] = files ret['comment'] = '{0} extracted to {1}'.format(source_match, name) if not keep: os.unlink(filename) if source_hash and source_hash_update: _update_checksum(hash_fname, name, hash[1]) else: __salt__['file.remove'](if_missing) ret['result'] = False ret['comment'] = 'Can\'t extract content of {0}'.format(source_match) return ret
def at(*args, **kwargs): # pylint: disable=C0103 ''' Add a job to the queue. The 'timespec' follows the format documented in the at(1) manpage. CLI Example: .. code-block:: bash salt '*' at.at <timespec> <cmd> [tag=<tag>] [runas=<user>] salt '*' at.at 12:05am '/sbin/reboot' tag=reboot salt '*' at.at '3:05am +3 days' 'bin/myscript' tag=nightly runas=jim ''' if len(args) < 2: return {'jobs': []} # Shim to produce output similar to what __virtual__() should do # but __salt__ isn't available in __virtual__() binary = salt.utils.which('at') if not binary: return '\'at.at\' is not available.' if __grains__['os_family'] == 'RedHat': echo_cmd = 'echo -e' else: echo_cmd = 'echo' if 'tag' in kwargs: cmd = '{4} "### SALT: {0}\n{1}" | {2} {3}'.format( _cmd_quote(kwargs['tag']), _cmd_quote(' '.join(args[1:])), binary, _cmd_quote(args[0]), echo_cmd) else: cmd = '{3} "{1}" | {2} {0}'.format(_cmd_quote(args[0]), _cmd_quote(' '.join(args[1:])), binary, echo_cmd) # Can't use _cmd here since we need to prepend 'echo_cmd' if 'runas' in kwargs: output = __salt__['cmd.run']('{0}'.format(cmd), python_shell=True, runas=kwargs['runas']) else: output = __salt__['cmd.run']('{0}'.format(cmd), python_shell=True) if output is None: return '\'at.at\' is not available.' if output.endswith('Garbled time'): return {'jobs': [], 'error': 'invalid timespec'} if output.startswith('warning: commands'): output = output.splitlines()[1] if output.startswith('commands will be executed'): output = output.splitlines()[1] output = output.split()[1] if __grains__['os'] in BSD: return atq(str(output)) else: return atq(int(output))
def extracted(name, source, archive_format, archive_user=None, user=None, group=None, tar_options=None, source_hash=None, if_missing=None, keep=False): ''' .. versionadded:: 2014.1.0 State that make sure an archive is extracted in a directory. The downloaded archive is erased if successfully extracted. The archive is downloaded only if necessary. .. note:: If ``if_missing`` is not defined, this state will check for ``name`` instead. If ``name`` exists, it will assume the archive was previously extracted successfully and will not extract it again. Example, tar with flag for lmza compression: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - tar_options: J - archive_format: tar - if_missing: /opt/graylog2-server-0.9.6p1/ Example, tar with flag for verbose output: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - archive_format: tar - tar_options: v - user: root - group: root - if_missing: /opt/graylog2-server-0.9.6p1/ name Location where archive should be extracted source Archive source, same syntax as file.managed source argument. source_hash Hash of source file, or file with list of hash-to-file mappings. It uses the same syntax as the file.managed source_hash argument. archive_format tar, zip or rar archive_user The user to own each extracted file. .. deprecated:: 2014.7.2 Replaced by ``user`` parameter user The user to own each extracted file. .. versionadded:: 2015.8.0 group The group to own each extracted file. .. versionadded:: 2015.8.0 if_missing If specified, this path will be checked, and if it exists then the archive will not be extracted. This can be helpful if the archive extracts all files into a subfolder. This path can be either a directory or a file, so this option can also be used to check for a semaphore file and conditionally skip extraction. tar_options If ``archive_format`` is set to ``tar``, this option can be used to specify a string of additional arguments to pass to the tar command. If ``archive_format`` is set to ``tar`` and this option is *not* used, then the minion will attempt to use Python's native tarfile_ support to extract it. Python's native tarfile_ support can only handle gzip and bzip2 compression, however. .. versionchanged:: 2015.8.11,2016.3.2 XZ-compressed archives no longer require ``J`` to manually be set in the ``tar_options``, they are now detected automatically and Salt will extract them using ``xz-utils``. This is a more platform-independent solution, as not all tar implementations support the ``J`` argument for extracting archives. .. note:: Main operators like -x, --extract, --get, -c and -f/--file **should not be used** here. Using this option means that the ``tar`` command will be used, which is less platform-independent, so keep this in mind when using this option; the options must be valid options for the ``tar`` implementation on the minion's OS. .. _tarfile: https://docs.python.org/2/library/tarfile.html keep Keep the archive in the minion's cache ''' ret = {'name': name, 'result': None, 'changes': {}, 'comment': ''} valid_archives = ('tar', 'rar', 'zip') if archive_format not in valid_archives: ret['result'] = False ret['comment'] = '{0} is not supported, valid formats are: {1}'.format( archive_format, ','.join(valid_archives)) return ret # remove this whole block after formal deprecation. if archive_user is not None: warn_until( 'Boron', 'Passing \'archive_user\' is deprecated.' 'Pass \'user\' instead.' ) if user is None: user = archive_user if not name.endswith('/'): name += '/' if if_missing is None: if_missing = name if ( __salt__['file.directory_exists'](if_missing) or __salt__['file.file_exists'](if_missing) ): ret['result'] = True ret['comment'] = '{0} already exists'.format(if_missing) return ret log.debug('Input seem valid so far') filename = os.path.join(__opts__['cachedir'], 'files', __env__, '{0}.{1}'.format(re.sub('[:/\\\\]', '_', if_missing), archive_format)) if __opts__['test']: source_match = source else: try: source_match = __salt__['file.source_list'](source, source_hash, __env__)[0] except CommandExecutionError as exc: ret['result'] = False ret['comment'] = exc.strerror return ret if not os.path.exists(filename): if __opts__['test']: ret['result'] = None ret['comment'] = \ '{0} {1} would be downloaded to cache'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match ) return ret log.debug('%s is not in cache, downloading it', source_match) file_result = __salt__['state.single']('file.managed', filename, source=source, source_hash=source_hash, makedirs=True, saltenv=__env__) log.debug('file.managed: {0}'.format(file_result)) # get value of first key try: file_result = file_result[next(six.iterkeys(file_result))] except AttributeError: pass try: if not file_result['result']: log.debug('failed to download {0}'.format(source)) return file_result except TypeError: if not file_result: log.debug('failed to download {0}'.format(source)) return file_result else: log.debug('Archive %s is already in cache', source) if __opts__['test']: ret['result'] = None ret['comment'] = '{0} {1} would be extracted to {2}'.format( 'One of' if not isinstance(source_match, six.string_types) else 'Archive', source_match, name ) return ret created_destdir = False if __salt__['file.file_exists'](name.rstrip('/')): ret['result'] = False ret['comment'] = ('{0} exists and is not a directory' .format(name.rstrip('/'))) return ret elif not __salt__['file.directory_exists'](name): __salt__['file.makedirs'](name, user=archive_user) created_destdir = True log.debug('Extracting {0} to {1}'.format(filename, name)) if archive_format == 'zip': files = __salt__['archive.unzip'](filename, name) elif archive_format == 'rar': files = __salt__['archive.unrar'](filename, name) else: if tar_options is None: try: with closing(tarfile.open(filename, 'r')) as tar: files = tar.getnames() tar.extractall(name) except tarfile.ReadError: if salt.utils.which('xz'): if __salt__['cmd.retcode'](['xz', '-l', filename], python_shell=False, ignore_retcode=True) == 0: # XZ-compressed data log.debug( 'Tar file is XZ-compressed, attempting ' 'decompression and extraction using xz-utils ' 'and the tar command' ) # Must use python_shell=True here because not all tar # implementations support the -J flag for decompressing # XZ-compressed data. We need to dump the decompressed # data to stdout and pipe it to tar for extraction. cmd = 'xz --decompress --stdout {0} | tar xvf -' results = __salt__['cmd.run_all']( cmd.format(_cmd_quote(filename)), cwd=name, python_shell=True) if results['retcode'] != 0: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] else: # Failed to open tar archive and it is not # XZ-compressed, gracefully fail the state if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive using Python\'s ' 'native tar file support. If archive is ' 'compressed using something other than gzip or ' 'bzip2, the \'tar_options\' parameter may be ' 'required to pass the correct options to the tar ' 'command in order to extract the archive.' ) return ret else: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive. If it is ' 'XZ-compressed, install xz-utils to attempt ' 'extraction.' ) return ret else: try: tar_opts = tar_options.split(' ') except AttributeError: tar_opts = str(tar_options).split(' ') tar_cmd = ['tar'] tar_shortopts = 'x' tar_longopts = [] for position, opt in enumerate(tar_opts): if opt.startswith('-'): tar_longopts.append(opt) else: if position > 0: tar_longopts.append(opt) else: append_opt = opt append_opt = append_opt.replace('x', '').replace('f', '') tar_shortopts = tar_shortopts + append_opt tar_cmd.append(tar_shortopts) tar_cmd.extend(tar_longopts) tar_cmd.extend(['-f', filename]) results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False) if results['retcode'] != 0: ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] if not files: files = 'no tar output so far' # Recursively set user and group ownership of files after extraction. # Note: We do this here because we might not have access to the cachedir. if user or group: dir_result = __salt__['state.single']('file.directory', name, user=user, group=group, recurse=['user', 'group']) log.debug('file.directory: {0}'.format(dir_result)) if len(files) > 0: ret['result'] = True ret['changes']['directories_created'] = [name] ret['changes']['extracted_files'] = files ret['comment'] = '{0} extracted to {1}'.format(source_match, name) if not keep: os.unlink(filename) else: __salt__['file.remove'](if_missing) ret['result'] = False ret['comment'] = 'Can\'t extract content of {0}'.format(source_match) return ret
def extracted(name, source, source_hash=None, source_hash_update=False, skip_verify=False, password=None, options=None, list_options=None, force=False, user=None, group=None, if_missing=None, keep=False, trim_output=False, use_cmd_unzip=None, extract_perms=True, enforce_toplevel=True, enforce_ownership_on=None, archive_format=None, **kwargs): ''' .. versionadded:: 2014.1.0 .. versionchanged:: 2016.11.0 This state has been rewritten. Some arguments are new to this release and will not be available in the 2016.3 release cycle (and earlier). Additionally, the **ZIP Archive Handling** section below applies specifically to the 2016.11.0 release (and newer). Ensure that an archive is extracted to a specific directory. .. important:: **ZIP Archive Handling** Salt has two different functions for extracting ZIP archives: 1. :py:func:`archive.unzip <salt.modules.archive.unzip>`, which uses Python's zipfile_ module to extract ZIP files. 2. :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>`, which uses the ``unzip`` CLI command to extract ZIP files. Salt will prefer the use of :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>` when CLI options are specified (via the ``options`` argument), and will otherwise prefer the :py:func:`archive.unzip <salt.modules.archive.unzip>` function. Use of :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>` can be forced however by setting the ``use_cmd_unzip`` argument to ``True``. By contrast, setting this argument to ``False`` will force usage of :py:func:`archive.unzip <salt.modules.archive.unzip>`. For example: .. code-block:: yaml /var/www: archive.extracted: - source: salt://foo/bar/myapp.zip - use_cmd_unzip: True When ``use_cmd_unzip`` is omitted, Salt will choose which extraction function to use based on the source archive and the arguments passed to the state. When in doubt, simply do not set this argument; it is provided as a means of overriding the logic Salt uses to decide which function to use. There are differences in the features available in both extraction functions. These are detailed below. - *Command-line options* (only supported by :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>`) - When the ``options`` argument is used, :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>` is the only function that can be used to extract the archive. Therefore, if ``use_cmd_unzip`` is specified and set to ``False``, and ``options`` is also set, the state will not proceed. - *Password-protected ZIP Archives* (only supported by :py:func:`archive.unzip <salt.modules.archive.unzip>`) - :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>` is not be permitted to extract password-protected ZIP archives, as attempting to do so will cause the unzip command to block on user input. The :py:func:`archive.is_encrypted <salt.modules.archive.unzip>` function will be used to determine if the archive is password-protected. If it is, then the ``password`` argument will be required for the state to proceed. If ``use_cmd_unzip`` is specified and set to ``True``, then the state will not proceed. - *Permissions* - Due to an `upstream bug in Python`_, permissions are not preserved when the zipfile_ module is used to extract an archive. As of the 2016.11.0 release, :py:func:`archive.unzip <salt.modules.archive.unzip>` (as well as this state) has an ``extract_perms`` argument which, when set to ``True`` (the default), will attempt to match the permissions of the extracted files/directories to those defined within the archive. To disable this functionality and have the state not attempt to preserve the permissions from the ZIP archive, set ``extract_perms`` to ``False``: .. code-block:: yaml /var/www: archive.extracted: - source: salt://foo/bar/myapp.zip - extract_perms: False .. _`upstream bug in Python`: https://bugs.python.org/issue15795 name Directory into which the archive should be extracted source Archive to be extracted .. note:: This argument uses the same syntax as its counterpart in the :py:func:`file.managed <salt.states.file.managed>` state. source_hash Hash of source file, or file with list of hash-to-file mappings .. note:: This argument uses the same syntax as its counterpart in the :py:func:`file.managed <salt.states.file.managed>` state. source_hash_update Set this to ``True`` if archive should be extracted if source_hash has changed. This would extract regardless of the ``if_missing`` parameter. .. versionadded:: 2016.3.0 skip_verify : False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. .. versionadded:: 2016.3.4 password **For ZIP archives only.** Password used for extraction. .. versionadded:: 2016.3.0 options **For tar and zip archives only.** This option can be used to specify a string of additional arguments to pass to the tar/zip command. If this argument is not used, then the minion will attempt to use Python's native tarfile_/zipfile_ support to extract it. For zip archives, this argument is mostly used to overwrite exsiting files with ``o``. Using this argument means that the ``tar`` or ``unzip`` command will be used, which is less platform-independent, so keep this in mind when using this option; the CLI options must be valid options for the ``tar``/``unzip`` implementation on the minion's OS. .. versionadded:: 2016.11.0 The ``tar_options`` and ``zip_options`` parameters have been deprecated in favor of a single argument name. .. versionchanged:: 2015.8.11,2016.3.2 XZ-compressed tar archives no longer require ``J`` to manually be set in the ``options``, they are now detected automatically and decompressed using xz-utils_ and extracted using ``tar xvf``. This is a more platform-independent solution, as not all tar implementations support the ``J`` argument for extracting archives. .. note:: For tar archives, main operators like ``-x``, ``--extract``, ``--get``, ``-c`` and ``-f``/``--file`` should *not* be used here. tar_options .. deprecated:: 2016.11.0 Use ``options`` instead. zip_options .. versionadded:: 2016.3.1 .. deprecated:: 2016.11.0 Use ``options`` instead. list_options **For tar archives only.** This state uses :py:func:`archive.list <salt.modules.archive.list_>` to discover the contents of the source archive so that it knows which file paths should exist on the minion if the archive has already been extracted. For the vast majority of tar archives, :py:func:`archive.list <salt.modules.archive.list_>` "just works". Archives compressed using gzip, bzip2, and xz/lzma (with the help of xz-utils_) are supported automatically. However, for archives compressed using other compression types, CLI options must be passed to :py:func:`archive.list <salt.modules.archive.list_>`. This argument will be passed through to :py:func:`archive.list <salt.modules.archive.list_>` as its ``options`` argument, to allow it to successfully list the archive's contents. For the vast majority of archives, this argument should not need to be used, it should only be needed in cases where the state fails with an error stating that the archive's contents could not be listed. .. versionadded:: 2016.11.0 force : False If a path that should be occupied by a file in the extracted result is instead a directory (or vice-versa), the state will fail. Set this argument to ``True`` to force these paths to be removed in order to allow the archive to be extracted. .. warning:: Use this option *very* carefully. .. versionadded:: 2016.11.0 user The user to own each extracted file. Not available on Windows. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. .. versionchanged:: 2016.11.0 Ownership will be enforced only on the file/directory paths found by running :py:func:`archive.list <salt.modules.archive.list_>` on the source archive. An alternative root directory on which to enforce ownership can be specified using the ``enforce_ownership_on`` argument. group The group to own each extracted file. Not available on Windows. .. versionadded:: 2015.8.0 .. versionchanged:: 2016.3.0 When used in combination with ``if_missing``, ownership will only be enforced if ``if_missing`` is a directory. .. versionchanged:: 2016.11.0 Ownership will be enforced only on the file/directory paths found by running :py:func:`archive.list <salt.modules.archive.list_>` on the source archive. An alternative root directory on which to enforce ownership can be specified using the ``enforce_ownership_on`` argument. if_missing If specified, this path will be checked, and if it exists then the archive will not be extracted. This path can be either a directory or a file, so this option can also be used to check for a semaphore file and conditionally skip extraction. .. versionchanged:: 2016.3.0 When used in combination with either ``user`` or ``group``, ownership will only be enforced when ``if_missing`` is a directory. .. versionchanged:: 2016.11.0 Ownership enforcement is no longer tied to this argument, it is simply checked for existence and extraction will be skipped if if is present. keep : False For ``source`` archives not local to the minion (i.e. from the Salt fileserver or a remote source such as ``http(s)`` or ``ftp``), Salt will need to download the archive to the minion cache before they can be extracted. After extraction, these source archives will be removed unless this argument is set to ``True``. trim_output : False Useful for archives with many files in them. This can either be set to ``True`` (in which case only the first 100 files extracted will be in the state results), or it can be set to an integer for more exact control over the max number of files to include in the state results. .. versionadded:: 2016.3.0 use_cmd_unzip : False Set to ``True`` for zip files to force usage of the :py:func:`archive.cmd_unzip <salt.modules.archive.cmd_unzip>` function to extract. .. versionadded:: 2016.11.0 extract_perms : True **For ZIP archives only.** When using :py:func:`archive.unzip <salt.modules.archive.unzip>` to extract ZIP archives, Salt works around an `upstream bug in Python`_ to set the permissions on extracted files/directories to match those encoded into the ZIP archive. Set this argument to ``False`` to skip this workaround. .. versionadded:: 2016.11.0 enforce_toplevel : True This option will enforce a single directory at the top level of the source archive, to prevent extracting a 'tar-bomb'. Set this argument to ``False`` to allow archives with files (or multiple directories) at the top level to be extracted. .. versionadded:: 2016.11.0 enforce_ownership_on When ``user`` or ``group`` is specified, Salt will default to enforcing permissions on the file/directory paths detected by running :py:func:`archive.list <salt.modules.archive.list_>` on the source archive. Use this argument to specify an alternate directory on which ownership should be enforced. .. note:: This path must be within the path specified by the ``name`` argument. .. versionadded:: 2016.11.0 archive_format One of ``tar``, ``zip``, or ``rar``. .. versionchanged:: 2016.11.0 If omitted, the archive format will be guessed based on the value of the ``source`` argument. .. _tarfile: https://docs.python.org/2/library/tarfile.html .. _zipfile: https://docs.python.org/2/library/zipfile.html .. _xz-utils: http://tukaani.org/xz/ **Examples** 1. tar with lmza (i.e. xz) compression: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 2. tar archive with flag for verbose output, and enforcement of user/group ownership: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.gz - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - tar_options: v - user: foo - group: foo 3. tar archive, with ``source_hash_update`` set to ``True`` to prevent state from attempting extraction unless the ``source_hash`` differs from the previous time the archive was extracted: .. code-block:: yaml graylog2-server: archive.extracted: - name: /opt/ - source: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.9.6p1.tar.lzma - source_hash: md5=499ae16dcae71eeb7c3a30c75ea7a1a6 - source_hash_update: True ''' ret = {'name': name, 'result': False, 'changes': {}, 'comment': ''} # Remove pub kwargs as they're irrelevant here. kwargs = salt.utils.clean_kwargs(**kwargs) if not _path_is_abs(name): ret['comment'] = '{0} is not an absolute path'.format(name) return ret else: if name is None: # Only way this happens is if some doofus specifies "- name: None" # in their SLS file. Prevent tracebacks by failing gracefully. ret['comment'] = 'None is not a valid directory path' return ret # os.path.isfile() returns False when there is a trailing slash, hence # our need for first stripping the slash and then adding it back later. # Otherwise, we can't properly check if the extraction location both a) # exists and b) is a file. # # >>> os.path.isfile('/tmp/foo.txt') # True # >>> os.path.isfile('/tmp/foo.txt/') # False name = name.rstrip('/') if os.path.isfile(name): ret['comment'] = '{0} exists and is not a directory'.format(name) return ret # Add back the slash so that file.makedirs properly creates the # destdir if it needs to be created. file.makedirs expects a trailing # slash in the directory path. name += '/' if not _path_is_abs(if_missing): ret['comment'] = 'Value for \'if_missing\' is not an absolute path' return ret if not _path_is_abs(enforce_ownership_on): ret['comment'] = ('Value for \'enforce_ownership_on\' is not an ' 'absolute path') return ret else: if enforce_ownership_on is not None: try: not_rel = os.path.relpath(enforce_ownership_on, name).startswith('..' + os.sep) except Exception: # A ValueError is raised on Windows when the paths passed to # os.path.relpath are not on the same drive letter. Using a # generic Exception here to keep other possible exception types # from making this state blow up with a traceback. not_rel = True if not_rel: ret['comment'] = ( 'Value for \'enforce_ownership_on\' must be within {0}'. format(name)) return ret if user or group: if salt.utils.is_windows(): ret['comment'] = \ 'User/group ownership cannot be enforced on Windows minions' return ret if user: uid = __salt__['file.user_to_uid'](user) if not uid: ret['comment'] = 'User {0} does not exist'.format(user) return ret else: uid = -1 if group: gid = __salt__['file.group_to_gid'](group) if not gid: ret['comment'] = 'Group {0} does not exist'.format(group) return ret else: gid = -1 else: # We should never hit the ownership enforcement code unless user or # group was specified, but just in case, set uid/gid to -1 to make the # os.chown() a no-op and avoid a NameError. uid = gid = -1 if source_hash_update and not source_hash: ret.setdefault('warnings', []).append( 'The \'source_hash_update\' argument is ignored when ' '\'source_hash\' is not also specified.') try: source_match = __salt__['file.source_list'](source, source_hash, __env__)[0] except CommandExecutionError as exc: ret['result'] = False ret['comment'] = exc.strerror return ret urlparsed_source = _urlparse(source_match) source_hash_name = urlparsed_source.path or urlparsed_source.netloc valid_archive_formats = ('tar', 'rar', 'zip') if not archive_format: archive_format = salt.utils.files.guess_archive_type(source_hash_name) if archive_format is None: ret['comment'] = ( 'Could not guess archive_format from the value of the ' '\'source\' argument. Please set this archive_format to one ' 'of the following: {0}'.format( ', '.join(valid_archive_formats))) return ret try: archive_format = archive_format.lower() except AttributeError: pass if archive_format not in valid_archive_formats: ret['comment'] = ( 'Invalid archive_format \'{0}\'. Either set it to a supported ' 'value ({1}) or remove this argument and the archive format will ' 'be guesseed based on file extension.'.format( archive_format, ', '.join(valid_archive_formats), )) return ret tar_options = kwargs.pop('tar_options', None) zip_options = kwargs.pop('zip_options', None) if tar_options: msg = ('The \'tar_options\' argument has been deprecated, please use ' '\'options\' instead.') salt.utils.warn_until('Oxygen', msg) ret.setdefault('warnings', []).append(msg) options = tar_options elif zip_options: msg = ('The \'zip_options\' argument has been deprecated, please use ' '\'options\' instead.') salt.utils.warn_until('Oxygen', msg) ret.setdefault('warnings', []).append(msg) options = zip_options if archive_format == 'zip': if options: if use_cmd_unzip is None: log.info( 'Presence of CLI options in archive.extracted state for ' '\'%s\' implies that use_cmd_unzip is set to True.', name) use_cmd_unzip = True elif not use_cmd_unzip: # use_cmd_unzip explicitly disabled ret['comment'] = ( '\'use_cmd_unzip\' cannot be set to False if CLI options ' 'are being specified (via the \'options\' argument). ' 'Either remove \'use_cmd_unzip\', or set it to True.') return ret if password: if use_cmd_unzip is None: log.info( 'Presence of a password in archive.extracted state for ' '\'%s\' implies that use_cmd_unzip is set to False.', name) use_cmd_unzip = False elif use_cmd_unzip: ret.setdefault('warnings', []).append( 'Using a password in combination with setting ' '\'use_cmd_unzip\' to True is considered insecure. It is ' 'recommended to remove the \'use_cmd_unzip\' argument (or ' 'set it to False) and allow Salt to extract the archive ' 'using Python\'s built-in ZIP file support.') else: if password: ret['comment'] = \ 'The \'password\' argument is only supported for zip archives' return ret supports_options = ('tar', 'zip') if options and archive_format not in supports_options: ret['comment'] = ( 'The \'options\' argument is only compatible with the following ' 'archive formats: {0}'.format(', '.join(supports_options))) return ret if trim_output and not isinstance(trim_output, (bool, six.integer_types)): try: # Try to handle cases where trim_output was passed as a # string-ified integer. trim_output = int(trim_output) except TypeError: ret['comment'] = ( 'Invalid value for trim_output, must be True/False or an ' 'integer') return ret cached_source = os.path.join( __opts__['cachedir'], 'files', __env__, re.sub(r'[:/\\]', '_', source_hash_name), ) if os.path.isdir(cached_source): # Prevent a traceback from attempting to read from a directory path salt.utils.rm_rf(cached_source) if source_hash: try: source_sum = __salt__['file.get_source_sum'](source_hash_name, source_hash, __env__) except CommandExecutionError as exc: ret['comment'] = exc.strerror return ret if source_hash_update: if _compare_checksum(cached_source, source_sum): ret['result'] = True ret['comment'] = \ 'Hash {0} has not changed'.format(source_sum['hsum']) return ret else: source_sum = {} if not os.path.isfile(cached_source): if __opts__['test']: ret['result'] = None ret['comment'] = \ 'Archive {0} would be downloaded to cache'.format(source_match) return ret log.debug('%s is not in cache, downloading it', source_match) file_result = __salt__['state.single']( 'file.managed', cached_source, source=source_match, source_hash=source_hash, makedirs=True, skip_verify=skip_verify, saltenv=__env__, source_hash_name=source_hash_name) log.debug('file.managed: {0}'.format(file_result)) # Get actual state result. The state.single return is a single-element # dictionary with the state's unique ID at the top level, and its value # being the state's return dictionary. next(iter(dict_name)) will give # us the value of the first key, so # file_result[next(iter(file_result))] will give us the results of the # state.single we just ran. try: file_result = file_result[next(iter(file_result))] except AttributeError: pass try: if not file_result['result']: log.debug('failed to download {0}'.format(source_match)) return file_result except TypeError: if not file_result: log.debug('failed to download {0}'.format(source_match)) return file_result else: log.debug('Archive %s is already in cache', source_match) if source_hash: _update_checksum(cached_source, source_sum) if archive_format == 'zip' and not password: log.debug('Checking %s to see if it is password-protected', source_match) # Either use_cmd_unzip was explicitly set to True, or was # implicitly enabled by setting the "options" argument. try: encrypted_zip = __salt__['archive.is_encrypted'](cached_source, clean=False, saltenv=__env__) except CommandExecutionError: # This would happen if archive_format=zip and the source archive is # not actually a zip file. pass else: if encrypted_zip: ret['comment'] = ( 'Archive {0} is password-protected, but no password was ' 'specified. Please set the \'password\' argument.'.format( source_match)) return ret try: contents = __salt__['archive.list'](cached_source, archive_format=archive_format, options=list_options, clean=False, verbose=True) except CommandExecutionError as exc: contents = None errors = [] if not if_missing: errors.append('\'if_missing\' must be set') if not enforce_ownership_on and (user or group): errors.append('Ownership cannot be managed without setting ' '\'enforce_ownership_on\'.') msg = exc.strerror if errors: msg += '\n\n' if archive_format == 'tar': msg += ( 'If the source archive is a tar archive compressed using ' 'a compression type not natively supported by the tar ' 'command, then setting the \'list_options\' argument may ' 'allow the contents to be listed. Otherwise, if Salt is ' 'unable to determine the files/directories in the ' 'archive, the following workaround(s) would need to be ' 'used for this state to proceed') else: msg += ( 'The following workarounds must be used for this state to ' 'proceed') msg += (' (assuming the source file is a valid {0} archive):\n'. format(archive_format)) for error in errors: msg += '\n- {0}'.format(error) ret['comment'] = msg return ret if enforce_toplevel and contents is not None \ and (len(contents['top_level_dirs']) > 1 or len(contents['top_level_files']) > 0): ret['comment'] = ( 'Archive does not have a single top-level directory. ' 'To allow this archive to be extracted, set ' '\'enforce_toplevel\' to False. To avoid a ' '\'{0}-bomb\' it may also be advisable to set a ' 'top-level directory by adding it to the \'name\' ' 'value (for example, setting \'name\' to {1} ' 'instead of {2}).'.format( archive_format, os.path.join(name, 'some_dir'), name, )) return ret # Check to see if we need to extract the archive. Using os.stat() in a # try/except is considerably faster than using os.path.exists(), and we # already need to catch an OSError to cover edge cases where the minion is # running as a non-privileged user and is trying to check for the existence # of a path to which it does not have permission. extraction_needed = False try: if_missing_path_exists = os.path.exists(if_missing) except TypeError: if_missing_path_exists = False if not if_missing_path_exists: if contents is None: try: os.stat(if_missing) extraction_needed = False except OSError as exc: if exc.errno == errno.ENOENT: extraction_needed = True else: ret['comment'] = ( 'Failed to check for existence of if_missing path ' '({0}): {1}'.format(if_missing, exc.__str__())) return ret else: incorrect_type = [] extraction_needed = False for path_list, func in ((contents['dirs'], stat.S_ISDIR), (contents['files'], stat.S_ISREG)): for path in path_list: full_path = os.path.join(name, path) try: path_mode = os.stat(full_path).st_mode if not func(path_mode): incorrect_type.append(path) except OSError as exc: if exc.errno == errno.ENOENT: extraction_needed = True else: ret['comment'] = exc.__str__() return ret if incorrect_type: if not force: msg = ( 'The below paths (relative to {0}) exist, but are the ' 'incorrect type (i.e. file instead of directory or ' 'vice-versa). To proceed with extraction, set ' '\'force\' to True.\n'.format(name)) for path in incorrect_type: msg += '\n- {0}'.format(path) ret['comment'] = msg else: errors = [] for path in incorrect_type: full_path = os.path.join(name, path) try: salt.utils.rm_rf(full_path) ret['changes'].setdefault('removed', []).append(full_path) except OSError as exc: if exc.errno != errno.ENOENT: errors.append(exc.__str__()) if errors: msg = ( 'One or more paths existed by were the incorrect ' 'type (i.e. file instead of directory or ' 'vice-versa), but could not be removed. The ' 'following errors were observed:\n') for error in errors: msg += '\n- {0}'.format(error) ret['comment'] = msg return ret created_destdir = False if extraction_needed: if __opts__['test']: ret['result'] = None ret['comment'] = \ 'Archive {0} would be extracted to {1}'.format( source_match, name ) return ret if not os.path.isdir(name): __salt__['file.makedirs'](name, user=user) created_destdir = True log.debug('Extracting {0} to {1}'.format(cached_source, name)) try: if archive_format == 'zip': if use_cmd_unzip: files = __salt__['archive.cmd_unzip']( cached_source, name, options=options, trim_output=trim_output, password=password, **kwargs) else: files = __salt__['archive.unzip'](cached_source, name, options=options, trim_output=trim_output, password=password, **kwargs) elif archive_format == 'rar': files = __salt__['archive.unrar'](cached_source, name, trim_output=trim_output, **kwargs) else: if options is None: try: with closing(tarfile.open(cached_source, 'r')) as tar: tar.extractall(name) files = tar.getnames() except tarfile.ReadError: if salt.utils.which('xz'): if __salt__['cmd.retcode']( ['xz', '-l', cached_source], python_shell=False, ignore_retcode=True) == 0: # XZ-compressed data log.debug( 'Tar file is XZ-compressed, attempting ' 'decompression and extraction using xz-utils ' 'and the tar command') # Must use python_shell=True here because not # all tar implementations support the -J flag # for decompressing XZ-compressed data. We need # to dump the decompressed data to stdout and # pipe it to tar for extraction. cmd = 'xz --decompress --stdout {0} | tar xvf -' results = __salt__['cmd.run_all']( cmd.format(_cmd_quote(cached_source)), cwd=name, python_shell=True) if results['retcode'] != 0: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] else: # Failed to open tar archive and it is not # XZ-compressed, gracefully fail the state if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive using ' 'Python\'s native tar file support. If ' 'archive is compressed using something ' 'other than gzip or bzip2, the ' '\'options\' argument may be required to ' 'pass the correct options to the tar ' 'command in order to extract the archive.') return ret else: if created_destdir: _cleanup_destdir(name) ret['result'] = False ret['comment'] = ( 'Failed to read from tar archive. If it is ' 'XZ-compressed, install xz-utils to attempt ' 'extraction.') return ret else: try: tar_opts = shlex.split(options) except AttributeError: tar_opts = shlex.split(str(options)) tar_cmd = ['tar'] tar_shortopts = 'x' tar_longopts = [] for position, opt in enumerate(tar_opts): if opt.startswith('-'): tar_longopts.append(opt) else: if position > 0: tar_longopts.append(opt) else: append_opt = opt append_opt = append_opt.replace('x', '') append_opt = append_opt.replace('f', '') tar_shortopts = tar_shortopts + append_opt if __grains__['os'].lower() == 'openbsd': tar_shortopts = '-' + tar_shortopts tar_cmd.append(tar_shortopts) tar_cmd.extend(tar_longopts) tar_cmd.extend(['-f', cached_source]) results = __salt__['cmd.run_all'](tar_cmd, cwd=name, python_shell=False) if results['retcode'] != 0: ret['result'] = False ret['changes'] = results return ret if _is_bsdtar(): files = results['stderr'] else: files = results['stdout'] if not files: files = 'no tar output so far' except CommandExecutionError as exc: ret['comment'] = exc.strerror return ret # Recursively set user and group ownership of files enforce_missing = [] enforce_failed = [] if user or group: if enforce_ownership_on: enforce_dirs = [enforce_ownership_on] enforce_files = [] else: if contents is not None: enforce_dirs = contents['top_level_dirs'] enforce_files = contents['top_level_files'] recurse = [] if user: recurse.append('user') if group: recurse.append('group') recurse_str = ', '.join(recurse) owner_changes = dict([(x, y) for x, y in (('user', user), ('group', group)) if y]) for dirname in enforce_dirs: full_path = os.path.join(name, dirname) if not os.path.isdir(full_path): if not __opts__['test']: enforce_missing.append(full_path) else: log.debug( 'Enforcing %s ownership on %s using a file.directory state%s', recurse_str, dirname, ' (dry-run only)' if __opts__['test'] else '') dir_result = __salt__['state.single']('file.directory', full_path, user=user, group=group, recurse=recurse, test=__opts__['test']) try: dir_result = dir_result[next(iter(dir_result))] except AttributeError: pass log.debug('file.directory: %s', dir_result) if __opts__['test']: if dir_result.get('pchanges'): ret['changes']['updated ownership'] = True else: try: if dir_result['result']: if dir_result['changes']: ret['changes']['updated ownership'] = True else: enforce_failed.append(full_path) except (KeyError, TypeError): log.warning( 'Bad state return %s for file.directory state on %s', dir_result, dirname) for filename in enforce_files: full_path = os.path.join(name, filename) try: # Using os.stat instead of calling out to # __salt__['file.stats'], since we may be doing this for a lot # of files, and simply calling os.stat directly will speed # things up a bit. file_stat = os.stat(full_path) except OSError as exc: if not __opts__['test']: if exc.errno == errno.ENOENT: enforce_missing.append(full_path) enforce_failed.append(full_path) else: # Earlier we set uid, gid to -1 if we're not enforcing # ownership on user, group, as passing -1 to os.chown will tell # it not to change that ownership. Since we've done that, we # can selectively compare the uid/gid from the values in # file_stat, _only if_ the "desired" uid/gid is something other # than -1. if (uid != -1 and uid != file_stat.st_uid) \ or (gid != -1 and gid != file_stat.st_gid): if __opts__['test']: ret['changes']['updated ownership'] = True else: try: os.chown(full_path, uid, gid) ret['changes']['updated ownership'] = True except OSError: enforce_failed.append(filename) if extraction_needed: if len(files) > 0: if created_destdir: ret['changes']['directories_created'] = [name] ret['changes']['extracted_files'] = files ret['comment'] = '{0} extracted to {1}'.format(source_match, name) if not keep: log.debug('Cleaning cached source file %s', cached_source) try: os.remove(cached_source) except OSError as exc: if exc.errno != errno.ENOENT: log.error('Failed to clean cached source file %s: %s', cached_source, exc.__str__()) ret['result'] = True else: ret['result'] = False ret['comment'] = 'Can\'t extract content of {0}'.format( source_match) else: ret['result'] = True if if_missing_path_exists: ret['comment'] = '{0} exists'.format(if_missing) else: ret['comment'] = 'All files in archive are already present' if __opts__['test']: if ret['changes'].get('updated ownership'): ret['result'] = None ret['comment'] += ( '. Ownership would be updated on one or more ' 'files/directories.') if enforce_missing: if not if_missing: # If is_missing was used, and both a) the archive had never been # extracted, and b) the path referred to by if_missing exists, then # enforce_missing would contain paths of top_levle dirs/files that # _would_ have been extracted. Since if_missing can be used as a # semaphore to conditionally extract, we don't want to make this a # case where the state fails, so we only fail the state if # is_missing is not used. ret['result'] = False ret['comment'] += ( '\n\nWhile trying to enforce user/group ownership, the following ' 'paths were missing:\n') for item in enforce_missing: ret['comment'] += '\n- {0}'.format(item) if enforce_failed: ret['result'] = False ret['comment'] += ( '\n\nWhile trying to enforce user/group ownership, Salt was ' 'unable to change ownership on the following paths:\n') for item in enforce_failed: ret['comment'] += '\n- {0}'.format(item) return ret