Exemplo n.º 1
0
def source_list(source, source_hash, env):
    '''
    Check the source list and return the source to use
    '''
    if isinstance(source, list):
        # get the master file list
        mfiles = __salt__['cp.list_master'](env)
        mdirs = __salt__['cp.list_master_dirs'](env)
        for single in source:
            if isinstance(single, dict):
                # check the proto, if it is http or ftp then download the file
                # to check, if it is salt then check the master list
                if len(single) != 1:
                    continue
                single_src = next(iter(single))
                single_hash = single[single_src]
                proto = urlparse(single_src).scheme
                if proto == 'salt':
                    if single_src in mfiles:
                        source = single_src
                        break
                elif proto.startswith('http') or proto == 'ftp':
                    dest = salt.utils.mkstemp()
                    fn_ = __salt__['cp.get_url'](single_src, dest)
                    os.remove(fn_)
                    if fn_:
                        source = single_src
                        source_hash = single_hash
                        break
            elif isinstance(single, string_types):
                if single[7:] in mfiles or single[7:] in mdirs:
                    source = single
                    break
    return source, source_hash
Exemplo n.º 2
0
    def is_cached(self, path, saltenv='base', env=None):
        '''
        Returns the full path to a file if it is cached locally on the minion
        otherwise returns a blank string
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        # support url
        url_data = urlparse(path)

        localsfilesdest = os.path.join(self.opts['cachedir'], 'localfiles',
                                       path.lstrip('/'))
        filesdest = os.path.join(self.opts['cachedir'], 'files', saltenv,
                                 path.lstrip('salt://'))
        # support url
        urlcacheddest = salt.utils.path_join(self.opts['cachedir'],
                                             'extrn_files', saltenv,
                                             url_data.netloc, url_data.path)

        if os.path.exists(filesdest):
            return filesdest
        elif os.path.exists(localsfilesdest):
            return localsfilesdest
        elif os.path.exists(urlcacheddest):
            return urlcacheddest

        return ''
Exemplo n.º 3
0
def source_list(source, source_hash, env):
    '''
    Check the source list and return the source to use
    '''
    if isinstance(source, list):
        # get the master file list
        mfiles = __salt__['cp.list_master'](env)
        mdirs = __salt__['cp.list_master_dirs'](env)
        for single in source:
            if isinstance(single, dict):
                # check the proto, if it is http or ftp then download the file
                # to check, if it is salt then check the master list
                if len(single) != 1:
                    continue
                single_src = next(iter(single))
                single_hash = single[single_src]
                proto = urlparse(single_src).scheme
                if proto == 'salt':
                    if single_src in mfiles:
                        source = single_src
                        break
                elif proto.startswith('http') or proto == 'ftp':
                    dest = salt.utils.mkstemp()
                    fn_ = __salt__['cp.get_url'](single_src, dest)
                    os.remove(fn_)
                    if fn_:
                        source = single_src
                        source_hash = single_hash
                        break
            elif isinstance(single, string_types):
                if single[7:] in mfiles or single[7:] in mdirs:
                    source = single
                    break
    return source, source_hash
Exemplo n.º 4
0
 def get_url(self, url, dest, makedirs=False, env='base'):
     '''
     Get a single file from a URL.
     '''
     url_data = urlparse(url)
     if url_data.scheme == 'salt':
         return self.get_file(url, dest, makedirs, env)
     if dest:
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
                 os.makedirs(destdir)
             else:
                 return ''
     else:
         dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                     env, url_data.netloc, url_data.path)
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             os.makedirs(destdir)
     try:
         with contextlib.closing(url_open(url)) as srcfp:
             with open(dest, 'wb') as destfp:
                 shutil.copyfileobj(srcfp, destfp)
         return dest
     except HTTPError as ex:
         raise MinionError('HTTP error {0} reading {1}: {3}'.format(
             ex.code, url,
             *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
     except URLError as ex:
         raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
     return ''
Exemplo n.º 5
0
    def is_cached(self, path, saltenv='base', env=None):
        '''
        Returns the full path to a file if it is cached locally on the minion
        otherwise returns a blank string
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        # support url
        url_data = urlparse(path)

        localsfilesdest = os.path.join(
            self.opts['cachedir'], 'localfiles', path.lstrip('/'))
        filesdest = os.path.join(
            self.opts['cachedir'], 'files', saltenv, path.lstrip('salt://'))
        # support url
        urlcacheddest = salt.utils.path_join(
            self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path)

        if os.path.exists(filesdest):
            return filesdest
        elif os.path.exists(localsfilesdest):
            return localsfilesdest
        elif os.path.exists(urlcacheddest):
            return urlcacheddest

        return ''
Exemplo n.º 6
0
 def get_template(self, url, dest, template="jinja", makedirs=False, env="base", **kwargs):
     """
     Cache a file then process it as a template
     """
     kwargs["env"] = env
     url_data = urlparse(url)
     sfn = self.cache_file(url, env)
     if not os.path.exists(sfn):
         return ""
     if template in salt.utils.templates.TEMPLATE_REGISTRY:
         data = salt.utils.templates.TEMPLATE_REGISTRY[template](sfn, **kwargs)
     else:
         log.error("Attempted to render template with unavailable engine " "{0}".format(template))
         salt.utils.safe_rm(data["data"])
         return ""
     if not data["result"]:
         # Failed to render the template
         log.error("Failed to render template with error: {0}".format(data["data"]))
         return ""
     if not dest:
         # No destination passed, set the dest as an extrn_files cache
         dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path)
     destdir = os.path.dirname(dest)
     if not os.path.isdir(destdir):
         if makedirs:
             os.makedirs(destdir)
         else:
             salt.utils.safe_rm(data["data"])
             return ""
     shutil.move(data["data"], dest)
     return dest
Exemplo n.º 7
0
 def get_url(self, url, dest, makedirs=False, env="base"):
     """
     Get a single file from a URL.
     """
     url_data = urlparse(url)
     if url_data.scheme == "salt":
         return self.get_file(url, dest, makedirs, env)
     if dest:
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
                 os.makedirs(destdir)
             else:
                 return ""
     else:
         dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path)
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             os.makedirs(destdir)
     try:
         with contextlib.closing(url_open(url)) as srcfp:
             with salt.utils.fopen(dest, "wb") as destfp:
                 shutil.copyfileobj(srcfp, destfp)
         return dest
     except HTTPError as ex:
         raise MinionError(
             "HTTP error {0} reading {1}: {3}".format(
                 ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]
             )
         )
     except URLError as ex:
         raise MinionError("Error reading {0}: {1}".format(url, ex.reason))
Exemplo n.º 8
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                        saltenv, url_data.netloc,
                                        url_data.path)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(None, fixed_url, url_data.username,
                                    url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 9
0
    def get_template(self,
                     url,
                     dest,
                     template='jinja',
                     makedirs=False,
                     saltenv='base',
                     env=None,
                     **kwargs):
        '''
        Cache a file then process it as a template
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        kwargs['saltenv'] = saltenv
        url_data = urlparse(url)
        sfn = self.cache_file(url, saltenv)
        if not os.path.exists(sfn):
            return ''
        if template in salt.utils.templates.TEMPLATE_REGISTRY:
            data = salt.utils.templates.TEMPLATE_REGISTRY[template](sfn,
                                                                    **kwargs)
        else:
            log.error('Attempted to render template with unavailable engine '
                      '{0}'.format(template))
            return ''
        if not data['result']:
            # Failed to render the template
            log.error('Failed to render template with error: {0}'.format(
                data['data']))
            return ''
        if not dest:
            # No destination passed, set the dest as an extrn_files cache
            dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                        saltenv, url_data.netloc,
                                        url_data.path)
            # If Salt generated the dest name, create any required dirs
            makedirs = True

        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            if makedirs:
                os.makedirs(destdir)
            else:
                salt.utils.safe_rm(data['data'])
                return ''
        shutil.move(data['data'], dest)
        return dest
Exemplo n.º 10
0
    def get_template(
            self,
            url,
            dest,
            template='jinja',
            makedirs=False,
            env='base',
            **kwargs):
        '''
        Cache a file then process it as a template
        '''
        kwargs['env'] = env
        url_data = urlparse(url)
        sfn = self.cache_file(url, env)
        if not os.path.exists(sfn):
            return ''
        if template in salt.utils.templates.TEMPLATE_REGISTRY:
            data = salt.utils.templates.TEMPLATE_REGISTRY[template](
                sfn,
                **kwargs
            )
        else:
            log.error('Attempted to render template with unavailable engine '
                      '{0}'.format(template))
            return ''
        if not data['result']:
            # Failed to render the template
            log.error(
                'Failed to render template with error: {0}'.format(
                    data['data']
                )
            )
            return ''
        if not dest:
            # No destination passed, set the dest as an extrn_files cache
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                env,
                url_data.netloc,
                url_data.path
            )
            # If Salt generated the dest name, create any required dirs
            makedirs = True

        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            if makedirs:
                os.makedirs(destdir)
            else:
                salt.utils.safe_rm(data['data'])
                return ''
        shutil.move(data['data'], dest)
        return dest
Exemplo n.º 11
0
 def get_template(
         self,
         url,
         dest,
         template='jinja',
         makedirs=False,
         env='base',
         **kwargs):
     '''
     Cache a file then process it as a template
     '''
     kwargs['env'] = env
     url_data = urlparse(url)
     sfn = self.cache_file(url, env)
     if not os.path.exists(sfn):
         return ''
     if template in salt.utils.templates.TEMPLATE_REGISTRY:
         data = salt.utils.templates.TEMPLATE_REGISTRY[template](
                 sfn,
                 **kwargs
                 )
     else:
         log.error('Attempted to render template with unavailable engine '
                   '{0}'.format(template))
         salt.utils.safe_rm(data['data'])
         return ''
     if not data['result']:
         # Failed to render the template
         log.error(
             'Failed to render template with error: {0}'.format(
                 data['data']
             )
         )
         return ''
     if not dest:
         # No destination passed, set the dest as an extrn_files cache
         dest = salt.utils.path_join(
             self.opts['cachedir'],
             'extrn_files',
             env,
             url_data.netloc,
             url_data.path
         )
     destdir = os.path.dirname(dest)
     if not os.path.isdir(destdir):
         if makedirs:
             os.makedirs(destdir)
         else:
             salt.utils.safe_rm(data['data'])
             return ''
     shutil.move(data['data'], dest)
     return dest
Exemplo n.º 12
0
 def get_url(self, url, dest, makedirs=False, env='base'):
     '''
     Get a single file from a URL.
     '''
     url_data = urlparse(url)
     if url_data.scheme == 'salt':
         return self.get_file(url, dest, makedirs, env)
     if dest:
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
                 os.makedirs(destdir)
             else:
                 return ''
     else:
         dest = salt.utils.path_join(
             self.opts['cachedir'],
             'extrn_files',
             env,
             url_data.netloc,
             url_data.path
         )
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             os.makedirs(destdir)
     if url_data.username is not None \
             and url_data.scheme in ('http', 'https'):
         _, netloc = url_data.netloc.split('@', 1)
         fixed_url = urlunparse(
             (url_data.scheme, netloc, url_data.path,
              url_data.params, url_data.query, url_data.fragment))
         passwd_mgr = url_passwd_mgr()
         passwd_mgr.add_password(
             None, fixed_url, url_data.username, url_data.password)
         auth_handler = url_auth_handler(passwd_mgr)
         opener = url_build_opener(auth_handler)
         url_install_opener(opener)
     else:
         fixed_url = url
     try:
         with contextlib.closing(url_open(fixed_url)) as srcfp:
             with salt.utils.fopen(dest, 'wb') as destfp:
                 shutil.copyfileobj(srcfp, destfp)
         return dest
     except HTTPError as ex:
         raise MinionError('HTTP error {0} reading {1}: {3}'.format(
             ex.code,
             url,
             *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
     except URLError as ex:
         raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 13
0
 def get_url(self, url, dest, makedirs=False, env='base'):
     '''
     Get a single file from a URL.
     '''
     url_data = urlparse(url)
     if url_data.scheme == 'salt':
         return self.get_file(url, dest, makedirs, env)
     if dest:
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
                 os.makedirs(destdir)
             else:
                 return ''
     else:
         dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                     env, url_data.netloc, url_data.path)
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             os.makedirs(destdir)
     if url_data.username is not None \
             and url_data.scheme in ('http', 'https'):
         _, netloc = url_data.netloc.split('@', 1)
         fixed_url = urlunparse(
             (url_data.scheme, netloc, url_data.path, url_data.params,
              url_data.query, url_data.fragment))
         passwd_mgr = url_passwd_mgr()
         passwd_mgr.add_password(None, fixed_url, url_data.username,
                                 url_data.password)
         auth_handler = url_auth_handler(passwd_mgr)
         opener = url_build_opener(auth_handler)
         url_install_opener(opener)
     else:
         fixed_url = url
     try:
         with contextlib.closing(url_open(fixed_url)) as srcfp:
             with salt.utils.fopen(dest, 'wb') as destfp:
                 shutil.copyfileobj(srcfp, destfp)
         return dest
     except HTTPError as ex:
         raise MinionError('HTTP error {0} reading {1}: {3}'.format(
             ex.code, url,
             *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
     except URLError as ex:
         raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 14
0
 def get_url(self, url, dest, makedirs=False, env='base'):
     '''
     Get a single file from a URL.
     '''
     url_data = urlparse(url)
     if url_data.scheme == 'salt':
         return self.get_file(url, dest, makedirs, env)
     if dest:
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
                 os.makedirs(destdir)
             else:
                 return ''
     else:
         dest = os.path.normpath(
             os.sep.join([
                 self.opts['cachedir'],
                 'extrn_files',
                 env,
                 url_data.netloc,
                 url_data.path]))
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             os.makedirs(destdir)
     try:
         with contextlib.closing(url_open(url)) as srcfp:
             with open(dest, 'wb') as destfp:
                 shutil.copyfileobj(srcfp, destfp)
         return dest
     except HTTPError as ex:
         raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                 ex.code,
                 url,
                 *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
     except URLError as ex:
         raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
     return ''
Exemplo n.º 15
0
def manage_file(name, sfn, ret, source, source_sum, user, group, mode, env,
                backup):
    '''
    Checks the destination against what was retrieved with get_managed and
    makes the appropriate modifications (if necessary).
    '''
    if not ret:
        ret = {'name': name, 'changes': {}, 'comment': '', 'result': True}
    # Check changes if the target file exists
    if os.path.isfile(name):
        # Only test the checksums on files with managed contents
        if source:
            name_sum = ''
            hash_func = getattr(hashlib, source_sum['hash_type'])
            with salt.utils.fopen(name, 'rb') as namefile:
                name_sum = hash_func(namefile.read()).hexdigest()

        # Check if file needs to be replaced
        if source and source_sum['hsum'] != name_sum:
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, env)
            if not sfn:
                return _error(ret, 'Source file {0} not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if urlparse(source).scheme != 'salt':
                with salt.utils.fopen(sfn, 'rb') as dlfile:
                    dl_sum = hash_func(dlfile.read()).hexdigest()
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = ('File sum set for file {0} of {1} does '
                                      'not match real sum of {2}').format(
                                          name, source_sum['hsum'], dl_sum)
                    ret['result'] = False
                    return ret

            # Check to see if the files are bins
            if _is_bin(sfn) or _is_bin(name):
                ret['changes']['diff'] = 'Replace binary file'
            else:
                with nested(salt.utils.fopen(sfn, 'rb'),
                            salt.utils.fopen(name, 'rb')) as (src, name_):
                    slines = src.readlines()
                    nlines = name_.readlines()
                    # Print a diff equivalent to diff -u old new
                    ret['changes']['diff'] = (''.join(
                        difflib.unified_diff(nlines, slines)))
            # Pre requisites are met, and the file needs to be replaced, do it
            try:
                salt.utils.copyfile(sfn, name,
                                    __salt__['config.backup_mode'](backup),
                                    __opts__['cachedir'])
            except IOError:
                __clean_tmp(sfn)
                return _error(ret, 'Failed to commit change, permission error')

        ret, perms = check_perms(name, ret, user, group, mode)

        if ret['changes']:
            ret['comment'] = 'File {0} updated'.format(name)

        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File {0} is in the correct state'.format(name)
        __clean_tmp(sfn)
        return ret
    else:
        # Only set the diff if the file contents is managed
        if source:
            # It is a new file, set the diff accordingly
            ret['changes']['diff'] = 'New file'
            # Apply the new file
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, env)
            if not sfn:
                return ret.error(ret,
                                 'Source file {0} not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if urlparse(source).scheme != 'salt':
                hash_func = getattr(hashlib, source_sum['hash_type'])
                with salt.utils.fopen(sfn, 'rb') as dlfile:
                    dl_sum = hash_func(dlfile.read()).hexdigest()
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = ('File sum set for file {0} of {1} does '
                                      'not match real sum of {2}').format(
                                          name, source_sum['hsum'], dl_sum)
                    ret['result'] = False
                    return ret

            if not os.path.isdir(os.path.dirname(name)):
                if makedirs:
                    makedirs(name, user=user, group=group, mode=mode)
                else:
                    __clean_tmp(sfn)
                    return _error(ret, 'Parent directory not present')
        else:
            if not os.path.isdir(os.path.dirname(name)):
                if makedirs:
                    makedirs(name, user=user, group=group, mode=mode)
                else:
                    __clean_tmp(sfn)
                    return _error(ret, 'Parent directory not present')

            # Create the file, user rw-only if mode will be set to prevent
            # a small security race problem before the permissions are set
            if mode:
                current_umask = os.umask(63)

            # Create a new file when test is False and source is None
            if not __opts__['test']:
                if __salt__['file.touch'](name):
                    ret['changes']['new'] = 'file {0} created'.format(name)
                    ret['comment'] = 'Empty file'
                else:
                    return _error(ret,
                                  'Empty file {0} not created'.format(name))

            if mode:
                os.umask(current_umask)

        # Now copy the file contents if there is a source file
        if sfn:
            salt.utils.copyfile(sfn, name,
                                __salt__['config.backup_mode'](backup),
                                __opts__['cachedir'])
            __clean_tmp(sfn)

        # Check and set the permissions if necessary
        ret, perms = check_perms(name, ret, user, group, mode)

        if not ret['comment']:
            ret['comment'] = 'File ' + name + ' updated'

        if __opts__['test']:
            ret['comment'] = 'File ' + name + ' not updated'
        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File ' + name + ' is in the correct state'
        __clean_tmp(sfn)
        return ret
Exemplo n.º 16
0
def get_managed(
        name,
        template,
        source,
        source_hash,
        user,
        group,
        mode,
        env,
        context,
        defaults,
        **kwargs):
    '''
    Return the managed file data for file.managed
    '''
    # If the file is a template and the contents is managed
    # then make sure to copy it down and templatize  things.
    sfn = ''
    source_sum = {}
    if template and source:
        sfn = __salt__['cp.cache_file'](source, env)
        if not os.path.exists(sfn):
            return sfn, {}, 'File "{0}" could not be found'.format(sfn)
        if template in salt.utils.templates.TEMPLATE_REGISTRY:
            context_dict = defaults if defaults else {}
            if context:
                context_dict.update(context)
            data = salt.utils.templates.TEMPLATE_REGISTRY[template](
                    sfn,
                    name=name,
                    source=source,
                    user=user,
                    group=group,
                    mode=mode,
                    env=env,
                    context=context_dict,
                    salt=__salt__,
                    pillar=__pillar__,
                    grains=__grains__,
                    opts=__opts__,
                    **kwargs
                    )
        else:
            return sfn, {}, ('Specified template format {0} is not supported'
                      ).format(template)

        if data['result']:
            sfn = data['data']
            hsum = get_hash(sfn)
            source_sum = {'hash_type': 'md5',
                          'hsum': hsum}
        else:
            __clean_tmp(sfn)
            return sfn, {}, data['data']
    else:
        # Copy the file down if there is a source
        if source:
            if urlparse(source).scheme == 'salt':
                source_sum = __salt__['cp.hash_file'](source, env)
                if not source_sum:
                    return '', {}, 'Source file {0} not found'.format(source)
            elif source_hash:
                protos = ['salt', 'http', 'ftp']
                if urlparse(source_hash).scheme in protos:
                    # The source_hash is a file on a server
                    hash_fn = __salt__['cp.cache_file'](source_hash)
                    if not hash_fn:
                        return '', {}, 'Source hash file {0} not found'.format(
                             source_hash)
                    hash_fn_fopen = salt.utils.fopen(hash_fn, 'r')
                    for line in hash_fn_fopen.read().splitlines():
                        line = line.strip()
                        if ' ' not in line:
                            hashstr = line
                            break
                        elif line.startswith('{0} '.format(name)):
                            hashstr = line.split()[1]
                            break
                    else:
                        hashstr = ''  # NOT FOUND
                    comps = hashstr.split('=')
                    if len(comps) < 2:
                        return '', {}, ('Source hash file {0} contains an '
                                  'invalid hash format, it must be in '
                                  'the format <hash type>=<hash>'
                                  ).format(source_hash)
                    source_sum['hsum'] = comps[1].strip()
                    source_sum['hash_type'] = comps[0].strip()
                else:
                    # The source_hash is a hash string
                    comps = source_hash.split('=')
                    if len(comps) < 2:
                        return '', {}, ('Source hash file {0} contains an '
                                  'invalid hash format, it must be in '
                                  'the format <hash type>=<hash>'
                                  ).format(source_hash)
                    source_sum['hsum'] = comps[1].strip()
                    source_sum['hash_type'] = comps[0].strip()
            else:
                return '', {}, ('Unable to determine upstream hash of'
                          ' source file {0}').format(source)
    return sfn, source_sum, ''
Exemplo n.º 17
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                url_data.netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(
                None, fixed_url, url_data.username, url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 18
0
def manage_file(name,
        sfn,
        ret,
        source,
        source_sum,
        user,
        group,
        mode,
        env,
        backup):
    '''
    Checks the destination against what was retrieved with get_managed and
    makes the appropriate modifications (if necessary).
    '''
    if not ret:
        ret = {'name': name,
               'changes': {},
               'comment': '',
               'result': True}
    # Check changes if the target file exists
    if os.path.isfile(name):
        # Only test the checksums on files with managed contents
        if source:
            name_sum = get_hash(name, source_sum['hash_type'])

        # Check if file needs to be replaced
        if source and source_sum['hsum'] != name_sum:
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, env)
            if not sfn:
                return _error(
                    ret, 'Source file {0} not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if urlparse(source).scheme != 'salt':
                dl_sum = get_hash(sfn, source_sum['hash_type'])
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = ('File sum set for file {0} of {1} does '
                                      'not match real sum of {2}'
                                      ).format(
                                              name,
                                              source_sum['hsum'],
                                              dl_sum
                                              )
                    ret['result'] = False
                    return ret

            # Check to see if the files are bins
            if _is_bin(sfn) or _is_bin(name):
                ret['changes']['diff'] = 'Replace binary file'
            else:
                with nested(salt.utils.fopen(sfn, 'rb'),
                            salt.utils.fopen(name, 'rb')) as (src, name_):
                    slines = src.readlines()
                    nlines = name_.readlines()
                # Print a diff equivalent to diff -u old new
                    ret['changes']['diff'] = (''.join(difflib
                                                      .unified_diff(nlines,
                                                                    slines)))
            # Pre requisites are met, and the file needs to be replaced, do it
            try:
                salt.utils.copyfile(
                        sfn,
                        name,
                        __salt__['config.backup_mode'](backup),
                        __opts__['cachedir'])
            except IOError:
                __clean_tmp(sfn)
                return _error(
                    ret, 'Failed to commit change, permission error')

        ret, perms = check_perms(name, ret, user, group, mode)

        if ret['changes']:
            ret['comment'] = 'File {0} updated'.format(name)

        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File {0} is in the correct state'.format(name)
        __clean_tmp(sfn)
        return ret
    else:
        # Only set the diff if the file contents is managed
        if source:
            # It is a new file, set the diff accordingly
            ret['changes']['diff'] = 'New file'
            # Apply the new file
            if not sfn:
                sfn = __salt__['cp.cache_file'](source, env)
            if not sfn:
                return ret.error(
                    ret, 'Source file {0} not found'.format(source))
            # If the downloaded file came from a non salt server source verify
            # that it matches the intended sum value
            if urlparse(source).scheme != 'salt':
                dl_sum = get_hash(name, source_sum['hash_type'])
                if dl_sum != source_sum['hsum']:
                    ret['comment'] = ('File sum set for file {0} of {1} does '
                                      'not match real sum of {2}'
                                      ).format(
                                              name,
                                              source_sum['hsum'],
                                              dl_sum
                                              )
                    ret['result'] = False
                    return ret

            if not os.path.isdir(os.path.dirname(name)):
                if makedirs:
                    makedirs(name, user=user, group=group, mode=mode)
                else:
                    __clean_tmp(sfn)
                    return _error(ret, 'Parent directory not present')
        else:
            if not os.path.isdir(os.path.dirname(name)):
                if makedirs:
                    makedirs(name, user=user, group=group, mode=mode)
                else:
                    __clean_tmp(sfn)
                    return _error(ret, 'Parent directory not present')

            # Create the file, user rw-only if mode will be set to prevent
            # a small security race problem before the permissions are set
            if mode:
                current_umask = os.umask(63)

            # Create a new file when test is False and source is None
            if not __opts__['test']:
                if __salt__['file.touch'](name):
                    ret['changes']['new'] = 'file {0} created'.format(name)
                    ret['comment'] = 'Empty file'
                else:
                    return _error(
                        ret, 'Empty file {0} not created'.format(name)
                    )

            if mode:
                os.umask(current_umask)

        # Now copy the file contents if there is a source file
        if sfn:
            salt.utils.copyfile(
                    sfn,
                    name,
                    __salt__['config.backup_mode'](backup),
                    __opts__['cachedir'])
            __clean_tmp(sfn)

        # Check and set the permissions if necessary
        ret, perms = check_perms(name, ret, user, group, mode)

        if not ret['comment']:
            ret['comment'] = 'File ' + name + ' updated'

        if __opts__['test']:
            ret['comment'] = 'File ' + name + ' not updated'
        elif not ret['changes'] and ret['result']:
            ret['comment'] = 'File ' + name + ' is in the correct state'
        __clean_tmp(sfn)
        return ret
Exemplo n.º 19
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True))
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(
                None, fixed_url, url_data.username, url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 20
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path)
                )
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url
        try:
            if requests.__version__[0] == '0':
                # 'stream' was called 'prefetch' before 1.0, with flipped meaning
                get_kwargs['prefetch'] = False
            else:
                get_kwargs['stream'] = True
            response = requests.get(fixed_url, **get_kwargs)
            response.raise_for_status()
            with salt.utils.fopen(dest, 'wb') as destfp:
                for chunk in response.iter_content(chunk_size=32*1024):
                    destfp.write(chunk)
            return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
Exemplo n.º 21
0
Arquivo: setup.py Projeto: jmdcal/salt
def setup_logfile_logger(log_path,
                         log_level='error',
                         log_format=None,
                         date_format=None):
    '''
    Setup the logfile logger

    Since version 0.10.6 we support logging to syslog, some examples:

        tcp://localhost:514/LOG_USER
        tcp://localhost/LOG_DAEMON
        udp://localhost:5145/LOG_KERN
        udp://localhost
        file:///dev/log
        file:///dev/log/LOG_SYSLOG
        file:///dev/log/LOG_DAEMON

    The above examples are self explanatory, but:
        <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility>

    If you're thinking on doing remote logging you might also be thinking that
    you could point salt's logging to the remote syslog. **Please Don't!**
    An issue has been reported when doing this over TCP when the logged lines
    get concatenated. See #3061.

    The preferred way to do remote logging is setup a local syslog, point
    salt's logging to the local syslog(unix socket is much faster) and then
    have the local syslog forward the log messages to the remote syslog.
    '''

    if is_logfile_configured():
        logging.getLogger(__name__).warn('Logfile logging already configured')
        return

    if log_path is None:
        logging.getLogger(__name__).warn(
            'log_path setting is set to `None`. Nothing else to do')
        return

    # Remove the temporary logging handler
    __remove_temp_logging_handler()

    if log_level is None:
        log_level = 'warning'

    level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)

    parsed_log_path = urlparse(log_path)

    root_logger = logging.getLogger()

    if parsed_log_path.scheme in ('tcp', 'udp', 'file'):
        syslog_opts = {
            'facility': SysLogHandler.LOG_USER,
            'socktype': socket.SOCK_DGRAM
        }

        if parsed_log_path.scheme == 'file' and parsed_log_path.path:
            facility_name = parsed_log_path.path.split(os.sep)[-1].upper()
            if not facility_name.startswith('LOG_'):
                # The user is not specifying a syslog facility
                facility_name = 'LOG_USER'  # Syslog default
                syslog_opts['address'] = parsed_log_path.path
            else:
                # The user has set a syslog facility, let's update the path to
                # the logging socket
                syslog_opts['address'] = os.sep.join(
                    parsed_log_path.path.split(os.sep)[:-1])
        elif parsed_log_path.path:
            # In case of udp or tcp with a facility specified
            facility_name = parsed_log_path.path.lstrip(os.sep).upper()
            if not facility_name.startswith('LOG_'):
                # Logging facilities start with LOG_ if this is not the case
                # fail right now!
                raise RuntimeError(
                    'The syslog facility {0!r} is not known'.format(
                        facility_name))
        else:
            # This is the case of udp or tcp without a facility specified
            facility_name = 'LOG_USER'  # Syslog default

        facility = getattr(SysLogHandler, facility_name, None)
        if facility is None:
            # This python syslog version does not know about the user provided
            # facility name
            raise RuntimeError(
                'The syslog facility {0!r} is not known'.format(facility_name))
        syslog_opts['facility'] = facility

        if parsed_log_path.scheme == 'tcp':
            # tcp syslog support was only added on python versions >= 2.7
            if sys.version_info < (2, 7):
                raise RuntimeError(
                    'Python versions lower than 2.7 do not support logging '
                    'to syslog using tcp sockets')
            syslog_opts['socktype'] = socket.SOCK_STREAM

        if parsed_log_path.scheme in ('tcp', 'udp'):
            syslog_opts['address'] = (parsed_log_path.hostname,
                                      parsed_log_path.port
                                      or logging.handlers.SYSLOG_UDP_PORT)

        if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file':
            # There's not socktype support on python versions lower than 2.7
            syslog_opts.pop('socktype', None)

        try:
            # Et voilá! Finally our syslog handler instance
            handler = SysLogHandler(**syslog_opts)
        except socket.error as err:
            logging.getLogger(__name__).error(
                'Failed to setup the Syslog logging handler: {0}'.format(err))
            sys.exit(2)
    else:
        try:
            # Logfile logging is UTF-8 on purpose.
            # Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a
            # user is not using plain ASCII, their system should be ready to
            # handle UTF-8.
            handler = WatchedFileHandler(log_path,
                                         mode='a',
                                         encoding='utf-8',
                                         delay=0)
        except (IOError, OSError):
            logging.getLogger(__name__).warning(
                'Failed to open log file, do you have permission to write to '
                '{0}?'.format(log_path))
            # Do not proceed with any more configuration since it will fail, we
            # have the console logging already setup and the user should see
            # the error.
            return

    handler.setLevel(level)

    # Set the default console formatter config
    if not log_format:
        log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s'
    if not date_format:
        date_format = '%Y-%m-%d %H:%M:%S'

    formatter = logging.Formatter(log_format, datefmt=date_format)

    handler.setFormatter(formatter)
    root_logger.addHandler(handler)

    global __LOGFILE_CONFIGURED
    __LOGFILE_CONFIGURED = True
Exemplo n.º 22
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                url_data.netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True))
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(
                None, fixed_url, url_data.username, url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Exemplo n.º 23
0
    def get_template(
            self,
            url,
            dest,
            template='jinja',
            makedirs=False,
            saltenv='base',
            env=None,
            **kwargs):
        '''
        Cache a file then process it as a template
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        kwargs['saltenv'] = saltenv
        url_data = urlparse(url)
        sfn = self.cache_file(url, saltenv)
        if not os.path.exists(sfn):
            return ''
        if template in salt.utils.templates.TEMPLATE_REGISTRY:
            data = salt.utils.templates.TEMPLATE_REGISTRY[template](
                sfn,
                **kwargs
            )
        else:
            log.error('Attempted to render template with unavailable engine '
                      '{0}'.format(template))
            return ''
        if not data['result']:
            # Failed to render the template
            log.error(
                'Failed to render template with error: {0}'.format(
                    data['data']
                )
            )
            return ''
        if not dest:
            # No destination passed, set the dest as an extrn_files cache
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                url_data.netloc,
                url_data.path
            )
            # If Salt generated the dest name, create any required dirs
            makedirs = True

        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            if makedirs:
                os.makedirs(destdir)
            else:
                salt.utils.safe_rm(data['data'])
                return ''
        shutil.move(data['data'], dest)
        return dest
Exemplo n.º 24
0
def get_managed(name, template, source, source_hash, user, group, mode, env,
                context, defaults, **kwargs):
    '''
    Return the managed file data for file.managed
    '''
    # If the file is a template and the contents is managed
    # then make sure to copy it down and templatize  things.
    sfn = ''
    source_sum = {}
    if template and source:
        sfn = __salt__['cp.cache_file'](source, env)
        if not os.path.exists(sfn):
            return sfn, {}, 'File "{0}" could not be found'.format(sfn)
        if template in salt.utils.templates.template_registry:
            context_dict = defaults if defaults else {}
            if context:
                context_dict.update(context)
            data = salt.utils.templates.template_registry[template](
                sfn,
                name=name,
                source=source,
                user=user,
                group=group,
                mode=mode,
                env=env,
                context=context_dict,
                salt=__salt__,
                pillar=__pillar__,
                grains=__grains__,
                opts=__opts__,
                **kwargs)
        else:
            return sfn, {}, ('Specified template format {0} is not supported'
                             ).format(template)

        if data['result']:
            sfn = data['data']
            with salt.utils.fopen(sfn, 'r') as source:
                hsum = hashlib.md5(source.read()).hexdigest()
            source_sum = {'hash_type': 'md5', 'hsum': hsum}
        else:
            __clean_tmp(sfn)
            return sfn, {}, data['data']
    else:
        # Copy the file down if there is a source
        if source:
            if urlparse(source).scheme == 'salt':
                source_sum = __salt__['cp.hash_file'](source, env)
                if not source_sum:
                    return '', {}, 'Source file {0} not found'.format(source)
            elif source_hash:
                protos = ['salt', 'http', 'ftp']
                if urlparse(source_hash).scheme in protos:
                    # The source_hash is a file on a server
                    hash_fn = __salt__['cp.cache_file'](source_hash)
                    if not hash_fn:
                        return '', {}, 'Source hash file {0} not found'.format(
                            source_hash)
                    hash_fn_fopen = salt.utils.fopen(hash_fn, 'r')
                    for line in hash_fn_fopen.read().splitlines():
                        line = line.strip()
                        if ' ' not in line:
                            hashstr = line
                            break
                        elif line.startswith('{0} '.format(name)):
                            hashstr = line.split()[1]
                            break
                    else:
                        hashstr = ''  # NOT FOUND
                    comps = hashstr.split('=')
                    if len(comps) < 2:
                        return '', {}, ('Source hash file {0} contains an '
                                        'invalid hash format, it must be in '
                                        'the format <hash type>=<hash>'
                                        ).format(source_hash)
                    source_sum['hsum'] = comps[1].strip()
                    source_sum['hash_type'] = comps[0].strip()
                else:
                    # The source_hash is a hash string
                    comps = source_hash.split('=')
                    if len(comps) < 2:
                        return '', {}, ('Source hash file {0} contains an '
                                        'invalid hash format, it must be in '
                                        'the format <hash type>=<hash>'
                                        ).format(source_hash)
                    source_sum['hsum'] = comps[1].strip()
                    source_sum['hash_type'] = comps[0].strip()
            else:
                return '', {}, ('Unable to determine upstream hash of'
                                ' source file {0}').format(source)
    return sfn, source_sum, ''
Exemplo n.º 25
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path))
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                        saltenv, netloc, url_data.path)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(
                    method='GET',
                    bucket=url_data.netloc,
                    path=url_data.path[1:],
                    return_bin=False,
                    local_file=dest,
                    action=None,
                    key=self.opts.get('s3.key', None),
                    keyid=self.opts.get('s3.keyid', None),
                    service_url=self.opts.get('s3.service_url', None),
                    verify_ssl=self.opts.get('s3.verify_ssl', True))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(
                    self.opts.get('keystone.user', None),
                    self.opts.get('keystone.tenant', None),
                    self.opts.get('keystone.auth_url', None),
                    self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url
        try:
            if requests.__version__[0] == '0':
                # 'stream' was called 'prefetch' before 1.0, with flipped meaning
                get_kwargs['prefetch'] = False
            else:
                get_kwargs['stream'] = True
            response = requests.get(fixed_url, **get_kwargs)
            response.raise_for_status()
            with salt.utils.fopen(dest, 'wb') as destfp:
                for chunk in response.iter_content(chunk_size=32 * 1024):
                    destfp.write(chunk)
            return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))