def source_list(source, source_hash, env): ''' Check the source list and return the source to use ''' if isinstance(source, list): # get the master file list mfiles = __salt__['cp.list_master'](env) mdirs = __salt__['cp.list_master_dirs'](env) for single in source: if isinstance(single, dict): # check the proto, if it is http or ftp then download the file # to check, if it is salt then check the master list if len(single) != 1: continue single_src = next(iter(single)) single_hash = single[single_src] proto = urlparse(single_src).scheme if proto == 'salt': if single_src in mfiles: source = single_src break elif proto.startswith('http') or proto == 'ftp': dest = salt.utils.mkstemp() fn_ = __salt__['cp.get_url'](single_src, dest) os.remove(fn_) if fn_: source = single_src source_hash = single_hash break elif isinstance(single, string_types): if single[7:] in mfiles or single[7:] in mdirs: source = single break return source, source_hash
def is_cached(self, path, saltenv='base', env=None): ''' Returns the full path to a file if it is cached locally on the minion otherwise returns a blank string ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env # support url url_data = urlparse(path) localsfilesdest = os.path.join(self.opts['cachedir'], 'localfiles', path.lstrip('/')) filesdest = os.path.join(self.opts['cachedir'], 'files', saltenv, path.lstrip('salt://')) # support url urlcacheddest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path) if os.path.exists(filesdest): return filesdest elif os.path.exists(localsfilesdest): return localsfilesdest elif os.path.exists(urlcacheddest): return urlcacheddest return ''
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with open(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) return ''
def is_cached(self, path, saltenv='base', env=None): ''' Returns the full path to a file if it is cached locally on the minion otherwise returns a blank string ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env # support url url_data = urlparse(path) localsfilesdest = os.path.join( self.opts['cachedir'], 'localfiles', path.lstrip('/')) filesdest = os.path.join( self.opts['cachedir'], 'files', saltenv, path.lstrip('salt://')) # support url urlcacheddest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path) if os.path.exists(filesdest): return filesdest elif os.path.exists(localsfilesdest): return localsfilesdest elif os.path.exists(urlcacheddest): return urlcacheddest return ''
def get_template(self, url, dest, template="jinja", makedirs=False, env="base", **kwargs): """ Cache a file then process it as a template """ kwargs["env"] = env url_data = urlparse(url) sfn = self.cache_file(url, env) if not os.path.exists(sfn): return "" if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template](sfn, **kwargs) else: log.error("Attempted to render template with unavailable engine " "{0}".format(template)) salt.utils.safe_rm(data["data"]) return "" if not data["result"]: # Failed to render the template log.error("Failed to render template with error: {0}".format(data["data"])) return "" if not dest: # No destination passed, set the dest as an extrn_files cache dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data["data"]) return "" shutil.move(data["data"], dest) return dest
def get_url(self, url, dest, makedirs=False, env="base"): """ Get a single file from a URL. """ url_data = urlparse(url) if url_data.scheme == "salt": return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return "" else: dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with salt.utils.fopen(dest, "wb") as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError( "HTTP error {0} reading {1}: {3}".format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code] ) ) except URLError as ex: raise MinionError("Error reading {0}: {1}".format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password(None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: req = requests.get(fixed_url) with salt.utils.fopen(dest, 'wb') as destfp: destfp.write(req.content) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_template(self, url, dest, template='jinja', makedirs=False, saltenv='base', env=None, **kwargs): ''' Cache a file then process it as a template ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env kwargs['saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template](sfn, **kwargs) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) return '' if not data['result']: # Failed to render the template log.error('Failed to render template with error: {0}'.format( data['data'])) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def get_template( self, url, dest, template='jinja', makedirs=False, env='base', **kwargs): ''' Cache a file then process it as a template ''' kwargs['env'] = env url_data = urlparse(url) sfn = self.cache_file(url, env) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) return '' if not data['result']: # Failed to render the template log.error( 'Failed to render template with error: {0}'.format( data['data'] ) ) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path ) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def get_template( self, url, dest, template='jinja', makedirs=False, env='base', **kwargs): ''' Cache a file then process it as a template ''' kwargs['env'] = env url_data = urlparse(url) sfn = self.cache_file(url, env) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) salt.utils.safe_rm(data['data']) return '' if not data['result']: # Failed to render the template log.error( 'Failed to render template with error: {0}'.format( data['data'] ) ) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: with contextlib.closing(url_open(fixed_url)) as srcfp: with salt.utils.fopen(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password(None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: with contextlib.closing(url_open(fixed_url)) as srcfp: with salt.utils.fopen(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = os.path.normpath( os.sep.join([ self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path])) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with open(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) return ''
def manage_file(name, sfn, ret, source, source_sum, user, group, mode, env, backup): ''' Checks the destination against what was retrieved with get_managed and makes the appropriate modifications (if necessary). ''' if not ret: ret = {'name': name, 'changes': {}, 'comment': '', 'result': True} # Check changes if the target file exists if os.path.isfile(name): # Only test the checksums on files with managed contents if source: name_sum = '' hash_func = getattr(hashlib, source_sum['hash_type']) with salt.utils.fopen(name, 'rb') as namefile: name_sum = hash_func(namefile.read()).hexdigest() # Check if file needs to be replaced if source and source_sum['hsum'] != name_sum: if not sfn: sfn = __salt__['cp.cache_file'](source, env) if not sfn: return _error(ret, 'Source file {0} not found'.format(source)) # If the downloaded file came from a non salt server source verify # that it matches the intended sum value if urlparse(source).scheme != 'salt': with salt.utils.fopen(sfn, 'rb') as dlfile: dl_sum = hash_func(dlfile.read()).hexdigest() if dl_sum != source_sum['hsum']: ret['comment'] = ('File sum set for file {0} of {1} does ' 'not match real sum of {2}').format( name, source_sum['hsum'], dl_sum) ret['result'] = False return ret # Check to see if the files are bins if _is_bin(sfn) or _is_bin(name): ret['changes']['diff'] = 'Replace binary file' else: with nested(salt.utils.fopen(sfn, 'rb'), salt.utils.fopen(name, 'rb')) as (src, name_): slines = src.readlines() nlines = name_.readlines() # Print a diff equivalent to diff -u old new ret['changes']['diff'] = (''.join( difflib.unified_diff(nlines, slines))) # Pre requisites are met, and the file needs to be replaced, do it try: salt.utils.copyfile(sfn, name, __salt__['config.backup_mode'](backup), __opts__['cachedir']) except IOError: __clean_tmp(sfn) return _error(ret, 'Failed to commit change, permission error') ret, perms = check_perms(name, ret, user, group, mode) if ret['changes']: ret['comment'] = 'File {0} updated'.format(name) elif not ret['changes'] and ret['result']: ret['comment'] = 'File {0} is in the correct state'.format(name) __clean_tmp(sfn) return ret else: # Only set the diff if the file contents is managed if source: # It is a new file, set the diff accordingly ret['changes']['diff'] = 'New file' # Apply the new file if not sfn: sfn = __salt__['cp.cache_file'](source, env) if not sfn: return ret.error(ret, 'Source file {0} not found'.format(source)) # If the downloaded file came from a non salt server source verify # that it matches the intended sum value if urlparse(source).scheme != 'salt': hash_func = getattr(hashlib, source_sum['hash_type']) with salt.utils.fopen(sfn, 'rb') as dlfile: dl_sum = hash_func(dlfile.read()).hexdigest() if dl_sum != source_sum['hsum']: ret['comment'] = ('File sum set for file {0} of {1} does ' 'not match real sum of {2}').format( name, source_sum['hsum'], dl_sum) ret['result'] = False return ret if not os.path.isdir(os.path.dirname(name)): if makedirs: makedirs(name, user=user, group=group, mode=mode) else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') else: if not os.path.isdir(os.path.dirname(name)): if makedirs: makedirs(name, user=user, group=group, mode=mode) else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') # Create the file, user rw-only if mode will be set to prevent # a small security race problem before the permissions are set if mode: current_umask = os.umask(63) # Create a new file when test is False and source is None if not __opts__['test']: if __salt__['file.touch'](name): ret['changes']['new'] = 'file {0} created'.format(name) ret['comment'] = 'Empty file' else: return _error(ret, 'Empty file {0} not created'.format(name)) if mode: os.umask(current_umask) # Now copy the file contents if there is a source file if sfn: salt.utils.copyfile(sfn, name, __salt__['config.backup_mode'](backup), __opts__['cachedir']) __clean_tmp(sfn) # Check and set the permissions if necessary ret, perms = check_perms(name, ret, user, group, mode) if not ret['comment']: ret['comment'] = 'File ' + name + ' updated' if __opts__['test']: ret['comment'] = 'File ' + name + ' not updated' elif not ret['changes'] and ret['result']: ret['comment'] = 'File ' + name + ' is in the correct state' __clean_tmp(sfn) return ret
def get_managed( name, template, source, source_hash, user, group, mode, env, context, defaults, **kwargs): ''' Return the managed file data for file.managed ''' # If the file is a template and the contents is managed # then make sure to copy it down and templatize things. sfn = '' source_sum = {} if template and source: sfn = __salt__['cp.cache_file'](source, env) if not os.path.exists(sfn): return sfn, {}, 'File "{0}" could not be found'.format(sfn) if template in salt.utils.templates.TEMPLATE_REGISTRY: context_dict = defaults if defaults else {} if context: context_dict.update(context) data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, name=name, source=source, user=user, group=group, mode=mode, env=env, context=context_dict, salt=__salt__, pillar=__pillar__, grains=__grains__, opts=__opts__, **kwargs ) else: return sfn, {}, ('Specified template format {0} is not supported' ).format(template) if data['result']: sfn = data['data'] hsum = get_hash(sfn) source_sum = {'hash_type': 'md5', 'hsum': hsum} else: __clean_tmp(sfn) return sfn, {}, data['data'] else: # Copy the file down if there is a source if source: if urlparse(source).scheme == 'salt': source_sum = __salt__['cp.hash_file'](source, env) if not source_sum: return '', {}, 'Source file {0} not found'.format(source) elif source_hash: protos = ['salt', 'http', 'ftp'] if urlparse(source_hash).scheme in protos: # The source_hash is a file on a server hash_fn = __salt__['cp.cache_file'](source_hash) if not hash_fn: return '', {}, 'Source hash file {0} not found'.format( source_hash) hash_fn_fopen = salt.utils.fopen(hash_fn, 'r') for line in hash_fn_fopen.read().splitlines(): line = line.strip() if ' ' not in line: hashstr = line break elif line.startswith('{0} '.format(name)): hashstr = line.split()[1] break else: hashstr = '' # NOT FOUND comps = hashstr.split('=') if len(comps) < 2: return '', {}, ('Source hash file {0} contains an ' 'invalid hash format, it must be in ' 'the format <hash type>=<hash>' ).format(source_hash) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: # The source_hash is a hash string comps = source_hash.split('=') if len(comps) < 2: return '', {}, ('Source hash file {0} contains an ' 'invalid hash format, it must be in ' 'the format <hash type>=<hash>' ).format(source_hash) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: return '', {}, ('Unable to determine upstream hash of' ' source file {0}').format(source) return sfn, source_sum, ''
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: req = requests.get(fixed_url) with salt.utils.fopen(dest, 'wb') as destfp: destfp.write(req.content) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def manage_file(name, sfn, ret, source, source_sum, user, group, mode, env, backup): ''' Checks the destination against what was retrieved with get_managed and makes the appropriate modifications (if necessary). ''' if not ret: ret = {'name': name, 'changes': {}, 'comment': '', 'result': True} # Check changes if the target file exists if os.path.isfile(name): # Only test the checksums on files with managed contents if source: name_sum = get_hash(name, source_sum['hash_type']) # Check if file needs to be replaced if source and source_sum['hsum'] != name_sum: if not sfn: sfn = __salt__['cp.cache_file'](source, env) if not sfn: return _error( ret, 'Source file {0} not found'.format(source)) # If the downloaded file came from a non salt server source verify # that it matches the intended sum value if urlparse(source).scheme != 'salt': dl_sum = get_hash(sfn, source_sum['hash_type']) if dl_sum != source_sum['hsum']: ret['comment'] = ('File sum set for file {0} of {1} does ' 'not match real sum of {2}' ).format( name, source_sum['hsum'], dl_sum ) ret['result'] = False return ret # Check to see if the files are bins if _is_bin(sfn) or _is_bin(name): ret['changes']['diff'] = 'Replace binary file' else: with nested(salt.utils.fopen(sfn, 'rb'), salt.utils.fopen(name, 'rb')) as (src, name_): slines = src.readlines() nlines = name_.readlines() # Print a diff equivalent to diff -u old new ret['changes']['diff'] = (''.join(difflib .unified_diff(nlines, slines))) # Pre requisites are met, and the file needs to be replaced, do it try: salt.utils.copyfile( sfn, name, __salt__['config.backup_mode'](backup), __opts__['cachedir']) except IOError: __clean_tmp(sfn) return _error( ret, 'Failed to commit change, permission error') ret, perms = check_perms(name, ret, user, group, mode) if ret['changes']: ret['comment'] = 'File {0} updated'.format(name) elif not ret['changes'] and ret['result']: ret['comment'] = 'File {0} is in the correct state'.format(name) __clean_tmp(sfn) return ret else: # Only set the diff if the file contents is managed if source: # It is a new file, set the diff accordingly ret['changes']['diff'] = 'New file' # Apply the new file if not sfn: sfn = __salt__['cp.cache_file'](source, env) if not sfn: return ret.error( ret, 'Source file {0} not found'.format(source)) # If the downloaded file came from a non salt server source verify # that it matches the intended sum value if urlparse(source).scheme != 'salt': dl_sum = get_hash(name, source_sum['hash_type']) if dl_sum != source_sum['hsum']: ret['comment'] = ('File sum set for file {0} of {1} does ' 'not match real sum of {2}' ).format( name, source_sum['hsum'], dl_sum ) ret['result'] = False return ret if not os.path.isdir(os.path.dirname(name)): if makedirs: makedirs(name, user=user, group=group, mode=mode) else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') else: if not os.path.isdir(os.path.dirname(name)): if makedirs: makedirs(name, user=user, group=group, mode=mode) else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') # Create the file, user rw-only if mode will be set to prevent # a small security race problem before the permissions are set if mode: current_umask = os.umask(63) # Create a new file when test is False and source is None if not __opts__['test']: if __salt__['file.touch'](name): ret['changes']['new'] = 'file {0} created'.format(name) ret['comment'] = 'Empty file' else: return _error( ret, 'Empty file {0} not created'.format(name) ) if mode: os.umask(current_umask) # Now copy the file contents if there is a source file if sfn: salt.utils.copyfile( sfn, name, __salt__['config.backup_mode'](backup), __opts__['cachedir']) __clean_tmp(sfn) # Check and set the permissions if necessary ret, perms = check_perms(name, ret, user, group, mode) if not ret['comment']: ret['comment'] = 'File ' + name + ' updated' if __opts__['test']: ret['comment'] = 'File ' + name + ' not updated' elif not ret['changes'] and ret['result']: ret['comment'] = 'File ' + name + ' is in the correct state' __clean_tmp(sfn) return ret
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception as ex: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift(self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception as ex: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: req = requests.get(fixed_url) with salt.utils.fopen(dest, 'wb') as destfp: destfp.write(req.content) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path {0!r} is not absolute'.format(url_data.path) ) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift(self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url try: if requests.__version__[0] == '0': # 'stream' was called 'prefetch' before 1.0, with flipped meaning get_kwargs['prefetch'] = False else: get_kwargs['stream'] = True response = requests.get(fixed_url, **get_kwargs) response.raise_for_status() with salt.utils.fopen(dest, 'wb') as destfp: for chunk in response.iter_content(chunk_size=32*1024): destfp.write(chunk) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def setup_logfile_logger(log_path, log_level='error', log_format=None, date_format=None): ''' Setup the logfile logger Since version 0.10.6 we support logging to syslog, some examples: tcp://localhost:514/LOG_USER tcp://localhost/LOG_DAEMON udp://localhost:5145/LOG_KERN udp://localhost file:///dev/log file:///dev/log/LOG_SYSLOG file:///dev/log/LOG_DAEMON The above examples are self explanatory, but: <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility> If you're thinking on doing remote logging you might also be thinking that you could point salt's logging to the remote syslog. **Please Don't!** An issue has been reported when doing this over TCP when the logged lines get concatenated. See #3061. The preferred way to do remote logging is setup a local syslog, point salt's logging to the local syslog(unix socket is much faster) and then have the local syslog forward the log messages to the remote syslog. ''' if is_logfile_configured(): logging.getLogger(__name__).warn('Logfile logging already configured') return if log_path is None: logging.getLogger(__name__).warn( 'log_path setting is set to `None`. Nothing else to do') return # Remove the temporary logging handler __remove_temp_logging_handler() if log_level is None: log_level = 'warning' level = LOG_LEVELS.get(log_level.lower(), logging.ERROR) parsed_log_path = urlparse(log_path) root_logger = logging.getLogger() if parsed_log_path.scheme in ('tcp', 'udp', 'file'): syslog_opts = { 'facility': SysLogHandler.LOG_USER, 'socktype': socket.SOCK_DGRAM } if parsed_log_path.scheme == 'file' and parsed_log_path.path: facility_name = parsed_log_path.path.split(os.sep)[-1].upper() if not facility_name.startswith('LOG_'): # The user is not specifying a syslog facility facility_name = 'LOG_USER' # Syslog default syslog_opts['address'] = parsed_log_path.path else: # The user has set a syslog facility, let's update the path to # the logging socket syslog_opts['address'] = os.sep.join( parsed_log_path.path.split(os.sep)[:-1]) elif parsed_log_path.path: # In case of udp or tcp with a facility specified facility_name = parsed_log_path.path.lstrip(os.sep).upper() if not facility_name.startswith('LOG_'): # Logging facilities start with LOG_ if this is not the case # fail right now! raise RuntimeError( 'The syslog facility {0!r} is not known'.format( facility_name)) else: # This is the case of udp or tcp without a facility specified facility_name = 'LOG_USER' # Syslog default facility = getattr(SysLogHandler, facility_name, None) if facility is None: # This python syslog version does not know about the user provided # facility name raise RuntimeError( 'The syslog facility {0!r} is not known'.format(facility_name)) syslog_opts['facility'] = facility if parsed_log_path.scheme == 'tcp': # tcp syslog support was only added on python versions >= 2.7 if sys.version_info < (2, 7): raise RuntimeError( 'Python versions lower than 2.7 do not support logging ' 'to syslog using tcp sockets') syslog_opts['socktype'] = socket.SOCK_STREAM if parsed_log_path.scheme in ('tcp', 'udp'): syslog_opts['address'] = (parsed_log_path.hostname, parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT) if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file': # There's not socktype support on python versions lower than 2.7 syslog_opts.pop('socktype', None) try: # Et voilá! Finally our syslog handler instance handler = SysLogHandler(**syslog_opts) except socket.error as err: logging.getLogger(__name__).error( 'Failed to setup the Syslog logging handler: {0}'.format(err)) sys.exit(2) else: try: # Logfile logging is UTF-8 on purpose. # Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a # user is not using plain ASCII, their system should be ready to # handle UTF-8. handler = WatchedFileHandler(log_path, mode='a', encoding='utf-8', delay=0) except (IOError, OSError): logging.getLogger(__name__).warning( 'Failed to open log file, do you have permission to write to ' '{0}?'.format(log_path)) # Do not proceed with any more configuration since it will fail, we # have the console logging already setup and the user should see # the error. return handler.setLevel(level) # Set the default console formatter config if not log_format: log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s' if not date_format: date_format = '%Y-%m-%d %H:%M:%S' formatter = logging.Formatter(log_format, datefmt=date_format) handler.setFormatter(formatter) root_logger.addHandler(handler) global __LOGFILE_CONFIGURED __LOGFILE_CONFIGURED = True
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception as ex: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift(self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception as ex: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: req = requests.get(fixed_url) with salt.utils.fopen(dest, 'wb') as destfp: destfp.write(req.content) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_template( self, url, dest, template='jinja', makedirs=False, saltenv='base', env=None, **kwargs): ''' Cache a file then process it as a template ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env kwargs['saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) return '' if not data['result']: # Failed to render the template log.error( 'Failed to render template with error: {0}'.format( data['data'] ) ) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path ) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def get_managed(name, template, source, source_hash, user, group, mode, env, context, defaults, **kwargs): ''' Return the managed file data for file.managed ''' # If the file is a template and the contents is managed # then make sure to copy it down and templatize things. sfn = '' source_sum = {} if template and source: sfn = __salt__['cp.cache_file'](source, env) if not os.path.exists(sfn): return sfn, {}, 'File "{0}" could not be found'.format(sfn) if template in salt.utils.templates.template_registry: context_dict = defaults if defaults else {} if context: context_dict.update(context) data = salt.utils.templates.template_registry[template]( sfn, name=name, source=source, user=user, group=group, mode=mode, env=env, context=context_dict, salt=__salt__, pillar=__pillar__, grains=__grains__, opts=__opts__, **kwargs) else: return sfn, {}, ('Specified template format {0} is not supported' ).format(template) if data['result']: sfn = data['data'] with salt.utils.fopen(sfn, 'r') as source: hsum = hashlib.md5(source.read()).hexdigest() source_sum = {'hash_type': 'md5', 'hsum': hsum} else: __clean_tmp(sfn) return sfn, {}, data['data'] else: # Copy the file down if there is a source if source: if urlparse(source).scheme == 'salt': source_sum = __salt__['cp.hash_file'](source, env) if not source_sum: return '', {}, 'Source file {0} not found'.format(source) elif source_hash: protos = ['salt', 'http', 'ftp'] if urlparse(source_hash).scheme in protos: # The source_hash is a file on a server hash_fn = __salt__['cp.cache_file'](source_hash) if not hash_fn: return '', {}, 'Source hash file {0} not found'.format( source_hash) hash_fn_fopen = salt.utils.fopen(hash_fn, 'r') for line in hash_fn_fopen.read().splitlines(): line = line.strip() if ' ' not in line: hashstr = line break elif line.startswith('{0} '.format(name)): hashstr = line.split()[1] break else: hashstr = '' # NOT FOUND comps = hashstr.split('=') if len(comps) < 2: return '', {}, ('Source hash file {0} contains an ' 'invalid hash format, it must be in ' 'the format <hash type>=<hash>' ).format(source_hash) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: # The source_hash is a hash string comps = source_hash.split('=') if len(comps) < 2: return '', {}, ('Source hash file {0} contains an ' 'invalid hash format, it must be in ' 'the format <hash type>=<hash>' ).format(source_hash) source_sum['hsum'] = comps[1].strip() source_sum['hash_type'] = comps[0].strip() else: return '', {}, ('Unable to determine upstream hash of' ' source file {0}').format(source) return sfn, source_sum, ''
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path {0!r} is not absolute'.format(url_data.path)) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query( method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift( self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url try: if requests.__version__[0] == '0': # 'stream' was called 'prefetch' before 1.0, with flipped meaning get_kwargs['prefetch'] = False else: get_kwargs['stream'] = True response = requests.get(fixed_url, **get_kwargs) response.raise_for_status() with salt.utils.fopen(dest, 'wb') as destfp: for chunk in response.iter_content(chunk_size=32 * 1024): destfp.write(chunk) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))