def _http_request(url, request_timeout=None): ''' PRIVATE METHOD Uses json.load to fetch the json results from the solr api. url : str a complete url that can be passed to urllib.open request_timeout : int (None) The number of seconds before the timeout should fail. Leave blank/None to use the default. __opts__['solr.request_timeout'] Return: dict<str,obj>:: {'success':boolean, 'data':dict, 'errors':list, 'warnings':list} ''' try: request_timeout = __salt__['config.option']('solr.request_timeout') if request_timeout is None: data = json.load(url_open(url)) else: data = json.load(url_open(url, timeout=request_timeout)) return _get_return_dict(True, data, []) except Exception as e: return _get_return_dict(False, {}, ["{0} : {1}".format(url, e)])
def _http_request(url, request_timeout=None): ''' PRIVATE METHOD Uses json.load to fetch the JSON results from the solr API. url : str a complete URL that can be passed to urllib.open request_timeout : int (None) The number of seconds before the timeout should fail. Leave blank/None to use the default. __opts__['solr.request_timeout'] Return: dict<str,obj>:: {'success':boolean, 'data':dict, 'errors':list, 'warnings':list} ''' _auth(url) try: request_timeout = __salt__['config.option']('solr.request_timeout') if request_timeout is None: data = json.load(url_open(url)) else: data = json.load(url_open(url, timeout=request_timeout)) return _get_return_dict(True, data, []) except Exception as err: return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)])
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with open(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) return ''
def get_url(self, url, dest, makedirs=False, env="base"): """ Get a single file from a URL. """ url_data = urlparse(url) if url_data.scheme == "salt": return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return "" else: dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with salt.utils.fopen(dest, "wb") as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError( "HTTP error {0} reading {1}: {3}".format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code] ) ) except URLError as ex: raise MinionError("Error reading {0}: {1}".format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password(None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: with contextlib.closing(url_open(fixed_url)) as srcfp: with salt.utils.fopen(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: with contextlib.closing(url_open(fixed_url)) as srcfp: with salt.utils.fopen(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'): ''' Get a single file from a URL. ''' url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, env) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = os.path.normpath( os.sep.join([ self.opts['cachedir'], 'extrn_files', env, url_data.netloc, url_data.path])) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) try: with contextlib.closing(url_open(url)) as srcfp: with open(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) return ''
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None)) return dest except Exception as ex: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() passwd_mgr.add_password( None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) else: fixed_url = url try: with contextlib.closing(url_open(fixed_url)) as srcfp: with salt.utils.fopen(dest, 'wb') as destfp: shutil.copyfileobj(srcfp, destfp) return dest except HTTPError as ex: raise MinionError('HTTP error {0} reading {1}: {3}'.format( ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) except URLError as ex: raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))