def _extrn_path(self, url, saltenv, cachedir=None): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc # Strip user:pass from URLs netloc = netloc.split('@')[-1] if cachedir is None: cachedir = self.opts['cachedir'] elif not os.path.isabs(cachedir): cachedir = os.path.join(self.opts['cachedir'], cachedir) if url_data.query: file_name = '-'.join([url_data.path, url_data.query]) else: file_name = url_data.path return salt.utils.path_join( cachedir, 'extrn_files', saltenv, netloc, file_name )
def add_http_basic_auth(url, user=None, password=None, https_only=False): ''' Return a string with http basic auth incorporated into it ''' if user is None and password is None: return url else: urltuple = urlparse(url) if https_only and urltuple.scheme != 'https': raise ValueError('Basic Auth only supported for HTTPS') if password is None: netloc = '{0}@{1}'.format( user, urltuple.netloc ) urltuple = urltuple._replace(netloc=netloc) return urlunparse(urltuple) else: netloc = '{0}:{1}@{2}'.format( user, password, urltuple.netloc ) urltuple = urltuple._replace(netloc=netloc) return urlunparse(urltuple)
def scan(filename): ''' scan function ''' parsed = urlparse(filename) if not parsed.scheme: filename = 'salt://' + filename cached_source = __salt__['cp.cache_file'](filename) ret = {'Vulnerabilities': []} cmd = '{0} oval eval {1}'.format(_OSCAP, cached_source) salt_ret = __salt__['cmd.run_all'](cmd, python_shell=False) items = salt_ret['stdout'].split('\n') for item in items: if 'true' in item: if 'rhsa' in item: rhsa = item.split(':')[3] year = item.split(':')[3][:4] num = item.split(':')[3][4:] url = 'https://rhn.redhat.com/errata/RHSA-' + year + '-' + num + '.html' ret['Vulnerabilities'].append('RHSA-' + rhsa + ' : ' + url) return ret
def test_hook_can_handle_get_parameters(self, get_event): self._app.mod_opts['webhook_disable_auth'] = True event = MagicMock() event.fire_event.return_value = True get_event.return_value = event response = self.fetch( '/hook/my_service/?param=1¶m=2', body=json.dumps({}), method='POST', headers={'Content-Type': self.content_type_map['json']}) self.assertEqual(response.code, 200, response.body) host = urlparse(response.effective_url).netloc event.fire_event.assert_called_once_with( { 'headers': { 'Content-Length': '2', 'Connection': 'close', 'Content-Type': 'application/json', 'Host': host, 'Accept-Encoding': 'gzip' }, 'post': {}, 'get': { 'param': ['1', '2'] } }, 'salt/netapi/hook/my_service/', )
def _extrn_path(self, url, saltenv, cachedir=None): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc if cachedir is None: cachedir = self.opts['cachedir'] elif not os.path.isabs(cachedir): cachedir = os.path.join(self.opts['cachedir'], cachedir) if url_data.query: file_name = '-'.join([url_data.path, url_data.query]) else: file_name = url_data.path return salt.utils.path_join( cachedir, 'extrn_files', saltenv, netloc, file_name )
def test_hook_can_handle_get_parameters(self): with patch("salt.utils.event.get_event") as get_event: with patch.dict(self._app.mod_opts, {"webhook_disable_auth": True}): event = MagicMock() event.fire_event.return_value = True get_event.return_value = event response = self.fetch( "/hook/my_service/?param=1¶m=2", body=salt.utils.json.dumps({}), method="POST", headers={"Content-Type": self.content_type_map["json"]}, ) self.assertEqual(response.code, 200, response.body) host = urlparse(response.effective_url).netloc event.fire_event.assert_called_once_with( { "headers": { "Content-Length": "2", "Connection": "close", "Content-Type": "application/json", "Host": host, "Accept-Encoding": "gzip", }, "post": {}, "get": { "param": ["1", "2"] }, }, "salt/netapi/hook/my_service/", )
def validate(url, protos): ''' Return true if the passed URL scheme is in the list of accepted protos ''' if urlparse(url).scheme in protos: return True return False
def get_template( self, url, dest, template='jinja', makedirs=False, saltenv='base', env=None, **kwargs): ''' Cache a file then process it as a template ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env kwargs['saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) return '' if not data['result']: # Failed to render the template log.error( 'Failed to render template with error: {0}'.format( data['data'] ) ) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = self._extrn_path(url, saltenv) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def adjust_uri(self, uri, filename): scheme = urlparse(uri).scheme if scheme in ("salt", "file"): return uri elif scheme: raise ValueError("Unsupported URL scheme({0}) in {1}".format( scheme, uri)) return self.lookup.adjust_uri(uri, filename)
def _extrn_path(self, url, saltenv): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) return salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path)
def get_template( self, url, dest, template='jinja', makedirs=False, saltenv='base', cachedir=None, **kwargs): ''' Cache a file then process it as a template ''' if 'env' in kwargs: salt.utils.warn_until( 'Oxygen', 'Parameter \'env\' has been detected in the argument list. This ' 'parameter is no longer used and has been replaced by \'saltenv\' ' 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' ) kwargs.pop('env') kwargs['saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv, cachedir=cachedir) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error('Attempted to render template with unavailable engine ' '{0}'.format(template)) return '' if not data['result']: # Failed to render the template log.error( 'Failed to render template with error: {0}'.format( data['data'] ) ) return '' if not dest: # No destination passed, set the dest as an extrn_files cache dest = self._extrn_path(url, saltenv, cachedir=cachedir) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.safe_rm(data['data']) return '' shutil.move(data['data'], dest) return dest
def setUp(self): domain = urlparse(TEST_REPO).netloc try: if hasattr(socket, "setdefaulttimeout"): # 10 second dns timeout socket.setdefaulttimeout(10) socket.gethostbyname(domain) except socket.error: msg = "error resolving {0}, possible network issue?" self.skipTest(msg.format(domain))
def _extrn_path(self, url, saltenv): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc return salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path)
def check_remote(cmdline_path): """ Checks to see if the path provided contains ftp, http, or https. Returns the full path if it is found. :param cmdline_path: The path to the initrd image or the kernel """ regex = re.compile('^(ht|f)tps?\\b') if regex.match(urlparse(cmdline_path).scheme): return True return False
def _extrn_path(self, url, saltenv): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) return salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, url_data.netloc, url_data.path )
def get_template( self, url, dest, template=u'jinja', makedirs=False, saltenv=u'base', cachedir=None, **kwargs): ''' Cache a file then process it as a template ''' if u'env' in kwargs: # "env" is not supported; Use "saltenv". kwargs.pop(u'env') kwargs[u'saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv, cachedir=cachedir) if not os.path.exists(sfn): return u'' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: log.error( u'Attempted to render template with unavailable engine %s', template ) return u'' if not data[u'result']: # Failed to render the template log.error(u'Failed to render template with error: %s', data[u'data']) return u'' if not dest: # No destination passed, set the dest as an extrn_files cache dest = self._extrn_path(url, saltenv, cachedir=cachedir) # If Salt generated the dest name, create any required dirs makedirs = True destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: salt.utils.files.safe_rm(data[u'data']) return u'' shutil.move(data[u'data'], dest) return dest
def unescape(url): ''' remove escape character `|` from `url` ''' if salt.utils.is_windows(): return url scheme = urlparse(url).scheme if not scheme: return url.lstrip('|') elif scheme == 'salt': path, saltenv = parse(url) return create(path.lstrip('|'), saltenv) else: return url
def is_escaped(url): ''' test whether `url` is escaped with `|` ''' if salt.utils.is_windows(): return False scheme = urlparse(url).scheme if not scheme: return url.startswith('|') elif scheme == 'salt': path, saltenv = parse(url) return path.startswith('|') else: return False
def unescape(url): ''' remove escape character `|` from `url` ''' if salt.utils.platform.is_windows(): return url scheme = urlparse(url).scheme if not scheme: return url.lstrip('|') elif scheme == 'salt': path, saltenv = parse(url) return create(path.lstrip('|'), saltenv) else: return url
def is_escaped(url): """ test whether `url` is escaped with `|` """ scheme = urlparse(url).scheme if not scheme: return url.startswith("|") elif scheme == "salt": path, saltenv = parse(url) if salt.utils.platform.is_windows() and "|" in url: return path.startswith("_") else: return path.startswith("|") else: return False
def unescape(url): """ remove escape character `|` from `url` """ scheme = urlparse(url).scheme if not scheme: return url.lstrip("|") elif scheme == "salt": path, saltenv = parse(url) if salt.utils.platform.is_windows() and "|" in url: return create(path.lstrip("_"), saltenv) else: return create(path.lstrip("|"), saltenv) else: return url
def is_escaped(url): ''' test whether `url` is escaped with `|` ''' if salt.utils.platform.is_windows(): return False scheme = urlparse(url).scheme if not scheme: return url.startswith('|') elif scheme == 'salt': path, saltenv = parse(url) return path.startswith('|') else: return False
def _get_hostname_and_port(url, default_port=None): parsed = urlparse(url) if parsed.hostname: hostname = parsed.hostname port = parsed.port else: splitted_url = url.split(":") hostname = splitted_url[0] if len(splitted_url) > 1: port = int(splitted_url[1]) else: port = None res = (hostname, port or default_port) __utils__["caasp_log.debug"]("%s parsed as %s", url, res) return res
def _extrn_path(self, url, saltenv): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc return salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path )
def _extrn_path(self, url, saltenv, cachedir=None): ''' Return the extn_filepath for a given url ''' url_data = urlparse(url) if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc if cachedir is None: cachedir = self.opts['cachedir'] elif not os.path.isabs(cachedir): cachedir = os.path.join(self.opts['cachedir'], cachedir) return salt.utils.path_join(cachedir, 'extrn_files', saltenv, netloc, url_data.path)
def escape(url): ''' add escape character `|` to `url` ''' if salt.utils.is_windows(): return url scheme = urlparse(url).scheme if not scheme: if url.startswith('|'): return url else: return u'|{0}'.format(url) elif scheme == 'salt': path, saltenv = parse(url) if path.startswith('|'): return create(path, saltenv) else: return create(u'|{0}'.format(path), saltenv) else: return url
def escape(url): ''' add escape character `|` to `url` ''' if salt.utils.platform.is_windows(): return url scheme = urlparse(url).scheme if not scheme: if url.startswith('|'): return url else: return u'|{0}'.format(url) elif scheme == 'salt': path, saltenv = parse(url) if path.startswith('|'): return create(path, saltenv) else: return create(u'|{0}'.format(path), saltenv) else: return url
def escape(url): """ add escape character `|` to `url` """ if salt.utils.platform.is_windows(): return url scheme = urlparse(url).scheme if not scheme: if url.startswith("|"): return url else: return "|{0}".format(url) elif scheme == "salt": path, saltenv = parse(url) if path.startswith("|"): return create(path, saltenv) else: return create("|{0}".format(path), saltenv) else: return url
def download_remote(url, dir): """ Attempts to download a file specified by 'url' :param url: The full remote path of the file which should be downloaded. :param dir: The path the file should be downloaded to. """ try: rand = hashlib.md5(os.urandom(32)).hexdigest() remote_filename = urlparse(url).path.split('/')[-1] full_directory = \ os.path.join(dir, "{}-{}".format(rand, remote_filename)) with salt.utils.files.fopen(full_directory, 'wb') as file,\ request.urlopen(url) as response: file.write(response.rease()) return full_directory except Exception as err: # pylint: disable=broad-except raise err
def _salt_send_event(opaque, conn, data): ''' Convenience function adding common data to the event and sending it on the salt event bus. :param opaque: the opaque data that is passed to the callback. This is a dict with 'prefix', 'object' and 'event' keys. :param conn: libvirt connection :param data: additional event data dict to send ''' tag_prefix = opaque['prefix'] object_type = opaque['object'] event_type = opaque['event'] # Prepare the connection URI to fit in the tag # qemu+ssh://user@host:1234/system -> qemu+ssh/user@host:1234/system uri = urlparse(conn.getURI()) uri_tag = [uri.scheme] if uri.netloc: uri_tag.append(uri.netloc) path = uri.path.strip('/') if path: uri_tag.append(path) uri_str = "/".join(uri_tag) # Append some common data all_data = { 'uri': conn.getURI() } all_data.update(data) tag = '/'.join((tag_prefix, uri_str, object_type, event_type)) # Actually send the event in salt if __opts__.get('__role') == 'master': salt.utils.event.get_master_event( __opts__, __opts__['sock_dir']).fire_event(all_data, tag) else: __salt__['event.send'](tag, all_data)
def test_hook_can_handle_get_parameters(self, get_event): self._app.mod_opts['webhook_disable_auth'] = True event = MagicMock() event.fire_event.return_value = True get_event.return_value = event response = self.fetch('/hook/my_service/?param=1¶m=2', body=json.dumps({}), method='POST', headers={'Content-Type': self.content_type_map['json']}) self.assertEqual(response.code, 200, response.body) host = urlparse(response.effective_url).netloc event.fire_event.assert_called_once_with( {'headers': {'Content-Length': '2', 'Connection': 'close', 'Content-Type': 'application/json', 'Host': host, 'Accept-Encoding': 'gzip'}, 'post': {}, 'get': {'param': ['1', '2']} }, 'salt/netapi/hook/my_service/', )
def query(url, method='GET', params=None, data=None, data_file=None, header_dict=None, header_list=None, header_file=None, username=None, password=None, auth=None, decode=False, decode_type='auto', status=False, headers=False, text=False, cookies=None, cookie_jar=None, cookie_format='lwp', persist_session=False, session_cookie_jar=None, data_render=False, data_renderer=None, header_render=False, header_renderer=None, template_dict=None, test=False, test_url=None, node='minion', port=80, opts=None, backend=None, ca_bundle=None, verify_ssl=None, cert=None, text_out=None, headers_out=None, decode_out=None, stream=False, streaming_callback=None, header_callback=None, handle=False, agent=USERAGENT, hide_fields=None, raise_error=True, **kwargs): ''' Query a resource, and decode the return data ''' ret = {} if opts is None: if node == 'master': opts = salt.config.master_config( os.path.join(salt.syspaths.CONFIG_DIR, 'master')) elif node == 'minion': opts = salt.config.minion_config( os.path.join(salt.syspaths.CONFIG_DIR, 'minion')) else: opts = {} if not backend: backend = opts.get('backend', 'tornado') match = re.match( r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)', url) if not match: salt.utils.network.refresh_dns() if backend == 'requests': if HAS_REQUESTS is False: ret['error'] = ('http.query has been set to use requests, but the ' 'requests library does not seem to be installed') log.error(ret['error']) return ret else: requests_log = logging.getLogger('requests') requests_log.setLevel(logging.WARNING) # Some libraries don't support separation of url and GET parameters # Don't need a try/except block, since Salt depends on tornado url_full = tornado.httputil.url_concat(url, params) if params else url if ca_bundle is None: ca_bundle = get_ca_bundle(opts) if verify_ssl is None: verify_ssl = opts.get('verify_ssl', True) if cert is None: cert = opts.get('cert', None) if data_file is not None: data = _render(data_file, data_render, data_renderer, template_dict, opts) # Make sure no secret fields show up in logs log_url = sanitize_url(url_full, hide_fields) log.debug('Requesting URL %s using %s method', log_url, method) log.debug("Using backend: %s", backend) if method == 'POST' and log.isEnabledFor(logging.TRACE): # Make sure no secret fields show up in logs if isinstance(data, dict): log_data = data.copy() if isinstance(hide_fields, list): for item in data: for field in hide_fields: if item == field: log_data[item] = 'XXXXXXXXXX' log.trace('Request POST Data: %s', pprint.pformat(log_data)) else: log.trace('Request POST Data: %s', pprint.pformat(data)) if header_file is not None: header_tpl = _render(header_file, header_render, header_renderer, template_dict, opts) if isinstance(header_tpl, dict): header_dict = header_tpl else: header_list = header_tpl.splitlines() if header_dict is None: header_dict = {} if header_list is None: header_list = [] if cookie_jar is None: cookie_jar = os.path.join( opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt') if session_cookie_jar is None: session_cookie_jar = os.path.join( opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p') if persist_session is True and HAS_MSGPACK: # TODO: This is hackish; it will overwrite the session cookie jar with # all cookies from this one connection, rather than behaving like a # proper cookie jar. Unfortunately, since session cookies do not # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: session_cookies = msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: msgpack.dump('', fh_) for header in header_list: comps = header.split(':') if len(comps) < 2: continue header_dict[comps[0].strip()] = comps[1].strip() if not auth: if username and password: auth = (username, password) if agent == USERAGENT: agent = '{0} http.query()'.format(agent) header_dict['User-agent'] = agent if backend == 'requests': sess = requests.Session() sess.auth = auth sess.headers.update(header_dict) log.trace('Request Headers: %s', sess.headers) sess_cookies = sess.cookies sess.verify = verify_ssl elif backend == 'urllib2': sess_cookies = None else: # Tornado sess_cookies = None if cookies is not None: if cookie_format == 'mozilla': sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar( cookie_jar) else: sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar( cookie_jar) if not os.path.isfile(cookie_jar): sess_cookies.save() sess_cookies.load() if test is True: if test_url is None: return {} else: url = test_url ret['test'] = True if backend == 'requests': req_kwargs = {} if stream is True: if requests.__version__[0] == '0': # 'stream' was called 'prefetch' before 1.0, with flipped meaning req_kwargs['prefetch'] = False else: req_kwargs['stream'] = True # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs['cert'] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs['cert'] = cert else: log.error( 'The client-side certificate path that' ' was passed is not valid: %s', cert) result = sess.request(method, url, params=params, data=data, **req_kwargs) result.raise_for_status() if stream is True: # fake a HTTP response header header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code)) # fake streaming the content streaming_callback(result.content) return { 'handle': result, } if handle is True: return { 'handle': result, 'body': result.content, } log.debug('Final URL location of Response: %s', sanitize_url(result.url, hide_fields)) result_status_code = result.status_code result_headers = result.headers result_text = result.content result_cookies = result.cookies body = result.content if not isinstance(body, six.text_type): body = body.decode(result.encoding or 'utf-8') ret['body'] = body elif backend == 'urllib2': request = urllib_request.Request(url_full, data) handlers = [ urllib_request.HTTPHandler, urllib_request.HTTPCookieProcessor(sess_cookies) ] if url.startswith('https'): hostname = request.get_host() handlers[0] = urllib_request.HTTPSHandler(1) if not HAS_MATCHHOSTNAME: log.warning( 'match_hostname() not available, SSL hostname checking ' 'not available. THIS CONNECTION MAY NOT BE SECURE!') elif verify_ssl is False: log.warning('SSL certificate verification has been explicitly ' 'disabled. THIS CONNECTION MAY NOT BE SECURE!') else: if ':' in hostname: hostname, port = hostname.split(':') else: port = 443 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((hostname, int(port))) sockwrap = ssl.wrap_socket(sock, ca_certs=ca_bundle, cert_reqs=ssl.CERT_REQUIRED) try: match_hostname(sockwrap.getpeercert(), hostname) except CertificateError as exc: ret['error'] = ('The certificate was invalid. ' 'Error returned was: %s', pprint.pformat(exc)) return ret # Client-side cert handling if cert is not None: cert_chain = None if isinstance(cert, six.string_types): if os.path.exists(cert): cert_chain = (cert) elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): cert_chain = cert else: log.error( 'The client-side certificate path that was ' 'passed is not valid: %s', cert) return if hasattr(ssl, 'SSLContext'): # Python >= 2.7.9 context = ssl.SSLContext.load_cert_chain(*cert_chain) handlers.append( urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123 else: # Python < 2.7.9 cert_kwargs = { 'host': request.get_host(), 'port': port, 'cert_file': cert_chain[0] } if len(cert_chain) > 1: cert_kwargs['key_file'] = cert_chain[1] handlers[ 0] = salt.ext.six.moves.http_client.HTTPSConnection( **cert_kwargs) opener = urllib_request.build_opener(*handlers) for header in header_dict: request.add_header(header, header_dict[header]) request.get_method = lambda: method try: result = opener.open(request) except URLError as exc: return {'Error': six.text_type(exc)} if stream is True or handle is True: return { 'handle': result, 'body': result.content, } result_status_code = result.code result_headers = dict(result.info()) result_text = result.read() if 'Content-Type' in result_headers: res_content_type, res_params = cgi.parse_header( result_headers['Content-Type']) if res_content_type.startswith('text/') and \ 'charset' in res_params and \ not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): result_text = result.body.decode('utf-8') ret['body'] = result_text else: # Tornado req_kwargs = {} # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs['client_cert'] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs['client_cert'] = cert[0] req_kwargs['client_key'] = cert[1] else: log.error( 'The client-side certificate path that ' 'was passed is not valid: %s', cert) if isinstance(data, dict): data = _urlencode(data) if verify_ssl: # tornado requires a str, cannot be unicode str in py2 req_kwargs['ca_certs'] = salt.utils.stringutils.to_str(ca_bundle) max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body']) connect_timeout = opts.get( 'http_connect_timeout', salt.config.DEFAULT_MINION_OPTS['http_connect_timeout']) timeout = opts.get( 'http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout']) client_argspec = None proxy_host = opts.get('proxy_host', None) if proxy_host: # tornado requires a str for proxy_host, cannot be a unicode str in py2 proxy_host = salt.utils.stringutils.to_str(proxy_host) proxy_port = opts.get('proxy_port', None) proxy_username = opts.get('proxy_username', None) if proxy_username: # tornado requires a str, cannot be unicode str in py2 proxy_username = salt.utils.stringutils.to_str(proxy_username) proxy_password = opts.get('proxy_password', None) if proxy_password: # tornado requires a str, cannot be unicode str in py2 proxy_password = salt.utils.stringutils.to_str(proxy_password) no_proxy = opts.get('no_proxy', []) # Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones # except we remove the valid ones if a url has a no_proxy hostname in it if urlparse(url_full).hostname in no_proxy: proxy_host = None proxy_port = None # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: ret['error'] = ( 'proxy_host and proxy_port has been set. This requires pycurl and tornado, ' 'but the libraries does not seem to be installed') log.error(ret['error']) return ret tornado.httpclient.AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient') client_argspec = salt.utils.args.get_function_argspec( tornado.curl_httpclient.CurlAsyncHTTPClient.initialize) else: client_argspec = salt.utils.args.get_function_argspec( tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize) supports_max_body_size = 'max_body_size' in client_argspec.args try: download_client = HTTPClient(max_body_size=max_body) \ if supports_max_body_size \ else HTTPClient() result = download_client.fetch( url_full, method=method, headers=header_dict, auth_username=username, auth_password=password, body=data, validate_cert=verify_ssl, allow_nonstandard_methods=True, streaming_callback=streaming_callback, header_callback=header_callback, connect_timeout=connect_timeout, request_timeout=timeout, proxy_host=proxy_host, proxy_port=proxy_port, proxy_username=proxy_username, proxy_password=proxy_password, raise_error=raise_error, decompress_response=False, **req_kwargs) except tornado.httpclient.HTTPError as exc: ret['status'] = exc.code ret['error'] = six.text_type(exc) return ret except socket.gaierror as exc: if status is True: ret['status'] = 0 ret['error'] = six.text_type(exc) return ret if stream is True or handle is True: return { 'handle': result, 'body': result.body, } result_status_code = result.code result_headers = result.headers result_text = result.body if 'Content-Type' in result_headers: res_content_type, res_params = cgi.parse_header( result_headers['Content-Type']) if res_content_type.startswith('text/') and \ 'charset' in res_params and \ not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): result_text = result_text.decode('utf-8') ret['body'] = result_text if 'Set-Cookie' in result_headers and cookies is not None: result_cookies = parse_cookie_header(result_headers['Set-Cookie']) for item in result_cookies: sess_cookies.set_cookie(item) else: result_cookies = None if isinstance(result_headers, list): result_headers_dict = {} for header in result_headers: comps = header.split(':') result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip() result_headers = result_headers_dict log.debug('Response Status Code: %s', result_status_code) log.trace('Response Headers: %s', result_headers) log.trace('Response Cookies: %s', sess_cookies) # log.trace("Content: %s", result_text) coding = result_headers.get('Content-Encoding', "identity") # Requests will always decompress the content, and working around that is annoying. if backend != 'requests': result_text = __decompressContent(coding, result_text) try: log.trace('Response Text: %s', result_text) except UnicodeEncodeError as exc: log.trace( 'Cannot Trace Log Response Text: %s. This may be due to ' 'incompatibilities between requests and logging.', exc) if text_out is not None: with salt.utils.files.fopen(text_out, 'w') as tof: tof.write(result_text) if headers_out is not None and os.path.exists(headers_out): with salt.utils.files.fopen(headers_out, 'w') as hof: hof.write(result_headers) if cookies is not None: sess_cookies.save() if persist_session is True and HAS_MSGPACK: # TODO: See persist_session above if 'set-cookie' in result_headers: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: msgpack.dump({'Cookie': session_cookies}, fh_) else: msgpack.dump('', fh_) if status is True: ret['status'] = result_status_code if headers is True: ret['headers'] = result_headers if decode is True: if decode_type == 'auto': content_type = result_headers.get('content-type', 'application/json') if 'xml' in content_type: decode_type = 'xml' elif 'json' in content_type: decode_type = 'json' elif 'yaml' in content_type: decode_type = 'yaml' else: decode_type = 'plain' valid_decodes = ('json', 'xml', 'yaml', 'plain') if decode_type not in valid_decodes: ret['error'] = ('Invalid decode_type specified. ' 'Valid decode types are: {0}'.format( pprint.pformat(valid_decodes))) log.error(ret['error']) return ret if decode_type == 'json': ret['dict'] = salt.utils.json.loads(result_text) elif decode_type == 'xml': ret['dict'] = [] items = ET.fromstring(result_text) for item in items: ret['dict'].append(xml.to_dict(item)) elif decode_type == 'yaml': ret['dict'] = salt.utils.data.decode( salt.utils.yaml.safe_load(result_text)) else: text = True if decode_out: with salt.utils.files.fopen(decode_out, 'w') as dof: dof.write(result_text) if text is True: ret['text'] = result_text return ret
def get_url(self, url, dest, makedirs=False, saltenv='base', no_cache=False, cachedir=None): ''' Get a single file from a URL. ''' url_data = urlparse(url) url_scheme = url_data.scheme url_path = os.path.join( url_data.netloc, url_data.path).rstrip(os.sep) if url_scheme and url_scheme.lower() in string.ascii_lowercase: url_path = ':'.join((url_scheme, url_path)) url_scheme = 'file' if url_scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_path): raise CommandExecutionError( 'Path \'{0}\' is not absolute'.format(url_path) ) if dest is None: with salt.utils.fopen(url_data.path, 'r') as fp_: data = fp_.read() return data return url_path if url_scheme == 'salt': result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir) if result and dest is None: with salt.utils.fopen(result, 'r') as fp_: data = fp_.read() return data return result if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: dest = self._extrn_path(url, saltenv, cachedir=cachedir) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: def s3_opt(key, default=None): '''Get value of s3.<key> from Minion config or from Pillar''' if 's3.' + key in self.opts: return self.opts['s3.' + key] try: return self.opts['pillar']['s3'][key] except (KeyError, TypeError): return default self.utils['s3.query'](method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=s3_opt('key'), keyid=s3_opt('keyid'), service_url=s3_opt('service_url'), verify_ssl=s3_opt('verify_ssl', True), location=s3_opt('location')) return dest except Exception as exc: raise MinionError( 'Could not fetch from {0}. Exception: {1}'.format(url, exc) ) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP(url_data.hostname) ftp.login() with salt.utils.fopen(dest, 'wb') as fp_: ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) return dest except Exception as exc: raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc)) if url_data.scheme == 'swift': try: def swift_opt(key, default): '''Get value of <key> from Minion config or from Pillar''' if key in self.opts: return self.opts[key] try: return self.opts['pillar'][key] except (KeyError, TypeError): return default swift_conn = SaltSwift(swift_opt('keystone.user', None), swift_opt('keystone.tenant', None), swift_opt('keystone.auth_url', None), swift_opt('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): netloc = url_data.netloc at_sign_pos = netloc.rfind('@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url destfp = None try: # Tornado calls streaming_callback on redirect response bodies. # But we need streaming to support fetching large files (> RAM avail). # Here we working this around by disabling recording the body for redirections. # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed # when we'll deprecate Tornado<4.3.0. # See #27093 and #30431 for details. # Use list here to make it writable inside the on_header callback. Simple bool doesn't # work here: on_header creates a new local variable instead. This could be avoided in # Py3 with 'nonlocal' statement. There is no Py2 alternative for this. write_body = [False] def on_header(hdr): try: hdr = parse_response_start_line(hdr) except HTTPInputError: # Not the first line, do nothing return write_body[0] = hdr.code not in [301, 302, 303, 307] if no_cache: result = [] def on_chunk(chunk): if write_body[0]: result.append(chunk) else: dest_tmp = "{0}.part".format(dest) # We need an open filehandle to use in the on_chunk callback, # that's why we're not using a with clause here. destfp = salt.utils.fopen(dest_tmp, 'wb') def on_chunk(chunk): if write_body[0]: destfp.write(chunk) query = salt.utils.http.query( fixed_url, stream=True, streaming_callback=on_chunk, header_callback=on_header, username=url_data.username, password=url_data.password, opts=self.opts, **get_kwargs ) if 'handle' not in query: raise MinionError('Error: {0} reading {1}'.format(query['error'], url)) if no_cache: return ''.join(result) else: destfp.close() destfp = None salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close()
def mod_repo(repo, **kwargs): ''' Modify one or more values for a repo. If the repo does not exist, it will be created, so long as the following values are specified: repo or alias alias by which the zypper refers to the repo url or mirrorlist the URL for zypper to reference enabled enable or disable (True or False) repository, but do not remove if disabled. refresh enable or disable (True or False) auto-refresh of the repository. cache Enable or disable (True or False) RPM files caching. gpgcheck Enable or disable (True or False) GOG check for this repository. Key/Value pairs may also be removed from a repo's configuration by setting a key to a blank value. Bear in mind that a name cannot be deleted, and a url can only be deleted if a mirrorlist is specified (or vice versa). CLI Examples: .. code-block:: bash salt '*' pkg.mod_repo alias alias=new_alias salt '*' pkg.mod_repo alias url= mirrorlist=http://host.com/ ''' repos_cfg = _get_configured_repos() added = False # An attempt to add new one? if repo not in repos_cfg.sections(): url = kwargs.get('url', kwargs.get('mirrorlist')) if not url: raise CommandExecutionError( 'Repository \'{0}\' not found and no URL passed to create one.'.format(repo)) if not urlparse(url).scheme: raise CommandExecutionError( 'Repository \'{0}\' not found and passed URL looks wrong.'.format(repo)) # Is there already such repo under different alias? for alias in repos_cfg.sections(): repo_meta = _get_repo_info(alias, repos_cfg=repos_cfg) # Complete user URL, in case it is not new_url = urlparse(url) if not new_url.path: new_url = urlparse.ParseResult(scheme=new_url.scheme, # pylint: disable=E1123 netloc=new_url.netloc, path='/', params=new_url.params, query=new_url.query, fragment=new_url.fragment) base_url = urlparse(repo_meta['baseurl']) if new_url == base_url: raise CommandExecutionError( 'Repository \'{0}\' already exists as \'{1}\'.'.format(repo, alias)) # Add new repo doc = None try: # Try to parse the output and find the error, # but this not always working (depends on Zypper version) doc = dom.parseString(__salt__['cmd.run'](('zypper -x ar {0} \'{1}\''.format(url, repo)), output_loglevel='trace')) except Exception: # No XML out available, but it is still unknown the state of the result. pass if doc: msg_nodes = doc.getElementsByTagName('message') if msg_nodes: msg_node = msg_nodes[0] if msg_node.getAttribute('type') == 'error': raise CommandExecutionError(msg_node.childNodes[0].nodeValue) # Verify the repository has been added repos_cfg = _get_configured_repos() if repo not in repos_cfg.sections(): raise CommandExecutionError( 'Failed add new repository \'{0}\' for unknown reason. ' 'Please look into Zypper logs.'.format(repo)) added = True # Modify added or existing repo according to the options cmd_opt = [] if 'enabled' in kwargs: cmd_opt.append(kwargs['enabled'] and '--enable' or '--disable') if 'refresh' in kwargs: cmd_opt.append(kwargs['refresh'] and '--refresh' or '--no-refresh') if 'cache' in kwargs: cmd_opt.append(kwargs['cache'] and '--keep-packages' or '--no-keep-packages') if 'gpgcheck' in kwargs: cmd_opt.append(kwargs['gpgcheck'] and '--gpgcheck' or '--no-gpgcheck') if cmd_opt: __salt__['cmd.run'](('zypper -x mr {0} \'{1}\''.format(' '.join(cmd_opt), repo)), output_loglevel='trace') # If repo nor added neither modified, error should be thrown if not added and not cmd_opt: raise CommandExecutionError( 'Modification of the repository \'{0}\' was not specified.'.format(repo)) return {}
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path \'{0}\' is not absolute'.format(url_data.path) ) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: dest = self._extrn_path(url, saltenv) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: def s3_opt(key, default=None): '''Get value of s3.<key> from Minion config or from Pillar''' if 's3.' + key in self.opts: return self.opts['s3.' + key] try: return self.opts['pillar']['s3'][key] except (KeyError, TypeError): return default salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=s3_opt('key'), keyid=s3_opt('keyid'), service_url=s3_opt('service_url'), verify_ssl=s3_opt('verify_ssl', True), location=s3_opt('location')) return dest except Exception as exc: raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc)) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP(url_data.hostname) ftp.login() with salt.utils.fopen(dest, 'wb') as fp_: ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) return dest except Exception as exc: raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc)) if url_data.scheme == 'swift': try: def swift_opt(key, default): '''Get value of <key> from Minion config or from Pillar''' if key in self.opts: return self.opts[key] try: return self.opts['pillar'][key] except (KeyError, TypeError): return default swift_conn = SaltSwift(swift_opt('keystone.user', None), swift_opt('keystone.tenant', None), swift_opt('keystone.auth_url', None), swift_opt('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): netloc = url_data.netloc at_sign_pos = netloc.rfind('@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url destfp = None try: query = salt.utils.http.query( fixed_url, text=True, username=url_data.username, password=url_data.password, **get_kwargs ) if 'text' not in query: raise MinionError('Error: {0}'.format(query['error'])) if no_cache: return query['body'] else: dest_tmp = "{0}.part".format(dest) with salt.utils.fopen(dest_tmp, 'wb') as destfp: destfp.write(query['body']) salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.' ) # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path {0!r} is not absolute'.format(url_data.path) ) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc dest = salt.utils.path_join( self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path ) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift(self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url try: query = salt.utils.http.query( fixed_url, stream=True, username=url_data.username, password=url_data.password, **get_kwargs ) response = query['handle'] chunk_size = 32 * 1024 if not no_cache: with salt.utils.fopen(dest, 'wb') as destfp: if hasattr(response, 'iter_content'): for chunk in response.iter_content(chunk_size=chunk_size): destfp.write(chunk) else: while True: chunk = response.buffer.read(chunk_size) destfp.write(chunk) if len(chunk) < chunk_size: break return dest else: if hasattr(response, 'text'): return response.text else: return response['text'] except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path {0!r} is not absolute'.format(url_data.path)) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' else: if salt.utils.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files', saltenv, netloc, url_data.path) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: salt.utils.s3.query( method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=self.opts.get('s3.key', None), keyid=self.opts.get('s3.keyid', None), service_url=self.opts.get('s3.service_url', None), verify_ssl=self.opts.get('s3.verify_ssl', True)) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) if url_data.scheme == 'swift': try: swift_conn = SaltSwift( self.opts.get('keystone.user', None), self.opts.get('keystone.tenant', None), self.opts.get('keystone.auth_url', None), self.opts.get('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url try: if requests.__version__[0] == '0': # 'stream' was called 'prefetch' before 1.0, with flipped meaning get_kwargs['prefetch'] = False else: get_kwargs['stream'] = True response = requests.get(fixed_url, **get_kwargs) with salt.utils.fopen(dest, 'wb') as destfp: for chunk in response.iter_content(chunk_size=32 * 1024): destfp.write(chunk) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Carbon.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path \'{0}\' is not absolute'.format(url_data.path)) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: dest = self._extrn_path(url, saltenv) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: def s3_opt(key, default=None): '''Get value of s3.<key> from Minion config or from Pillar''' if 's3.' + key in self.opts: return self.opts['s3.' + key] try: return self.opts['pillar']['s3'][key] except (KeyError, TypeError): return default salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=s3_opt('key'), keyid=s3_opt('keyid'), service_url=s3_opt('service_url'), verify_ssl=s3_opt('verify_ssl', True), location=s3_opt('location')) return dest except Exception as exc: raise MinionError( 'Could not fetch from {0}. Exception: {1}'.format( url, exc)) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP(url_data.hostname) ftp.login() with salt.utils.fopen(dest, 'wb') as fp_: ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) return dest except Exception as exc: raise MinionError( 'Could not retrieve {0} from FTP server. Exception: {1}'. format(url, exc)) if url_data.scheme == 'swift': try: def swift_opt(key, default): '''Get value of <key> from Minion config or from Pillar''' if key in self.opts: return self.opts[key] try: return self.opts['pillar'][key] except (KeyError, TypeError): return default swift_conn = SaltSwift(swift_opt('keystone.user', None), swift_opt('keystone.tenant', None), swift_opt('keystone.auth_url', None), swift_opt('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): netloc = url_data.netloc at_sign_pos = netloc.rfind('@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url destfp = None try: # Tornado calls streaming_callback on redirect response bodies. # But we need streaming to support fetching large files (> RAM avail). # Here we working this around by disabling recording the body for redirections. # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed # when we'll deprecate Tornado<4.3.0. # See #27093 and #30431 for details. # Use list here to make it writable inside the on_header callback. Simple bool doesn't # work here: on_header creates a new local variable instead. This could be avoided in # Py3 with 'nonlocal' statement. There is no Py2 alternative for this. write_body = [False] def on_header(hdr): try: hdr = parse_response_start_line(hdr) except HTTPInputError: # Not the first line, do nothing return write_body[0] = hdr.code not in [301, 302, 303, 307] if no_cache: result = [] def on_chunk(chunk): if write_body[0]: result.append(chunk) else: dest_tmp = "{0}.part".format(dest) destfp = salt.utils.fopen(dest_tmp, 'wb') def on_chunk(chunk): if write_body[0]: destfp.write(chunk) query = salt.utils.http.query(fixed_url, stream=True, streaming_callback=on_chunk, header_callback=on_header, username=url_data.username, password=url_data.password, **get_kwargs) if 'handle' not in query: raise MinionError('Error: {0}'.format(query['error'])) if no_cache: return ''.join(result) else: destfp.close() destfp = None salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close()
def setup_logfile_logger(log_path, log_level='error', log_format=None, date_format=None): ''' Setup the logfile logger Since version 0.10.6 we support logging to syslog, some examples: tcp://localhost:514/LOG_USER tcp://localhost/LOG_DAEMON udp://localhost:5145/LOG_KERN udp://localhost file:///dev/log file:///dev/log/LOG_SYSLOG file:///dev/log/LOG_DAEMON The above examples are self explanatory, but: <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility> If you're thinking on doing remote logging you might also be thinking that you could point salt's logging to the remote syslog. **Please Don't!** An issue has been reported when doing this over TCP when the logged lines get concatenated. See #3061. The preferred way to do remote logging is setup a local syslog, point salt's logging to the local syslog(unix socket is much faster) and then have the local syslog forward the log messages to the remote syslog. ''' if is_logfile_configured(): logging.getLogger(__name__).warn('Logfile logging already configured') return if log_path is None: logging.getLogger(__name__).warn( 'log_path setting is set to `None`. Nothing else to do' ) return # Remove the temporary logging handler __remove_temp_logging_handler() if log_level is None: log_level = 'warning' level = LOG_LEVELS.get(log_level.lower(), logging.ERROR) parsed_log_path = urlparse(log_path) root_logger = logging.getLogger() if parsed_log_path.scheme in ('tcp', 'udp', 'file'): syslog_opts = { 'facility': SysLogHandler.LOG_USER, 'socktype': socket.SOCK_DGRAM } if parsed_log_path.scheme == 'file' and parsed_log_path.path: facility_name = parsed_log_path.path.split(os.sep)[-1].upper() if not facility_name.startswith('LOG_'): # The user is not specifying a syslog facility facility_name = 'LOG_USER' # Syslog default syslog_opts['address'] = parsed_log_path.path else: # The user has set a syslog facility, let's update the path to # the logging socket syslog_opts['address'] = os.sep.join( parsed_log_path.path.split(os.sep)[:-1] ) elif parsed_log_path.path: # In case of udp or tcp with a facility specified facility_name = parsed_log_path.path.lstrip(os.sep).upper() if not facility_name.startswith('LOG_'): # Logging facilities start with LOG_ if this is not the case # fail right now! raise RuntimeError( 'The syslog facility {0!r} is not known'.format( facility_name ) ) else: # This is the case of udp or tcp without a facility specified facility_name = 'LOG_USER' # Syslog default facility = getattr( SysLogHandler, facility_name, None ) if facility is None: # This python syslog version does not know about the user provided # facility name raise RuntimeError( 'The syslog facility {0!r} is not known'.format( facility_name ) ) syslog_opts['facility'] = facility if parsed_log_path.scheme == 'tcp': # tcp syslog support was only added on python versions >= 2.7 if sys.version_info < (2, 7): raise RuntimeError( 'Python versions lower than 2.7 do not support logging ' 'to syslog using tcp sockets' ) syslog_opts['socktype'] = socket.SOCK_STREAM if parsed_log_path.scheme in ('tcp', 'udp'): syslog_opts['address'] = ( parsed_log_path.hostname, parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT ) if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file': # There's not socktype support on python versions lower than 2.7 syslog_opts.pop('socktype', None) try: # Et voilá! Finally our syslog handler instance handler = SysLogHandler(**syslog_opts) except socket.error as err: logging.getLogger(__name__).error( 'Failed to setup the Syslog logging handler: {0}'.format( err ) ) sys.exit(2) else: try: # Logfile logging is UTF-8 on purpose. # Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a # user is not using plain ASCII, their system should be ready to # handle UTF-8. handler = WatchedFileHandler(log_path, mode='a', encoding='utf-8', delay=0) except (IOError, OSError): logging.getLogger(__name__).warning( 'Failed to open log file, do you have permission to write to ' '{0}?'.format(log_path) ) # Do not proceed with any more configuration since it will fail, we # have the console logging already setup and the user should see # the error. return handler.setLevel(level) # Set the default console formatter config if not log_format: log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s' if not date_format: date_format = '%Y-%m-%d %H:%M:%S' formatter = logging.Formatter(log_format, datefmt=date_format) handler.setFormatter(formatter) root_logger.addHandler(handler) global __LOGFILE_CONFIGURED __LOGFILE_CONFIGURED = True
def query(params=None, setname=None, requesturl=None, location=None, return_url=False, return_root=False, opts=None, provider=None, endpoint=None, product='ec2', sigver='2'): ''' Perform a query against AWS services using Signature Version 2 Signing Process. This is documented at: http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html Regions and endpoints are documented at: http://docs.aws.amazon.com/general/latest/gr/rande.html Default ``product`` is ``ec2``. Valid ``product`` names are: .. code-block: yaml - autoscaling (Auto Scaling) - cloudformation (CloudFormation) - ec2 (Elastic Compute Cloud) - elasticache (ElastiCache) - elasticbeanstalk (Elastic BeanStalk) - elasticloadbalancing (Elastic Load Balancing) - elasticmapreduce (Elastic MapReduce) - iam (Identity and Access Management) - importexport (Import/Export) - monitoring (CloudWatch) - rds (Relational Database Service) - simpledb (SimpleDB) - sns (Simple Notification Service) - sqs (Simple Queue Service) ''' if params is None: params = {} if opts is None: opts = {} function = opts.get('function', (None, product)) providers = opts.get('providers', {}) if provider is None: prov_dict = providers.get(function[1], {}).get(product, {}) if prov_dict: driver = list(list(prov_dict.keys()))[0] provider = providers.get(driver, product) else: prov_dict = providers.get(provider, {}).get(product, {}) service_url = prov_dict.get('service_url', 'amazonaws.com') if not location: location = get_location(opts, provider) if endpoint is None: if not requesturl: endpoint = prov_dict.get( 'endpoint', '{0}.{1}.{2}'.format(product, location, service_url)) requesturl = 'https://{0}/'.format(endpoint) else: endpoint = urlparse(requesturl).netloc if endpoint == '': endpoint_err = ( 'Could not find a valid endpoint in the ' 'requesturl: {0}. Looking for something ' 'like https://some.aws.endpoint/?args').format(requesturl) LOG.error(endpoint_err) if return_url is True: return {'error': endpoint_err}, requesturl return {'error': endpoint_err} LOG.debug('Using AWS endpoint: {0}'.format(endpoint)) method = 'GET' aws_api_version = prov_dict.get( 'aws_api_version', prov_dict.get('{0}_api_version'.format(product), DEFAULT_AWS_API_VERSION)) if sigver == '4': headers, requesturl = sig4(method, endpoint, params, prov_dict, aws_api_version, location, product, requesturl=requesturl) params_with_headers = {} else: params_with_headers = sig2(method, endpoint, params, prov_dict, aws_api_version) headers = {} attempts = 5 while attempts > 0: LOG.debug('AWS Request: {0}'.format(requesturl)) LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers)) try: result = requests.get(requesturl, headers=headers, params=params_with_headers) LOG.debug('AWS Response Status Code: {0}'.format( result.status_code)) LOG.trace('AWS Response Text: {0}'.format(result.text)) result.raise_for_status() break except requests.exceptions.HTTPError as exc: root = ET.fromstring(exc.response.content) data = xml.to_dict(root) # check to see if we should retry the query err_code = data.get('Errors', {}).get('Error', {}).get('Code', '') if attempts > 0 and err_code and err_code in AWS_RETRY_CODES: attempts -= 1 LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}; ' 'Attempts remaining: {3}'.format( exc.response.status_code, exc, data, attempts)) # Wait a bit before continuing to prevent throttling time.sleep(2) continue LOG.error( 'AWS Response Status Code and Error: [{0} {1}] {2}'.format( exc.response.status_code, exc, data)) if return_url is True: return {'error': data}, requesturl return {'error': data} else: LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}'.format( exc.response.status_code, exc, data)) if return_url is True: return {'error': data}, requesturl return {'error': data} response = result.text root = ET.fromstring(response) items = root[1] if return_root is True: items = root if setname: if sys.version_info < (2, 7): children_len = len(root.getchildren()) else: children_len = len(root) for item in range(0, children_len): comps = root[item].tag.split('}') if comps[1] == setname: items = root[item] ret = [] for item in items: ret.append(xml.to_dict(item)) if return_url is True: return ret, requesturl return ret
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False): ''' Get a single file from a URL. ''' if env is not None: salt.utils.warn_until( 'Boron', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' 'Boron.') # Backwards compatibility saltenv = env url_data = urlparse(url) if url_data.scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_data.path): raise CommandExecutionError( 'Path \'{0}\' is not absolute'.format(url_data.path)) return url_data.path if url_data.scheme == 'salt': return self.get_file(url, dest, makedirs, saltenv) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: dest = self._extrn_path(url, saltenv) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: def s3_opt(key, default=None): '''Get value of s3.<key> from Minion config or from Pillar''' if 's3.' + key in self.opts: return self.opts['s3.' + key] try: return self.opts['pillar']['s3'][key] except (KeyError, TypeError): return default salt.utils.s3.query(method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=s3_opt('key'), keyid=s3_opt('keyid'), service_url=s3_opt('service_url'), verify_ssl=s3_opt('verify_ssl', True), location=s3_opt('location')) return dest except Exception as exc: raise MinionError( 'Could not fetch from {0}. Exception: {1}'.format( url, exc)) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP(url_data.hostname) ftp.login() with salt.utils.fopen(dest, 'wb') as fp_: ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) return dest except Exception as exc: raise MinionError( 'Could not retrieve {0} from FTP server. Exception: {1}'. format(url, exc)) if url_data.scheme == 'swift': try: def swift_opt(key, default): '''Get value of <key> from Minion config or from Pillar''' if key in self.opts: return self.opts[key] try: return self.opts['pillar'][key] except (KeyError, TypeError): return default swift_conn = SaltSwift(swift_opt('keystone.user', None), swift_opt('keystone.tenant', None), swift_opt('keystone.auth_url', None), swift_opt('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): netloc = url_data.netloc at_sign_pos = netloc.rfind('@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url destfp = None try: query = salt.utils.http.query(fixed_url, text=True, username=url_data.username, password=url_data.password, **get_kwargs) if 'text' not in query: raise MinionError('Error: {0}'.format(query['error'])) if no_cache: return query['body'] else: dest_tmp = "{0}.part".format(dest) with salt.utils.fopen(dest_tmp, 'wb') as destfp: destfp.write(query['body']) salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close()
def setup_logfile_logger(log_path, log_level='error', log_format=None, date_format=None): ''' Setup the logfile logger Since version 0.10.6 we support logging to syslog, some examples: tcp://localhost:514/LOG_USER tcp://localhost/LOG_DAEMON udp://localhost:5145/LOG_KERN udp://localhost file:///dev/log file:///dev/log/LOG_SYSLOG file:///dev/log/LOG_DAEMON The above examples are self explanatory, but: <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility> If you're thinking on doing remote logging you might also be thinking that you could point salt's logging to the remote syslog. **Please Don't!** An issue has been reported when doing this over TCP when the logged lines get concatenated. See #3061. The preferred way to do remote logging is setup a local syslog, point salt's logging to the local syslog(unix socket is much faster) and then have the local syslog forward the log messages to the remote syslog. ''' if is_logfile_configured(): logging.getLogger(__name__).warn('Logfile logging already configured') return if log_path is None: logging.getLogger(__name__).warn( 'log_path setting is set to `None`. Nothing else to do') return # Remove the temporary logging handler __remove_temp_logging_handler() if log_level is None: log_level = 'warning' level = LOG_LEVELS.get(log_level.lower(), logging.ERROR) parsed_log_path = urlparse(log_path) root_logger = logging.getLogger() if parsed_log_path.scheme in ('tcp', 'udp', 'file'): syslog_opts = { 'facility': SysLogHandler.LOG_USER, 'socktype': socket.SOCK_DGRAM } if parsed_log_path.scheme == 'file' and parsed_log_path.path: facility_name = parsed_log_path.path.split(os.sep)[-1].upper() if not facility_name.startswith('LOG_'): # The user is not specifying a syslog facility facility_name = 'LOG_USER' # Syslog default syslog_opts['address'] = parsed_log_path.path else: # The user has set a syslog facility, let's update the path to # the logging socket syslog_opts['address'] = os.sep.join( parsed_log_path.path.split(os.sep)[:-1]) elif parsed_log_path.path: # In case of udp or tcp with a facility specified facility_name = parsed_log_path.path.lstrip(os.sep).upper() if not facility_name.startswith('LOG_'): # Logging facilities start with LOG_ if this is not the case # fail right now! raise RuntimeError( 'The syslog facility {0!r} is not known'.format( facility_name)) else: # This is the case of udp or tcp without a facility specified facility_name = 'LOG_USER' # Syslog default facility = getattr(SysLogHandler, facility_name, None) if facility is None: # This python syslog version does not know about the user provided # facility name raise RuntimeError( 'The syslog facility {0!r} is not known'.format(facility_name)) syslog_opts['facility'] = facility if parsed_log_path.scheme == 'tcp': # tcp syslog support was only added on python versions >= 2.7 if sys.version_info < (2, 7): raise RuntimeError( 'Python versions lower than 2.7 do not support logging ' 'to syslog using tcp sockets') syslog_opts['socktype'] = socket.SOCK_STREAM if parsed_log_path.scheme in ('tcp', 'udp'): syslog_opts['address'] = (parsed_log_path.hostname, parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT) if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file': # There's not socktype support on python versions lower than 2.7 syslog_opts.pop('socktype', None) try: # Et voilá! Finally our syslog handler instance handler = SysLogHandler(**syslog_opts) except socket.error as err: logging.getLogger(__name__).error( 'Failed to setup the Syslog logging handler: {0}'.format(err)) sys.exit(2) else: try: # Logfile logging is UTF-8 on purpose. # Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a # user is not using plain ASCII, their system should be ready to # handle UTF-8. handler = WatchedFileHandler(log_path, mode='a', encoding='utf-8', delay=0) except (IOError, OSError): logging.getLogger(__name__).warning( 'Failed to open log file, do you have permission to write to ' '{0}?'.format(log_path)) # Do not proceed with any more configuration since it will fail, we # have the console logging already setup and the user should see # the error. return handler.setLevel(level) # Set the default console formatter config if not log_format: log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s' if not date_format: date_format = '%Y-%m-%d %H:%M:%S' formatter = logging.Formatter(log_format, datefmt=date_format) handler.setFormatter(formatter) root_logger.addHandler(handler) global __LOGFILE_CONFIGURED __LOGFILE_CONFIGURED = True
def query(params=None, setname=None, requesturl=None, location=None, return_url=False, return_root=False, opts=None, provider=None, endpoint=None, product='ec2', sigver='2'): ''' Perform a query against AWS services using Signature Version 2 Signing Process. This is documented at: http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html Regions and endpoints are documented at: http://docs.aws.amazon.com/general/latest/gr/rande.html Default ``product`` is ``ec2``. Valid ``product`` names are: .. code-block: yaml - autoscaling (Auto Scaling) - cloudformation (CloudFormation) - ec2 (Elastic Compute Cloud) - elasticache (ElastiCache) - elasticbeanstalk (Elastic BeanStalk) - elasticloadbalancing (Elastic Load Balancing) - elasticmapreduce (Elastic MapReduce) - iam (Identity and Access Management) - importexport (Import/Export) - monitoring (CloudWatch) - rds (Relational Database Service) - simpledb (SimpleDB) - sns (Simple Notification Service) - sqs (Simple Queue Service) ''' if params is None: params = {} if opts is None: opts = {} function = opts.get('function', (None, product)) providers = opts.get('providers', {}) if provider is None: prov_dict = providers.get(function[1], {}).get(product, {}) if prov_dict: driver = list(list(prov_dict.keys()))[0] provider = providers.get(driver, product) else: prov_dict = providers.get(provider, {}).get(product, {}) service_url = prov_dict.get('service_url', 'amazonaws.com') if not location: location = get_location(opts, provider) if endpoint is None: if not requesturl: endpoint = prov_dict.get( 'endpoint', '{0}.{1}.{2}'.format(product, location, service_url) ) requesturl = 'https://{0}/'.format(endpoint) else: endpoint = urlparse(requesturl).netloc if endpoint == '': endpoint_err = ('Could not find a valid endpoint in the ' 'requesturl: {0}. Looking for something ' 'like https://some.aws.endpoint/?args').format( requesturl ) LOG.error(endpoint_err) if return_url is True: return {'error': endpoint_err}, requesturl return {'error': endpoint_err} LOG.debug('Using AWS endpoint: {0}'.format(endpoint)) method = 'GET' aws_api_version = prov_dict.get( 'aws_api_version', prov_dict.get( '{0}_api_version'.format(product), DEFAULT_AWS_API_VERSION ) ) if sigver == '4': headers, requesturl = sig4( method, endpoint, params, prov_dict, aws_api_version, location, product, requesturl=requesturl ) params_with_headers = {} else: params_with_headers = sig2( method, endpoint, params, prov_dict, aws_api_version ) headers = {} attempts = 5 while attempts > 0: LOG.debug('AWS Request: {0}'.format(requesturl)) LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers)) try: result = requests.get(requesturl, headers=headers, params=params_with_headers) LOG.debug( 'AWS Response Status Code: {0}'.format( result.status_code ) ) LOG.trace( 'AWS Response Text: {0}'.format( result.text ) ) result.raise_for_status() break except requests.exceptions.HTTPError as exc: root = ET.fromstring(exc.response.content) data = xml.to_dict(root) # check to see if we should retry the query err_code = data.get('Errors', {}).get('Error', {}).get('Code', '') if attempts > 0 and err_code and err_code in AWS_RETRY_CODES: attempts -= 1 LOG.error( 'AWS Response Status Code and Error: [{0} {1}] {2}; ' 'Attempts remaining: {3}'.format( exc.response.status_code, exc, data, attempts ) ) # Wait a bit before continuing to prevent throttling time.sleep(2) continue LOG.error( 'AWS Response Status Code and Error: [{0} {1}] {2}'.format( exc.response.status_code, exc, data ) ) if return_url is True: return {'error': data}, requesturl return {'error': data} else: LOG.error( 'AWS Response Status Code and Error: [{0} {1}] {2}'.format( exc.response.status_code, exc, data ) ) if return_url is True: return {'error': data}, requesturl return {'error': data} response = result.text root = ET.fromstring(response) items = root[1] if return_root is True: items = root if setname: if sys.version_info < (2, 7): children_len = len(root.getchildren()) else: children_len = len(root) for item in range(0, children_len): comps = root[item].tag.split('}') if comps[1] == setname: items = root[item] ret = [] for item in items: ret.append(xml.to_dict(item)) if return_url is True: return ret, requesturl return ret