def compress(data, compresslevel=9): ''' Returns the data compressed at gzip level compression. ''' buf = StringIO() with open_fileobj(buf, 'wb', compresslevel) as ogz: ogz.write(data) compressed = buf.getvalue() return compressed
def _read_temp(data): tout = StringIO() tout.write(data) tout.seek(0) output = tout.read().splitlines() # Discard newlines tout.close() return output
def close(self): # Don't save unless there's something there. In Windows # the class gets initialized the first time with mode = w # which sets the initial value to ''. When the class closes # it clears out data and causes the test to fail. # I don't know why it get's initialized with a mode of 'w' # For the purposes of this test data shouldn't be empty # This is a problem with this class and not with the hosts # module if self.getvalue(): data[0] = self.getvalue() StringIO.close(self)
def _produce_output(report, failed, setup): ''' Produce output from the report dictionary generated by _generate_report ''' report_format = setup.get('report_format', 'yaml') log.debug('highstate output format: %s', report_format) if report_format == 'json': report_text = salt.utils.json.dumps(report) elif report_format == 'yaml': string_file = StringIO() salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False) string_file.seek(0) report_text = string_file.read() else: string_file = StringIO() _generate_html(report, string_file) string_file.seek(0) report_text = string_file.read() report_delivery = setup.get('report_delivery', 'file') log.debug('highstate report_delivery: %s', report_delivery) if report_delivery == 'file': output_file = _sprinkle(setup.get('file_output', '/tmp/test.rpt')) with salt.utils.files.fopen(output_file, 'w') as out: out.write(salt.utils.stringutils.to_str(report_text)) else: msg = MIMEText(report_text, report_format) sender = setup.get('smtp_sender', '') recipients = setup.get('smtp_recipients', '') if failed: subject = setup.get('smtp_failure_subject', 'Installation failure') else: subject = setup.get('smtp_success_subject', 'Installation success') subject = _sprinkle(subject) msg['Subject'] = subject msg['From'] = sender msg['To'] = recipients smtp = smtplib.SMTP(host=setup.get('smtp_server', '')) smtp.sendmail(sender, [x.strip() for x in recipients.split(',')], msg.as_string()) smtp.quit()
def _produce_output(report, failed, setup): """ Produce output from the report dictionary generated by _generate_report """ report_format = setup.get("report_format", "yaml") log.debug("highstate output format: %s", report_format) if report_format == "json": report_text = salt.utils.json.dumps(report) elif report_format == "yaml": string_file = StringIO() salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False) string_file.seek(0) report_text = string_file.read() else: string_file = StringIO() _generate_html(report, string_file) string_file.seek(0) report_text = string_file.read() report_delivery = setup.get("report_delivery", "file") log.debug("highstate report_delivery: %s", report_delivery) if report_delivery == "file": output_file = _sprinkle(setup.get("file_output", "/tmp/test.rpt")) with salt.utils.files.fopen(output_file, "w") as out: out.write(salt.utils.stringutils.to_str(report_text)) else: msg = MIMEText(report_text, report_format) sender = setup.get("smtp_sender", "") recipients = setup.get("smtp_recipients", "") if failed: subject = setup.get("smtp_failure_subject", "Installation failure") else: subject = setup.get("smtp_success_subject", "Installation success") subject = _sprinkle(subject) msg["Subject"] = subject msg["From"] = sender msg["To"] = recipients smtp = smtplib.SMTP(host=setup.get("smtp_server", "")) smtp.sendmail( sender, [x.strip() for x in recipients.split(",")], msg.as_string() ) smtp.quit()
def render(template_file, saltenv='base', sls='', argline='', context=None, tmplpath=None, **kws): ''' Render the template_file, passing the functions and grains into the Jinja rendering system. :rtype: string ''' from_str = argline == '-s' if not from_str and argline: raise SaltRenderError( 'Unknown renderer option: {opt}'.format(opt=argline)) tmp_data = salt.utils.templates.JINJA(template_file, to_str=True, salt=_split_module_dicts(), grains=__grains__, opts=__opts__, pillar=__pillar__, saltenv=saltenv, sls=sls, context=context, tmplpath=tmplpath, **kws) if not tmp_data.get('result', False): raise SaltRenderError( tmp_data.get('data', 'Unknown render error in jinja renderer')) return StringIO(tmp_data['data'])
def string_io(data=None): # cStringIO can't handle unicode ''' Pass data through to stringIO module and return result ''' try: return cStringIO(bytes(data)) except (UnicodeEncodeError, TypeError): return StringIO(data)
def render_sls(self, content, sls='', saltenv='base', **kws): if 'env' in kws: # "env" is not supported; Use "saltenv". kws.pop('env') return self.HIGHSTATE.state.rend['pydsl']( StringIO(content), saltenv=saltenv, sls=sls, **kws )
def render_sls(self, content, sls="", saltenv="base", **kws): if "env" in kws: # "env" is not supported; Use "saltenv". kws.pop("env") return self.HIGHSTATE.state.rend["pydsl"]( StringIO(content), saltenv=saltenv, sls=sls, **kws )
class _DelegateIO(object): ''' This class defines an object that captures whatever is written to a stream or file. ''' def __init__(self, delegate): self._captured = StringIO() self.delegate = delegate def write(self, text): if six.PY2 and isinstance(text, six.text_type): text = text.encode(__salt_system_encoding__) self._captured.write(text) self.delegate.write(text) def __getattr__(self, attr): try: return getattr(self._captured, attr) except AttributeError: return getattr(self.delegate, attr)
def process_sls_data(data, context=None, extract=False): sls_dir = os.path.dirname(sls.replace(".", os.path.sep)) if "." in sls else sls ctx = dict(sls_dir=sls_dir if sls_dir else ".") if context: ctx.update(context) tmplout = render_template( StringIO(data), saltenv, sls, context=ctx, argline=rt_argline.strip(), **kws ) high = render_data(tmplout, saltenv, sls, argline=rd_argline.strip()) return process_high_data(high, extract)
def __test_html(): ''' HTML generation test only used when called from the command line: python ./highstate.py Typical options for generating the report file: highstate: report_format: yaml report_delivery: file file_output: '/srv/salt/_returners/test.rpt' ''' with salt.utils.fopen('test.rpt', 'r') as input_file: data_text = input_file.read() data = yaml.safe_load(data_text) string_file = StringIO() _generate_html(data, string_file) string_file.seek(0) result = string_file.read() with salt.utils.fopen('test.html', 'w') as output: output.write(result)
def __test_html(): """ HTML generation test only used when called from the command line: python ./highstate.py Typical options for generating the report file: highstate: report_format: yaml report_delivery: file file_output: '/srv/salt/_returners/test.rpt' """ with salt.utils.files.fopen("test.rpt", "r") as input_file: data_text = salt.utils.stringutils.to_unicode(input_file.read()) data = salt.utils.yaml.safe_load(data_text) string_file = StringIO() _generate_html(data, string_file) string_file.seek(0) result = string_file.read() with salt.utils.files.fopen("test.html", "w") as output: output.write(salt.utils.stringutils.to_str(result))
def _render_sls(self, content, sls='', saltenv='base', argline='-G yaml . jinja', **kws): return self._renderers['stateconf']( StringIO(content), saltenv=saltenv, sls=sls, argline=argline, renderers=salt.loader.render(self.config, {}), **kws )
def render_sls(self, content, sls='', saltenv='base', **kws): if 'env' in kws: salt.utils.warn_until( 'Oxygen', 'Parameter \'env\' has been detected in the argument list. This ' 'parameter is no longer used and has been replaced by \'saltenv\' ' 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' ) kws.pop('env') return self.HIGHSTATE.state.rend['pydsl']( StringIO(content), saltenv=saltenv, sls=sls, **kws )
def mock_open(data=None): ''' Mock "open" function in a simple way. :param data: :return: ''' data = StringIO(data) mock = MagicMock(spec=io.FileIO) handle = MagicMock(spec=io.FileIO) handle.write.return_value = None handle.__enter__.return_value = data or handle mock.return_value = handle return mock
def render( template_file, saltenv="base", sls="", argline="", context=None, tmplpath=None, **kws ): """ Render the template_file, passing the functions and grains into the Jinja rendering system. :rtype: string """ from_str = argline == "-s" if not from_str and argline: raise SaltRenderError("Unknown renderer option: {opt}".format(opt=argline)) tmp_data = salt.utils.templates.JINJA( template_file, to_str=True, salt=_split_module_dicts(), grains=__grains__, opts=__opts__, pillar=__pillar__, saltenv=saltenv, sls=sls, context=context, tmplpath=tmplpath, proxy=__proxy__, **kws ) if not tmp_data.get("result", False): raise SaltRenderError( tmp_data.get("data", "Unknown render error in jinja renderer") ) if isinstance(tmp_data["data"], bytes): tmp_data["data"] = tmp_data["data"].decode(__salt_system_encoding__) return StringIO(tmp_data["data"])
def __init__(self, data=None): if data: StringIO.__init__(self, data) else: StringIO.__init__(self) self.data = []
def _produce_output(report, failed, setup): """ Produce output from the report dictionary generated by _generate_report """ report_format = setup.get("report_format", "yaml") log.debug("highstate output format: %s", report_format) if report_format == "json": report_text = salt.utils.json.dumps(report) elif report_format == "yaml": string_file = StringIO() salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False) string_file.seek(0) report_text = string_file.read() else: string_file = StringIO() _generate_html(report, string_file) string_file.seek(0) report_text = string_file.read() report_delivery = setup.get("report_delivery", "file") log.debug("highstate report_delivery: %s", report_delivery) if report_delivery == "file": output_file = _sprinkle(setup.get("file_output", "/tmp/test.rpt")) with salt.utils.files.fopen(output_file, "w") as out: out.write(salt.utils.stringutils.to_str(report_text)) else: msg = MIMEText(report_text, report_format) sender = setup.get("smtp_sender", "") recipients = setup.get("smtp_recipients", "") host = setup.get('smtp_server', '') port = int(setup.get('smtp_port', 25)) tls = setup.get('smtp_tls') username = setup.get('smtp_username') password = setup.get('smtp_password') if failed: subject = setup.get("smtp_failure_subject", "Installation failure") else: subject = setup.get("smtp_success_subject", "Installation success") subject = _sprinkle(subject) msg["Subject"] = subject msg["From"] = sender msg["To"] = recipients log.debug('highstate smtp port: %d', port) smtp = smtplib.SMTP(host=host, port=port) if tls is True: smtp.starttls() log.debug('highstate smtp tls enabled') if username and password: smtp.login(username, password) log.debug('highstate smtp authenticated') smtp.sendmail(sender, [x.strip() for x in recipients.split(',')], msg.as_string()) log.debug('highstate message sent.') smtp.quit()
def query(url, method="GET", params=None, data=None, data_file=None, header_dict=None, header_list=None, header_file=None, username=None, password=None, auth=None, decode=False, decode_type="auto", status=False, headers=False, text=False, cookies=None, cookie_jar=None, cookie_format="lwp", persist_session=False, session_cookie_jar=None, data_render=False, data_renderer=None, header_render=False, header_renderer=None, template_dict=None, test=False, test_url=None, node="minion", port=80, opts=None, backend=None, ca_bundle=None, verify_ssl=None, cert=None, text_out=None, headers_out=None, decode_out=None, stream=False, streaming_callback=None, header_callback=None, handle=False, agent=USERAGENT, hide_fields=None, raise_error=True, formdata=False, formdata_fieldname=None, formdata_filename=None, **kwargs): """ Query a resource, and decode the return data """ ret = {} if opts is None: if node == "master": opts = salt.config.master_config( os.path.join(salt.syspaths.CONFIG_DIR, "master")) elif node == "minion": opts = salt.config.minion_config( os.path.join(salt.syspaths.CONFIG_DIR, "minion")) else: opts = {} if not backend: backend = opts.get("backend", "tornado") match = re.match( r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)", url, ) if not match: salt.utils.network.refresh_dns() if backend == "requests": if HAS_REQUESTS is False: ret["error"] = ("http.query has been set to use requests, but the " "requests library does not seem to be installed") log.error(ret["error"]) return ret else: requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Some libraries don't support separation of url and GET parameters # Don't need a try/except block, since Salt depends on tornado url_full = salt.ext.tornado.httputil.url_concat(url, params) if params else url if ca_bundle is None: ca_bundle = get_ca_bundle(opts) if verify_ssl is None: verify_ssl = opts.get("verify_ssl", True) if cert is None: cert = opts.get("cert", None) if data_file is not None: data = _render(data_file, data_render, data_renderer, template_dict, opts) # Make sure no secret fields show up in logs log_url = sanitize_url(url_full, hide_fields) log.debug("Requesting URL %s using %s method", log_url, method) log.debug("Using backend: %s", backend) if method == "POST" and log.isEnabledFor(logging.TRACE): # Make sure no secret fields show up in logs if isinstance(data, dict): log_data = data.copy() if isinstance(hide_fields, list): for item in data: for field in hide_fields: if item == field: log_data[item] = "XXXXXXXXXX" log.trace("Request POST Data: %s", pprint.pformat(log_data)) else: log.trace("Request POST Data: %s", pprint.pformat(data)) if header_file is not None: header_tpl = _render(header_file, header_render, header_renderer, template_dict, opts) if isinstance(header_tpl, dict): header_dict = header_tpl else: header_list = header_tpl.splitlines() if header_dict is None: header_dict = {} if header_list is None: header_list = [] if cookie_jar is None: cookie_jar = os.path.join( opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.txt") if session_cookie_jar is None: session_cookie_jar = os.path.join( opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.session.p") if persist_session is True and salt.utils.msgpack.HAS_MSGPACK: # TODO: This is hackish; it will overwrite the session cookie jar with # all cookies from this one connection, rather than behaving like a # proper cookie jar. Unfortunately, since session cookies do not # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: session_cookies = salt.utils.msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: salt.utils.msgpack.dump('', fh_) for header in header_list: comps = header.split(":") if len(comps) < 2: continue header_dict[comps[0].strip()] = comps[1].strip() if not auth: if username and password: auth = (username, password) if agent == USERAGENT: user_agent = opts.get('user_agent', None) if user_agent: agent = user_agent agent = '{0} http.query()'.format(agent) header_dict['User-agent'] = agent if backend == "requests": sess = requests.Session() sess.auth = auth sess.headers.update(header_dict) log.trace("Request Headers: %s", sess.headers) sess_cookies = sess.cookies sess.verify = verify_ssl elif backend == "urllib2": sess_cookies = None else: # Tornado sess_cookies = None if cookies is not None: if cookie_format == "mozilla": sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar( cookie_jar) else: sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar( cookie_jar) if not os.path.isfile(cookie_jar): sess_cookies.save() sess_cookies.load() if test is True: if test_url is None: return {} else: url = test_url ret["test"] = True if backend == "requests": req_kwargs = {} if stream is True: if requests.__version__[0] == "0": # 'stream' was called 'prefetch' before 1.0, with flipped meaning req_kwargs["prefetch"] = False else: req_kwargs["stream"] = True # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs["cert"] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs["cert"] = cert else: log.error( "The client-side certificate path that" " was passed is not valid: %s", cert, ) if formdata: if not formdata_fieldname: ret["error"] = "formdata_fieldname is required when formdata=True" log.error(ret["error"]) return ret result = sess.request(method, url, params=params, files={ formdata_fieldname: (formdata_filename, StringIO(data)) }, **req_kwargs) else: result = sess.request(method, url, params=params, data=data, **req_kwargs) result.raise_for_status() if stream is True: # fake a HTTP response header header_callback("HTTP/1.0 {0} MESSAGE".format(result.status_code)) # fake streaming the content streaming_callback(result.content) return { "handle": result, } if handle is True: return { "handle": result, "body": result.content, } log.debug("Final URL location of Response: %s", sanitize_url(result.url, hide_fields)) result_status_code = result.status_code result_headers = result.headers result_text = result.content result_cookies = result.cookies body = result.content if not isinstance(body, six.text_type): body = body.decode(result.encoding or "utf-8") ret["body"] = body elif backend == "urllib2": request = urllib_request.Request(url_full, data) handlers = [ urllib_request.HTTPHandler, urllib_request.HTTPCookieProcessor(sess_cookies), ] if url.startswith("https"): hostname = request.get_host() handlers[0] = urllib_request.HTTPSHandler(1) if not HAS_MATCHHOSTNAME: log.warning( "match_hostname() not available, SSL hostname checking " "not available. THIS CONNECTION MAY NOT BE SECURE!") elif verify_ssl is False: log.warning("SSL certificate verification has been explicitly " "disabled. THIS CONNECTION MAY NOT BE SECURE!") else: if ":" in hostname: hostname, port = hostname.split(":") else: port = 443 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((hostname, int(port))) sockwrap = ssl.wrap_socket(sock, ca_certs=ca_bundle, cert_reqs=ssl.CERT_REQUIRED) try: match_hostname(sockwrap.getpeercert(), hostname) except CertificateError as exc: ret["error"] = ( "The certificate was invalid. Error returned was: %s", pprint.pformat(exc), ) return ret # Client-side cert handling if cert is not None: cert_chain = None if isinstance(cert, six.string_types): if os.path.exists(cert): cert_chain = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): cert_chain = cert else: log.error( "The client-side certificate path that was " "passed is not valid: %s", cert, ) return if hasattr(ssl, "SSLContext"): # Python >= 2.7.9 context = ssl.SSLContext.load_cert_chain(*cert_chain) handlers.append( urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123 else: # Python < 2.7.9 cert_kwargs = { "host": request.get_host(), "port": port, "cert_file": cert_chain[0], } if len(cert_chain) > 1: cert_kwargs["key_file"] = cert_chain[1] handlers[ 0] = salt.ext.six.moves.http_client.HTTPSConnection( **cert_kwargs) opener = urllib_request.build_opener(*handlers) for header in header_dict: request.add_header(header, header_dict[header]) request.get_method = lambda: method try: result = opener.open(request) except URLError as exc: return {"Error": six.text_type(exc)} if stream is True or handle is True: return { "handle": result, "body": result.content, } result_status_code = result.code result_headers = dict(result.info()) result_text = result.read() if "Content-Type" in result_headers: res_content_type, res_params = cgi.parse_header( result_headers["Content-Type"]) if (res_content_type.startswith("text/") and "charset" in res_params and not isinstance(result_text, six.text_type)): result_text = result_text.decode(res_params["charset"]) if six.PY3 and isinstance(result_text, bytes): result_text = result_text.decode('utf-8') ret['body'] = result_text else: # Tornado req_kwargs = {} # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs["client_cert"] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs["client_cert"] = cert[0] req_kwargs["client_key"] = cert[1] else: log.error( "The client-side certificate path that " "was passed is not valid: %s", cert, ) if isinstance(data, dict): data = _urlencode(data) if verify_ssl: req_kwargs["ca_certs"] = ca_bundle max_body = opts.get("http_max_body", salt.config.DEFAULT_MINION_OPTS["http_max_body"]) connect_timeout = opts.get( "http_connect_timeout", salt.config.DEFAULT_MINION_OPTS["http_connect_timeout"], ) timeout = opts.get( "http_request_timeout", salt.config.DEFAULT_MINION_OPTS["http_request_timeout"], ) client_argspec = None proxy_host = opts.get("proxy_host", None) if proxy_host: # tornado requires a str for proxy_host, cannot be a unicode str in py2 proxy_host = salt.utils.stringutils.to_str(proxy_host) proxy_port = opts.get("proxy_port", None) proxy_username = opts.get("proxy_username", None) if proxy_username: # tornado requires a str, cannot be unicode str in py2 proxy_username = salt.utils.stringutils.to_str(proxy_username) proxy_password = opts.get("proxy_password", None) if proxy_password: # tornado requires a str, cannot be unicode str in py2 proxy_password = salt.utils.stringutils.to_str(proxy_password) no_proxy = opts.get("no_proxy", []) # Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones # except we remove the valid ones if a url has a no_proxy hostname in it if urlparse(url_full).hostname in no_proxy: proxy_host = None proxy_port = None # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: ret["error"] = ( "proxy_host and proxy_port has been set. This requires pycurl and tornado, " "but the libraries does not seem to be installed") log.error(ret["error"]) return ret salt.ext.tornado.httpclient.AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") client_argspec = salt.utils.args.get_function_argspec( salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize ) else: client_argspec = salt.utils.args.get_function_argspec( salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient. initialize) supports_max_body_size = "max_body_size" in client_argspec.args req_kwargs.update({ "method": method, "headers": header_dict, "auth_username": username, "auth_password": password, "body": data, "validate_cert": verify_ssl, "allow_nonstandard_methods": True, "streaming_callback": streaming_callback, "header_callback": header_callback, "connect_timeout": connect_timeout, "request_timeout": timeout, "proxy_host": proxy_host, "proxy_port": proxy_port, "proxy_username": proxy_username, "proxy_password": proxy_password, "raise_error": raise_error, "decompress_response": False, }) # Unicode types will cause a TypeError when Tornado's curl HTTPClient # invokes setopt. Therefore, make sure all arguments we pass which # contain strings are str types. req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True) try: download_client = (HTTPClient(max_body_size=max_body) if supports_max_body_size else HTTPClient()) result = download_client.fetch(url_full, **req_kwargs) except salt.ext.tornado.httpclient.HTTPError as exc: ret["status"] = exc.code ret["error"] = six.text_type(exc) return ret except (socket.herror, socket.error, socket.timeout, socket.gaierror) as exc: if status is True: ret["status"] = 0 ret["error"] = six.text_type(exc) log.debug("Cannot perform 'http.query': {0} - {1}".format( url_full, ret["error"])) return ret if stream is True or handle is True: return { "handle": result, "body": result.body, } result_status_code = result.code result_headers = result.headers result_text = result.body if "Content-Type" in result_headers: res_content_type, res_params = cgi.parse_header( result_headers["Content-Type"]) if (res_content_type.startswith("text/") and "charset" in res_params and not isinstance(result_text, six.text_type)): result_text = result_text.decode(res_params["charset"]) if six.PY3 and isinstance(result_text, bytes): result_text = result_text.decode("utf-8") ret["body"] = result_text if "Set-Cookie" in result_headers and cookies is not None: result_cookies = parse_cookie_header(result_headers["Set-Cookie"]) for item in result_cookies: sess_cookies.set_cookie(item) else: result_cookies = None if isinstance(result_headers, list): result_headers_dict = {} for header in result_headers: comps = header.split(":") result_headers_dict[comps[0].strip()] = ":".join(comps[1:]).strip() result_headers = result_headers_dict log.debug("Response Status Code: %s", result_status_code) log.trace("Response Headers: %s", result_headers) log.trace("Response Cookies: %s", sess_cookies) # log.trace("Content: %s", result_text) coding = result_headers.get("Content-Encoding", "identity") # Requests will always decompress the content, and working around that is annoying. if backend != "requests": result_text = __decompressContent(coding, result_text) try: log.trace("Response Text: %s", result_text) except UnicodeEncodeError as exc: log.trace( "Cannot Trace Log Response Text: %s. This may be due to " "incompatibilities between requests and logging.", exc, ) if text_out is not None: with salt.utils.files.fopen(text_out, "w") as tof: tof.write(result_text) if headers_out is not None and os.path.exists(headers_out): with salt.utils.files.fopen(headers_out, "w") as hof: hof.write(result_headers) if cookies is not None: sess_cookies.save() if persist_session is True and salt.utils.msgpack.HAS_MSGPACK: # TODO: See persist_session above if "set-cookie" in result_headers: with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_: session_cookies = result_headers.get("set-cookie", None) if session_cookies is not None: salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_) else: salt.utils.msgpack.dump('', fh_) if status is True: ret["status"] = result_status_code if headers is True: ret["headers"] = result_headers if decode is True: if decode_type == "auto": content_type = result_headers.get("content-type", "application/json") if "xml" in content_type: decode_type = "xml" elif "json" in content_type: decode_type = "json" elif "yaml" in content_type: decode_type = "yaml" else: decode_type = "plain" valid_decodes = ("json", "xml", "yaml", "plain") if decode_type not in valid_decodes: ret["error"] = ("Invalid decode_type specified. " "Valid decode types are: {0}".format( pprint.pformat(valid_decodes))) log.error(ret["error"]) return ret if decode_type == "json": ret["dict"] = salt.utils.json.loads(result_text) elif decode_type == "xml": ret["dict"] = [] items = ET.fromstring(result_text) for item in items: ret["dict"].append(xml.to_dict(item)) elif decode_type == "yaml": ret["dict"] = salt.utils.data.decode( salt.utils.yaml.safe_load(result_text)) else: text = True if decode_out: with salt.utils.files.fopen(decode_out, "w") as dof: dof.write(result_text) if text is True: ret["text"] = result_text return ret
def render_sls(self, content, sls='', env='base', **kws): return self.HIGHSTATE.state.rend['pydsl'](StringIO(content), env=env, sls=sls, **kws)
def _get_rend(renderer, value): ''' We need a new MagicMock each time since we're dealing with StringIO objects which are read like files. ''' return {renderer: MagicMock(return_value=StringIO(value))}
def compile_template(template, renderers, default, blacklist, whitelist, saltenv='base', sls='', input_data='', **kwargs): ''' Take the path to a template and return the high data structure derived from the template. ''' # if any error occurs, we return an empty dictionary ret = {} log.debug('compile template: %s', template) if 'env' in kwargs: # "env" is not supported; Use "saltenv". kwargs.pop('env') if template != ':string:': # Template was specified incorrectly if not isinstance(template, six.string_types): log.error('Template was specified incorrectly: %s', template) return ret # Template does not exist if not os.path.isfile(template): log.error('Template does not exist: %s', template) return ret # Template is an empty file if salt.utils.files.is_empty(template): log.debug('Template is an empty file: %s', template) return ret with codecs.open(template, encoding=SLS_ENCODING) as ifile: # data input to the first render function in the pipe input_data = ifile.read() if not input_data.strip(): # Template is nothing but whitespace log.error('Template is nothing but whitespace: %s', template) return ret # Get the list of render funcs in the render pipe line. render_pipe = template_shebang(template, renderers, default, blacklist, whitelist, input_data) windows_newline = '\r\n' in input_data input_data = StringIO(input_data) for render, argline in render_pipe: if salt.utils.stringio.is_readable(input_data): input_data.seek(0) # pylint: disable=no-member render_kwargs = dict(renderers=renderers, tmplpath=template) render_kwargs.update(kwargs) if argline: render_kwargs['argline'] = argline start = time.time() ret = render(input_data, saltenv, sls, **render_kwargs) log.profile( 'Time (in seconds) to render \'%s\' using \'%s\' renderer: %s', template, render.__module__.split('.')[-1], time.time() - start ) if ret is None: # The file is empty or is being written elsewhere time.sleep(0.01) ret = render(input_data, saltenv, sls, **render_kwargs) input_data = ret if log.isEnabledFor(logging.GARBAGE): # pylint: disable=no-member # If ret is not a StringIO (which means it was rendered using # yaml, mako, or another engine which renders to a data # structure) we don't want to log this. if salt.utils.stringio.is_readable(ret): log.debug( 'Rendered data from file: %s:\n%s', template, salt.utils.locales.sdecode(ret.read())) # pylint: disable=no-member ret.seek(0) # pylint: disable=no-member # Preserve newlines from original template if windows_newline: if salt.utils.stringio.is_readable(ret): is_stringio = True contents = ret.read() else: is_stringio = False contents = ret if isinstance(contents, six.string_types): if '\r\n' not in contents: contents = contents.replace('\n', '\r\n') ret = StringIO(contents) if is_stringio else contents else: if is_stringio: ret.seek(0) return ret
def close(self): data[0] = self.getvalue() StringIO.close(self)
def __init__(self, fn, mode='r'): initial_value = data[0] if 'w' in mode: initial_value = '' StringIO.__init__(self, initial_value)
def test_exc_info_on_loglevel(self): def raise_exception_on_purpose(): 1/0 # pylint: disable=pointless-statement log = saltlog.SaltLoggingClass(__name__) # Only stream2 should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.INFO) handler2.setLevel(logging.DEBUG) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.DEBUG) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue() ) self.assertNotIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue() ) self.assertIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2) # Both streams should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.INFO) handler2.setLevel(logging.DEBUG) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.INFO) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue() ) self.assertIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue() ) self.assertIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2) # No streams should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.ERROR) handler2.setLevel(logging.INFO) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.DEBUG) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue() ) self.assertNotIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue() ) self.assertNotIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2)
def test_exc_info_on_loglevel(self): def raise_exception_on_purpose(): 1 / 0 # pylint: disable=pointless-statement log = saltlog.SaltLoggingClass(__name__) # Only stream2 should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.INFO) handler2.setLevel(logging.DEBUG) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.DEBUG) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue()) self.assertNotIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue()) self.assertIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2) # Both streams should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.INFO) handler2.setLevel(logging.DEBUG) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.INFO) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue()) self.assertIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue()) self.assertIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2) # No streams should contain the traceback stream1 = StringIO() stream2 = StringIO() handler1 = StreamHandler(stream1) handler2 = StreamHandler(stream2) handler1.setLevel(logging.ERROR) handler2.setLevel(logging.INFO) log.addHandler(handler1) log.addHandler(handler2) try: raise_exception_on_purpose() except ZeroDivisionError as exc: log.error('Exception raised on purpose caught: ZeroDivisionError', exc_info_on_loglevel=logging.DEBUG) try: self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream1.getvalue()) self.assertNotIn('Traceback (most recent call last)', stream1.getvalue()) self.assertIn( 'Exception raised on purpose caught: ZeroDivisionError', stream2.getvalue()) self.assertNotIn('Traceback (most recent call last)', stream2.getvalue()) finally: log.removeHandler(handler1) log.removeHandler(handler2)
def request(self, path='/', method='GET', app_path='', scheme='http', proto='HTTP/1.1', body=None, qs=None, headers=None, **kwargs): """ CherryPy does not have a facility for serverless unit testing. However this recipe demonstrates a way of doing it by calling its internal API to simulate an incoming request. This will exercise the whole stack from there. Remember a couple of things: * CherryPy is multithreaded. The response you will get from this method is a thread-data object attached to the current thread. Unless you use many threads from within a unit test, you can mostly forget about the thread data aspect of the response. * Responses are dispatched to a mounted application's page handler, if found. This is the reason why you must indicate which app you are targeting with this request by specifying its mount point. You can simulate various request settings by setting the `headers` parameter to a dictionary of headers, the request's `scheme` or `protocol`. .. seealso: http://docs.cherrypy.org/stable/refman/_cprequest.html#cherrypy._cprequest.Response """ # This is a required header when running HTTP/1.1 h = {'Host': '127.0.0.1'} # if we had some data passed as the request entity # let's make sure we have the content-length set fd = None if body is not None: h['content-length'] = '{0}'.format(len(body)) fd = StringIO(body) if headers is not None: h.update(headers) # Get our application and run the request against it app = cherrypy.tree.apps.get(app_path) if not app: # XXX: perhaps not the best exception to raise? raise AssertionError( "No application mounted at '{0}'".format(app_path)) # Cleanup any previous returned response # between calls to this method app.release_serving() # Let's fake the local and remote addresses request, response = app.get_serving(local, remote, scheme, proto) try: h = [(k, v) for k, v in six.iteritems(h)] response = request.run(method, path, qs, proto, h, fd) finally: if fd: fd.close() fd = None if response.output_status.startswith(six.b('500')): response_body = response.collapse_body() if six.PY3: response_body = response_body.decode(__salt_system_encoding__) print(response_body) raise AssertionError("Unexpected error") # collapse the response into a bytestring response.collapse_body() return request, response
def query(url, method='GET', params=None, data=None, data_file=None, header_dict=None, header_list=None, header_file=None, username=None, password=None, auth=None, decode=False, decode_type='auto', status=False, headers=False, text=False, cookies=None, cookie_jar=None, cookie_format='lwp', persist_session=False, session_cookie_jar=None, data_render=False, data_renderer=None, header_render=False, header_renderer=None, template_dict=None, test=False, test_url=None, node='minion', port=80, opts=None, backend=None, ca_bundle=None, verify_ssl=None, cert=None, text_out=None, headers_out=None, decode_out=None, stream=False, streaming_callback=None, header_callback=None, handle=False, agent=USERAGENT, hide_fields=None, raise_error=True, formdata=False, formdata_fieldname=None, formdata_filename=None, **kwargs): ''' Query a resource, and decode the return data ''' ret = {} if opts is None: if node == 'master': opts = salt.config.master_config( os.path.join(salt.syspaths.CONFIG_DIR, 'master') ) elif node == 'minion': opts = salt.config.minion_config( os.path.join(salt.syspaths.CONFIG_DIR, 'minion') ) else: opts = {} if not backend: backend = opts.get('backend', 'tornado') match = re.match(r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)', url) if not match: salt.utils.network.refresh_dns() if backend == 'requests': if HAS_REQUESTS is False: ret['error'] = ('http.query has been set to use requests, but the ' 'requests library does not seem to be installed') log.error(ret['error']) return ret else: requests_log = logging.getLogger('requests') requests_log.setLevel(logging.WARNING) # Some libraries don't support separation of url and GET parameters # Don't need a try/except block, since Salt depends on tornado url_full = salt.ext.tornado.httputil.url_concat(url, params) if params else url if ca_bundle is None: ca_bundle = get_ca_bundle(opts) if verify_ssl is None: verify_ssl = opts.get('verify_ssl', True) if cert is None: cert = opts.get('cert', None) if data_file is not None: data = _render( data_file, data_render, data_renderer, template_dict, opts ) # Make sure no secret fields show up in logs log_url = sanitize_url(url_full, hide_fields) log.debug('Requesting URL %s using %s method', log_url, method) log.debug("Using backend: %s", backend) if method == 'POST' and log.isEnabledFor(logging.TRACE): # Make sure no secret fields show up in logs if isinstance(data, dict): log_data = data.copy() if isinstance(hide_fields, list): for item in data: for field in hide_fields: if item == field: log_data[item] = 'XXXXXXXXXX' log.trace('Request POST Data: %s', pprint.pformat(log_data)) else: log.trace('Request POST Data: %s', pprint.pformat(data)) if header_file is not None: header_tpl = _render( header_file, header_render, header_renderer, template_dict, opts ) if isinstance(header_tpl, dict): header_dict = header_tpl else: header_list = header_tpl.splitlines() if header_dict is None: header_dict = {} if header_list is None: header_list = [] if cookie_jar is None: cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt') if session_cookie_jar is None: session_cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p') if persist_session is True and salt.utils.msgpack.HAS_MSGPACK: # TODO: This is hackish; it will overwrite the session cookie jar with # all cookies from this one connection, rather than behaving like a # proper cookie jar. Unfortunately, since session cookies do not # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: session_cookies = salt.utils.msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: salt.utils.msgpack.dump('', fh_) for header in header_list: comps = header.split(':') if len(comps) < 2: continue header_dict[comps[0].strip()] = comps[1].strip() if not auth: if username and password: auth = (username, password) if agent == USERAGENT: agent = '{0} http.query()'.format(agent) header_dict['User-agent'] = agent if backend == 'requests': sess = requests.Session() sess.auth = auth sess.headers.update(header_dict) log.trace('Request Headers: %s', sess.headers) sess_cookies = sess.cookies sess.verify = verify_ssl elif backend == 'urllib2': sess_cookies = None else: # Tornado sess_cookies = None if cookies is not None: if cookie_format == 'mozilla': sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar) else: sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar) if not os.path.isfile(cookie_jar): sess_cookies.save() sess_cookies.load() if test is True: if test_url is None: return {} else: url = test_url ret['test'] = True if backend == 'requests': req_kwargs = {} if stream is True: if requests.__version__[0] == '0': # 'stream' was called 'prefetch' before 1.0, with flipped meaning req_kwargs['prefetch'] = False else: req_kwargs['stream'] = True # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs['cert'] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs['cert'] = cert else: log.error('The client-side certificate path that' ' was passed is not valid: %s', cert) if formdata: if not formdata_fieldname: ret['error'] = ('formdata_fieldname is required when formdata=True') log.error(ret['error']) return ret result = sess.request( method, url, params=params, files={formdata_fieldname: (formdata_filename, StringIO(data))}, **req_kwargs ) else: result = sess.request( method, url, params=params, data=data, **req_kwargs ) result.raise_for_status() if stream is True: # fake a HTTP response header header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code)) # fake streaming the content streaming_callback(result.content) return { 'handle': result, } if handle is True: return { 'handle': result, 'body': result.content, } log.debug('Final URL location of Response: %s', sanitize_url(result.url, hide_fields)) result_status_code = result.status_code result_headers = result.headers result_text = result.content result_cookies = result.cookies body = result.content if not isinstance(body, six.text_type): body = body.decode(result.encoding or 'utf-8') ret['body'] = body elif backend == 'urllib2': request = urllib_request.Request(url_full, data) handlers = [ urllib_request.HTTPHandler, urllib_request.HTTPCookieProcessor(sess_cookies) ] if url.startswith('https'): hostname = request.get_host() handlers[0] = urllib_request.HTTPSHandler(1) if not HAS_MATCHHOSTNAME: log.warning('match_hostname() not available, SSL hostname checking ' 'not available. THIS CONNECTION MAY NOT BE SECURE!') elif verify_ssl is False: log.warning('SSL certificate verification has been explicitly ' 'disabled. THIS CONNECTION MAY NOT BE SECURE!') else: if ':' in hostname: hostname, port = hostname.split(':') else: port = 443 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((hostname, int(port))) sockwrap = ssl.wrap_socket( sock, ca_certs=ca_bundle, cert_reqs=ssl.CERT_REQUIRED ) try: match_hostname(sockwrap.getpeercert(), hostname) except CertificateError as exc: ret['error'] = ( 'The certificate was invalid. ' 'Error returned was: %s', pprint.pformat(exc) ) return ret # Client-side cert handling if cert is not None: cert_chain = None if isinstance(cert, six.string_types): if os.path.exists(cert): cert_chain = (cert) elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): cert_chain = cert else: log.error('The client-side certificate path that was ' 'passed is not valid: %s', cert) return if hasattr(ssl, 'SSLContext'): # Python >= 2.7.9 context = ssl.SSLContext.load_cert_chain(*cert_chain) handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123 else: # Python < 2.7.9 cert_kwargs = { 'host': request.get_host(), 'port': port, 'cert_file': cert_chain[0] } if len(cert_chain) > 1: cert_kwargs['key_file'] = cert_chain[1] handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs) opener = urllib_request.build_opener(*handlers) for header in header_dict: request.add_header(header, header_dict[header]) request.get_method = lambda: method try: result = opener.open(request) except URLError as exc: return {'Error': six.text_type(exc)} if stream is True or handle is True: return { 'handle': result, 'body': result.content, } result_status_code = result.code result_headers = dict(result.info()) result_text = result.read() if 'Content-Type' in result_headers: res_content_type, res_params = cgi.parse_header(result_headers['Content-Type']) if res_content_type.startswith('text/') and \ 'charset' in res_params and \ not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): result_text = result_text.decode('utf-8') ret['body'] = result_text else: # Tornado req_kwargs = {} # Client-side cert handling if cert is not None: if isinstance(cert, six.string_types): if os.path.exists(cert): req_kwargs['client_cert'] = cert elif isinstance(cert, list): if os.path.exists(cert[0]) and os.path.exists(cert[1]): req_kwargs['client_cert'] = cert[0] req_kwargs['client_key'] = cert[1] else: log.error('The client-side certificate path that ' 'was passed is not valid: %s', cert) if isinstance(data, dict): data = _urlencode(data) if verify_ssl: req_kwargs['ca_certs'] = ca_bundle max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body']) connect_timeout = opts.get('http_connect_timeout', salt.config.DEFAULT_MINION_OPTS['http_connect_timeout']) timeout = opts.get('http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout']) client_argspec = None proxy_host = opts.get('proxy_host', None) if proxy_host: # tornado requires a str for proxy_host, cannot be a unicode str in py2 proxy_host = salt.utils.stringutils.to_str(proxy_host) proxy_port = opts.get('proxy_port', None) proxy_username = opts.get('proxy_username', None) if proxy_username: # tornado requires a str, cannot be unicode str in py2 proxy_username = salt.utils.stringutils.to_str(proxy_username) proxy_password = opts.get('proxy_password', None) if proxy_password: # tornado requires a str, cannot be unicode str in py2 proxy_password = salt.utils.stringutils.to_str(proxy_password) no_proxy = opts.get('no_proxy', []) # Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones # except we remove the valid ones if a url has a no_proxy hostname in it if urlparse(url_full).hostname in no_proxy: proxy_host = None proxy_port = None # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, ' 'but the libraries does not seem to be installed') log.error(ret['error']) return ret salt.ext.tornado.httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient') client_argspec = salt.utils.args.get_function_argspec( salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize) else: client_argspec = salt.utils.args.get_function_argspec( salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize) supports_max_body_size = 'max_body_size' in client_argspec.args req_kwargs.update({ 'method': method, 'headers': header_dict, 'auth_username': username, 'auth_password': password, 'body': data, 'validate_cert': verify_ssl, 'allow_nonstandard_methods': True, 'streaming_callback': streaming_callback, 'header_callback': header_callback, 'connect_timeout': connect_timeout, 'request_timeout': timeout, 'proxy_host': proxy_host, 'proxy_port': proxy_port, 'proxy_username': proxy_username, 'proxy_password': proxy_password, 'raise_error': raise_error, 'decompress_response': False, }) # Unicode types will cause a TypeError when Tornado's curl HTTPClient # invokes setopt. Therefore, make sure all arguments we pass which # contain strings are str types. req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True) try: download_client = HTTPClient(max_body_size=max_body) \ if supports_max_body_size \ else HTTPClient() result = download_client.fetch(url_full, **req_kwargs) except salt.ext.tornado.httpclient.HTTPError as exc: ret['status'] = exc.code ret['error'] = six.text_type(exc) return ret except socket.gaierror as exc: if status is True: ret['status'] = 0 ret['error'] = six.text_type(exc) return ret if stream is True or handle is True: return { 'handle': result, 'body': result.body, } result_status_code = result.code result_headers = result.headers result_text = result.body if 'Content-Type' in result_headers: res_content_type, res_params = cgi.parse_header(result_headers['Content-Type']) if res_content_type.startswith('text/') and \ 'charset' in res_params and \ not isinstance(result_text, six.text_type): result_text = result_text.decode(res_params['charset']) if six.PY3 and isinstance(result_text, bytes): result_text = result_text.decode('utf-8') ret['body'] = result_text if 'Set-Cookie' in result_headers and cookies is not None: result_cookies = parse_cookie_header(result_headers['Set-Cookie']) for item in result_cookies: sess_cookies.set_cookie(item) else: result_cookies = None if isinstance(result_headers, list): result_headers_dict = {} for header in result_headers: comps = header.split(':') result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip() result_headers = result_headers_dict log.debug('Response Status Code: %s', result_status_code) log.trace('Response Headers: %s', result_headers) log.trace('Response Cookies: %s', sess_cookies) # log.trace("Content: %s", result_text) coding = result_headers.get('Content-Encoding', "identity") # Requests will always decompress the content, and working around that is annoying. if backend != 'requests': result_text = __decompressContent(coding, result_text) try: log.trace('Response Text: %s', result_text) except UnicodeEncodeError as exc: log.trace('Cannot Trace Log Response Text: %s. This may be due to ' 'incompatibilities between requests and logging.', exc) if text_out is not None: with salt.utils.files.fopen(text_out, 'w') as tof: tof.write(result_text) if headers_out is not None and os.path.exists(headers_out): with salt.utils.files.fopen(headers_out, 'w') as hof: hof.write(result_headers) if cookies is not None: sess_cookies.save() if persist_session is True and salt.utils.msgpack.HAS_MSGPACK: # TODO: See persist_session above if 'set-cookie' in result_headers: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_) else: salt.utils.msgpack.dump('', fh_) if status is True: ret['status'] = result_status_code if headers is True: ret['headers'] = result_headers if decode is True: if decode_type == 'auto': content_type = result_headers.get( 'content-type', 'application/json' ) if 'xml' in content_type: decode_type = 'xml' elif 'json' in content_type: decode_type = 'json' elif 'yaml' in content_type: decode_type = 'yaml' else: decode_type = 'plain' valid_decodes = ('json', 'xml', 'yaml', 'plain') if decode_type not in valid_decodes: ret['error'] = ( 'Invalid decode_type specified. ' 'Valid decode types are: {0}'.format( pprint.pformat(valid_decodes) ) ) log.error(ret['error']) return ret if decode_type == 'json': ret['dict'] = salt.utils.json.loads(result_text) elif decode_type == 'xml': ret['dict'] = [] items = ET.fromstring(result_text) for item in items: ret['dict'].append(xml.to_dict(item)) elif decode_type == 'yaml': ret['dict'] = salt.utils.data.decode(salt.utils.yaml.safe_load(result_text)) else: text = True if decode_out: with salt.utils.files.fopen(decode_out, 'w') as dof: dof.write(result_text) if text is True: ret['text'] = result_text return ret
def compile_template( template, renderers, default, blacklist, whitelist, saltenv="base", sls="", input_data="", **kwargs ): """ Take the path to a template and return the high data structure derived from the template. Helpers: :param mask_value: Mask value for debugging purposes (prevent sensitive information etc) example: "mask_value="pass*". All "passwd", "password", "pass" will be masked (as text). """ # if any error occurs, we return an empty dictionary ret = {} log.debug("compile template: %s", template) if "env" in kwargs: # "env" is not supported; Use "saltenv". kwargs.pop("env") if template != ":string:": # Template was specified incorrectly if not isinstance(template, six.string_types): log.error("Template was specified incorrectly: %s", template) return ret # Template does not exist if not os.path.isfile(template): log.error("Template does not exist: %s", template) return ret # Template is an empty file if salt.utils.files.is_empty(template): log.debug("Template is an empty file: %s", template) return ret with codecs.open(template, encoding=SLS_ENCODING) as ifile: # data input to the first render function in the pipe input_data = ifile.read() if not input_data.strip(): # Template is nothing but whitespace log.error("Template is nothing but whitespace: %s", template) return ret # Get the list of render funcs in the render pipe line. render_pipe = template_shebang( template, renderers, default, blacklist, whitelist, input_data ) windows_newline = "\r\n" in input_data input_data = StringIO(input_data) for render, argline in render_pipe: if salt.utils.stringio.is_readable(input_data): input_data.seek(0) # pylint: disable=no-member render_kwargs = dict(renderers=renderers, tmplpath=template) render_kwargs.update(kwargs) if argline: render_kwargs["argline"] = argline start = time.time() ret = render(input_data, saltenv, sls, **render_kwargs) log.profile( "Time (in seconds) to render '%s' using '%s' renderer: %s", template, render.__module__.split(".")[-1], time.time() - start, ) if ret is None: # The file is empty or is being written elsewhere time.sleep(0.01) ret = render(input_data, saltenv, sls, **render_kwargs) input_data = ret if log.isEnabledFor(logging.GARBAGE): # pylint: disable=no-member # If ret is not a StringIO (which means it was rendered using # yaml, mako, or another engine which renders to a data # structure) we don't want to log this. if salt.utils.stringio.is_readable(ret): log.debug( "Rendered data from file: %s:\n%s", template, salt.utils.sanitizers.mask_args_value( salt.utils.data.decode(ret.read()), kwargs.get("mask_value") ), ) # pylint: disable=no-member ret.seek(0) # pylint: disable=no-member # Preserve newlines from original template if windows_newline: if salt.utils.stringio.is_readable(ret): is_stringio = True contents = ret.read() else: is_stringio = False contents = ret if isinstance(contents, six.string_types): if "\r\n" not in contents: contents = contents.replace("\n", "\r\n") ret = StringIO(contents) if is_stringio else contents else: if is_stringio: ret.seek(0) return ret
def __init__(self, fn, mode="r"): initial_value = data[0] if "w" in mode: initial_value = "" StringIO.__init__(self, initial_value)
def uncompress(data): buf = StringIO(data) with open_fileobj(buf, 'rb') as igz: unc = igz.read() return unc
STUB_CRON_TIMESTAMP = { 'minute': '1', 'hour': '2', 'daymonth': '3', 'month': '4', 'dayweek': '5'} STUB_SIMPLE_RAW_CRON = '5 0 * * * /tmp/no_script.sh' STUB_SIMPLE_CRON_DICT = { 'pre': ['5 0 * * * /tmp/no_script.sh'], 'crons': [], 'env': [], 'special': []} CRONTAB = StringIO() def get_crontab(*args, **kw): return CRONTAB.getvalue() def set_crontab(val): CRONTAB.seek(0) CRONTAB.truncate(0) CRONTAB.write(val) def write_crontab(*args, **kw): set_crontab('\n'.join( [a.strip() for a in args[1]]))
def request(self, path='/', method='GET', app_path='', scheme='http', proto='HTTP/1.1', body=None, qs=None, headers=None, **kwargs): """ CherryPy does not have a facility for serverless unit testing. However this recipe demonstrates a way of doing it by calling its internal API to simulate an incoming request. This will exercise the whole stack from there. Remember a couple of things: * CherryPy is multithreaded. The response you will get from this method is a thread-data object attached to the current thread. Unless you use many threads from within a unit test, you can mostly forget about the thread data aspect of the response. * Responses are dispatched to a mounted application's page handler, if found. This is the reason why you must indicate which app you are targeting with this request by specifying its mount point. You can simulate various request settings by setting the `headers` parameter to a dictionary of headers, the request's `scheme` or `protocol`. .. seealso: http://docs.cherrypy.org/stable/refman/_cprequest.html#cherrypy._cprequest.Response """ # This is a required header when running HTTP/1.1 h = {'Host': '127.0.0.1'} # if we had some data passed as the request entity # let's make sure we have the content-length set fd = None if body is not None: h['content-length'] = '{0}'.format(len(body)) fd = StringIO(body) if headers is not None: h.update(headers) # Get our application and run the request against it app = cherrypy.tree.apps.get(app_path) if not app: # XXX: perhaps not the best exception to raise? raise AssertionError("No application mounted at '{0}'".format(app_path)) # Cleanup any previous returned response # between calls to this method app.release_serving() # Let's fake the local and remote addresses request, response = app.get_serving(local, remote, scheme, proto) try: h = [(k, v) for k, v in six.iteritems(h)] response = request.run(method, path, qs, proto, h, fd) finally: if fd: fd.close() fd = None if response.output_status.startswith('500'): print(response.body) raise AssertionError("Unexpected error") # collapse the response into a bytestring response.collapse_body() return request, response