def compress(data, compresslevel=9):
'''
Returns the data compressed at gzip level compression.
'''
buf = StringIO()
with open_fileobj(buf, 'wb', compresslevel) as ogz:
ogz.write(data)
compressed = buf.getvalue()
return compressed
def compress(data, compresslevel=9):
'''
Returns the data compressed at gzip level compression.
'''
buf = StringIO()
with open_fileobj(buf, 'wb', compresslevel) as ogz:
ogz.write(data)
compressed = buf.getvalue()
return compressed
def _read_temp(data):
tout = StringIO()
tout.write(data)
tout.seek(0)
output = tout.read().splitlines() # Discard newlines
tout.close()
return output
def close(self):
# Don't save unless there's something there. In Windows
# the class gets initialized the first time with mode = w
# which sets the initial value to ''. When the class closes
# it clears out data and causes the test to fail.
# I don't know why it get's initialized with a mode of 'w'
# For the purposes of this test data shouldn't be empty
# This is a problem with this class and not with the hosts
# module
if self.getvalue():
data[0] = self.getvalue()
StringIO.close(self)
def _produce_output(report, failed, setup):
'''
Produce output from the report dictionary generated by _generate_report
'''
report_format = setup.get('report_format', 'yaml')
log.debug('highstate output format: %s', report_format)
if report_format == 'json':
report_text = salt.utils.json.dumps(report)
elif report_format == 'yaml':
string_file = StringIO()
salt.utils.yaml.safe_dump(report,
string_file,
default_flow_style=False)
string_file.seek(0)
report_text = string_file.read()
else:
string_file = StringIO()
_generate_html(report, string_file)
string_file.seek(0)
report_text = string_file.read()
report_delivery = setup.get('report_delivery', 'file')
log.debug('highstate report_delivery: %s', report_delivery)
if report_delivery == 'file':
output_file = _sprinkle(setup.get('file_output', '/tmp/test.rpt'))
with salt.utils.files.fopen(output_file, 'w') as out:
out.write(salt.utils.stringutils.to_str(report_text))
else:
msg = MIMEText(report_text, report_format)
sender = setup.get('smtp_sender', '')
recipients = setup.get('smtp_recipients', '')
if failed:
subject = setup.get('smtp_failure_subject', 'Installation failure')
else:
subject = setup.get('smtp_success_subject', 'Installation success')
subject = _sprinkle(subject)
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = recipients
smtp = smtplib.SMTP(host=setup.get('smtp_server', ''))
smtp.sendmail(sender, [x.strip() for x in recipients.split(',')],
msg.as_string())
smtp.quit()
def _produce_output(report, failed, setup):
"""
Produce output from the report dictionary generated by _generate_report
"""
report_format = setup.get("report_format", "yaml")
log.debug("highstate output format: %s", report_format)
if report_format == "json":
report_text = salt.utils.json.dumps(report)
elif report_format == "yaml":
string_file = StringIO()
salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False)
string_file.seek(0)
report_text = string_file.read()
else:
string_file = StringIO()
_generate_html(report, string_file)
string_file.seek(0)
report_text = string_file.read()
report_delivery = setup.get("report_delivery", "file")
log.debug("highstate report_delivery: %s", report_delivery)
if report_delivery == "file":
output_file = _sprinkle(setup.get("file_output", "/tmp/test.rpt"))
with salt.utils.files.fopen(output_file, "w") as out:
out.write(salt.utils.stringutils.to_str(report_text))
else:
msg = MIMEText(report_text, report_format)
sender = setup.get("smtp_sender", "")
recipients = setup.get("smtp_recipients", "")
if failed:
subject = setup.get("smtp_failure_subject", "Installation failure")
else:
subject = setup.get("smtp_success_subject", "Installation success")
subject = _sprinkle(subject)
msg["Subject"] = subject
msg["From"] = sender
msg["To"] = recipients
smtp = smtplib.SMTP(host=setup.get("smtp_server", ""))
smtp.sendmail(
sender, [x.strip() for x in recipients.split(",")], msg.as_string()
)
smtp.quit()
def _read_temp(data):
tout = StringIO()
tout.write(data)
tout.seek(0)
output = tout.read().splitlines() # Discard newlines
tout.close()
return output
def render(template_file,
saltenv='base',
sls='',
argline='',
context=None,
tmplpath=None,
**kws):
'''
Render the template_file, passing the functions and grains into the
Jinja rendering system.
:rtype: string
'''
from_str = argline == '-s'
if not from_str and argline:
raise SaltRenderError(
'Unknown renderer option: {opt}'.format(opt=argline))
tmp_data = salt.utils.templates.JINJA(template_file,
to_str=True,
salt=_split_module_dicts(),
grains=__grains__,
opts=__opts__,
pillar=__pillar__,
saltenv=saltenv,
sls=sls,
context=context,
tmplpath=tmplpath,
**kws)
if not tmp_data.get('result', False):
raise SaltRenderError(
tmp_data.get('data', 'Unknown render error in jinja renderer'))
return StringIO(tmp_data['data'])
def string_io(data=None): # cStringIO can't handle unicode
'''
Pass data through to stringIO module and return result
'''
try:
return cStringIO(bytes(data))
except (UnicodeEncodeError, TypeError):
return StringIO(data)
def render_sls(self, content, sls='', saltenv='base', **kws):
if 'env' in kws:
# "env" is not supported; Use "saltenv".
kws.pop('env')
return self.HIGHSTATE.state.rend['pydsl'](
StringIO(content), saltenv=saltenv, sls=sls, **kws
)
def render_sls(self, content, sls="", saltenv="base", **kws):
if "env" in kws:
# "env" is not supported; Use "saltenv".
kws.pop("env")
return self.HIGHSTATE.state.rend["pydsl"](
StringIO(content), saltenv=saltenv, sls=sls, **kws
)
class _DelegateIO(object):
'''
This class defines an object that captures whatever is written to
a stream or file.
'''
def __init__(self, delegate):
self._captured = StringIO()
self.delegate = delegate
def write(self, text):
if six.PY2 and isinstance(text, six.text_type):
text = text.encode(__salt_system_encoding__)
self._captured.write(text)
self.delegate.write(text)
def __getattr__(self, attr):
try:
return getattr(self._captured, attr)
except AttributeError:
return getattr(self.delegate, attr)
def process_sls_data(data, context=None, extract=False):
sls_dir = os.path.dirname(sls.replace(".", os.path.sep)) if "." in sls else sls
ctx = dict(sls_dir=sls_dir if sls_dir else ".")
if context:
ctx.update(context)
tmplout = render_template(
StringIO(data), saltenv, sls, context=ctx, argline=rt_argline.strip(), **kws
)
high = render_data(tmplout, saltenv, sls, argline=rd_argline.strip())
return process_high_data(high, extract)
def __test_html():
'''
HTML generation test only used when called from the command line:
python ./highstate.py
Typical options for generating the report file:
highstate:
report_format: yaml
report_delivery: file
file_output: '/srv/salt/_returners/test.rpt'
'''
with salt.utils.fopen('test.rpt', 'r') as input_file:
data_text = input_file.read()
data = yaml.safe_load(data_text)
string_file = StringIO()
_generate_html(data, string_file)
string_file.seek(0)
result = string_file.read()
with salt.utils.fopen('test.html', 'w') as output:
output.write(result)
def __test_html():
"""
HTML generation test only used when called from the command line:
python ./highstate.py
Typical options for generating the report file:
highstate:
report_format: yaml
report_delivery: file
file_output: '/srv/salt/_returners/test.rpt'
"""
with salt.utils.files.fopen("test.rpt", "r") as input_file:
data_text = salt.utils.stringutils.to_unicode(input_file.read())
data = salt.utils.yaml.safe_load(data_text)
string_file = StringIO()
_generate_html(data, string_file)
string_file.seek(0)
result = string_file.read()
with salt.utils.files.fopen("test.html", "w") as output:
output.write(salt.utils.stringutils.to_str(result))
def _render_sls(self,
content,
sls='',
saltenv='base',
argline='-G yaml . jinja',
**kws):
return self._renderers['stateconf'](
StringIO(content), saltenv=saltenv, sls=sls,
argline=argline,
renderers=salt.loader.render(self.config, {}),
**kws
)
def render_sls(self, content, sls='', saltenv='base', **kws):
if 'env' in kws:
salt.utils.warn_until(
'Oxygen',
'Parameter \'env\' has been detected in the argument list. This '
'parameter is no longer used and has been replaced by \'saltenv\' '
'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.'
)
kws.pop('env')
return self.HIGHSTATE.state.rend['pydsl'](
StringIO(content), saltenv=saltenv, sls=sls, **kws
)
def mock_open(data=None):
'''
Mock "open" function in a simple way.
:param data:
:return:
'''
data = StringIO(data)
mock = MagicMock(spec=io.FileIO)
handle = MagicMock(spec=io.FileIO)
handle.write.return_value = None
handle.__enter__.return_value = data or handle
mock.return_value = handle
return mock
def render(
template_file,
saltenv="base",
sls="",
argline="",
context=None,
tmplpath=None,
**kws
):
"""
Render the template_file, passing the functions and grains into the
Jinja rendering system.
:rtype: string
"""
from_str = argline == "-s"
if not from_str and argline:
raise SaltRenderError("Unknown renderer option: {opt}".format(opt=argline))
tmp_data = salt.utils.templates.JINJA(
template_file,
to_str=True,
salt=_split_module_dicts(),
grains=__grains__,
opts=__opts__,
pillar=__pillar__,
saltenv=saltenv,
sls=sls,
context=context,
tmplpath=tmplpath,
proxy=__proxy__,
**kws
)
if not tmp_data.get("result", False):
raise SaltRenderError(
tmp_data.get("data", "Unknown render error in jinja renderer")
)
if isinstance(tmp_data["data"], bytes):
tmp_data["data"] = tmp_data["data"].decode(__salt_system_encoding__)
return StringIO(tmp_data["data"])
def __init__(self, data=None):
if data:
StringIO.__init__(self, data)
else:
StringIO.__init__(self)
self.data = []
def _produce_output(report, failed, setup):
"""
Produce output from the report dictionary generated by _generate_report
"""
report_format = setup.get("report_format", "yaml")
log.debug("highstate output format: %s", report_format)
if report_format == "json":
report_text = salt.utils.json.dumps(report)
elif report_format == "yaml":
string_file = StringIO()
salt.utils.yaml.safe_dump(report,
string_file,
default_flow_style=False)
string_file.seek(0)
report_text = string_file.read()
else:
string_file = StringIO()
_generate_html(report, string_file)
string_file.seek(0)
report_text = string_file.read()
report_delivery = setup.get("report_delivery", "file")
log.debug("highstate report_delivery: %s", report_delivery)
if report_delivery == "file":
output_file = _sprinkle(setup.get("file_output", "/tmp/test.rpt"))
with salt.utils.files.fopen(output_file, "w") as out:
out.write(salt.utils.stringutils.to_str(report_text))
else:
msg = MIMEText(report_text, report_format)
sender = setup.get("smtp_sender", "")
recipients = setup.get("smtp_recipients", "")
host = setup.get('smtp_server', '')
port = int(setup.get('smtp_port', 25))
tls = setup.get('smtp_tls')
username = setup.get('smtp_username')
password = setup.get('smtp_password')
if failed:
subject = setup.get("smtp_failure_subject", "Installation failure")
else:
subject = setup.get("smtp_success_subject", "Installation success")
subject = _sprinkle(subject)
msg["Subject"] = subject
msg["From"] = sender
msg["To"] = recipients
log.debug('highstate smtp port: %d', port)
smtp = smtplib.SMTP(host=host, port=port)
if tls is True:
smtp.starttls()
log.debug('highstate smtp tls enabled')
if username and password:
smtp.login(username, password)
log.debug('highstate smtp authenticated')
smtp.sendmail(sender, [x.strip() for x in recipients.split(',')],
msg.as_string())
log.debug('highstate message sent.')
smtp.quit()
def query(url,
method="GET",
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type="auto",
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format="lwp",
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node="minion",
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
formdata=False,
formdata_fieldname=None,
formdata_filename=None,
**kwargs):
"""
Query a resource, and decode the return data
"""
ret = {}
if opts is None:
if node == "master":
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, "master"))
elif node == "minion":
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, "minion"))
else:
opts = {}
if not backend:
backend = opts.get("backend", "tornado")
match = re.match(
r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)",
url,
)
if not match:
salt.utils.network.refresh_dns()
if backend == "requests":
if HAS_REQUESTS is False:
ret["error"] = ("http.query has been set to use requests, but the "
"requests library does not seem to be installed")
log.error(ret["error"])
return ret
else:
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = salt.ext.tornado.httputil.url_concat(url,
params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get("verify_ssl", True)
if cert is None:
cert = opts.get("cert", None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug("Requesting URL %s using %s method", log_url, method)
log.debug("Using backend: %s", backend)
if method == "POST" and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = "XXXXXXXXXX"
log.trace("Request POST Data: %s", pprint.pformat(log_data))
else:
log.trace("Request POST Data: %s", pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.txt")
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.session.p")
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = salt.utils.msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
salt.utils.msgpack.dump('', fh_)
for header in header_list:
comps = header.split(":")
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
user_agent = opts.get('user_agent', None)
if user_agent:
agent = user_agent
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == "requests":
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace("Request Headers: %s", sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == "urllib2":
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == "mozilla":
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret["test"] = True
if backend == "requests":
req_kwargs = {}
if stream is True:
if requests.__version__[0] == "0":
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs["prefetch"] = False
else:
req_kwargs["stream"] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs["cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["cert"] = cert
else:
log.error(
"The client-side certificate path that"
" was passed is not valid: %s",
cert,
)
if formdata:
if not formdata_fieldname:
ret["error"] = "formdata_fieldname is required when formdata=True"
log.error(ret["error"])
return ret
result = sess.request(method,
url,
params=params,
files={
formdata_fieldname:
(formdata_filename, StringIO(data))
},
**req_kwargs)
else:
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback("HTTP/1.0 {0} MESSAGE".format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
"handle": result,
}
if handle is True:
return {
"handle": result,
"body": result.content,
}
log.debug("Final URL location of Response: %s",
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type):
body = body.decode(result.encoding or "utf-8")
ret["body"] = body
elif backend == "urllib2":
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies),
]
if url.startswith("https"):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
"match_hostname() not available, SSL hostname checking "
"not available. THIS CONNECTION MAY NOT BE SECURE!")
elif verify_ssl is False:
log.warning("SSL certificate verification has been explicitly "
"disabled. THIS CONNECTION MAY NOT BE SECURE!")
else:
if ":" in hostname:
hostname, port = hostname.split(":")
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret["error"] = (
"The certificate was invalid. Error returned was: %s",
pprint.pformat(exc),
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
"The client-side certificate path that was "
"passed is not valid: %s",
cert,
)
return
if hasattr(ssl, "SSLContext"):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
"host": request.get_host(),
"port": port,
"cert_file": cert_chain[0],
}
if len(cert_chain) > 1:
cert_kwargs["key_file"] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {"Error": six.text_type(exc)}
if stream is True or handle is True:
return {
"handle": result,
"body": result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, six.text_type)):
result_text = result_text.decode(res_params["charset"])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs["client_cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["client_cert"] = cert[0]
req_kwargs["client_key"] = cert[1]
else:
log.error(
"The client-side certificate path that "
"was passed is not valid: %s",
cert,
)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
req_kwargs["ca_certs"] = ca_bundle
max_body = opts.get("http_max_body",
salt.config.DEFAULT_MINION_OPTS["http_max_body"])
connect_timeout = opts.get(
"http_connect_timeout",
salt.config.DEFAULT_MINION_OPTS["http_connect_timeout"],
)
timeout = opts.get(
"http_request_timeout",
salt.config.DEFAULT_MINION_OPTS["http_request_timeout"],
)
client_argspec = None
proxy_host = opts.get("proxy_host", None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get("proxy_port", None)
proxy_username = opts.get("proxy_username", None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get("proxy_password", None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get("no_proxy", [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret["error"] = (
"proxy_host and proxy_port has been set. This requires pycurl and tornado, "
"but the libraries does not seem to be installed")
log.error(ret["error"])
return ret
salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
)
else:
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.
initialize)
supports_max_body_size = "max_body_size" in client_argspec.args
req_kwargs.update({
"method": method,
"headers": header_dict,
"auth_username": username,
"auth_password": password,
"body": data,
"validate_cert": verify_ssl,
"allow_nonstandard_methods": True,
"streaming_callback": streaming_callback,
"header_callback": header_callback,
"connect_timeout": connect_timeout,
"request_timeout": timeout,
"proxy_host": proxy_host,
"proxy_port": proxy_port,
"proxy_username": proxy_username,
"proxy_password": proxy_password,
"raise_error": raise_error,
"decompress_response": False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = (HTTPClient(max_body_size=max_body)
if supports_max_body_size else HTTPClient())
result = download_client.fetch(url_full, **req_kwargs)
except salt.ext.tornado.httpclient.HTTPError as exc:
ret["status"] = exc.code
ret["error"] = six.text_type(exc)
return ret
except (socket.herror, socket.error, socket.timeout,
socket.gaierror) as exc:
if status is True:
ret["status"] = 0
ret["error"] = six.text_type(exc)
log.debug("Cannot perform 'http.query': {0} - {1}".format(
url_full, ret["error"]))
return ret
if stream is True or handle is True:
return {
"handle": result,
"body": result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, six.text_type)):
result_text = result_text.decode(res_params["charset"])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode("utf-8")
ret["body"] = result_text
if "Set-Cookie" in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers["Set-Cookie"])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(":")
result_headers_dict[comps[0].strip()] = ":".join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug("Response Status Code: %s", result_status_code)
log.trace("Response Headers: %s", result_headers)
log.trace("Response Cookies: %s", sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get("Content-Encoding", "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != "requests":
result_text = __decompressContent(coding, result_text)
try:
log.trace("Response Text: %s", result_text)
except UnicodeEncodeError as exc:
log.trace(
"Cannot Trace Log Response Text: %s. This may be due to "
"incompatibilities between requests and logging.",
exc,
)
if text_out is not None:
with salt.utils.files.fopen(text_out, "w") as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, "w") as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: See persist_session above
if "set-cookie" in result_headers:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
session_cookies = result_headers.get("set-cookie", None)
if session_cookies is not None:
salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_)
else:
salt.utils.msgpack.dump('', fh_)
if status is True:
ret["status"] = result_status_code
if headers is True:
ret["headers"] = result_headers
if decode is True:
if decode_type == "auto":
content_type = result_headers.get("content-type",
"application/json")
if "xml" in content_type:
decode_type = "xml"
elif "json" in content_type:
decode_type = "json"
elif "yaml" in content_type:
decode_type = "yaml"
else:
decode_type = "plain"
valid_decodes = ("json", "xml", "yaml", "plain")
if decode_type not in valid_decodes:
ret["error"] = ("Invalid decode_type specified. "
"Valid decode types are: {0}".format(
pprint.pformat(valid_decodes)))
log.error(ret["error"])
return ret
if decode_type == "json":
ret["dict"] = salt.utils.json.loads(result_text)
elif decode_type == "xml":
ret["dict"] = []
items = ET.fromstring(result_text)
for item in items:
ret["dict"].append(xml.to_dict(item))
elif decode_type == "yaml":
ret["dict"] = salt.utils.data.decode(
salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, "w") as dof:
dof.write(result_text)
if text is True:
ret["text"] = result_text
return ret
def render_sls(self, content, sls='', env='base', **kws):
return self.HIGHSTATE.state.rend['pydsl'](StringIO(content),
env=env,
sls=sls,
**kws)
def _get_rend(renderer, value):
'''
We need a new MagicMock each time since we're dealing with StringIO
objects which are read like files.
'''
return {renderer: MagicMock(return_value=StringIO(value))}
def compile_template(template,
renderers,
default,
blacklist,
whitelist,
saltenv='base',
sls='',
input_data='',
**kwargs):
'''
Take the path to a template and return the high data structure
derived from the template.
'''
# if any error occurs, we return an empty dictionary
ret = {}
log.debug('compile template: %s', template)
if 'env' in kwargs:
# "env" is not supported; Use "saltenv".
kwargs.pop('env')
if template != ':string:':
# Template was specified incorrectly
if not isinstance(template, six.string_types):
log.error('Template was specified incorrectly: %s', template)
return ret
# Template does not exist
if not os.path.isfile(template):
log.error('Template does not exist: %s', template)
return ret
# Template is an empty file
if salt.utils.files.is_empty(template):
log.debug('Template is an empty file: %s', template)
return ret
with codecs.open(template, encoding=SLS_ENCODING) as ifile:
# data input to the first render function in the pipe
input_data = ifile.read()
if not input_data.strip():
# Template is nothing but whitespace
log.error('Template is nothing but whitespace: %s', template)
return ret
# Get the list of render funcs in the render pipe line.
render_pipe = template_shebang(template, renderers, default, blacklist, whitelist, input_data)
windows_newline = '\r\n' in input_data
input_data = StringIO(input_data)
for render, argline in render_pipe:
if salt.utils.stringio.is_readable(input_data):
input_data.seek(0) # pylint: disable=no-member
render_kwargs = dict(renderers=renderers, tmplpath=template)
render_kwargs.update(kwargs)
if argline:
render_kwargs['argline'] = argline
start = time.time()
ret = render(input_data, saltenv, sls, **render_kwargs)
log.profile(
'Time (in seconds) to render \'%s\' using \'%s\' renderer: %s',
template,
render.__module__.split('.')[-1],
time.time() - start
)
if ret is None:
# The file is empty or is being written elsewhere
time.sleep(0.01)
ret = render(input_data, saltenv, sls, **render_kwargs)
input_data = ret
if log.isEnabledFor(logging.GARBAGE): # pylint: disable=no-member
# If ret is not a StringIO (which means it was rendered using
# yaml, mako, or another engine which renders to a data
# structure) we don't want to log this.
if salt.utils.stringio.is_readable(ret):
log.debug(
'Rendered data from file: %s:\n%s',
template,
salt.utils.locales.sdecode(ret.read())) # pylint: disable=no-member
ret.seek(0) # pylint: disable=no-member
# Preserve newlines from original template
if windows_newline:
if salt.utils.stringio.is_readable(ret):
is_stringio = True
contents = ret.read()
else:
is_stringio = False
contents = ret
if isinstance(contents, six.string_types):
if '\r\n' not in contents:
contents = contents.replace('\n', '\r\n')
ret = StringIO(contents) if is_stringio else contents
else:
if is_stringio:
ret.seek(0)
return ret
def close(self):
data[0] = self.getvalue()
StringIO.close(self)
def __init__(self, fn, mode='r'):
initial_value = data[0]
if 'w' in mode:
initial_value = ''
StringIO.__init__(self, initial_value)
def test_exc_info_on_loglevel(self):
def raise_exception_on_purpose():
1/0 # pylint: disable=pointless-statement
log = saltlog.SaltLoggingClass(__name__)
# Only stream2 should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.INFO)
handler2.setLevel(logging.DEBUG)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.DEBUG)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue()
)
self.assertNotIn('Traceback (most recent call last)', stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue()
)
self.assertIn('Traceback (most recent call last)', stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
# Both streams should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.INFO)
handler2.setLevel(logging.DEBUG)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.INFO)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue()
)
self.assertIn('Traceback (most recent call last)', stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue()
)
self.assertIn('Traceback (most recent call last)', stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
# No streams should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.ERROR)
handler2.setLevel(logging.INFO)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.DEBUG)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue()
)
self.assertNotIn('Traceback (most recent call last)', stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue()
)
self.assertNotIn('Traceback (most recent call last)', stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
def test_exc_info_on_loglevel(self):
def raise_exception_on_purpose():
1 / 0 # pylint: disable=pointless-statement
log = saltlog.SaltLoggingClass(__name__)
# Only stream2 should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.INFO)
handler2.setLevel(logging.DEBUG)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.DEBUG)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue())
self.assertNotIn('Traceback (most recent call last)',
stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue())
self.assertIn('Traceback (most recent call last)',
stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
# Both streams should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.INFO)
handler2.setLevel(logging.DEBUG)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.INFO)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue())
self.assertIn('Traceback (most recent call last)',
stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue())
self.assertIn('Traceback (most recent call last)',
stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
# No streams should contain the traceback
stream1 = StringIO()
stream2 = StringIO()
handler1 = StreamHandler(stream1)
handler2 = StreamHandler(stream2)
handler1.setLevel(logging.ERROR)
handler2.setLevel(logging.INFO)
log.addHandler(handler1)
log.addHandler(handler2)
try:
raise_exception_on_purpose()
except ZeroDivisionError as exc:
log.error('Exception raised on purpose caught: ZeroDivisionError',
exc_info_on_loglevel=logging.DEBUG)
try:
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream1.getvalue())
self.assertNotIn('Traceback (most recent call last)',
stream1.getvalue())
self.assertIn(
'Exception raised on purpose caught: ZeroDivisionError',
stream2.getvalue())
self.assertNotIn('Traceback (most recent call last)',
stream2.getvalue())
finally:
log.removeHandler(handler1)
log.removeHandler(handler2)
def request(self,
path='/',
method='GET',
app_path='',
scheme='http',
proto='HTTP/1.1',
body=None,
qs=None,
headers=None,
**kwargs):
"""
CherryPy does not have a facility for serverless unit testing.
However this recipe demonstrates a way of doing it by
calling its internal API to simulate an incoming request.
This will exercise the whole stack from there.
Remember a couple of things:
* CherryPy is multithreaded. The response you will get
from this method is a thread-data object attached to
the current thread. Unless you use many threads from
within a unit test, you can mostly forget
about the thread data aspect of the response.
* Responses are dispatched to a mounted application's
page handler, if found. This is the reason why you
must indicate which app you are targeting with
this request by specifying its mount point.
You can simulate various request settings by setting
the `headers` parameter to a dictionary of headers,
the request's `scheme` or `protocol`.
.. seealso: http://docs.cherrypy.org/stable/refman/_cprequest.html#cherrypy._cprequest.Response
"""
# This is a required header when running HTTP/1.1
h = {'Host': '127.0.0.1'}
# if we had some data passed as the request entity
# let's make sure we have the content-length set
fd = None
if body is not None:
h['content-length'] = '{0}'.format(len(body))
fd = StringIO(body)
if headers is not None:
h.update(headers)
# Get our application and run the request against it
app = cherrypy.tree.apps.get(app_path)
if not app:
# XXX: perhaps not the best exception to raise?
raise AssertionError(
"No application mounted at '{0}'".format(app_path))
# Cleanup any previous returned response
# between calls to this method
app.release_serving()
# Let's fake the local and remote addresses
request, response = app.get_serving(local, remote, scheme, proto)
try:
h = [(k, v) for k, v in six.iteritems(h)]
response = request.run(method, path, qs, proto, h, fd)
finally:
if fd:
fd.close()
fd = None
if response.output_status.startswith(six.b('500')):
response_body = response.collapse_body()
if six.PY3:
response_body = response_body.decode(__salt_system_encoding__)
print(response_body)
raise AssertionError("Unexpected error")
# collapse the response into a bytestring
response.collapse_body()
return request, response
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
formdata=False,
formdata_fieldname=None,
formdata_filename=None,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'master')
)
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'minion')
)
else:
opts = {}
if not backend:
backend = opts.get('backend', 'tornado')
match = re.match(r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)', url)
if not match:
salt.utils.network.refresh_dns()
if backend == 'requests':
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = salt.ext.tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL %s using %s method', log_url, method)
log.debug("Using backend: %s", backend)
if method == 'POST' and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: %s', pprint.pformat(log_data))
else:
log.trace('Request POST Data: %s', pprint.pformat(data))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = salt.utils.msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
salt.utils.msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: %s', sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error('The client-side certificate path that'
' was passed is not valid: %s', cert)
if formdata:
if not formdata_fieldname:
ret['error'] = ('formdata_fieldname is required when formdata=True')
log.error(ret['error'])
return ret
result = sess.request(
method,
url,
params=params,
files={formdata_fieldname: (formdata_filename, StringIO(data))},
**req_kwargs
)
else:
result = sess.request(
method, url, params=params, data=data, **req_kwargs
)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
'handle': result,
}
if handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: %s',
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type):
body = body.decode(result.encoding or 'utf-8')
ret['body'] = body
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!')
elif verify_ssl is False:
log.warning('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!')
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(
sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = (
'The certificate was invalid. '
'Error returned was: %s',
pprint.pformat(exc)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: %s', cert)
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': six.text_type(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error('The client-side certificate path that '
'was passed is not valid: %s', cert)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
req_kwargs['ca_certs'] = ca_bundle
max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body'])
connect_timeout = opts.get('http_connect_timeout', salt.config.DEFAULT_MINION_OPTS['http_connect_timeout'])
timeout = opts.get('http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get('proxy_password', None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get('no_proxy', [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, '
'but the libraries does not seem to be installed')
log.error(ret['error'])
return ret
salt.ext.tornado.httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
req_kwargs.update({
'method': method,
'headers': header_dict,
'auth_username': username,
'auth_password': password,
'body': data,
'validate_cert': verify_ssl,
'allow_nonstandard_methods': True,
'streaming_callback': streaming_callback,
'header_callback': header_callback,
'connect_timeout': connect_timeout,
'request_timeout': timeout,
'proxy_host': proxy_host,
'proxy_port': proxy_port,
'proxy_username': proxy_username,
'proxy_password': proxy_password,
'raise_error': raise_error,
'decompress_response': False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = HTTPClient(max_body_size=max_body) \
if supports_max_body_size \
else HTTPClient()
result = download_client.fetch(url_full, **req_kwargs)
except salt.ext.tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = six.text_type(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = six.text_type(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
if 'Set-Cookie' in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: %s', result_status_code)
log.trace('Response Headers: %s', result_headers)
log.trace('Response Cookies: %s', sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get('Content-Encoding', "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != 'requests':
result_text = __decompressContent(coding, result_text)
try:
log.trace('Response Text: %s', result_text)
except UnicodeEncodeError as exc:
log.trace('Cannot Trace Log Response Text: %s. This may be due to '
'incompatibilities between requests and logging.', exc)
if text_out is not None:
with salt.utils.files.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_)
else:
salt.utils.msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = salt.utils.json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = salt.utils.data.decode(salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def close(self):
data[0] = self.getvalue()
StringIO.close(self)
def compile_template(
template,
renderers,
default,
blacklist,
whitelist,
saltenv="base",
sls="",
input_data="",
**kwargs
):
"""
Take the path to a template and return the high data structure
derived from the template.
Helpers:
:param mask_value:
Mask value for debugging purposes (prevent sensitive information etc)
example: "mask_value="pass*". All "passwd", "password", "pass" will
be masked (as text).
"""
# if any error occurs, we return an empty dictionary
ret = {}
log.debug("compile template: %s", template)
if "env" in kwargs:
# "env" is not supported; Use "saltenv".
kwargs.pop("env")
if template != ":string:":
# Template was specified incorrectly
if not isinstance(template, six.string_types):
log.error("Template was specified incorrectly: %s", template)
return ret
# Template does not exist
if not os.path.isfile(template):
log.error("Template does not exist: %s", template)
return ret
# Template is an empty file
if salt.utils.files.is_empty(template):
log.debug("Template is an empty file: %s", template)
return ret
with codecs.open(template, encoding=SLS_ENCODING) as ifile:
# data input to the first render function in the pipe
input_data = ifile.read()
if not input_data.strip():
# Template is nothing but whitespace
log.error("Template is nothing but whitespace: %s", template)
return ret
# Get the list of render funcs in the render pipe line.
render_pipe = template_shebang(
template, renderers, default, blacklist, whitelist, input_data
)
windows_newline = "\r\n" in input_data
input_data = StringIO(input_data)
for render, argline in render_pipe:
if salt.utils.stringio.is_readable(input_data):
input_data.seek(0) # pylint: disable=no-member
render_kwargs = dict(renderers=renderers, tmplpath=template)
render_kwargs.update(kwargs)
if argline:
render_kwargs["argline"] = argline
start = time.time()
ret = render(input_data, saltenv, sls, **render_kwargs)
log.profile(
"Time (in seconds) to render '%s' using '%s' renderer: %s",
template,
render.__module__.split(".")[-1],
time.time() - start,
)
if ret is None:
# The file is empty or is being written elsewhere
time.sleep(0.01)
ret = render(input_data, saltenv, sls, **render_kwargs)
input_data = ret
if log.isEnabledFor(logging.GARBAGE): # pylint: disable=no-member
# If ret is not a StringIO (which means it was rendered using
# yaml, mako, or another engine which renders to a data
# structure) we don't want to log this.
if salt.utils.stringio.is_readable(ret):
log.debug(
"Rendered data from file: %s:\n%s",
template,
salt.utils.sanitizers.mask_args_value(
salt.utils.data.decode(ret.read()), kwargs.get("mask_value")
),
) # pylint: disable=no-member
ret.seek(0) # pylint: disable=no-member
# Preserve newlines from original template
if windows_newline:
if salt.utils.stringio.is_readable(ret):
is_stringio = True
contents = ret.read()
else:
is_stringio = False
contents = ret
if isinstance(contents, six.string_types):
if "\r\n" not in contents:
contents = contents.replace("\n", "\r\n")
ret = StringIO(contents) if is_stringio else contents
else:
if is_stringio:
ret.seek(0)
return ret
def __init__(self, fn, mode="r"):
initial_value = data[0]
if "w" in mode:
initial_value = ""
StringIO.__init__(self, initial_value)
def uncompress(data):
buf = StringIO(data)
with open_fileobj(buf, 'rb') as igz:
unc = igz.read()
return unc
STUB_CRON_TIMESTAMP = {
'minute': '1',
'hour': '2',
'daymonth': '3',
'month': '4',
'dayweek': '5'}
STUB_SIMPLE_RAW_CRON = '5 0 * * * /tmp/no_script.sh'
STUB_SIMPLE_CRON_DICT = {
'pre': ['5 0 * * * /tmp/no_script.sh'],
'crons': [],
'env': [],
'special': []}
CRONTAB = StringIO()
def get_crontab(*args, **kw):
return CRONTAB.getvalue()
def set_crontab(val):
CRONTAB.seek(0)
CRONTAB.truncate(0)
CRONTAB.write(val)
def write_crontab(*args, **kw):
set_crontab('\n'.join(
[a.strip() for a in args[1]]))
def request(self, path='/', method='GET', app_path='', scheme='http',
proto='HTTP/1.1', body=None, qs=None, headers=None, **kwargs):
"""
CherryPy does not have a facility for serverless unit testing.
However this recipe demonstrates a way of doing it by
calling its internal API to simulate an incoming request.
This will exercise the whole stack from there.
Remember a couple of things:
* CherryPy is multithreaded. The response you will get
from this method is a thread-data object attached to
the current thread. Unless you use many threads from
within a unit test, you can mostly forget
about the thread data aspect of the response.
* Responses are dispatched to a mounted application's
page handler, if found. This is the reason why you
must indicate which app you are targeting with
this request by specifying its mount point.
You can simulate various request settings by setting
the `headers` parameter to a dictionary of headers,
the request's `scheme` or `protocol`.
.. seealso: http://docs.cherrypy.org/stable/refman/_cprequest.html#cherrypy._cprequest.Response
"""
# This is a required header when running HTTP/1.1
h = {'Host': '127.0.0.1'}
# if we had some data passed as the request entity
# let's make sure we have the content-length set
fd = None
if body is not None:
h['content-length'] = '{0}'.format(len(body))
fd = StringIO(body)
if headers is not None:
h.update(headers)
# Get our application and run the request against it
app = cherrypy.tree.apps.get(app_path)
if not app:
# XXX: perhaps not the best exception to raise?
raise AssertionError("No application mounted at '{0}'".format(app_path))
# Cleanup any previous returned response
# between calls to this method
app.release_serving()
# Let's fake the local and remote addresses
request, response = app.get_serving(local, remote, scheme, proto)
try:
h = [(k, v) for k, v in six.iteritems(h)]
response = request.run(method, path, qs, proto, h, fd)
finally:
if fd:
fd.close()
fd = None
if response.output_status.startswith('500'):
print(response.body)
raise AssertionError("Unexpected error")
# collapse the response into a bytestring
response.collapse_body()
return request, response
