def assert_xml_equals(expected, actual):
"""
Assert that two ElementTree nodes are equal
"""
assert xmlutil.to_dict(xmlutil.strip_spaces(expected),
True) == xmlutil.to_dict(
xmlutil.strip_spaces(actual), True)
def mksls(src, dst=None):
'''
Convert an AutoYAST file to an SLS file
'''
with salt.utils.fopen(src, 'r') as fh_:
ps_opts = xml.to_dict(ET.fromstring(fh_.read()))
if dst is not None:
with salt.utils.fopen(dst, 'w') as fh_:
fh_.write(yaml.safe_dump(ps_opts, default_flow_style=False))
else:
return yaml.safe_dump(ps_opts, default_flow_style=False)
def mksls(src, dst=None):
'''
Convert an AutoYAST file to an SLS file
'''
with salt.utils.files.fopen(src, 'r') as fh_:
ps_opts = xml.to_dict(ET.fromstring(fh_.read()))
if dst is not None:
with salt.utils.files.fopen(dst, 'w') as fh_:
fh_.write(yaml.safe_dump(ps_opts, default_flow_style=False))
else:
return yaml.safe_dump(ps_opts, default_flow_style=False)
def mksls(src, dst=None):
"""
Convert an AutoYAST file to an SLS file
"""
with salt.utils.files.fopen(src, "r") as fh_:
ps_opts = xml.to_dict(ET.fromstring(fh_.read()))
if dst is not None:
with salt.utils.files.fopen(dst, "w") as fh_:
salt.utils.yaml.safe_dump(ps_opts, fh_, default_flow_style=False)
else:
return salt.utils.yaml.safe_dump(ps_opts, default_flow_style=False)
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, location=None, full_headers=False):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key or not keyid:
key = salt.utils.aws.IROLE_CODE
keyid = salt.utils.aws.IROLE_CODE
if not location:
location = iam.get_iam_region()
if not location:
location = DEFAULT_LOCATION
data = ''
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={'id': keyid, 'key': key},
location=location,
product='s3',
requesturl=requesturl,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(
key,
keyid,
method="GET",
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path=None,
return_bin=False,
action=None,
local_file=None,
):
"""
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
A service_url may also be specified in the configuration::
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
"""
if not headers:
headers = {}
if not params:
params = {}
if path is None:
path = ""
if not service_url:
service_url = "s3.amazonaws.com"
if bucket:
endpoint = "{0}.{1}".format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
token = None
if not key or not keyid:
iam_creds = get_iam_metadata()
key = iam_creds["secret_key"]
keyid = iam_creds["access_key"]
token = iam_creds["security_token"]
if not requesturl:
x_amz_date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
content_type = "text/plain"
if method == "GET":
if bucket:
can_resource = "/{0}/{1}".format(bucket, path)
else:
can_resource = "/"
elif method == "PUT" or method == "HEAD" or method == "DELETE":
if path:
can_resource = "/{0}/{1}".format(bucket, path)
else:
can_resource = "/{0}/".format(bucket)
if action:
can_resource += "?{0}".format(action)
log.debug("CanonicalizedResource: {0}".format(can_resource))
headers["Host"] = endpoint
headers["Content-type"] = content_type
headers["Date"] = x_amz_date
if token:
headers["x-amz-security-token"] = token
string_to_sign = "{0}\n".format(method)
new_headers = []
for header in sorted(headers.keys()):
if header.lower().startswith("x-amz"):
log.debug(header.lower())
new_headers.append("{0}:{1}".format(header.lower(), headers[header]))
can_headers = "\n".join(new_headers)
log.debug("CanonicalizedAmzHeaders: {0}".format(can_headers))
string_to_sign += "\n{0}".format(content_type)
string_to_sign += "\n{0}".format(x_amz_date)
if can_headers:
string_to_sign += "\n{0}".format(can_headers)
string_to_sign += "\n{0}".format(can_resource)
log.debug("String To Sign:: \n{0}".format(string_to_sign))
hashed = hmac.new(key, string_to_sign, hashlib.sha1)
sig = binascii.b2a_base64(hashed.digest())
headers["Authorization"] = "AWS {0}:{1}".format(keyid, sig.strip())
querystring = urllib.urlencode(params)
if action:
if querystring:
querystring = "{0}&{1}".format(action, querystring)
else:
querystring = action
requesturl = "https://{0}/".format(endpoint)
if path:
requesturl += path
if querystring:
requesturl += "?{0}".format(querystring)
req = urllib2.Request(url=requesturl)
if method == "PUT":
if local_file:
with salt.utils.fopen(local_file, "r") as ifile:
data = ifile.read()
req = urllib2.Request(url=requesturl, data=data)
req.get_method = lambda: "PUT"
elif method == "HEAD":
req.get_method = lambda: "HEAD"
elif method == "DELETE":
req.get_method = lambda: "DELETE"
log.debug("S3 Request: {0}".format(requesturl))
log.debug("S3 Headers::")
for header in sorted(headers.keys()):
if header == "Authorization":
continue
req.add_header(header, headers[header])
log.debug(" {0}: {1}".format(header, headers[header]))
log.debug(" Authorization: {0}".format(headers["Authorization"]))
req.add_header("Authorization", headers["Authorization"])
try:
result = urllib2.urlopen(req)
response = result.read()
except Exception as exc:
log.error("There was an error::")
if hasattr(exc, "code") and hasattr(exc, "msg"):
log.error(" Code: {0}: {1}".format(exc.code, exc.msg))
log.error(" Content: \n{0}".format(exc.read()))
return False
log.debug("S3 Response Status Code: {0}".format(result.getcode()))
result.close()
if method == "PUT":
if result.getcode() == 200:
if local_file:
log.debug("Uploaded from {0} to {1}".format(local_file, path))
else:
log.debug("Created bucket {0}".format(bucket))
else:
if local_file:
log.debug("Failed to upload from {0} to {1}: {2}".format(local_file, path, result.getcode()))
else:
log.debug("Failed to create bucket {0}".format(bucket))
return
if method == "DELETE":
if str(result.getcode()).startswith("2"):
if path:
log.debug("Deleted {0} from bucket {1}".format(path, bucket))
else:
log.debug("Deleted bucket {0}".format(bucket))
else:
if path:
log.debug("Failed to delete {0} from bucket {1}: {2}".format(path, bucket, result.getcode()))
else:
log.debug("Failed to delete bucket {0}".format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == "GET":
log.debug("Saving to local file: {0}".format(local_file))
with salt.utils.fopen(local_file, "w") as out:
out.write(response)
return "Saved to local file: {0}".format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
ret = {"headers": []}
for header in result.headers.headers:
ret["headers"].append(header.strip())
return ret
def query(url,
method="GET",
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type="auto",
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format="lwp",
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node="minion",
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
formdata=False,
formdata_fieldname=None,
formdata_filename=None,
decode_body=True,
**kwargs):
"""
Query a resource, and decode the return data
"""
ret = {}
if opts is None:
if node == "master":
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, "master"))
elif node == "minion":
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, "minion"))
else:
opts = {}
if not backend:
backend = opts.get("backend", "tornado")
match = re.match(
r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)",
url,
)
if not match:
salt.utils.network.refresh_dns()
if backend == "requests":
if HAS_REQUESTS is False:
ret["error"] = ("http.query has been set to use requests, but the "
"requests library does not seem to be installed")
log.error(ret["error"])
return ret
else:
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = salt.ext.tornado.httputil.url_concat(url,
params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get("verify_ssl", True)
if cert is None:
cert = opts.get("cert", None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug("Requesting URL %s using %s method", log_url, method)
log.debug("Using backend: %s", backend)
if method == "POST" and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = "XXXXXXXXXX"
log.trace("Request POST Data: %s", pprint.pformat(log_data))
else:
log.trace("Request POST Data: %s", pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.txt")
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get("cachedir", salt.syspaths.CACHE_DIR), "cookies.session.p")
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, "rb") as fh_:
session_cookies = salt.utils.msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
salt.utils.msgpack.dump("", fh_)
for header in header_list:
comps = header.split(":")
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = "{} http.query()".format(agent)
header_dict["User-agent"] = agent
if backend == "requests":
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace("Request Headers: %s", sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == "urllib2":
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == "mozilla":
sess_cookies = http.cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = http.cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret["test"] = True
if backend == "requests":
req_kwargs = {}
if stream is True:
if requests.__version__[0] == "0":
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs["prefetch"] = False
else:
req_kwargs["stream"] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs["cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["cert"] = cert
else:
log.error(
"The client-side certificate path that"
" was passed is not valid: %s",
cert,
)
if formdata:
if not formdata_fieldname:
ret["error"] = "formdata_fieldname is required when formdata=True"
log.error(ret["error"])
return ret
result = sess.request(method,
url,
params=params,
files={
formdata_fieldname:
(formdata_filename, io.StringIO(data))
},
**req_kwargs)
else:
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback("HTTP/1.0 {} MESSAGE".format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
"handle": result,
}
if handle is True:
return {
"handle": result,
"body": result.content,
}
log.debug("Final URL location of Response: %s",
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, str) and decode_body:
body = body.decode(result.encoding or "utf-8")
ret["body"] = body
elif backend == "urllib2":
request = urllib.request.Request(url_full, data)
handlers = [
urllib.request.HTTPHandler,
urllib.request.HTTPCookieProcessor(sess_cookies),
]
if url.startswith("https"):
hostname = request.get_host()
handlers[0] = urllib.request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
"match_hostname() not available, SSL hostname checking "
"not available. THIS CONNECTION MAY NOT BE SECURE!")
elif verify_ssl is False:
log.warning("SSL certificate verification has been explicitly "
"disabled. THIS CONNECTION MAY NOT BE SECURE!")
else:
if ":" in hostname:
hostname, port = hostname.split(":")
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret["error"] = "The certificate was invalid. Error returned was: {}".format(
pprint.pformat(exc))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, str):
if os.path.exists(cert):
cert_chain = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
"The client-side certificate path that was "
"passed is not valid: %s",
cert,
)
return
if hasattr(ssl, "SSLContext"):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib.request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
"host": request.get_host(),
"port": port,
"cert_file": cert_chain[0],
}
if len(cert_chain) > 1:
cert_kwargs["key_file"] = cert_chain[1]
handlers[0] = http.client.HTTPSConnection(
**cert_kwargs)
opener = urllib.request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except urllib.error.URLError as exc:
return {"Error": str(exc)}
if stream is True or handle is True:
return {
"handle": result,
"body": result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, str)):
result_text = result_text.decode(res_params["charset"])
if isinstance(result_text, bytes) and decode_body:
result_text = result_text.decode("utf-8")
ret["body"] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, str):
if os.path.exists(cert):
req_kwargs["client_cert"] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs["client_cert"] = cert[0]
req_kwargs["client_key"] = cert[1]
else:
log.error(
"The client-side certificate path that "
"was passed is not valid: %s",
cert,
)
if isinstance(data, dict):
data = urllib.parse.urlencode(data)
if verify_ssl:
req_kwargs["ca_certs"] = ca_bundle
max_body = opts.get("http_max_body",
salt.config.DEFAULT_MINION_OPTS["http_max_body"])
connect_timeout = opts.get(
"http_connect_timeout",
salt.config.DEFAULT_MINION_OPTS["http_connect_timeout"],
)
timeout = opts.get(
"http_request_timeout",
salt.config.DEFAULT_MINION_OPTS["http_request_timeout"],
)
client_argspec = None
proxy_host = opts.get("proxy_host", None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get("proxy_port", None)
proxy_username = opts.get("proxy_username", None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get("proxy_password", None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get("no_proxy", [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urllib.parse.urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
proxy_username = None
proxy_password = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret["error"] = (
"proxy_host and proxy_port has been set. This requires pycurl and tornado, "
"but the libraries does not seem to be installed")
log.error(ret["error"])
return ret
salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
)
else:
salt.ext.tornado.httpclient.AsyncHTTPClient.configure(None)
client_argspec = salt.utils.args.get_function_argspec(
salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.
initialize)
supports_max_body_size = "max_body_size" in client_argspec.args
req_kwargs.update({
"method": method,
"headers": header_dict,
"auth_username": username,
"auth_password": password,
"body": data,
"validate_cert": verify_ssl,
"allow_nonstandard_methods": True,
"streaming_callback": streaming_callback,
"header_callback": header_callback,
"connect_timeout": connect_timeout,
"request_timeout": timeout,
"proxy_host": proxy_host,
"proxy_port": proxy_port,
"proxy_username": proxy_username,
"proxy_password": proxy_password,
"raise_error": raise_error,
"decompress_response": False,
})
# Unicode types will cause a TypeError when Tornado's curl HTTPClient
# invokes setopt. Therefore, make sure all arguments we pass which
# contain strings are str types.
req_kwargs = salt.utils.data.decode(req_kwargs, to_str=True)
try:
download_client = (HTTPClient(max_body_size=max_body)
if supports_max_body_size else HTTPClient())
result = download_client.fetch(url_full, **req_kwargs)
except salt.ext.tornado.httpclient.HTTPError as exc:
ret["status"] = exc.code
ret["error"] = str(exc)
return ret
except (socket.herror, OSError, socket.timeout,
socket.gaierror) as exc:
if status is True:
ret["status"] = 0
ret["error"] = str(exc)
log.debug("Cannot perform 'http.query': %s - %s", url_full,
ret["error"])
return ret
if stream is True or handle is True:
return {
"handle": result,
"body": result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if "Content-Type" in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers["Content-Type"])
if (res_content_type.startswith("text/")
and "charset" in res_params
and not isinstance(result_text, str)):
result_text = result_text.decode(res_params["charset"])
if isinstance(result_text, bytes) and decode_body:
result_text = result_text.decode("utf-8")
ret["body"] = result_text
if "Set-Cookie" in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers["Set-Cookie"])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(":")
result_headers_dict[comps[0].strip()] = ":".join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug("Response Status Code: %s", result_status_code)
log.trace("Response Headers: %s", result_headers)
log.trace("Response Cookies: %s", sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get("Content-Encoding", "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != "requests":
result_text = __decompressContent(coding, result_text)
try:
log.trace("Response Text: %s", result_text)
except UnicodeEncodeError as exc:
log.trace(
"Cannot Trace Log Response Text: %s. This may be due to "
"incompatibilities between requests and logging.",
exc,
)
if text_out is not None:
with salt.utils.files.fopen(text_out, "w") as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, "w") as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and salt.utils.msgpack.HAS_MSGPACK:
# TODO: See persist_session above
if "set-cookie" in result_headers:
with salt.utils.files.fopen(session_cookie_jar, "wb") as fh_:
session_cookies = result_headers.get("set-cookie", None)
if session_cookies is not None:
salt.utils.msgpack.dump({"Cookie": session_cookies}, fh_)
else:
salt.utils.msgpack.dump("", fh_)
if status is True:
ret["status"] = result_status_code
if headers is True:
ret["headers"] = result_headers
if decode is True:
if decode_type == "auto":
content_type = result_headers.get("content-type",
"application/json")
if "xml" in content_type:
decode_type = "xml"
elif "json" in content_type:
decode_type = "json"
elif "yaml" in content_type:
decode_type = "yaml"
else:
decode_type = "plain"
valid_decodes = ("json", "xml", "yaml", "plain")
if decode_type not in valid_decodes:
ret["error"] = "Invalid decode_type specified. Valid decode types are: {}".format(
pprint.pformat(valid_decodes))
log.error(ret["error"])
return ret
if decode_type == "json":
ret["dict"] = salt.utils.json.loads(result_text)
elif decode_type == "xml":
ret["dict"] = []
items = ET.fromstring(result_text)
for item in items:
ret["dict"].append(xml.to_dict(item))
elif decode_type == "yaml":
ret["dict"] = salt.utils.data.decode(
salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, "w") as dof:
dof.write(result_text)
if text is True:
ret["text"] = result_text
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path='', return_bin=False, action=None, local_file=None,
verify_ssl=True, full_headers=False, kms_keyid=None,
location=None, role_arn=None, chunk_size=16384):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ('PUT', 'POST'):
headers['x-amz-server-side-encryption'] = 'aws:kms'
headers['x-amz-server-side-encryption-aws-kms-key-id'] = kms_keyid
if not location:
location = salt.utils.aws.get_location()
data = ''
payload_hash = None
if method == 'PUT':
if local_file:
payload_hash = salt.utils.get_hash(local_file, form='sha256')
if path is None:
path = ''
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={'id': keyid, 'key': key},
role_arn=role_arn,
location=location,
product='s3',
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
response = None
if method == 'PUT':
if local_file:
data = salt.utils.fopen(local_file, 'r')
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
response = result.content
elif method == 'GET' and local_file and not return_bin:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
else:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl)
response = result.content
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = response or result.content or 'Unknown error'
log.debug(' Response content: {0}'.format(err_text))
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data['Code']
err_msg = err_data['Message']
except (KeyError, ET.ParseError) as err:
log.debug('Failed to parse s3 err response. {0}: {1}'.format(
type(err).__name__, err))
err_code = 'http-{0}'.format(result.status_code)
err_msg = err_text
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
'Failed to upload from {0} to {1}. {2}: {3}'.format(
local_file, path, err_code, err_msg))
raise CommandExecutionError(
'Failed to create bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
return
if method == 'DELETE':
if not str(result.status_code).startswith('2'):
if path:
raise CommandExecutionError(
'Failed to delete {0} from bucket {1}. {2}: {3}'.format(
path, bucket, err_code, err_msg))
raise CommandExecutionError(
'Failed to delete bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
'Failed to get file. {0}: {1}'.format(err_code, err_msg))
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'wb') as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return 'Saved to local file: {0}'.format(local_file)
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
'Failed s3 operation. {0}: {1}'.format(err_code, err_msg))
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def test_xml_case_f_legacy(self):
xmldata = ET.fromstring(self.cases["f"]["xml"])
defaultdict = xml.to_dict(xmldata, False)
self.assertEqual(defaultdict, self.cases["f"]["legacy"])
def query(
key,
keyid,
method="GET",
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path="",
return_bin=False,
action=None,
local_file=None,
verify_ssl=True,
full_headers=False,
kms_keyid=None,
location=None,
role_arn=None,
chunk_size=16384,
path_style=False,
https_enable=True,
):
"""
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
Path style can be enabled:
s3.path_style: True
This can be useful if you need to use salt with a proxy for an s3 compatible storage
You can use either https protocol or http protocol:
s3.https_enable: True
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
"""
if not HAS_REQUESTS:
log.error("There was an error: requests is required for s3 access")
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = "s3.amazonaws.com"
if not bucket or path_style:
endpoint = service_url
else:
endpoint = "{}.{}".format(bucket, service_url)
if path_style and bucket:
path = "{}/{}".format(bucket, path)
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ("PUT", "POST"):
headers["x-amz-server-side-encryption"] = "aws:kms"
headers["x-amz-server-side-encryption-aws-kms-key-id"] = kms_keyid
if not location:
location = salt.utils.aws.get_location()
data = ""
fh = None
payload_hash = None
if method == "PUT":
if local_file:
payload_hash = salt.utils.hashutils.get_hash(local_file, form="sha256")
if path is None:
path = ""
path = urllib.parse.quote(path)
if not requesturl:
requesturl = (("https" if https_enable else "http") + "://{0}/{1}").format(
endpoint, path
)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri="/{}".format(path),
prov_dict={"id": keyid, "key": key},
role_arn=role_arn,
location=location,
product="s3",
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug("S3 Request: %s", requesturl)
log.debug("S3 Headers::")
log.debug(" Authorization: %s", headers["Authorization"])
if not data:
data = None
try:
if method == "PUT":
if local_file:
# pylint: disable=resource-leakage
fh = salt.utils.files.fopen(local_file, "rb")
# pylint: enable=resource-leakage
data = fh.read() # pylint: disable=resource-leakage
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300,
)
elif method == "GET" and local_file and not return_bin:
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True,
timeout=300,
)
else:
result = requests.request(
method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
timeout=300,
)
finally:
if fh is not None:
fh.close()
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = result.content or "Unknown error"
log.debug(" Response content: %s", err_text)
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data["Code"]
err_msg = err_data["Message"]
except (KeyError, ET.ParseError) as err:
log.debug(
"Failed to parse s3 err response. %s: %s", type(err).__name__, err
)
err_code = "http-{}".format(result.status_code)
err_msg = err_text
log.debug("S3 Response Status Code: %s", result.status_code)
if method == "PUT":
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
"Failed to upload from {} to {}. {}: {}".format(
local_file, path, err_code, err_msg
)
)
raise CommandExecutionError(
"Failed to create bucket {}. {}: {}".format(bucket, err_code, err_msg)
)
if local_file:
log.debug("Uploaded from %s to %s", local_file, path)
else:
log.debug("Created bucket %s", bucket)
return
if method == "DELETE":
if not str(result.status_code).startswith("2"):
if path:
raise CommandExecutionError(
"Failed to delete {} from bucket {}. {}: {}".format(
path, bucket, err_code, err_msg
)
)
raise CommandExecutionError(
"Failed to delete bucket {}. {}: {}".format(bucket, err_code, err_msg)
)
if path:
log.debug("Deleted %s from bucket %s", path, bucket)
else:
log.debug("Deleted bucket %s", bucket)
return
# This can be used to save a binary object to disk
if local_file and method == "GET":
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
"Failed to get file. {}: {}".format(err_code, err_msg)
)
log.debug("Saving to local file: %s", local_file)
with salt.utils.files.fopen(local_file, "wb") as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return "Saved to local file: {}".format(local_file)
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError(
"Failed s3 operation. {}: {}".format(err_code, err_msg)
)
# This can be used to return a binary object wholesale
if return_bin:
return result.content
if result.content:
items = ET.fromstring(result.content)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {"headers": []}
if full_headers:
ret["headers"] = dict(result.headers)
else:
for header in result.headers:
ret["headers"].append(header.strip())
return ret
def query(params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product='ec2',
sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, prov_dict)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint', '{0}.{1}.{2}'.format(product, location,
service_url))
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = (
'Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(requesturl)
log.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
log.debug('Using AWS endpoint: %s', endpoint)
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version',
prov_dict.get('{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION))
# Fallback to ec2's id & key if none is found, for this component
if not prov_dict.get('id', None):
prov_dict['id'] = providers.get(provider, {}).get('ec2',
{}).get('id', {})
prov_dict['key'] = providers.get(provider, {}).get('ec2',
{}).get('key', {})
if sigver == '4':
headers, requesturl = sig4(method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
MAX_RETRIES = 6
attempts = 0
while attempts < MAX_RETRIES:
log.debug('AWS Request: %s', requesturl)
log.trace('AWS Request Parameters: %s', params_with_headers)
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
log.debug('AWS Response Status Code: %s', result.status_code)
log.trace(
'AWS Response Text: %s',
result.text.encode(
result.encoding if result.encoding else 'utf-8'))
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts < MAX_RETRIES and err_code and err_code in AWS_RETRY_CODES:
attempts += 1
log.error(
'AWS Response Status Code and Error: [%s %s] %s; '
'Attempts remaining: %s', exc.response.status_code, exc,
data, attempts)
# backoff an exponential amount of time to throttle requests
# during "API Rate Exceeded" failures as suggested by the AWS documentation here:
# https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html
# and also here:
# https://docs.aws.amazon.com/general/latest/gr/api-retries.html
# Failure to implement this approach results in a failure rate of >30% when using salt-cloud with
# "--parallel" when creating 50 or more instances with a fixed delay of 2 seconds.
# A failure rate of >10% is observed when using the salt-api with an asyncronous client
# specified (runner_async).
time.sleep(random.uniform(1, 2**attempts))
continue
log.error('AWS Response Status Code and Error: [%s %s] %s',
exc.response.status_code, exc, data)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
log.error('AWS Response Status Code and Error: [%s %s] %s',
exc.response.status_code, exc, data)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text.encode(
result.encoding if result.encoding else 'utf-8')
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=JARFILE,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=SESSIONJARFILE,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
handle=False,
agent=USERAGENT,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master')
)
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion')
)
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
log.debug('Using {0} Method'.format(method))
if method == 'POST':
log.trace('POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'r') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if username and password:
auth = (username, password)
else:
auth = None
if requests_lib is True:
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
else:
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
else:
sess_cookies.load()
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if requests_lib is True:
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error('The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(
method, url, params=params, data=data, **req_kwargs
)
result.raise_for_status()
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.status_code
result_headers = result.headers
result_text = result.text
result_cookies = result.cookies
else:
request = urllib_request.Request(url, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https') or port == 443:
if not HAS_MATCHHOSTNAME:
log.warn(('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
hostname = request.get_host()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, 443))
sockwrap = ssl.wrap_socket(
sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = (
'The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)
)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
result = opener.open(request)
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def call(payload=None):
'''
This function captures the query string and sends it to the Palo Alto device.
'''
r = None
try:
if DETAILS['method'] == 'dev_key':
# Pass the api key without the target declaration
conditional_payload = {'key': DETAILS['apikey']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'dev_pass':
# Pass credentials without the target declaration
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'pan_key':
# Pass the api key with the target declaration
conditional_payload = {
'key': DETAILS['apikey'],
'target': DETAILS['serial']
}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
elif DETAILS['method'] == 'pan_pass':
# Pass credentials with the target declaration
conditional_payload = {'target': DETAILS['serial']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=DETAILS["verify_ssl"],
raise_error=True)
except KeyError as err:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
if not r:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
xmldata = ET.fromstring(r['text'])
# If we are pulling the candidate configuration, we need to strip the dirtyId
if payload['type'] == 'config' and payload['action'] == 'get':
xmldata = (_strip_dirty(xmldata))
return xml.to_dict(xmldata, True)
def test_xml_case_f_full(self):
xmldata = ET.fromstring(self.cases['f']['xml'])
defaultdict = xml.to_dict(xmldata, True)
self.assertEqual(defaultdict, self.cases['f']['full'])
def test_xml_case_f_legacy(self):
xmldata = ET.fromstring(self.cases['f']['xml'])
defaultdict = xml.to_dict(xmldata, False)
self.assertEqual(defaultdict, self.cases['f']['legacy'])
def service_group_exists(name,
groupname=None,
vsys=1,
members=None,
description=None,
commit=False):
'''
Ensures that a service group object exists in the configured state. If it does not exist or is not configured with
the specified attributes, it will be adjusted to match the specified values.
This module will enforce group membership. If a group exists and contains members this state does not include,
those members will be removed and replaced with the specified members in the state.
name: The name of the module function to execute.
groupname(str): The name of the service group object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
members(str, list): The members of the service group. These must be valid service objects or service groups on the
system that already exist prior to the execution of this state.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/service-group/my-group:
panos.service_group_exists:
- groupname: my-group
- vsys: 1
- members:
- tcp-80
- custom-port-group
- description: A group that needs to exist
- commit: False
'''
ret = _default_ret(name)
if not groupname:
ret.update({'comment': "The group name field must be provided."})
return ret
# Check if service group object currently exists
group = __salt__['panos.get_service_group'](groupname, vsys)['result']
if group and 'entry' in group:
group = group['entry']
else:
group = {}
# Verify the arguments
if members:
element = "<members>{0}</members>".format(_build_members(
members, True))
else:
ret.update({'comment': "The group members must be provided."})
return ret
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(groupname, element)
new_group = xml.to_dict(ET.fromstring(full_element), True)
if group == new_group:
ret.update({
'comment':
'Service group object already exists. No changes required.',
'result': True
})
return ret
else:
xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/service-group/" \
"entry[@name=\'{1}\']".format(vsys, groupname)
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({'comment': msg})
return ret
if commit is True:
ret.update({
'changes': {
'before': group,
'after': new_group
},
'commit': __salt__['panos.commit'](),
'comment': 'Service group object successfully configured.',
'result': True
})
else:
ret.update({
'changes': {
'before': group,
'after': new_group
},
'comment': 'Service group object successfully configured.',
'result': True
})
return ret
def call(payload=None):
"""
This function captures the query string and sends it to the Palo Alto device.
"""
r = None
try:
if DETAILS["method"] == "dev_key":
# Pass the api key without the target declaration
conditional_payload = {"key": DETAILS["apikey"]}
payload.update(conditional_payload)
r = __utils__["http.query"](
DETAILS["url"],
data=payload,
method="POST",
decode_type="plain",
decode=True,
verify_ssl=False,
status=True,
raise_error=True,
)
elif DETAILS["method"] == "dev_pass":
# Pass credentials without the target declaration
r = __utils__["http.query"](
DETAILS["url"],
username=DETAILS["username"],
password=DETAILS["password"],
data=payload,
method="POST",
decode_type="plain",
decode=True,
verify_ssl=False,
status=True,
raise_error=True,
)
elif DETAILS["method"] == "pan_key":
# Pass the api key with the target declaration
conditional_payload = {
"key": DETAILS["apikey"],
"target": DETAILS["serial"],
}
payload.update(conditional_payload)
r = __utils__["http.query"](
DETAILS["url"],
data=payload,
method="POST",
decode_type="plain",
decode=True,
verify_ssl=False,
status=True,
raise_error=True,
)
elif DETAILS["method"] == "pan_pass":
# Pass credentials with the target declaration
conditional_payload = {"target": DETAILS["serial"]}
payload.update(conditional_payload)
r = __utils__["http.query"](
DETAILS["url"],
username=DETAILS["username"],
password=DETAILS["password"],
data=payload,
method="POST",
decode_type="plain",
decode=True,
verify_ssl=False,
status=True,
raise_error=True,
)
except KeyError as err:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host."
)
if not r:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host."
)
if six.text_type(r["status"]) not in ["200", "201", "204"]:
if six.text_type(r["status"]) == "400":
raise salt.exceptions.CommandExecutionError(
"The server cannot process the request due to a client error."
)
elif six.text_type(r["status"]) == "401":
raise salt.exceptions.CommandExecutionError(
"The server cannot process the request because it lacks valid authentication "
"credentials for the target resource."
)
elif six.text_type(r["status"]) == "403":
raise salt.exceptions.CommandExecutionError(
"The server refused to authorize the request."
)
elif six.text_type(r["status"]) == "404":
raise salt.exceptions.CommandExecutionError(
"The requested resource could not be found."
)
else:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host."
)
xmldata = ET.fromstring(r["text"])
# If we are pulling the candidate configuration, we need to strip the dirtyId
if payload["type"] == "config" and payload["action"] == "get":
xmldata = _strip_dirty(xmldata)
return xml.to_dict(xmldata, True)
def query(params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product='ec2',
sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint', '{0}.{1}.{2}'.format(product, location,
service_url))
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = (
'Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(requesturl)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version',
prov_dict.get('{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION))
if sigver == '4':
headers, requesturl = sig4(method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
LOG.debug('AWS Response Status Code: {0}'.format(
result.status_code))
LOG.trace('AWS Response Text: {0}'.format(result.text))
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts))
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def service_exists(
name,
servicename=None,
vsys=1,
protocol=None,
port=None,
description=None,
commit=False,
):
"""
Ensures that a service object exists in the configured state. If it does not exist or is not configured with the
specified attributes, it will be adjusted to match the specified values.
name: The name of the module function to execute.
servicename(str): The name of the security object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
protocol(str): The protocol that is used by the service object. The only valid options are tcp and udp.
port(str): The port number that is used by the service object. This can be specified as a single integer or a
valid range of ports.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/service/tcp-80:
panos.service_exists:
- servicename: tcp-80
- vsys: 1
- protocol: tcp
- port: 80
- description: Hypertext Transfer Protocol
- commit: False
panos/service/udp-500-550:
panos.service_exists:
- servicename: udp-500-550
- vsys: 3
- protocol: udp
- port: 500-550
- commit: False
"""
ret = _default_ret(name)
if not servicename:
ret.update({"comment": "The service name field must be provided."})
return ret
# Check if service object currently exists
service = __salt__["panos.get_service"](servicename, vsys)["result"]
if service and "entry" in service:
service = service["entry"]
else:
service = {}
# Verify the arguments
if not protocol and protocol not in ["tcp", "udp"]:
ret.update({
"comment":
"The protocol must be provided and must be tcp or udp."
})
return ret
if not port:
ret.update({"comment": "The port field must be provided."})
return ret
element = "<protocol><{0}><port>{1}</port></{0}></protocol>".format(
protocol, port)
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(servicename, element)
new_service = xml.to_dict(ET.fromstring(full_element), True)
if service == new_service:
ret.update({
"comment": "Service object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/service/"
"entry[@name='{1}']".format(vsys, servicename))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": service,
"after": new_service
},
"commit": __salt__["panos.commit"](),
"comment": "Service object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": service,
"after": new_service
},
"comment": "Service object successfully configured.",
"result": True,
})
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
decode=True,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=JARFILE,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=SESSIONJARFILE,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
opts=None,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
if opts is None:
if node == 'master':
opts = salt.config.master_config('/etc/salt/master')
elif node == 'minion':
opts = salt.config.master_config('/etc/salt/minion')
else:
opts = {}
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
log.trace('POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if persist_session is True:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'r') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if username and password:
auth = (username, password)
else:
auth = None
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
if cookies is not None:
if cookie_format == 'mozilla':
sess.cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess.cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess.cookies.save()
else:
sess.cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
result = sess.request(
method, url, params=params, data=data
)
log.debug('Response Status Code: {0}'.format(result.status_code))
log.trace('Response Headers: {0}'.format(result.headers))
log.trace('Response Text: {0}'.format(result.text))
log.trace('Response Cookies: {0}'.format(result.cookies.get_dict()))
if cookies is not None:
sess.cookies.save()
if persist_session is True:
# TODO: See persist_session above
if 'set-cookie' in result.headers:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
session_cookies = result.headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result.status_code
if headers is True:
ret['headers'] = result.headers
if decode is True:
if decode_type == 'auto':
content_type = result.headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(result.text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result.text)
for item in items:
ret['dict'].append(xml.to_dict(item))
else:
text = True
if text is True:
ret['text'] = result.text
return ret
def query(key,
keyid,
method='GET',
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path='',
return_bin=False,
action=None,
local_file=None,
verify_ssl=True,
full_headers=False,
kms_keyid=None,
location=None,
role_arn=None,
chunk_size=16384,
path_style=False,
https_enable=True):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
Path style can be enabled:
s3.path_style: True
This can be useful if you need to use salt with a proxy for an s3 compatible storage
You can use either https protocol or http protocol:
s3.https_enable: True
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if not bucket or path_style:
endpoint = service_url
else:
endpoint = '{0}.{1}'.format(bucket, service_url)
if path_style and bucket:
path = '{0}/{1}'.format(bucket, path)
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key:
key = salt.utils.aws.IROLE_CODE
if not keyid:
keyid = salt.utils.aws.IROLE_CODE
if kms_keyid is not None and method in ('PUT', 'POST'):
headers['x-amz-server-side-encryption'] = 'aws:kms'
headers['x-amz-server-side-encryption-aws-kms-key-id'] = kms_keyid
if not location:
location = salt.utils.aws.get_location()
data = ''
payload_hash = None
if method == 'PUT':
if local_file:
payload_hash = salt.utils.hashutils.get_hash(local_file,
form='sha256')
if path is None:
path = ''
if not requesturl:
requesturl = (('https' if https_enable else 'http') +
'://{0}/{1}').format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={
'id': keyid,
'key': key
},
role_arn=role_arn,
location=location,
product='s3',
requesturl=requesturl,
headers=headers,
payload_hash=payload_hash,
)
log.debug('S3 Request: %s', requesturl)
log.debug('S3 Headers::')
log.debug(' Authorization: %s', headers['Authorization'])
if not data:
data = None
try:
if method == 'PUT':
if local_file:
data = salt.utils.files.fopen(local_file, 'r') # pylint: disable=resource-leakage
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
elif method == 'GET' and local_file and not return_bin:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl,
stream=True)
else:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl)
finally:
if data is not None:
data.close()
err_code = None
err_msg = None
if result.status_code >= 400:
# On error the S3 API response should contain error message
err_text = result.content or 'Unknown error'
log.debug(' Response content: %s', err_text)
# Try to get err info from response xml
try:
err_data = xml.to_dict(ET.fromstring(err_text))
err_code = err_data['Code']
err_msg = err_data['Message']
except (KeyError, ET.ParseError) as err:
log.debug('Failed to parse s3 err response. %s: %s',
type(err).__name__, err)
err_code = 'http-{0}'.format(result.status_code)
err_msg = err_text
log.debug('S3 Response Status Code: %s', result.status_code)
if method == 'PUT':
if result.status_code != 200:
if local_file:
raise CommandExecutionError(
'Failed to upload from {0} to {1}. {2}: {3}'.format(
local_file, path, err_code, err_msg))
raise CommandExecutionError(
'Failed to create bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if local_file:
log.debug('Uploaded from %s to %s', local_file, path)
else:
log.debug('Created bucket %s', bucket)
return
if method == 'DELETE':
if not six.text_type(result.status_code).startswith('2'):
if path:
raise CommandExecutionError(
'Failed to delete {0} from bucket {1}. {2}: {3}'.format(
path, bucket, err_code, err_msg))
raise CommandExecutionError(
'Failed to delete bucket {0}. {1}: {2}'.format(
bucket, err_code, err_msg))
if path:
log.debug('Deleted %s from bucket %s', path, bucket)
else:
log.debug('Deleted bucket %s', bucket)
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError('Failed to get file. {0}: {1}'.format(
err_code, err_msg))
log.debug('Saving to local file: %s', local_file)
with salt.utils.files.fopen(local_file, 'wb') as out:
for chunk in result.iter_content(chunk_size=chunk_size):
out.write(chunk)
return 'Saved to local file: {0}'.format(local_file)
if result.status_code < 200 or result.status_code >= 300:
raise CommandExecutionError('Failed s3 operation. {0}: {1}'.format(
err_code, err_msg))
# This can be used to return a binary object wholesale
if return_bin:
return result.content
if result.content:
items = ET.fromstring(result.content)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def address_exists(
name,
addressname=None,
vsys=1,
ipnetmask=None,
iprange=None,
fqdn=None,
description=None,
commit=False,
):
"""
Ensures that an address object exists in the configured state. If it does not exist or is not configured with the
specified attributes, it will be adjusted to match the specified values.
This module will only process a single address type (ip-netmask, ip-range, or fqdn). It will process the specified
value if the following order: ip-netmask, ip-range, fqdn. For proper execution, only specify a single address
type.
name: The name of the module function to execute.
addressname(str): The name of the address object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
ipnetmask(str): The IPv4 or IPv6 address or IP address range using the format ip_address/mask or ip_address where
the mask is the number of significant binary digits used for the network portion of the address. Ideally, for IPv6,
you specify only the network portion, not the host portion.
iprange(str): A range of addresses using the format ip_address–ip_address where both addresses can be IPv4 or both
can be IPv6.
fqdn(str): A fully qualified domain name format. The FQDN initially resolves at commit time. Entries are
subsequently refreshed when the firewall performs a check every 30 minutes; all changes in the IP address for the
entries are picked up at the refresh cycle.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/address/h-10.10.10.10:
panos.address_exists:
- addressname: h-10.10.10.10
- vsys: 1
- ipnetmask: 10.10.10.10
- commit: False
panos/address/10.0.0.1-10.0.0.50:
panos.address_exists:
- addressname: r-10.0.0.1-10.0.0.50
- vsys: 1
- iprange: 10.0.0.1-10.0.0.50
- commit: False
panos/address/foo.bar.com:
panos.address_exists:
- addressname: foo.bar.com
- vsys: 1
- fqdn: foo.bar.com
- description: My fqdn object
- commit: False
"""
ret = _default_ret(name)
if not addressname:
ret.update({"comment": "The service name field must be provided."})
return ret
# Check if address object currently exists
address = __salt__["panos.get_address"](addressname, vsys)["result"]
if address and "entry" in address:
address = address["entry"]
else:
address = {}
element = ""
# Verify the arguments
if ipnetmask:
element = "<ip-netmask>{0}</ip-netmask>".format(ipnetmask)
elif iprange:
element = "<ip-range>{0}</ip-range>".format(iprange)
elif fqdn:
element = "<fqdn>{0}</fqdn>".format(fqdn)
else:
ret.update({"comment": "A valid address type must be specified."})
return ret
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(addressname, element)
new_address = xml.to_dict(ET.fromstring(full_element), True)
if address == new_address:
ret.update({
"comment": "Address object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/address/"
"entry[@name='{1}']".format(vsys, addressname))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": address,
"after": new_address
},
"commit": __salt__["panos.commit"](),
"comment": "Address object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": address,
"after": new_address
},
"comment": "Service object successfully configured.",
"result": True,
})
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend='tornado',
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
handle=False,
agent=USERAGENT,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
log.warn('Please set "backend" to "requests" instead of setting '
'"requests_lib" to "True"')
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
backend = 'requests'
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
log.debug('Using {0} Method'.format(method))
if method == 'POST':
log.trace('POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR),
'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'r') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if username and password:
auth = (username, password)
else:
auth = None
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True or handle is True:
return {'handle': result}
log.debug(result.url)
log.trace(data)
result_status_code = result.status_code
result_headers = result.headers
result_text = result.text
result_cookies = result.cookies
elif backend == 'urllib2':
request = urllib_request.Request(url, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https') or port == 443:
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warn(
('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, 443))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': str(exc)}
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error(
'The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
try:
result = HTTPClient().fetch(tornado.httputil.url_concat(
url, params),
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
**req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {'handle': result}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Set-Cookie' in result_headers.keys() and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(
('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'w') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def address_group_exists(name,
groupname=None,
vsys=1,
members=None,
description=None,
commit=False):
"""
Ensures that an address group object exists in the configured state. If it does not exist or is not configured with
the specified attributes, it will be adjusted to match the specified values.
This module will enforce group membership. If a group exists and contains members this state does not include,
those members will be removed and replaced with the specified members in the state.
name: The name of the module function to execute.
groupname(str): The name of the address group object. The name is case-sensitive and can have up to 31 characters,
which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on
Panorama, unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
members(str, list): The members of the address group. These must be valid address objects or address groups on the
system that already exist prior to the execution of this state.
description(str): A description for the policy (up to 255 characters).
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/address-group/my-group:
panos.address_group_exists:
- groupname: my-group
- vsys: 1
- members:
- my-address-object
- my-other-address-group
- description: A group that needs to exist
- commit: False
"""
ret = _default_ret(name)
if not groupname:
ret.update({"comment": "The group name field must be provided."})
return ret
# Check if address group object currently exists
group = __salt__["panos.get_address_group"](groupname, vsys)["result"]
if group and "entry" in group:
group = group["entry"]
else:
group = {}
# Verify the arguments
if members:
element = "<static>{0}</static>".format(_build_members(members, True))
else:
ret.update({"comment": "The group members must be provided."})
return ret
if description:
element += "<description>{0}</description>".format(description)
full_element = "<entry name='{0}'>{1}</entry>".format(groupname, element)
new_group = xml.to_dict(ET.fromstring(full_element), True)
if group == new_group:
ret.update({
"comment":
"Address group object already exists. No changes required.",
"result": True,
})
return ret
else:
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/address-group/"
"entry[@name='{1}']".format(vsys, groupname))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
if commit is True:
ret.update({
"changes": {
"before": group,
"after": new_group
},
"commit": __salt__["panos.commit"](),
"comment": "Address group object successfully configured.",
"result": True,
})
else:
ret.update({
"changes": {
"before": group,
"after": new_group
},
"comment": "Address group object successfully configured.",
"result": True,
})
return ret
def test_xml_case_f_full(self):
xmldata = ET.fromstring(self.cases["f"]["xml"])
defaultdict = xml.to_dict(xmldata, True)
self.assertEqual(defaultdict, self.cases["f"]["full"])
def edit_config(name, xpath=None, value=None, commit=False):
"""
Edits a Palo Alto XPATH to a specific value. This will always overwrite the existing value, even if it is not
changed.
You can replace an existing object hierarchy at a specified location in the configuration with a new value. Use
the xpath parameter to specify the location of the object, including the node to be replaced.
This is the recommended state to enforce configurations on a xpath.
name: The name of the module function to execute.
xpath(str): The XPATH of the configuration API tree to control.
value(str): The XML value to edit. This must be a child to the XPATH.
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/addressgroup:
panos.edit_config:
- xpath: /config/devices/entry/vsys/entry[@name='vsys1']/address-group/entry[@name='test']
- value: <static><entry name='test'><member>abc</member><member>xyz</member></entry></static>
- commit: True
"""
ret = _default_ret(name)
# Verify if the current XPATH is equal to the specified value.
# If we are equal, no changes required.
xpath_split = xpath.split("/")
# Retrieve the head of the xpath for validation.
if xpath_split:
head = xpath_split[-1]
if "[" in head:
head = head.split("[")[0]
current_element = __salt__["panos.get_xpath"](xpath)["result"]
if head and current_element and head in current_element:
current_element = current_element[head]
else:
current_element = {}
new_element = xml.to_dict(ET.fromstring(value), True)
if current_element == new_element:
ret.update({
"comment": "XPATH is already equal to the specified value.",
"result": True,
})
return ret
result, msg = _edit_config(xpath, value)
ret.update({"comment": msg, "result": result})
if not result:
return ret
if commit is True:
ret.update({
"changes": {
"before": current_element,
"after": new_element
},
"commit": __salt__["panos.commit"](),
"result": True,
})
else:
ret.update({
"changes": {
"before": current_element,
"after": new_element
},
"result": True,
})
return ret
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend='tornado',
requests_lib=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(syspaths.CONFIG_DIR, 'master')
)
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(syspaths.CONFIG_DIR, 'minion')
)
else:
opts = {}
if requests_lib is None:
requests_lib = opts.get('requests_lib', False)
if requests_lib is True:
log.warn('Please set "backend" to "requests" instead of setting '
'"requests_lib" to "True"')
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
backend = 'requests'
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params)
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(
data_file, data_render, data_renderer, template_dict, opts
)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL {0} using {1} method'.format(log_url, method))
if method == 'POST':
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: {0}'.format(pprint.pformat(log_data)))
else:
log.trace('Request POST Data: {0}'.format(pprint.pformat(data)))
if header_file is not None:
header_tpl = _render(
header_file, header_render, header_renderer, template_dict, opts
)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: {0}'.format(sess.headers))
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error('The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
result = sess.request(
method, url, params=params, data=data, **req_kwargs
)
result.raise_for_status()
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: {0}'.format(sanitize_url(result.url, hide_fields)))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
ret['body'] = result.content
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warn(('match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!'))
elif verify_ssl is False:
log.warn(('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!'))
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(
sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED
)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = (
'The certificate was invalid. '
'Error returned was: {0}'.format(
pprint.pformat(exc)
)
)
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error('The client-side certificate path that was '
'passed is not valid: {0}'.format(cert))
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[0] = salt.ext.six.moves.http_client.HTTPSConnection(**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': str(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = result.headers.headers
result_text = result.read()
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, tuple):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error('The client-side certificate path that was passed is '
'not valid: {0}'.format(cert))
if isinstance(data, dict):
data = urllib.urlencode(data)
if verify_ssl:
req_kwargs['ca_certs'] = ca_bundle
max_body = opts.get('http_max_body', salt.config.DEFAULT_MINION_OPTS['http_max_body'])
timeout = opts.get('http_request_timeout', salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
proxy_password = opts.get('proxy_password', None)
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl, but the '
'pycurl library does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = inspect.getargspec(tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = inspect.getargspec(tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
try:
if supports_max_body_size:
result = HTTPClient(max_body_size=max_body).fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
**req_kwargs
)
else:
result = HTTPClient().fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
**req_kwargs
)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = str(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
ret['body'] = result.body
if 'Set-Cookie' in result_headers.keys() and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: {0}'.format(result_status_code))
log.trace('Response Headers: {0}'.format(result_headers))
log.trace('Response Cookies: {0}'.format(sess_cookies))
try:
log.trace('Response Text: {0}'.format(result_text))
except UnicodeEncodeError as exc:
log.trace(('Cannot Trace Log Response Text: {0}. This may be due to '
'incompatibilities between requests and logging.').format(exc))
if text_out is not None and os.path.exists(text_out):
with salt.utils.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get(
'content-type', 'application/json'
)
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = (
'Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)
)
)
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = json.loads(salt.utils.to_str(result_text))
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = yaml.safe_load(result_text)
else:
text = True
if decode_out and os.path.exists(decode_out):
with salt.utils.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def security_rule_exists(
name,
rulename=None,
vsys="1",
action=None,
disabled=None,
sourcezone=None,
destinationzone=None,
source=None,
destination=None,
application=None,
service=None,
description=None,
logsetting=None,
logstart=None,
logend=None,
negatesource=None,
negatedestination=None,
profilegroup=None,
datafilter=None,
fileblock=None,
spyware=None,
urlfilter=None,
virus=None,
vulnerability=None,
wildfire=None,
move=None,
movetarget=None,
commit=False,
):
"""
Ensures that a security rule exists on the device. Also, ensure that all configurations are set appropriately.
This method will create the rule if it does not exist. If the rule does exist, it will ensure that the
configurations are set appropriately.
If the rule does not exist and is created, any value that is not provided will be provided as the default.
The action, to, from, source, destination, application, and service fields are mandatory and must be provided.
This will enforce the exact match of the rule. For example, if the rule is currently configured with the log-end
option, but this option is not specified in the state method, it will be removed and reset to the system default.
It is strongly recommended to specify all options to ensure proper operation.
When defining the profile group settings, the device can only support either a profile group or individual settings.
If both are specified, the profile group will be preferred and the individual settings are ignored. If neither are
specified, the value will be set to system default of none.
name: The name of the module function to execute.
rulename(str): The name of the security rule. The name is case-sensitive and can have up to 31 characters, which
can be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on Panorama,
unique within its device group and any ancestor or descendant device groups.
vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1.
action(str): The action that the security rule will enforce. Valid options are: allow, deny, drop, reset-client,
reset-server, reset-both.
disabled(bool): Controls if the rule is disabled. Set 'True' to disable and 'False' to enable.
sourcezone(str, list): The source zone(s). The value 'any' will match all zones.
destinationzone(str, list): The destination zone(s). The value 'any' will match all zones.
source(str, list): The source address(es). The value 'any' will match all addresses.
destination(str, list): The destination address(es). The value 'any' will match all addresses.
application(str, list): The application(s) matched. The value 'any' will match all applications.
service(str, list): The service(s) matched. The value 'any' will match all services. The value
'application-default' will match based upon the application defined ports.
description(str): A description for the policy (up to 255 characters).
logsetting(str): The name of a valid log forwarding profile.
logstart(bool): Generates a traffic log entry for the start of a session (disabled by default).
logend(bool): Generates a traffic log entry for the end of a session (enabled by default).
negatesource(bool): Match all but the specified source addresses.
negatedestination(bool): Match all but the specified destination addresses.
profilegroup(str): A valid profile group name.
datafilter(str): A valid data filter profile name. Ignored with the profilegroup option set.
fileblock(str): A valid file blocking profile name. Ignored with the profilegroup option set.
spyware(str): A valid spyware profile name. Ignored with the profilegroup option set.
urlfilter(str): A valid URL filtering profile name. Ignored with the profilegroup option set.
virus(str): A valid virus profile name. Ignored with the profilegroup option set.
vulnerability(str): A valid vulnerability profile name. Ignored with the profilegroup option set.
wildfire(str): A valid vulnerability profile name. Ignored with the profilegroup option set.
move(str): An optional argument that ensure the rule is moved to a specific location. Valid options are 'top',
'bottom', 'before', or 'after'. The 'before' and 'after' options require the use of the 'movetarget' argument
to define the location of the move request.
movetarget(str): An optional argument that defines the target of the move operation if the move argument is
set to 'before' or 'after'.
commit(bool): If true the firewall will commit the changes, if false do not commit changes.
SLS Example:
.. code-block:: yaml
panos/rulebase/security/rule01:
panos.security_rule_exists:
- rulename: rule01
- vsys: 1
- action: allow
- disabled: False
- sourcezone: untrust
- destinationzone: trust
- source:
- 10.10.10.0/24
- 1.1.1.1
- destination:
- 2.2.2.2-2.2.2.4
- application:
- any
- service:
- tcp-25
- description: My test security rule
- logsetting: logprofile
- logstart: False
- logend: True
- negatesource: False
- negatedestination: False
- profilegroup: myprofilegroup
- move: top
- commit: False
panos/rulebase/security/rule01:
panos.security_rule_exists:
- rulename: rule01
- vsys: 1
- action: allow
- disabled: False
- sourcezone: untrust
- destinationzone: trust
- source:
- 10.10.10.0/24
- 1.1.1.1
- destination:
- 2.2.2.2-2.2.2.4
- application:
- any
- service:
- tcp-25
- description: My test security rule
- logsetting: logprofile
- logstart: False
- logend: False
- datafilter: foobar
- fileblock: foobar
- spyware: foobar
- urlfilter: foobar
- virus: foobar
- vulnerability: foobar
- wildfire: foobar
- move: after
- movetarget: rule02
- commit: False
"""
ret = _default_ret(name)
if not rulename:
return ret
# Check if rule currently exists
rule = __salt__["panos.get_security_rule"](rulename, vsys)["result"]
if rule and "entry" in rule:
rule = rule["entry"]
else:
rule = {}
# Build the rule element
element = ""
if sourcezone:
element += "<from>{0}</from>".format(_build_members(sourcezone, True))
else:
ret.update({"comment": "The sourcezone field must be provided."})
return ret
if destinationzone:
element += "<to>{0}</to>".format(_build_members(destinationzone, True))
else:
ret.update({"comment": "The destinationzone field must be provided."})
return ret
if source:
element += "<source>{0}</source>".format(_build_members(source, True))
else:
ret.update({"comment": "The source field must be provided."})
return
if destination:
element += "<destination>{0}</destination>".format(
_build_members(destination, True))
else:
ret.update({"comment": "The destination field must be provided."})
return ret
if application:
element += "<application>{0}</application>".format(
_build_members(application, True))
else:
ret.update({"comment": "The application field must be provided."})
return ret
if service:
element += "<service>{0}</service>".format(
_build_members(service, True))
else:
ret.update({"comment": "The service field must be provided."})
return ret
if action:
element += "<action>{0}</action>".format(action)
else:
ret.update({"comment": "The action field must be provided."})
return ret
if disabled is not None:
if disabled:
element += "<disabled>yes</disabled>"
else:
element += "<disabled>no</disabled>"
if description:
element += "<description>{0}</description>".format(description)
if logsetting:
element += "<log-setting>{0}</log-setting>".format(logsetting)
if logstart is not None:
if logstart:
element += "<log-start>yes</log-start>"
else:
element += "<log-start>no</log-start>"
if logend is not None:
if logend:
element += "<log-end>yes</log-end>"
else:
element += "<log-end>no</log-end>"
if negatesource is not None:
if negatesource:
element += "<negate-source>yes</negate-source>"
else:
element += "<negate-source>no</negate-source>"
if negatedestination is not None:
if negatedestination:
element += "<negate-destination>yes</negate-destination>"
else:
element += "<negate-destination>no</negate-destination>"
# Build the profile settings
profile_string = None
if profilegroup:
profile_string = "<group><member>{0}</member></group>".format(
profilegroup)
else:
member_string = ""
if datafilter:
member_string += "<data-filtering><member>{0}</member></data-filtering>".format(
datafilter)
if fileblock:
member_string += "<file-blocking><member>{0}</member></file-blocking>".format(
fileblock)
if spyware:
member_string += "<spyware><member>{0}</member></spyware>".format(
spyware)
if urlfilter:
member_string += "<url-filtering><member>{0}</member></url-filtering>".format(
urlfilter)
if virus:
member_string += "<virus><member>{0}</member></virus>".format(
virus)
if vulnerability:
member_string += "<vulnerability><member>{0}</member></vulnerability>".format(
vulnerability)
if wildfire:
member_string += "<wildfire-analysis><member>{0}</member></wildfire-analysis>".format(
wildfire)
if member_string != "":
profile_string = "<profiles>{0}</profiles>".format(member_string)
if profile_string:
element += "<profile-setting>{0}</profile-setting>".format(
profile_string)
full_element = "<entry name='{0}'>{1}</entry>".format(rulename, element)
new_rule = xml.to_dict(ET.fromstring(full_element), True)
config_change = False
if rule == new_rule:
ret.update(
{"comment": "Security rule already exists. No changes required."})
else:
config_change = True
xpath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/rulebase/"
"security/rules/entry[@name='{1}']".format(vsys, rulename))
result, msg = _edit_config(xpath, full_element)
if not result:
ret.update({"comment": msg})
return ret
ret.update({
"changes": {
"before": rule,
"after": new_rule
},
"comment": "Security rule verified successfully.",
})
if move:
movepath = (
"/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys{0}']/rulebase/"
"security/rules/entry[@name='{1}']".format(vsys, rulename))
move_result = False
move_msg = ""
if move == "before" and movetarget:
move_result, move_msg = _move_before(movepath, movetarget)
elif move == "after":
move_result, move_msg = _move_after(movepath, movetarget)
elif move == "top":
move_result, move_msg = _move_top(movepath)
elif move == "bottom":
move_result, move_msg = _move_bottom(movepath)
if config_change:
ret.update({
"changes": {
"before": rule,
"after": new_rule,
"move": move_msg
}
})
else:
ret.update({"changes": {"move": move_msg}})
if not move_result:
ret.update({"comment": move_msg})
return ret
if commit is True:
ret.update({"commit": __salt__["panos.commit"](), "result": True})
else:
ret.update({"result": True})
return ret
def query(key,
keyid,
method='GET',
params=None,
headers=None,
requesturl=None,
return_url=False,
bucket=None,
service_url=None,
path='',
return_bin=False,
action=None,
local_file=None,
verify_ssl=True,
location=None,
full_headers=False):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
If keyid or key is not specified, an attempt to fetch them from EC2 IAM
metadata service will be made.
A service_url may also be specified in the configuration:
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
A region may be specified:
s3.location: eu-central-1
If region is not specified, an attempt to fetch the region from EC2 IAM
metadata service will be made. Failing that, default is us-east-1
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
if not key or not keyid:
key = salt.utils.aws.IROLE_CODE
keyid = salt.utils.aws.IROLE_CODE
data = ''
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
if not requesturl:
requesturl = 'https://{0}/{1}'.format(endpoint, path)
headers, requesturl = salt.utils.aws.sig4(
method,
endpoint,
params,
data=data,
uri='/{0}'.format(path),
prov_dict={
'id': keyid,
'key': key
},
location=location,
product='s3',
requesturl=requesturl,
)
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
if not data:
data = None
try:
result = requests.request(method,
requesturl,
headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('Failed to {0} {1}::'.format(method, requesturl))
log.error(' Exception: {0}'.format(exc))
if exc.response:
log.error(' Response content: {0}'.format(exc.response.content))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def query(params=None, setname=None, requesturl=None, location=None,
return_url=False, return_root=False, opts=None, provider=None,
endpoint=None, product='ec2', sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint',
'{0}.{1}.{2}'.format(product, location, service_url)
)
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = ('Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(
requesturl
)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version', prov_dict.get(
'{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION
)
)
if sigver == '4':
headers, requesturl = sig4(
method, endpoint, params, prov_dict, aws_api_version, location, product, requesturl=requesturl
)
params_with_headers = {}
else:
params_with_headers = sig2(
method, endpoint, params, prov_dict, aws_api_version
)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl, headers=headers, params=params_with_headers)
LOG.debug(
'AWS Response Status Code: {0}'.format(
result.status_code
)
)
LOG.trace(
'AWS Response Text: {0}'.format(
result.text
)
)
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts
)
)
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path=None, return_bin=False, action=None, local_file=None,
verify_ssl=True):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
A service_url may also be specified in the configuration::
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
'''
if not headers:
headers = {}
if not params:
params = {}
if path is None:
path = ''
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
token = None
if not key or not keyid:
iam_creds = iam.get_iam_metadata()
key = iam_creds['secret_key']
keyid = iam_creds['access_key']
token = iam_creds['security_token']
if not requesturl:
x_amz_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
content_type = 'text/plain'
if method == 'GET':
if bucket:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/'
elif method == 'PUT' or method == 'HEAD' or method == 'DELETE':
if path:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/{0}/'.format(bucket)
if action:
can_resource += '?{0}'.format(action)
log.debug('CanonicalizedResource: {0}'.format(can_resource))
headers['Host'] = endpoint
headers['Content-type'] = content_type
headers['Date'] = x_amz_date
if token:
headers['x-amz-security-token'] = token
string_to_sign = '{0}\n'.format(method)
new_headers = []
for header in sorted(headers):
if header.lower().startswith('x-amz'):
log.debug(header.lower())
new_headers.append('{0}:{1}'.format(header.lower(),
headers[header]))
can_headers = '\n'.join(new_headers)
log.debug('CanonicalizedAmzHeaders: {0}'.format(can_headers))
string_to_sign += '\n{0}'.format(content_type)
string_to_sign += '\n{0}'.format(x_amz_date)
if can_headers:
string_to_sign += '\n{0}'.format(can_headers)
string_to_sign += '\n{0}'.format(can_resource)
log.debug('String To Sign:: \n{0}'.format(string_to_sign))
hashed = hmac.new(key, string_to_sign, hashlib.sha1)
sig = binascii.b2a_base64(hashed.digest())
headers['Authorization'] = 'AWS {0}:{1}'.format(keyid, sig.strip())
querystring = urllib.urlencode(params)
if action:
if querystring:
querystring = '{0}&{1}'.format(action, querystring)
else:
querystring = action
requesturl = 'https://{0}/'.format(endpoint)
if path:
requesturl += path
if querystring:
requesturl += '?{0}'.format(querystring)
data = None
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def call(payload=None):
'''
This function captures the query string and sends it to the Palo Alto device.
'''
r = None
try:
if DETAILS['method'] == 'dev_key':
# Pass the api key without the target declaration
conditional_payload = {'key': DETAILS['apikey']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=False,
status=True,
raise_error=True)
elif DETAILS['method'] == 'dev_pass':
# Pass credentials without the target declaration
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=False,
status=True,
raise_error=True)
elif DETAILS['method'] == 'pan_key':
# Pass the api key with the target declaration
conditional_payload = {
'key': DETAILS['apikey'],
'target': DETAILS['serial']
}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=False,
status=True,
raise_error=True)
elif DETAILS['method'] == 'pan_pass':
# Pass credentials with the target declaration
conditional_payload = {'target': DETAILS['serial']}
payload.update(conditional_payload)
r = __utils__['http.query'](DETAILS['url'],
username=DETAILS['username'],
password=DETAILS['password'],
data=payload,
method='POST',
decode_type='plain',
decode=True,
verify_ssl=False,
status=True,
raise_error=True)
except KeyError as err:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
if not r:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
if six.text_type(r['status']) not in ['200', '201', '204']:
if six.text_type(r['status']) == '400':
raise salt.exceptions.CommandExecutionError(
"The server cannot process the request due to a client error.")
elif six.text_type(r['status']) == '401':
raise salt.exceptions.CommandExecutionError(
"The server cannot process the request because it lacks valid authentication "
"credentials for the target resource.")
elif six.text_type(r['status']) == '403':
raise salt.exceptions.CommandExecutionError(
"The server refused to authorize the request.")
elif six.text_type(r['status']) == '404':
raise salt.exceptions.CommandExecutionError(
"The requested resource could not be found.")
else:
raise salt.exceptions.CommandExecutionError(
"Did not receive a valid response from host.")
xmldata = ET.fromstring(r['text'])
# If we are pulling the candidate configuration, we need to strip the dirtyId
if payload['type'] == 'config' and payload['action'] == 'get':
xmldata = (_strip_dirty(xmldata))
return xml.to_dict(xmldata, True)
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if not backend:
backend = opts.get('backend', 'tornado')
match = re.match(
r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)',
url)
if not match:
salt.utils.network.refresh_dns()
if backend == 'requests':
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL %s using %s method', log_url, method)
log.debug("Using backend: %s", backend)
if method == 'POST' and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: %s', pprint.pformat(log_data))
else:
log.trace('Request POST Data: %s', pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: %s', sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that'
' was passed is not valid: %s', cert)
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
'handle': result,
}
if handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: %s',
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type):
body = body.decode(result.encoding or 'utf-8')
ret['body'] = body
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
'match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!')
elif verify_ssl is False:
log.warning('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!')
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: %s',
pprint.pformat(exc))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
'The client-side certificate path that was '
'passed is not valid: %s', cert)
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': six.text_type(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result.body.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error(
'The client-side certificate path that '
'was passed is not valid: %s', cert)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
# tornado requires a str, cannot be unicode str in py2
req_kwargs['ca_certs'] = salt.utils.stringutils.to_str(ca_bundle)
max_body = opts.get('http_max_body',
salt.config.DEFAULT_MINION_OPTS['http_max_body'])
connect_timeout = opts.get(
'http_connect_timeout',
salt.config.DEFAULT_MINION_OPTS['http_connect_timeout'])
timeout = opts.get(
'http_request_timeout',
salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get('proxy_password', None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get('no_proxy', [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = (
'proxy_host and proxy_port has been set. This requires pycurl and tornado, '
'but the libraries does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = salt.utils.args.get_function_argspec(
tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = salt.utils.args.get_function_argspec(
tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
try:
download_client = HTTPClient(max_body_size=max_body) \
if supports_max_body_size \
else HTTPClient()
result = download_client.fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
header_callback=header_callback,
connect_timeout=connect_timeout,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
raise_error=raise_error,
decompress_response=False,
**req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = six.text_type(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = six.text_type(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
if 'Set-Cookie' in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: %s', result_status_code)
log.trace('Response Headers: %s', result_headers)
log.trace('Response Cookies: %s', sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get('Content-Encoding', "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != 'requests':
result_text = __decompressContent(coding, result_text)
try:
log.trace('Response Text: %s', result_text)
except UnicodeEncodeError as exc:
log.trace(
'Cannot Trace Log Response Text: %s. This may be due to '
'incompatibilities between requests and logging.', exc)
if text_out is not None:
with salt.utils.files.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = salt.utils.json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = salt.utils.data.decode(
salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def query(
params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product="ec2",
sigver="2",
):
"""
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block:: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
"""
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get("function", (None, product))
providers = opts.get("providers", {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get("service_url", "amazonaws.com")
if not location:
location = get_location(opts, prov_dict)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
"endpoint", "{}.{}.{}".format(product, location, service_url))
requesturl = "https://{}/".format(endpoint)
else:
endpoint = urllib.parse.urlparse(requesturl).netloc
if endpoint == "":
endpoint_err = (
"Could not find a valid endpoint in the "
"requesturl: {}. Looking for something "
"like https://some.aws.endpoint/?args").format(requesturl)
log.error(endpoint_err)
if return_url is True:
return {"error": endpoint_err}, requesturl
return {"error": endpoint_err}
log.debug("Using AWS endpoint: %s", endpoint)
method = "GET"
aws_api_version = prov_dict.get(
"aws_api_version",
prov_dict.get("{}_api_version".format(product),
DEFAULT_AWS_API_VERSION),
)
# Fallback to ec2's id & key if none is found, for this component
if not prov_dict.get("id", None):
prov_dict["id"] = providers.get(provider, {}).get("ec2",
{}).get("id", {})
prov_dict["key"] = providers.get(provider, {}).get("ec2",
{}).get("key", {})
if sigver == "4":
headers, requesturl = sig4(
method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl,
)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
attempts = 0
while attempts < AWS_MAX_RETRIES:
log.debug("AWS Request: %s", requesturl)
log.trace("AWS Request Parameters: %s", params_with_headers)
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
log.debug("AWS Response Status Code: %s", result.status_code)
log.trace("AWS Response Text: %s", result.text)
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get("Errors", {}).get("Error", {}).get("Code", "")
if attempts < AWS_MAX_RETRIES and err_code and err_code in AWS_RETRY_CODES:
attempts += 1
log.error(
"AWS Response Status Code and Error: [%s %s] %s; "
"Attempts remaining: %s",
exc.response.status_code,
exc,
data,
attempts,
)
sleep_exponential_backoff(attempts)
continue
log.error(
"AWS Response Status Code and Error: [%s %s] %s",
exc.response.status_code,
exc,
data,
)
if return_url is True:
return {"error": data}, requesturl
return {"error": data}
else:
log.error(
"AWS Response Status Code and Error: [%s %s] %s",
exc.response.status_code,
exc,
data,
)
if return_url is True:
return {"error": data}, requesturl
return {"error": data}
root = ET.fromstring(result.text)
items = root[1]
if return_root is True:
items = root
if setname:
for idx, item in enumerate(root):
comps = item.tag.split("}")
if comps[1] == setname:
items = root[idx]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
