def _extrn_path(self, url, saltenv, cachedir=None):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
# Strip user:pass from URLs
netloc = netloc.split('@')[-1]
if cachedir is None:
cachedir = self.opts['cachedir']
elif not os.path.isabs(cachedir):
cachedir = os.path.join(self.opts['cachedir'], cachedir)
if url_data.query:
file_name = '-'.join([url_data.path, url_data.query])
else:
file_name = url_data.path
return salt.utils.path_join(
cachedir,
'extrn_files',
saltenv,
netloc,
file_name
)
def add_http_basic_auth(url,
user=None,
password=None,
https_only=False):
'''
Return a string with http basic auth incorporated into it
'''
if user is None and password is None:
return url
else:
urltuple = urlparse(url)
if https_only and urltuple.scheme != 'https':
raise ValueError('Basic Auth only supported for HTTPS')
if password is None:
netloc = '{0}@{1}'.format(
user,
urltuple.netloc
)
urltuple = urltuple._replace(netloc=netloc)
return urlunparse(urltuple)
else:
netloc = '{0}:{1}@{2}'.format(
user,
password,
urltuple.netloc
)
urltuple = urltuple._replace(netloc=netloc)
return urlunparse(urltuple)
def scan(filename):
'''
scan function
'''
parsed = urlparse(filename)
if not parsed.scheme:
filename = 'salt://' + filename
cached_source = __salt__['cp.cache_file'](filename)
ret = {'Vulnerabilities': []}
cmd = '{0} oval eval {1}'.format(_OSCAP, cached_source)
salt_ret = __salt__['cmd.run_all'](cmd, python_shell=False)
items = salt_ret['stdout'].split('\n')
for item in items:
if 'true' in item:
if 'rhsa' in item:
rhsa = item.split(':')[3]
year = item.split(':')[3][:4]
num = item.split(':')[3][4:]
url = 'https://rhn.redhat.com/errata/RHSA-' + year + '-' + num + '.html'
ret['Vulnerabilities'].append('RHSA-' + rhsa + ' : ' + url)
return ret
def test_hook_can_handle_get_parameters(self, get_event):
self._app.mod_opts['webhook_disable_auth'] = True
event = MagicMock()
event.fire_event.return_value = True
get_event.return_value = event
response = self.fetch(
'/hook/my_service/?param=1¶m=2',
body=json.dumps({}),
method='POST',
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(response.code, 200, response.body)
host = urlparse(response.effective_url).netloc
event.fire_event.assert_called_once_with(
{
'headers': {
'Content-Length': '2',
'Connection': 'close',
'Content-Type': 'application/json',
'Host': host,
'Accept-Encoding': 'gzip'
},
'post': {},
'get': {
'param': ['1', '2']
}
},
'salt/netapi/hook/my_service/',
)
def _extrn_path(self, url, saltenv, cachedir=None):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
if cachedir is None:
cachedir = self.opts['cachedir']
elif not os.path.isabs(cachedir):
cachedir = os.path.join(self.opts['cachedir'], cachedir)
if url_data.query:
file_name = '-'.join([url_data.path, url_data.query])
else:
file_name = url_data.path
return salt.utils.path_join(
cachedir,
'extrn_files',
saltenv,
netloc,
file_name
)
def test_hook_can_handle_get_parameters(self):
with patch("salt.utils.event.get_event") as get_event:
with patch.dict(self._app.mod_opts,
{"webhook_disable_auth": True}):
event = MagicMock()
event.fire_event.return_value = True
get_event.return_value = event
response = self.fetch(
"/hook/my_service/?param=1¶m=2",
body=salt.utils.json.dumps({}),
method="POST",
headers={"Content-Type": self.content_type_map["json"]},
)
self.assertEqual(response.code, 200, response.body)
host = urlparse(response.effective_url).netloc
event.fire_event.assert_called_once_with(
{
"headers": {
"Content-Length": "2",
"Connection": "close",
"Content-Type": "application/json",
"Host": host,
"Accept-Encoding": "gzip",
},
"post": {},
"get": {
"param": ["1", "2"]
},
},
"salt/netapi/hook/my_service/",
)
def validate(url, protos):
'''
Return true if the passed URL scheme is in the list of accepted protos
'''
if urlparse(url).scheme in protos:
return True
return False
def validate(url, protos):
'''
Return true if the passed URL scheme is in the list of accepted protos
'''
if urlparse(url).scheme in protos:
return True
return False
def get_template(
self,
url,
dest,
template='jinja',
makedirs=False,
saltenv='base',
env=None,
**kwargs):
'''
Cache a file then process it as a template
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
kwargs['saltenv'] = saltenv
url_data = urlparse(url)
sfn = self.cache_file(url, saltenv)
if not os.path.exists(sfn):
return ''
if template in salt.utils.templates.TEMPLATE_REGISTRY:
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
sfn,
**kwargs
)
else:
log.error('Attempted to render template with unavailable engine '
'{0}'.format(template))
return ''
if not data['result']:
# Failed to render the template
log.error(
'Failed to render template with error: {0}'.format(
data['data']
)
)
return ''
if not dest:
# No destination passed, set the dest as an extrn_files cache
dest = self._extrn_path(url, saltenv)
# If Salt generated the dest name, create any required dirs
makedirs = True
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
salt.utils.safe_rm(data['data'])
return ''
shutil.move(data['data'], dest)
return dest
def get_template(
self,
url,
dest,
template='jinja',
makedirs=False,
saltenv='base',
env=None,
**kwargs):
'''
Cache a file then process it as a template
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
kwargs['saltenv'] = saltenv
url_data = urlparse(url)
sfn = self.cache_file(url, saltenv)
if not os.path.exists(sfn):
return ''
if template in salt.utils.templates.TEMPLATE_REGISTRY:
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
sfn,
**kwargs
)
else:
log.error('Attempted to render template with unavailable engine '
'{0}'.format(template))
return ''
if not data['result']:
# Failed to render the template
log.error(
'Failed to render template with error: {0}'.format(
data['data']
)
)
return ''
if not dest:
# No destination passed, set the dest as an extrn_files cache
dest = self._extrn_path(url, saltenv)
# If Salt generated the dest name, create any required dirs
makedirs = True
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
salt.utils.safe_rm(data['data'])
return ''
shutil.move(data['data'], dest)
return dest
def adjust_uri(self, uri, filename):
scheme = urlparse(uri).scheme
if scheme in ("salt", "file"):
return uri
elif scheme:
raise ValueError("Unsupported URL scheme({0}) in {1}".format(
scheme, uri))
return self.lookup.adjust_uri(uri, filename)
def _extrn_path(self, url, saltenv):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
return salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, url_data.netloc, url_data.path)
def get_template(
self,
url,
dest,
template='jinja',
makedirs=False,
saltenv='base',
cachedir=None,
**kwargs):
'''
Cache a file then process it as a template
'''
if 'env' in kwargs:
salt.utils.warn_until(
'Oxygen',
'Parameter \'env\' has been detected in the argument list. This '
'parameter is no longer used and has been replaced by \'saltenv\' '
'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.'
)
kwargs.pop('env')
kwargs['saltenv'] = saltenv
url_data = urlparse(url)
sfn = self.cache_file(url, saltenv, cachedir=cachedir)
if not os.path.exists(sfn):
return ''
if template in salt.utils.templates.TEMPLATE_REGISTRY:
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
sfn,
**kwargs
)
else:
log.error('Attempted to render template with unavailable engine '
'{0}'.format(template))
return ''
if not data['result']:
# Failed to render the template
log.error(
'Failed to render template with error: {0}'.format(
data['data']
)
)
return ''
if not dest:
# No destination passed, set the dest as an extrn_files cache
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
# If Salt generated the dest name, create any required dirs
makedirs = True
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
salt.utils.safe_rm(data['data'])
return ''
shutil.move(data['data'], dest)
return dest
def get_template(
self,
url,
dest,
template='jinja',
makedirs=False,
saltenv='base',
cachedir=None,
**kwargs):
'''
Cache a file then process it as a template
'''
if 'env' in kwargs:
salt.utils.warn_until(
'Oxygen',
'Parameter \'env\' has been detected in the argument list. This '
'parameter is no longer used and has been replaced by \'saltenv\' '
'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.'
)
kwargs.pop('env')
kwargs['saltenv'] = saltenv
url_data = urlparse(url)
sfn = self.cache_file(url, saltenv, cachedir=cachedir)
if not os.path.exists(sfn):
return ''
if template in salt.utils.templates.TEMPLATE_REGISTRY:
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
sfn,
**kwargs
)
else:
log.error('Attempted to render template with unavailable engine '
'{0}'.format(template))
return ''
if not data['result']:
# Failed to render the template
log.error(
'Failed to render template with error: {0}'.format(
data['data']
)
)
return ''
if not dest:
# No destination passed, set the dest as an extrn_files cache
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
# If Salt generated the dest name, create any required dirs
makedirs = True
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
salt.utils.safe_rm(data['data'])
return ''
shutil.move(data['data'], dest)
return dest
def setUp(self):
domain = urlparse(TEST_REPO).netloc
try:
if hasattr(socket, "setdefaulttimeout"):
# 10 second dns timeout
socket.setdefaulttimeout(10)
socket.gethostbyname(domain)
except socket.error:
msg = "error resolving {0}, possible network issue?"
self.skipTest(msg.format(domain))
def _extrn_path(self, url, saltenv):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
return salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, netloc, url_data.path)
def check_remote(cmdline_path):
"""
Checks to see if the path provided contains ftp, http, or https. Returns
the full path if it is found.
:param cmdline_path: The path to the initrd image or the kernel
"""
regex = re.compile('^(ht|f)tps?\\b')
if regex.match(urlparse(cmdline_path).scheme):
return True
return False
def _extrn_path(self, url, saltenv):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
return salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
url_data.netloc,
url_data.path
)
def get_template(
self,
url,
dest,
template=u'jinja',
makedirs=False,
saltenv=u'base',
cachedir=None,
**kwargs):
'''
Cache a file then process it as a template
'''
if u'env' in kwargs:
# "env" is not supported; Use "saltenv".
kwargs.pop(u'env')
kwargs[u'saltenv'] = saltenv
url_data = urlparse(url)
sfn = self.cache_file(url, saltenv, cachedir=cachedir)
if not os.path.exists(sfn):
return u''
if template in salt.utils.templates.TEMPLATE_REGISTRY:
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
sfn,
**kwargs
)
else:
log.error(
u'Attempted to render template with unavailable engine %s',
template
)
return u''
if not data[u'result']:
# Failed to render the template
log.error(u'Failed to render template with error: %s', data[u'data'])
return u''
if not dest:
# No destination passed, set the dest as an extrn_files cache
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
# If Salt generated the dest name, create any required dirs
makedirs = True
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
salt.utils.files.safe_rm(data[u'data'])
return u''
shutil.move(data[u'data'], dest)
return dest
def unescape(url):
'''
remove escape character `|` from `url`
'''
if salt.utils.is_windows():
return url
scheme = urlparse(url).scheme
if not scheme:
return url.lstrip('|')
elif scheme == 'salt':
path, saltenv = parse(url)
return create(path.lstrip('|'), saltenv)
else:
return url
def is_escaped(url):
'''
test whether `url` is escaped with `|`
'''
if salt.utils.is_windows():
return False
scheme = urlparse(url).scheme
if not scheme:
return url.startswith('|')
elif scheme == 'salt':
path, saltenv = parse(url)
return path.startswith('|')
else:
return False
def unescape(url):
'''
remove escape character `|` from `url`
'''
if salt.utils.platform.is_windows():
return url
scheme = urlparse(url).scheme
if not scheme:
return url.lstrip('|')
elif scheme == 'salt':
path, saltenv = parse(url)
return create(path.lstrip('|'), saltenv)
else:
return url
def is_escaped(url):
"""
test whether `url` is escaped with `|`
"""
scheme = urlparse(url).scheme
if not scheme:
return url.startswith("|")
elif scheme == "salt":
path, saltenv = parse(url)
if salt.utils.platform.is_windows() and "|" in url:
return path.startswith("_")
else:
return path.startswith("|")
else:
return False
def unescape(url):
"""
remove escape character `|` from `url`
"""
scheme = urlparse(url).scheme
if not scheme:
return url.lstrip("|")
elif scheme == "salt":
path, saltenv = parse(url)
if salt.utils.platform.is_windows() and "|" in url:
return create(path.lstrip("_"), saltenv)
else:
return create(path.lstrip("|"), saltenv)
else:
return url
def is_escaped(url):
'''
test whether `url` is escaped with `|`
'''
if salt.utils.platform.is_windows():
return False
scheme = urlparse(url).scheme
if not scheme:
return url.startswith('|')
elif scheme == 'salt':
path, saltenv = parse(url)
return path.startswith('|')
else:
return False
def _get_hostname_and_port(url, default_port=None):
parsed = urlparse(url)
if parsed.hostname:
hostname = parsed.hostname
port = parsed.port
else:
splitted_url = url.split(":")
hostname = splitted_url[0]
if len(splitted_url) > 1:
port = int(splitted_url[1])
else:
port = None
res = (hostname, port or default_port)
__utils__["caasp_log.debug"]("%s parsed as %s", url, res)
return res
def _extrn_path(self, url, saltenv):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
return salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
def _extrn_path(self, url, saltenv, cachedir=None):
'''
Return the extn_filepath for a given url
'''
url_data = urlparse(url)
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
if cachedir is None:
cachedir = self.opts['cachedir']
elif not os.path.isabs(cachedir):
cachedir = os.path.join(self.opts['cachedir'], cachedir)
return salt.utils.path_join(cachedir, 'extrn_files', saltenv, netloc,
url_data.path)
def escape(url):
'''
add escape character `|` to `url`
'''
if salt.utils.is_windows():
return url
scheme = urlparse(url).scheme
if not scheme:
if url.startswith('|'):
return url
else:
return u'|{0}'.format(url)
elif scheme == 'salt':
path, saltenv = parse(url)
if path.startswith('|'):
return create(path, saltenv)
else:
return create(u'|{0}'.format(path), saltenv)
else:
return url
def escape(url):
'''
add escape character `|` to `url`
'''
if salt.utils.platform.is_windows():
return url
scheme = urlparse(url).scheme
if not scheme:
if url.startswith('|'):
return url
else:
return u'|{0}'.format(url)
elif scheme == 'salt':
path, saltenv = parse(url)
if path.startswith('|'):
return create(path, saltenv)
else:
return create(u'|{0}'.format(path), saltenv)
else:
return url
def escape(url):
"""
add escape character `|` to `url`
"""
if salt.utils.platform.is_windows():
return url
scheme = urlparse(url).scheme
if not scheme:
if url.startswith("|"):
return url
else:
return "|{0}".format(url)
elif scheme == "salt":
path, saltenv = parse(url)
if path.startswith("|"):
return create(path, saltenv)
else:
return create("|{0}".format(path), saltenv)
else:
return url
def download_remote(url, dir):
"""
Attempts to download a file specified by 'url'
:param url: The full remote path of the file which should be downloaded.
:param dir: The path the file should be downloaded to.
"""
try:
rand = hashlib.md5(os.urandom(32)).hexdigest()
remote_filename = urlparse(url).path.split('/')[-1]
full_directory = \
os.path.join(dir, "{}-{}".format(rand, remote_filename))
with salt.utils.files.fopen(full_directory, 'wb') as file,\
request.urlopen(url) as response:
file.write(response.rease())
return full_directory
except Exception as err: # pylint: disable=broad-except
raise err
def _salt_send_event(opaque, conn, data):
'''
Convenience function adding common data to the event and sending it
on the salt event bus.
:param opaque: the opaque data that is passed to the callback.
This is a dict with 'prefix', 'object' and 'event' keys.
:param conn: libvirt connection
:param data: additional event data dict to send
'''
tag_prefix = opaque['prefix']
object_type = opaque['object']
event_type = opaque['event']
# Prepare the connection URI to fit in the tag
# qemu+ssh://user@host:1234/system -> qemu+ssh/user@host:1234/system
uri = urlparse(conn.getURI())
uri_tag = [uri.scheme]
if uri.netloc:
uri_tag.append(uri.netloc)
path = uri.path.strip('/')
if path:
uri_tag.append(path)
uri_str = "/".join(uri_tag)
# Append some common data
all_data = {
'uri': conn.getURI()
}
all_data.update(data)
tag = '/'.join((tag_prefix, uri_str, object_type, event_type))
# Actually send the event in salt
if __opts__.get('__role') == 'master':
salt.utils.event.get_master_event(
__opts__,
__opts__['sock_dir']).fire_event(all_data, tag)
else:
__salt__['event.send'](tag, all_data)
def test_hook_can_handle_get_parameters(self, get_event):
self._app.mod_opts['webhook_disable_auth'] = True
event = MagicMock()
event.fire_event.return_value = True
get_event.return_value = event
response = self.fetch('/hook/my_service/?param=1¶m=2',
body=json.dumps({}),
method='POST',
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(response.code, 200, response.body)
host = urlparse(response.effective_url).netloc
event.fire_event.assert_called_once_with(
{'headers': {'Content-Length': '2',
'Connection': 'close',
'Content-Type': 'application/json',
'Host': host,
'Accept-Encoding': 'gzip'},
'post': {},
'get': {'param': ['1', '2']}
},
'salt/netapi/hook/my_service/',
)
def query(url,
method='GET',
params=None,
data=None,
data_file=None,
header_dict=None,
header_list=None,
header_file=None,
username=None,
password=None,
auth=None,
decode=False,
decode_type='auto',
status=False,
headers=False,
text=False,
cookies=None,
cookie_jar=None,
cookie_format='lwp',
persist_session=False,
session_cookie_jar=None,
data_render=False,
data_renderer=None,
header_render=False,
header_renderer=None,
template_dict=None,
test=False,
test_url=None,
node='minion',
port=80,
opts=None,
backend=None,
ca_bundle=None,
verify_ssl=None,
cert=None,
text_out=None,
headers_out=None,
decode_out=None,
stream=False,
streaming_callback=None,
header_callback=None,
handle=False,
agent=USERAGENT,
hide_fields=None,
raise_error=True,
**kwargs):
'''
Query a resource, and decode the return data
'''
ret = {}
if opts is None:
if node == 'master':
opts = salt.config.master_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'master'))
elif node == 'minion':
opts = salt.config.minion_config(
os.path.join(salt.syspaths.CONFIG_DIR, 'minion'))
else:
opts = {}
if not backend:
backend = opts.get('backend', 'tornado')
match = re.match(
r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)',
url)
if not match:
salt.utils.network.refresh_dns()
if backend == 'requests':
if HAS_REQUESTS is False:
ret['error'] = ('http.query has been set to use requests, but the '
'requests library does not seem to be installed')
log.error(ret['error'])
return ret
else:
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARNING)
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
if verify_ssl is None:
verify_ssl = opts.get('verify_ssl', True)
if cert is None:
cert = opts.get('cert', None)
if data_file is not None:
data = _render(data_file, data_render, data_renderer, template_dict,
opts)
# Make sure no secret fields show up in logs
log_url = sanitize_url(url_full, hide_fields)
log.debug('Requesting URL %s using %s method', log_url, method)
log.debug("Using backend: %s", backend)
if method == 'POST' and log.isEnabledFor(logging.TRACE):
# Make sure no secret fields show up in logs
if isinstance(data, dict):
log_data = data.copy()
if isinstance(hide_fields, list):
for item in data:
for field in hide_fields:
if item == field:
log_data[item] = 'XXXXXXXXXX'
log.trace('Request POST Data: %s', pprint.pformat(log_data))
else:
log.trace('Request POST Data: %s', pprint.pformat(data))
if header_file is not None:
header_tpl = _render(header_file, header_render, header_renderer,
template_dict, opts)
if isinstance(header_tpl, dict):
header_dict = header_tpl
else:
header_list = header_tpl.splitlines()
if header_dict is None:
header_dict = {}
if header_list is None:
header_list = []
if cookie_jar is None:
cookie_jar = os.path.join(
opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt')
if session_cookie_jar is None:
session_cookie_jar = os.path.join(
opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p')
if persist_session is True and HAS_MSGPACK:
# TODO: This is hackish; it will overwrite the session cookie jar with
# all cookies from this one connection, rather than behaving like a
# proper cookie jar. Unfortunately, since session cookies do not
# contain expirations, they can't be stored in a proper cookie jar.
if os.path.isfile(session_cookie_jar):
with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_:
session_cookies = msgpack.load(fh_)
if isinstance(session_cookies, dict):
header_dict.update(session_cookies)
else:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
msgpack.dump('', fh_)
for header in header_list:
comps = header.split(':')
if len(comps) < 2:
continue
header_dict[comps[0].strip()] = comps[1].strip()
if not auth:
if username and password:
auth = (username, password)
if agent == USERAGENT:
agent = '{0} http.query()'.format(agent)
header_dict['User-agent'] = agent
if backend == 'requests':
sess = requests.Session()
sess.auth = auth
sess.headers.update(header_dict)
log.trace('Request Headers: %s', sess.headers)
sess_cookies = sess.cookies
sess.verify = verify_ssl
elif backend == 'urllib2':
sess_cookies = None
else:
# Tornado
sess_cookies = None
if cookies is not None:
if cookie_format == 'mozilla':
sess_cookies = salt.ext.six.moves.http_cookiejar.MozillaCookieJar(
cookie_jar)
else:
sess_cookies = salt.ext.six.moves.http_cookiejar.LWPCookieJar(
cookie_jar)
if not os.path.isfile(cookie_jar):
sess_cookies.save()
sess_cookies.load()
if test is True:
if test_url is None:
return {}
else:
url = test_url
ret['test'] = True
if backend == 'requests':
req_kwargs = {}
if stream is True:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
req_kwargs['prefetch'] = False
else:
req_kwargs['stream'] = True
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['cert'] = cert
else:
log.error(
'The client-side certificate path that'
' was passed is not valid: %s', cert)
result = sess.request(method,
url,
params=params,
data=data,
**req_kwargs)
result.raise_for_status()
if stream is True:
# fake a HTTP response header
header_callback('HTTP/1.0 {0} MESSAGE'.format(result.status_code))
# fake streaming the content
streaming_callback(result.content)
return {
'handle': result,
}
if handle is True:
return {
'handle': result,
'body': result.content,
}
log.debug('Final URL location of Response: %s',
sanitize_url(result.url, hide_fields))
result_status_code = result.status_code
result_headers = result.headers
result_text = result.content
result_cookies = result.cookies
body = result.content
if not isinstance(body, six.text_type):
body = body.decode(result.encoding or 'utf-8')
ret['body'] = body
elif backend == 'urllib2':
request = urllib_request.Request(url_full, data)
handlers = [
urllib_request.HTTPHandler,
urllib_request.HTTPCookieProcessor(sess_cookies)
]
if url.startswith('https'):
hostname = request.get_host()
handlers[0] = urllib_request.HTTPSHandler(1)
if not HAS_MATCHHOSTNAME:
log.warning(
'match_hostname() not available, SSL hostname checking '
'not available. THIS CONNECTION MAY NOT BE SECURE!')
elif verify_ssl is False:
log.warning('SSL certificate verification has been explicitly '
'disabled. THIS CONNECTION MAY NOT BE SECURE!')
else:
if ':' in hostname:
hostname, port = hostname.split(':')
else:
port = 443
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((hostname, int(port)))
sockwrap = ssl.wrap_socket(sock,
ca_certs=ca_bundle,
cert_reqs=ssl.CERT_REQUIRED)
try:
match_hostname(sockwrap.getpeercert(), hostname)
except CertificateError as exc:
ret['error'] = ('The certificate was invalid. '
'Error returned was: %s',
pprint.pformat(exc))
return ret
# Client-side cert handling
if cert is not None:
cert_chain = None
if isinstance(cert, six.string_types):
if os.path.exists(cert):
cert_chain = (cert)
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
cert_chain = cert
else:
log.error(
'The client-side certificate path that was '
'passed is not valid: %s', cert)
return
if hasattr(ssl, 'SSLContext'):
# Python >= 2.7.9
context = ssl.SSLContext.load_cert_chain(*cert_chain)
handlers.append(
urllib_request.HTTPSHandler(context=context)) # pylint: disable=E1123
else:
# Python < 2.7.9
cert_kwargs = {
'host': request.get_host(),
'port': port,
'cert_file': cert_chain[0]
}
if len(cert_chain) > 1:
cert_kwargs['key_file'] = cert_chain[1]
handlers[
0] = salt.ext.six.moves.http_client.HTTPSConnection(
**cert_kwargs)
opener = urllib_request.build_opener(*handlers)
for header in header_dict:
request.add_header(header, header_dict[header])
request.get_method = lambda: method
try:
result = opener.open(request)
except URLError as exc:
return {'Error': six.text_type(exc)}
if stream is True or handle is True:
return {
'handle': result,
'body': result.content,
}
result_status_code = result.code
result_headers = dict(result.info())
result_text = result.read()
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result.body.decode('utf-8')
ret['body'] = result_text
else:
# Tornado
req_kwargs = {}
# Client-side cert handling
if cert is not None:
if isinstance(cert, six.string_types):
if os.path.exists(cert):
req_kwargs['client_cert'] = cert
elif isinstance(cert, list):
if os.path.exists(cert[0]) and os.path.exists(cert[1]):
req_kwargs['client_cert'] = cert[0]
req_kwargs['client_key'] = cert[1]
else:
log.error(
'The client-side certificate path that '
'was passed is not valid: %s', cert)
if isinstance(data, dict):
data = _urlencode(data)
if verify_ssl:
# tornado requires a str, cannot be unicode str in py2
req_kwargs['ca_certs'] = salt.utils.stringutils.to_str(ca_bundle)
max_body = opts.get('http_max_body',
salt.config.DEFAULT_MINION_OPTS['http_max_body'])
connect_timeout = opts.get(
'http_connect_timeout',
salt.config.DEFAULT_MINION_OPTS['http_connect_timeout'])
timeout = opts.get(
'http_request_timeout',
salt.config.DEFAULT_MINION_OPTS['http_request_timeout'])
client_argspec = None
proxy_host = opts.get('proxy_host', None)
if proxy_host:
# tornado requires a str for proxy_host, cannot be a unicode str in py2
proxy_host = salt.utils.stringutils.to_str(proxy_host)
proxy_port = opts.get('proxy_port', None)
proxy_username = opts.get('proxy_username', None)
if proxy_username:
# tornado requires a str, cannot be unicode str in py2
proxy_username = salt.utils.stringutils.to_str(proxy_username)
proxy_password = opts.get('proxy_password', None)
if proxy_password:
# tornado requires a str, cannot be unicode str in py2
proxy_password = salt.utils.stringutils.to_str(proxy_password)
no_proxy = opts.get('no_proxy', [])
# Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
# except we remove the valid ones if a url has a no_proxy hostname in it
if urlparse(url_full).hostname in no_proxy:
proxy_host = None
proxy_port = None
# We want to use curl_http if we have a proxy defined
if proxy_host and proxy_port:
if HAS_CURL_HTTPCLIENT is False:
ret['error'] = (
'proxy_host and proxy_port has been set. This requires pycurl and tornado, '
'but the libraries does not seem to be installed')
log.error(ret['error'])
return ret
tornado.httpclient.AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient')
client_argspec = salt.utils.args.get_function_argspec(
tornado.curl_httpclient.CurlAsyncHTTPClient.initialize)
else:
client_argspec = salt.utils.args.get_function_argspec(
tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize)
supports_max_body_size = 'max_body_size' in client_argspec.args
try:
download_client = HTTPClient(max_body_size=max_body) \
if supports_max_body_size \
else HTTPClient()
result = download_client.fetch(
url_full,
method=method,
headers=header_dict,
auth_username=username,
auth_password=password,
body=data,
validate_cert=verify_ssl,
allow_nonstandard_methods=True,
streaming_callback=streaming_callback,
header_callback=header_callback,
connect_timeout=connect_timeout,
request_timeout=timeout,
proxy_host=proxy_host,
proxy_port=proxy_port,
proxy_username=proxy_username,
proxy_password=proxy_password,
raise_error=raise_error,
decompress_response=False,
**req_kwargs)
except tornado.httpclient.HTTPError as exc:
ret['status'] = exc.code
ret['error'] = six.text_type(exc)
return ret
except socket.gaierror as exc:
if status is True:
ret['status'] = 0
ret['error'] = six.text_type(exc)
return ret
if stream is True or handle is True:
return {
'handle': result,
'body': result.body,
}
result_status_code = result.code
result_headers = result.headers
result_text = result.body
if 'Content-Type' in result_headers:
res_content_type, res_params = cgi.parse_header(
result_headers['Content-Type'])
if res_content_type.startswith('text/') and \
'charset' in res_params and \
not isinstance(result_text, six.text_type):
result_text = result_text.decode(res_params['charset'])
if six.PY3 and isinstance(result_text, bytes):
result_text = result_text.decode('utf-8')
ret['body'] = result_text
if 'Set-Cookie' in result_headers and cookies is not None:
result_cookies = parse_cookie_header(result_headers['Set-Cookie'])
for item in result_cookies:
sess_cookies.set_cookie(item)
else:
result_cookies = None
if isinstance(result_headers, list):
result_headers_dict = {}
for header in result_headers:
comps = header.split(':')
result_headers_dict[comps[0].strip()] = ':'.join(comps[1:]).strip()
result_headers = result_headers_dict
log.debug('Response Status Code: %s', result_status_code)
log.trace('Response Headers: %s', result_headers)
log.trace('Response Cookies: %s', sess_cookies)
# log.trace("Content: %s", result_text)
coding = result_headers.get('Content-Encoding', "identity")
# Requests will always decompress the content, and working around that is annoying.
if backend != 'requests':
result_text = __decompressContent(coding, result_text)
try:
log.trace('Response Text: %s', result_text)
except UnicodeEncodeError as exc:
log.trace(
'Cannot Trace Log Response Text: %s. This may be due to '
'incompatibilities between requests and logging.', exc)
if text_out is not None:
with salt.utils.files.fopen(text_out, 'w') as tof:
tof.write(result_text)
if headers_out is not None and os.path.exists(headers_out):
with salt.utils.files.fopen(headers_out, 'w') as hof:
hof.write(result_headers)
if cookies is not None:
sess_cookies.save()
if persist_session is True and HAS_MSGPACK:
# TODO: See persist_session above
if 'set-cookie' in result_headers:
with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_:
session_cookies = result_headers.get('set-cookie', None)
if session_cookies is not None:
msgpack.dump({'Cookie': session_cookies}, fh_)
else:
msgpack.dump('', fh_)
if status is True:
ret['status'] = result_status_code
if headers is True:
ret['headers'] = result_headers
if decode is True:
if decode_type == 'auto':
content_type = result_headers.get('content-type',
'application/json')
if 'xml' in content_type:
decode_type = 'xml'
elif 'json' in content_type:
decode_type = 'json'
elif 'yaml' in content_type:
decode_type = 'yaml'
else:
decode_type = 'plain'
valid_decodes = ('json', 'xml', 'yaml', 'plain')
if decode_type not in valid_decodes:
ret['error'] = ('Invalid decode_type specified. '
'Valid decode types are: {0}'.format(
pprint.pformat(valid_decodes)))
log.error(ret['error'])
return ret
if decode_type == 'json':
ret['dict'] = salt.utils.json.loads(result_text)
elif decode_type == 'xml':
ret['dict'] = []
items = ET.fromstring(result_text)
for item in items:
ret['dict'].append(xml.to_dict(item))
elif decode_type == 'yaml':
ret['dict'] = salt.utils.data.decode(
salt.utils.yaml.safe_load(result_text))
else:
text = True
if decode_out:
with salt.utils.files.fopen(decode_out, 'w') as dof:
dof.write(result_text)
if text is True:
ret['text'] = result_text
return ret
def get_url(self, url, dest, makedirs=False, saltenv='base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = ':'.join((url_scheme, url_path))
url_scheme = 'file'
if url_scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.fopen(url_data.path, 'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == 'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.fopen(result, 'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
self.utils['s3.query'](method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [False]
def on_header(hdr):
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def mod_repo(repo, **kwargs):
'''
Modify one or more values for a repo. If the repo does not exist, it will
be created, so long as the following values are specified:
repo or alias
alias by which the zypper refers to the repo
url or mirrorlist
the URL for zypper to reference
enabled
enable or disable (True or False) repository,
but do not remove if disabled.
refresh
enable or disable (True or False) auto-refresh of the repository.
cache
Enable or disable (True or False) RPM files caching.
gpgcheck
Enable or disable (True or False) GOG check for this repository.
Key/Value pairs may also be removed from a repo's configuration by setting
a key to a blank value. Bear in mind that a name cannot be deleted, and a
url can only be deleted if a mirrorlist is specified (or vice versa).
CLI Examples:
.. code-block:: bash
salt '*' pkg.mod_repo alias alias=new_alias
salt '*' pkg.mod_repo alias url= mirrorlist=http://host.com/
'''
repos_cfg = _get_configured_repos()
added = False
# An attempt to add new one?
if repo not in repos_cfg.sections():
url = kwargs.get('url', kwargs.get('mirrorlist'))
if not url:
raise CommandExecutionError(
'Repository \'{0}\' not found and no URL passed to create one.'.format(repo))
if not urlparse(url).scheme:
raise CommandExecutionError(
'Repository \'{0}\' not found and passed URL looks wrong.'.format(repo))
# Is there already such repo under different alias?
for alias in repos_cfg.sections():
repo_meta = _get_repo_info(alias, repos_cfg=repos_cfg)
# Complete user URL, in case it is not
new_url = urlparse(url)
if not new_url.path:
new_url = urlparse.ParseResult(scheme=new_url.scheme, # pylint: disable=E1123
netloc=new_url.netloc,
path='/',
params=new_url.params,
query=new_url.query,
fragment=new_url.fragment)
base_url = urlparse(repo_meta['baseurl'])
if new_url == base_url:
raise CommandExecutionError(
'Repository \'{0}\' already exists as \'{1}\'.'.format(repo, alias))
# Add new repo
doc = None
try:
# Try to parse the output and find the error,
# but this not always working (depends on Zypper version)
doc = dom.parseString(__salt__['cmd.run'](('zypper -x ar {0} \'{1}\''.format(url, repo)),
output_loglevel='trace'))
except Exception:
# No XML out available, but it is still unknown the state of the result.
pass
if doc:
msg_nodes = doc.getElementsByTagName('message')
if msg_nodes:
msg_node = msg_nodes[0]
if msg_node.getAttribute('type') == 'error':
raise CommandExecutionError(msg_node.childNodes[0].nodeValue)
# Verify the repository has been added
repos_cfg = _get_configured_repos()
if repo not in repos_cfg.sections():
raise CommandExecutionError(
'Failed add new repository \'{0}\' for unknown reason. '
'Please look into Zypper logs.'.format(repo))
added = True
# Modify added or existing repo according to the options
cmd_opt = []
if 'enabled' in kwargs:
cmd_opt.append(kwargs['enabled'] and '--enable' or '--disable')
if 'refresh' in kwargs:
cmd_opt.append(kwargs['refresh'] and '--refresh' or '--no-refresh')
if 'cache' in kwargs:
cmd_opt.append(kwargs['cache'] and '--keep-packages' or '--no-keep-packages')
if 'gpgcheck' in kwargs:
cmd_opt.append(kwargs['gpgcheck'] and '--gpgcheck' or '--no-gpgcheck')
if cmd_opt:
__salt__['cmd.run'](('zypper -x mr {0} \'{1}\''.format(' '.join(cmd_opt), repo)),
output_loglevel='trace')
# If repo nor added neither modified, error should be thrown
if not added and not cmd_opt:
raise CommandExecutionError(
'Modification of the repository \'{0}\' was not specified.'.format(repo))
return {}
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
query = salt.utils.http.query(
fixed_url,
text=True,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
if 'text' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return query['body']
else:
dest_tmp = "{0}.part".format(dest)
with salt.utils.fopen(dest_tmp, 'wb') as destfp:
destfp.write(query['body'])
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
query = salt.utils.http.query(
fixed_url,
stream=True,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
response = query['handle']
chunk_size = 32 * 1024
if not no_cache:
with salt.utils.fopen(dest, 'wb') as destfp:
if hasattr(response, 'iter_content'):
for chunk in response.iter_content(chunk_size=chunk_size):
destfp.write(chunk)
else:
while True:
chunk = response.buffer.read(chunk_size)
destfp.write(chunk)
if len(chunk) < chunk_size:
break
return dest
else:
if hasattr(response, 'text'):
return response.text
else:
return response['text']
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
get_kwargs['prefetch'] = False
else:
get_kwargs['stream'] = True
response = requests.get(fixed_url, **get_kwargs)
with salt.utils.fopen(dest, 'wb') as destfp:
for chunk in response.iter_content(chunk_size=32 * 1024):
destfp.write(chunk)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Carbon',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Carbon.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [False]
def on_header(hdr):
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def setup_logfile_logger(log_path, log_level='error', log_format=None,
date_format=None):
'''
Setup the logfile logger
Since version 0.10.6 we support logging to syslog, some examples:
tcp://localhost:514/LOG_USER
tcp://localhost/LOG_DAEMON
udp://localhost:5145/LOG_KERN
udp://localhost
file:///dev/log
file:///dev/log/LOG_SYSLOG
file:///dev/log/LOG_DAEMON
The above examples are self explanatory, but:
<file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility>
If you're thinking on doing remote logging you might also be thinking that
you could point salt's logging to the remote syslog. **Please Don't!**
An issue has been reported when doing this over TCP when the logged lines
get concatenated. See #3061.
The preferred way to do remote logging is setup a local syslog, point
salt's logging to the local syslog(unix socket is much faster) and then
have the local syslog forward the log messages to the remote syslog.
'''
if is_logfile_configured():
logging.getLogger(__name__).warn('Logfile logging already configured')
return
if log_path is None:
logging.getLogger(__name__).warn(
'log_path setting is set to `None`. Nothing else to do'
)
return
# Remove the temporary logging handler
__remove_temp_logging_handler()
if log_level is None:
log_level = 'warning'
level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)
parsed_log_path = urlparse(log_path)
root_logger = logging.getLogger()
if parsed_log_path.scheme in ('tcp', 'udp', 'file'):
syslog_opts = {
'facility': SysLogHandler.LOG_USER,
'socktype': socket.SOCK_DGRAM
}
if parsed_log_path.scheme == 'file' and parsed_log_path.path:
facility_name = parsed_log_path.path.split(os.sep)[-1].upper()
if not facility_name.startswith('LOG_'):
# The user is not specifying a syslog facility
facility_name = 'LOG_USER' # Syslog default
syslog_opts['address'] = parsed_log_path.path
else:
# The user has set a syslog facility, let's update the path to
# the logging socket
syslog_opts['address'] = os.sep.join(
parsed_log_path.path.split(os.sep)[:-1]
)
elif parsed_log_path.path:
# In case of udp or tcp with a facility specified
facility_name = parsed_log_path.path.lstrip(os.sep).upper()
if not facility_name.startswith('LOG_'):
# Logging facilities start with LOG_ if this is not the case
# fail right now!
raise RuntimeError(
'The syslog facility {0!r} is not known'.format(
facility_name
)
)
else:
# This is the case of udp or tcp without a facility specified
facility_name = 'LOG_USER' # Syslog default
facility = getattr(
SysLogHandler, facility_name, None
)
if facility is None:
# This python syslog version does not know about the user provided
# facility name
raise RuntimeError(
'The syslog facility {0!r} is not known'.format(
facility_name
)
)
syslog_opts['facility'] = facility
if parsed_log_path.scheme == 'tcp':
# tcp syslog support was only added on python versions >= 2.7
if sys.version_info < (2, 7):
raise RuntimeError(
'Python versions lower than 2.7 do not support logging '
'to syslog using tcp sockets'
)
syslog_opts['socktype'] = socket.SOCK_STREAM
if parsed_log_path.scheme in ('tcp', 'udp'):
syslog_opts['address'] = (
parsed_log_path.hostname,
parsed_log_path.port or logging.handlers.SYSLOG_UDP_PORT
)
if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file':
# There's not socktype support on python versions lower than 2.7
syslog_opts.pop('socktype', None)
try:
# Et voilá! Finally our syslog handler instance
handler = SysLogHandler(**syslog_opts)
except socket.error as err:
logging.getLogger(__name__).error(
'Failed to setup the Syslog logging handler: {0}'.format(
err
)
)
sys.exit(2)
else:
try:
# Logfile logging is UTF-8 on purpose.
# Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a
# user is not using plain ASCII, their system should be ready to
# handle UTF-8.
handler = WatchedFileHandler(log_path, mode='a', encoding='utf-8', delay=0)
except (IOError, OSError):
logging.getLogger(__name__).warning(
'Failed to open log file, do you have permission to write to '
'{0}?'.format(log_path)
)
# Do not proceed with any more configuration since it will fail, we
# have the console logging already setup and the user should see
# the error.
return
handler.setLevel(level)
# Set the default console formatter config
if not log_format:
log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s'
if not date_format:
date_format = '%Y-%m-%d %H:%M:%S'
formatter = logging.Formatter(log_format, datefmt=date_format)
handler.setFormatter(formatter)
root_logger.addHandler(handler)
global __LOGFILE_CONFIGURED
__LOGFILE_CONFIGURED = True
def query(params=None,
setname=None,
requesturl=None,
location=None,
return_url=False,
return_root=False,
opts=None,
provider=None,
endpoint=None,
product='ec2',
sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint', '{0}.{1}.{2}'.format(product, location,
service_url))
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = (
'Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(requesturl)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version',
prov_dict.get('{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION))
if sigver == '4':
headers, requesturl = sig4(method,
endpoint,
params,
prov_dict,
aws_api_version,
location,
product,
requesturl=requesturl)
params_with_headers = {}
else:
params_with_headers = sig2(method, endpoint, params, prov_dict,
aws_api_version)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl,
headers=headers,
params=params_with_headers)
LOG.debug('AWS Response Status Code: {0}'.format(
result.status_code))
LOG.trace('AWS Response Text: {0}'.format(result.text))
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts))
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error('AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data))
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
query = salt.utils.http.query(fixed_url,
text=True,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'text' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return query['body']
else:
dest_tmp = "{0}.part".format(dest)
with salt.utils.fopen(dest_tmp, 'wb') as destfp:
destfp.write(query['body'])
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def setup_logfile_logger(log_path,
log_level='error',
log_format=None,
date_format=None):
'''
Setup the logfile logger
Since version 0.10.6 we support logging to syslog, some examples:
tcp://localhost:514/LOG_USER
tcp://localhost/LOG_DAEMON
udp://localhost:5145/LOG_KERN
udp://localhost
file:///dev/log
file:///dev/log/LOG_SYSLOG
file:///dev/log/LOG_DAEMON
The above examples are self explanatory, but:
<file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility>
If you're thinking on doing remote logging you might also be thinking that
you could point salt's logging to the remote syslog. **Please Don't!**
An issue has been reported when doing this over TCP when the logged lines
get concatenated. See #3061.
The preferred way to do remote logging is setup a local syslog, point
salt's logging to the local syslog(unix socket is much faster) and then
have the local syslog forward the log messages to the remote syslog.
'''
if is_logfile_configured():
logging.getLogger(__name__).warn('Logfile logging already configured')
return
if log_path is None:
logging.getLogger(__name__).warn(
'log_path setting is set to `None`. Nothing else to do')
return
# Remove the temporary logging handler
__remove_temp_logging_handler()
if log_level is None:
log_level = 'warning'
level = LOG_LEVELS.get(log_level.lower(), logging.ERROR)
parsed_log_path = urlparse(log_path)
root_logger = logging.getLogger()
if parsed_log_path.scheme in ('tcp', 'udp', 'file'):
syslog_opts = {
'facility': SysLogHandler.LOG_USER,
'socktype': socket.SOCK_DGRAM
}
if parsed_log_path.scheme == 'file' and parsed_log_path.path:
facility_name = parsed_log_path.path.split(os.sep)[-1].upper()
if not facility_name.startswith('LOG_'):
# The user is not specifying a syslog facility
facility_name = 'LOG_USER' # Syslog default
syslog_opts['address'] = parsed_log_path.path
else:
# The user has set a syslog facility, let's update the path to
# the logging socket
syslog_opts['address'] = os.sep.join(
parsed_log_path.path.split(os.sep)[:-1])
elif parsed_log_path.path:
# In case of udp or tcp with a facility specified
facility_name = parsed_log_path.path.lstrip(os.sep).upper()
if not facility_name.startswith('LOG_'):
# Logging facilities start with LOG_ if this is not the case
# fail right now!
raise RuntimeError(
'The syslog facility {0!r} is not known'.format(
facility_name))
else:
# This is the case of udp or tcp without a facility specified
facility_name = 'LOG_USER' # Syslog default
facility = getattr(SysLogHandler, facility_name, None)
if facility is None:
# This python syslog version does not know about the user provided
# facility name
raise RuntimeError(
'The syslog facility {0!r} is not known'.format(facility_name))
syslog_opts['facility'] = facility
if parsed_log_path.scheme == 'tcp':
# tcp syslog support was only added on python versions >= 2.7
if sys.version_info < (2, 7):
raise RuntimeError(
'Python versions lower than 2.7 do not support logging '
'to syslog using tcp sockets')
syslog_opts['socktype'] = socket.SOCK_STREAM
if parsed_log_path.scheme in ('tcp', 'udp'):
syslog_opts['address'] = (parsed_log_path.hostname,
parsed_log_path.port
or logging.handlers.SYSLOG_UDP_PORT)
if sys.version_info < (2, 7) or parsed_log_path.scheme == 'file':
# There's not socktype support on python versions lower than 2.7
syslog_opts.pop('socktype', None)
try:
# Et voilá! Finally our syslog handler instance
handler = SysLogHandler(**syslog_opts)
except socket.error as err:
logging.getLogger(__name__).error(
'Failed to setup the Syslog logging handler: {0}'.format(err))
sys.exit(2)
else:
try:
# Logfile logging is UTF-8 on purpose.
# Since salt uses YAML and YAML uses either UTF-8 or UTF-16, if a
# user is not using plain ASCII, their system should be ready to
# handle UTF-8.
handler = WatchedFileHandler(log_path,
mode='a',
encoding='utf-8',
delay=0)
except (IOError, OSError):
logging.getLogger(__name__).warning(
'Failed to open log file, do you have permission to write to '
'{0}?'.format(log_path))
# Do not proceed with any more configuration since it will fail, we
# have the console logging already setup and the user should see
# the error.
return
handler.setLevel(level)
# Set the default console formatter config
if not log_format:
log_format = '%(asctime)s [%(name)-15s][%(levelname)-8s] %(message)s'
if not date_format:
date_format = '%Y-%m-%d %H:%M:%S'
formatter = logging.Formatter(log_format, datefmt=date_format)
handler.setFormatter(formatter)
root_logger.addHandler(handler)
global __LOGFILE_CONFIGURED
__LOGFILE_CONFIGURED = True
def query(params=None, setname=None, requesturl=None, location=None,
return_url=False, return_root=False, opts=None, provider=None,
endpoint=None, product='ec2', sigver='2'):
'''
Perform a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
Regions and endpoints are documented at:
http://docs.aws.amazon.com/general/latest/gr/rande.html
Default ``product`` is ``ec2``. Valid ``product`` names are:
.. code-block: yaml
- autoscaling (Auto Scaling)
- cloudformation (CloudFormation)
- ec2 (Elastic Compute Cloud)
- elasticache (ElastiCache)
- elasticbeanstalk (Elastic BeanStalk)
- elasticloadbalancing (Elastic Load Balancing)
- elasticmapreduce (Elastic MapReduce)
- iam (Identity and Access Management)
- importexport (Import/Export)
- monitoring (CloudWatch)
- rds (Relational Database Service)
- simpledb (SimpleDB)
- sns (Simple Notification Service)
- sqs (Simple Queue Service)
'''
if params is None:
params = {}
if opts is None:
opts = {}
function = opts.get('function', (None, product))
providers = opts.get('providers', {})
if provider is None:
prov_dict = providers.get(function[1], {}).get(product, {})
if prov_dict:
driver = list(list(prov_dict.keys()))[0]
provider = providers.get(driver, product)
else:
prov_dict = providers.get(provider, {}).get(product, {})
service_url = prov_dict.get('service_url', 'amazonaws.com')
if not location:
location = get_location(opts, provider)
if endpoint is None:
if not requesturl:
endpoint = prov_dict.get(
'endpoint',
'{0}.{1}.{2}'.format(product, location, service_url)
)
requesturl = 'https://{0}/'.format(endpoint)
else:
endpoint = urlparse(requesturl).netloc
if endpoint == '':
endpoint_err = ('Could not find a valid endpoint in the '
'requesturl: {0}. Looking for something '
'like https://some.aws.endpoint/?args').format(
requesturl
)
LOG.error(endpoint_err)
if return_url is True:
return {'error': endpoint_err}, requesturl
return {'error': endpoint_err}
LOG.debug('Using AWS endpoint: {0}'.format(endpoint))
method = 'GET'
aws_api_version = prov_dict.get(
'aws_api_version', prov_dict.get(
'{0}_api_version'.format(product),
DEFAULT_AWS_API_VERSION
)
)
if sigver == '4':
headers, requesturl = sig4(
method, endpoint, params, prov_dict, aws_api_version, location, product, requesturl=requesturl
)
params_with_headers = {}
else:
params_with_headers = sig2(
method, endpoint, params, prov_dict, aws_api_version
)
headers = {}
attempts = 5
while attempts > 0:
LOG.debug('AWS Request: {0}'.format(requesturl))
LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
try:
result = requests.get(requesturl, headers=headers, params=params_with_headers)
LOG.debug(
'AWS Response Status Code: {0}'.format(
result.status_code
)
)
LOG.trace(
'AWS Response Text: {0}'.format(
result.text
)
)
result.raise_for_status()
break
except requests.exceptions.HTTPError as exc:
root = ET.fromstring(exc.response.content)
data = xml.to_dict(root)
# check to see if we should retry the query
err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
attempts -= 1
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}; '
'Attempts remaining: {3}'.format(
exc.response.status_code, exc, data, attempts
)
)
# Wait a bit before continuing to prevent throttling
time.sleep(2)
continue
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
else:
LOG.error(
'AWS Response Status Code and Error: [{0} {1}] {2}'.format(
exc.response.status_code, exc, data
)
)
if return_url is True:
return {'error': data}, requesturl
return {'error': data}
response = result.text
root = ET.fromstring(response)
items = root[1]
if return_root is True:
items = root
if setname:
if sys.version_info < (2, 7):
children_len = len(root.getchildren())
else:
children_len = len(root)
for item in range(0, children_len):
comps = root[item].tag.split('}')
if comps[1] == setname:
items = root[item]
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
return ret
