def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse.urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = os.path.normpath(
os.sep.join([
self.opts['cachedir'], 'extrn_files', env, url_data.netloc,
url_data.path
]))
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
try:
with contextlib.closing(urllib2.urlopen(url)) as srcfp:
with open(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except urllib2.HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except urllib2.URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
return ''
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, url_data.netloc,
url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(None, fixed_url, url_data.username,
url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def init(cnf):
'''
Initialize the module.
'''
CONFIG['host'] = cnf.get('proxy', {}).get('host')
if not CONFIG['host']:
raise MinionError(message="Cannot find 'host' parameter in the proxy configuration")
CONFIG['user'] = cnf.get('proxy', {}).get('user')
if not CONFIG['user']:
raise MinionError(message="Cannot find 'user' parameter in the proxy configuration")
CONFIG['uri'] = "/api/{0}".format(CONFIG['user'])
def init(cnf):
"""
Initialize the module.
"""
CONFIG["host"] = cnf.get("proxy", {}).get("host")
if not CONFIG["host"]:
raise MinionError(
message="Cannot find 'host' parameter in the proxy configuration")
CONFIG["user"] = cnf.get("proxy", {}).get("user")
if not CONFIG["user"]:
raise MinionError(
message="Cannot find 'user' parameter in the proxy configuration")
CONFIG["uri"] = "/api/{}".format(CONFIG["user"])
def _check_proto(self, path):
'''
Make sure that this path is intended for the salt master and trim it
'''
if not path.startswith('salt://'):
raise MinionError('Unsupported path: {0}'.format(path))
return path[7:]
def _check_proto(self, path):
'''
Make sure that this path is intended for the salt master and trim it
'''
if not path.startswith('salt://'):
raise MinionError(u'Unsupported path: {0}'.format(path))
file_path, saltenv = salt.utils.url.parse(path)
return file_path
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
env, url_data.netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(None, fixed_url, url_data.username,
url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def rename(src, dst):
'''
On Windows, os.rename() will fail with a WindowsError exception if a file
exists at the destination path. This function removes the destination path
first before attempting the os.rename().
'''
try:
os.remove(dst)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise MinionError(
'Error: Unable to remove {0}: {1}'.format(dst, exc.strerror)
)
os.rename(src, dst)
def rename(src, dst):
'''
On Windows, os.rename() will fail with a WindowsError exception if a file
exists at the destination path. This function checks for this error and if
found, it deletes the destination path first.
'''
try:
os.rename(src, dst)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
try:
os.remove(dst)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise MinionError('Error: Unable to remove {0}: {1}'.format(
dst, exc.strerror))
os.rename(src, dst)
def test_pkg_install_minion_error_https():
"""
Test pkg.install when cp.cache_file encounters a minion error
"""
ret__get_package_info = {
"3.03": {
"uninstaller": "%program.exe",
"reboot": False,
"msiexec": False,
"installer": "https://repo.test.com/runme.exe",
"uninstall_flags": "/S",
"locale": "en_US",
"install_flags": "/s",
"full_name": "Firebox 3.03 (x86 en-US)",
}
}
err_msg = ("Error: [Errno 11001] getaddrinfo failed reading"
" https://repo.test.com/runme.exe")
mock_none = MagicMock(return_value=None)
mock_minion_error = MagicMock(side_effect=MinionError(err_msg))
mock_parse = MagicMock(return_value=[{"firebox": "3.03"}, None])
with patch.object(
salt.utils.data, "is_true",
MagicMock(return_value=True)), patch.object(
win_pkg, "_get_package_info",
MagicMock(return_value=ret__get_package_info)), patch.dict(
win_pkg.__salt__,
{
"pkg_resource.parse_targets": mock_parse,
"cp.is_cached": mock_none,
"cp.cache_file": mock_minion_error,
},
):
ret = win_pkg.install(
name="firebox",
version="3.03",
)
expected = (
"Failed to cache https://repo.test.com/runme.exe\nError: [Errno 11001]"
" getaddrinfo failed reading https://repo.test.com/runme.exe")
assert ret == expected
def test_pkg_remove_minion_error_salt(self):
"""
Test pkg.remove when cp.cache_file encounters a minion error
"""
ret__get_package_info = {
"3.03": {
"uninstaller": "salt://software/runme.exe",
"reboot": False,
"msiexec": False,
"installer": "salt://software/runme.exe",
"uninstall_flags": "/U /S",
"locale": "en_US",
"install_flags": "/s",
"full_name": "Firebox 3.03 (x86 en-US)",
}
}
err_msg = "Error: [Errno 1] failed reading salt://software/runme.exe"
mock_minion_error = MagicMock(side_effect=MinionError(err_msg))
mock_none = MagicMock(return_value=None)
mock_parse = MagicMock(return_value=[{"firebox": "3.03"}, None])
se_list_pkgs = {"firebox": ["3.03"]}
with patch.object(
win_pkg, "list_pkgs", return_value=se_list_pkgs), patch.object(
salt.utils.data, "is_true",
MagicMock(return_value=True)), patch.object(
win_pkg, "_get_package_info",
MagicMock(
return_value=ret__get_package_info)), patch.dict(
win_pkg.__salt__,
{
"pkg_resource.parse_targets": mock_parse,
"cp.is_cached": mock_none,
"cp.cache_file": mock_minion_error,
},
):
ret = win_pkg.remove(name="firebox")
expected = (
"Failed to cache salt://software/runme.exe\n"
"Error: [Errno 1] failed reading salt://software/runme.exe")
self.assertEqual(ret, expected)
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
if urlparse.urlparse(url).scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return False
try:
with contextlib.closing(urllib2.urlopen(url)) as srcfp:
with open(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except urllib2.HTTPError, ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
def test_pkg_install_minion_error_salt_cache_dir():
"""
Test pkg.install when cp.cache_dir encounters a minion error
"""
ret__get_package_info = {
"3.03": {
"uninstaller": "%program.exe",
"reboot": False,
"msiexec": False,
"installer": "salt://software/runme.exe",
"cache_dir": True,
"uninstall_flags": "/S",
"locale": "en_US",
"install_flags": "/s",
"full_name": "Firebox 3.03 (x86 en-US)",
}
}
err_msg = "Error: [Errno 1] failed reading salt://software"
mock_none = MagicMock(return_value=None)
mock_minion_error = MagicMock(side_effect=MinionError(err_msg))
mock_parse = MagicMock(return_value=[{"firebox": "3.03"}, None])
with patch.object(
salt.utils.data, "is_true",
MagicMock(return_value=True)), patch.object(
win_pkg, "_get_package_info",
MagicMock(return_value=ret__get_package_info)), patch.dict(
win_pkg.__salt__,
{"cp.cache_dir": mock_minion_error},
):
ret = win_pkg.install(
name="firebox",
version="3.03",
)
expected = ("Failed to cache salt://software\n"
"Error: [Errno 1] failed reading salt://software")
assert ret == expected
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True),
location=self.opts.get('s3.location', None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
if no_cache:
result = []
def on_chunk(chunk):
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
try:
content_length = int(query['handle'].headers['Content-Length'])
except (AttributeError, KeyError, ValueError):
# Shouldn't happen but don't let this raise an exception.
# Instead, just don't do content length checking below.
log.warning(
'No Content-Length header in HTTP response from fetch of '
'{0}, or Content-Length is non-numeric'.format(fixed_url))
content_length = None
if no_cache:
content = ''.join(result)
if content_length is not None \
and len(content) > content_length:
return content[-content_length:]
else:
return content
else:
destfp.close()
destfp = None
dest_tmp_size = os.path.getsize(dest_tmp)
if content_length is not None \
and dest_tmp_size > content_length:
log.warning(
'Size of file downloaded from {0} ({1}) does not '
'match the Content-Length ({2}). This is probably due '
'to an upstream bug in tornado '
'(https://github.com/tornadoweb/tornado/issues/1518). '
'Re-writing the file to correct this.'.format(
fixed_url, dest_tmp_size, content_length))
dest_tmp_bak = dest_tmp + '.bak'
salt.utils.files.rename(dest_tmp, dest_tmp_bak)
with salt.utils.fopen(dest_tmp_bak, 'rb') as fp_bak:
fp_bak.seek(dest_tmp_size - content_length)
with salt.utils.fopen(dest_tmp, 'wb') as fp_new:
while True:
chunk = fp_bak.read(
self.opts['file_buffer_size'])
if not chunk:
break
fp_new.write(chunk)
os.remove(dest_tmp_bak)
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
class FileClient(object):
'''
Interact with the salt master file server.
'''
def __init__(self, opts):
self.opts = opts
self.serial = salt.payload.Serial(self.opts)
self.auth = salt.crypt.SAuth(opts)
self.socket = self.__get_socket()
def __get_socket(self):
'''
Return the ZeroMQ socket to use
'''
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect(self.opts['master_uri'])
return socket
def _check_proto(self, path):
'''
Make sure that this path is intended for the salt master and trim it
'''
if not path.startswith('salt://'):
raise MinionError('Unsupported path: {0}'.format(path))
return path[7:]
def _file_local_list(self, dest):
'''
Helper util to return a list of files in a directory
'''
if os.path.isdir(dest):
destdir = dest
else:
destdir = os.path.dirname(dest)
filelist = []
for root, dirs, files in os.walk(destdir):
for name in files:
path = os.path.join(root, name)
filelist.append(path)
return filelist
def get_file(self, path, dest='', makedirs=False, env='base'):
'''
Get a single file from the salt-master
'''
path = self._check_proto(path)
payload = {'enc': 'aes'}
fn_ = None
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return False
fn_ = open(dest, 'w+')
load = {'path': path, 'env': env, 'cmd': '_serve_file'}
while True:
if not fn_:
load['loc'] = 0
else:
load['loc'] = fn_.tell()
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send(self.serial.dumps(payload))
data = self.auth.crypticle.loads(
self.serial.loads(self.socket.recv()))
if not data['data']:
if not fn_ and data['dest']:
# This is a 0 byte file on the master
dest = os.path.join(self.opts['cachedir'], 'files', env,
data['dest'])
destdir = os.path.dirname(dest)
cumask = os.umask(191)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if not os.path.exists(dest):
open(dest, 'w+').write(data['data'])
os.chmod(dest, 384)
os.umask(cumask)
break
if not fn_:
dest = os.path.join(self.opts['cachedir'], 'files', env,
data['dest'])
destdir = os.path.dirname(dest)
cumask = os.umask(191)
if not os.path.isdir(destdir):
os.makedirs(destdir)
fn_ = open(dest, 'w+')
os.chmod(dest, 384)
os.umask(cumask)
fn_.write(data['data'])
return dest
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
if urlparse.urlparse(url).scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return False
try:
with contextlib.closing(urllib2.urlopen(url)) as srcfp:
with open(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except urllib2.HTTPError, ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except urllib2.URLError, ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Carbon',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Carbon.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [False]
def on_header(hdr):
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
query = salt.utils.http.query(fixed_url,
text=True,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'text' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return query['body']
else:
dest_tmp = "{0}.part".format(dest)
with salt.utils.fopen(dest_tmp, 'wb') as destfp:
destfp.write(query['body'])
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
get_kwargs['prefetch'] = False
else:
get_kwargs['stream'] = True
response = requests.get(fixed_url, **get_kwargs)
with salt.utils.fopen(dest, 'wb') as destfp:
for chunk in response.iter_content(chunk_size=32 * 1024):
destfp.write(chunk)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv=u'base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
# If dest is a directory, rewrite dest with filename
if dest is not None \
and (os.path.isdir(dest) or dest.endswith((u'/', u'\\'))):
if url_data.query or len(url_data.path) > 1 and not url_data.path.endswith(u'/'):
strpath = url.split(u'/')[-1]
else:
strpath = u'index.html'
if salt.utils.platform.is_windows():
strpath = salt.utils.sanitize_win_path_string(strpath)
dest = os.path.join(dest, strpath)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = u':'.join((url_scheme, url_path))
url_scheme = u'file'
if url_scheme in (u'file', u''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
u'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.files.fopen(url_path, u'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == u'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.files.fopen(result, u'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return u''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == u's3':
try:
def s3_opt(key, default=None):
u'''Get value of s3.<key> from Minion config or from Pillar'''
if u's3.' + key in self.opts:
return self.opts[u's3.' + key]
try:
return self.opts[u'pillar'][u's3'][key]
except (KeyError, TypeError):
return default
self.utils[u's3.query'](method=u'GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt(u'key'),
keyid=s3_opt(u'keyid'),
service_url=s3_opt(u'service_url'),
verify_ssl=s3_opt(u'verify_ssl', True),
location=s3_opt(u'location'),
path_style=s3_opt(u'path_style', False),
https_enable=s3_opt(u'https_enable', True))
return dest
except Exception as exc:
raise MinionError(
u'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == u'ftp':
try:
ftp = ftplib.FTP()
ftp.connect(url_data.hostname, url_data.port)
ftp.login(url_data.username, url_data.password)
with salt.utils.files.fopen(dest, u'wb') as fp_:
ftp.retrbinary(u'RETR {0}'.format(url_data.path), fp_.write)
ftp.quit()
return dest
except Exception as exc:
raise MinionError(u'Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == u'swift':
try:
def swift_opt(key, default):
'''
Get value of <key> from Minion config or from Pillar
'''
if key in self.opts:
return self.opts[key]
try:
return self.opts[u'pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt(u'keystone.user', None),
swift_opt(u'keystone.tenant', None),
swift_opt(u'keystone.auth_url', None),
swift_opt(u'keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError(u'Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in (u'http', u'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind(u'@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs[u'auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM
# avail). Here we are working around this by disabling recording
# the body for redirections. The issue is fixed in Tornado 4.3.0
# so on_header callback could be removed when we'll deprecate
# Tornado<4.3.0. See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback.
# Simple bool doesn't work here: on_header creates a new local
# variable instead. This could be avoided in Py3 with 'nonlocal'
# statement. There is no Py2 alternative for this.
#
# write_body[0] is used by the on_chunk callback to tell it whether
# or not we need to write the body of the request to disk. For
# 30x redirects we set this to False because we don't want to
# write the contents to disk, as we will need to wait until we
# get to the redirected URL.
#
# write_body[1] will contain a tornado.httputil.HTTPHeaders
# instance that we will use to parse each header line. We
# initialize this to False, and after we parse the status line we
# will replace it with the HTTPHeaders instance. If/when we have
# found the encoding used in the request, we set this value to
# False to signify that we are done parsing.
#
# write_body[2] is where the encoding will be stored
write_body = [None, False, None]
def on_header(hdr):
if write_body[1] is not False and write_body[2] is None:
if not hdr.strip() and 'Content-Type' not in write_body[1]:
# We've reached the end of the headers and not yet
# found the Content-Type. Reset the values we're
# tracking so that we properly follow the redirect.
write_body[0] = None
write_body[1] = False
return
# Try to find out what content type encoding is used if
# this is a text file
write_body[1].parse_line(hdr) # pylint: disable=no-member
if u'Content-Type' in write_body[1]:
content_type = write_body[1].get(u'Content-Type') # pylint: disable=no-member
if not content_type.startswith(u'text'):
write_body[1] = write_body[2] = False
else:
encoding = u'utf-8'
fields = content_type.split(u';')
for field in fields:
if u'encoding' in field:
encoding = field.split(u'encoding=')[-1]
write_body[2] = encoding
# We have found our encoding. Stop processing headers.
write_body[1] = False
# If write_body[0] is False, this means that this
# header is a 30x redirect, so we need to reset
# write_body[0] to None so that we parse the HTTP
# status code from the redirect target.
if write_body[0] is write_body[1] is False:
write_body[0] = None
# Check the status line of the HTTP request
if write_body[0] is None:
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
write_body[1] = HTTPHeaders()
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
if write_body[2]:
chunk = chunk.decode(write_body[2])
result.append(chunk)
else:
dest_tmp = u"{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.files.fopen(dest_tmp, u'wb') # pylint: disable=resource-leakage
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if u'handle' not in query:
raise MinionError(u'Error: {0} reading {1}'.format(query[u'error'], url))
if no_cache:
if write_body[2]:
return u''.join(result)
return six.b(u'').join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError(u'HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError(u'Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True),
location=self.opts.get('s3.location',
None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
if no_cache:
result = []
def on_chunk(chunk):
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
# Can't just do an os.rename() here, this results in a
# WindowsError being raised when the destination path exists on
# a Windows machine. Have to remove the file.
try:
os.remove(dest)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise MinionError(
'Error: Unable to remove {0}: {1}'.format(
dest,
exc.strerror
)
)
os.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True),
location=self.opts.get('s3.location',
None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
query = salt.utils.http.query(
fixed_url,
stream=True,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
response = query['handle']
chunk_size = 32 * 1024
if not no_cache:
with salt.utils.fopen(dest, 'wb') as destfp:
if hasattr(response, 'iter_content'):
for chunk in response.iter_content(chunk_size=chunk_size):
destfp.write(chunk)
else:
while True:
chunk = response.read(chunk_size)
destfp.write(chunk)
if len(chunk) < chunk_size:
break
return dest
else:
if hasattr(response, 'text'):
return response.text
else:
return response['text']
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = ':'.join((url_scheme, url_path))
url_scheme = 'file'
if url_scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.fopen(url_path, 'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == 'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.fopen(result, 'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
self.utils['s3.query'](method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP()
ftp.connect(url_data.hostname, url_data.port)
ftp.login(url_data.username, url_data.password)
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
ftp.quit()
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [None, False, None]
def on_header(hdr):
if write_body[1] is not False and write_body[2] is None:
# Try to find out what content type encoding is used if this is a text file
write_body[1].parse_line(hdr) # pylint: disable=no-member
if 'Content-Type' in write_body[1]:
content_type = write_body[1].get('Content-Type') # pylint: disable=no-member
if not content_type.startswith('text'):
write_body[1] = write_body[2] = False
else:
encoding = 'utf-8'
fields = content_type.split(';')
for field in fields:
if 'encoding' in field:
encoding = field.split('encoding=')[-1]
write_body[2] = encoding
# We have found our encoding. Stop processing headers.
write_body[1] = False
if write_body[0] is not None:
# We already parsed the first line. No need to run the code below again
return
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
write_body[1] = HTTPHeaders()
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
if write_body[2]:
chunk = chunk.decode(write_body[2])
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
if no_cache:
if write_body[2]:
return six.u('').join(result)
return six.b('').join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
