def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
query = salt.utils.http.query(fixed_url,
text=True,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'text' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return query['body']
else:
dest_tmp = "{0}.part".format(dest)
with salt.utils.fopen(dest_tmp, 'wb') as destfp:
destfp.write(query['body'])
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Carbon',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Carbon.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [False]
def on_header(hdr):
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
get_kwargs['prefetch'] = False
else:
get_kwargs['stream'] = True
response = requests.get(fixed_url, **get_kwargs)
with salt.utils.fopen(dest, 'wb') as destfp:
for chunk in response.iter_content(chunk_size=32 * 1024):
destfp.write(chunk)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = ':'.join((url_scheme, url_path))
url_scheme = 'file'
if url_scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.fopen(url_path, 'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == 'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.fopen(result, 'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
self.utils['s3.query'](method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP()
ftp.connect(url_data.hostname, url_data.port)
ftp.login(url_data.username, url_data.password)
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
ftp.quit()
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [None, False, None]
def on_header(hdr):
if write_body[1] is not False and write_body[2] is None:
# Try to find out what content type encoding is used if this is a text file
write_body[1].parse_line(hdr) # pylint: disable=no-member
if 'Content-Type' in write_body[1]:
content_type = write_body[1].get('Content-Type') # pylint: disable=no-member
if not content_type.startswith('text'):
write_body[1] = write_body[2] = False
else:
encoding = 'utf-8'
fields = content_type.split(';')
for field in fields:
if 'encoding' in field:
encoding = field.split('encoding=')[-1]
write_body[2] = encoding
# We have found our encoding. Stop processing headers.
write_body[1] = False
if write_body[0] is not None:
# We already parsed the first line. No need to run the code below again
return
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
write_body[1] = HTTPHeaders()
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
if write_body[2]:
chunk = chunk.decode(write_body[2])
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
if no_cache:
if write_body[2]:
return six.u('').join(result)
return six.b('').join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True),
location=self.opts.get('s3.location',
None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
query = salt.utils.http.query(
fixed_url,
stream=True,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
response = query['handle']
chunk_size = 32 * 1024
if not no_cache:
with salt.utils.fopen(dest, 'wb') as destfp:
if hasattr(response, 'iter_content'):
for chunk in response.iter_content(chunk_size=chunk_size):
destfp.write(chunk)
else:
while True:
chunk = response.read(chunk_size)
destfp.write(chunk)
if len(chunk) < chunk_size:
break
return dest
else:
if hasattr(response, 'text'):
return response.text
else:
return response['text']
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
query = salt.utils.http.query(
fixed_url,
stream=True,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
response = query['handle']
chunk_size = 32 * 1024
if not no_cache:
with salt.utils.fopen(dest, 'wb') as destfp:
if hasattr(response, 'iter_content'):
for chunk in response.iter_content(chunk_size=chunk_size):
destfp.write(chunk)
else:
while True:
chunk = response.buffer.read(chunk_size)
destfp.write(chunk)
if len(chunk) < chunk_size:
break
return dest
else:
if hasattr(response, 'text'):
return response.text
else:
return response['text']
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = ':'.join((url_scheme, url_path))
url_scheme = 'file'
if url_scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.fopen(url_data.path, 'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == 'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.fopen(result, 'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
self.utils['s3.query'](method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM avail).
# Here we working this around by disabling recording the body for redirections.
# The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
# when we'll deprecate Tornado<4.3.0.
# See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback. Simple bool doesn't
# work here: on_header creates a new local variable instead. This could be avoided in
# Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
write_body = [False]
def on_header(hdr):
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True),
location=self.opts.get('s3.location',
None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
if no_cache:
result = []
def on_chunk(chunk):
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return ''.join(result)
else:
destfp.close()
destfp = None
# Can't just do an os.rename() here, this results in a
# WindowsError being raised when the destination path exists on
# a Windows machine. Have to remove the file.
try:
os.remove(dest)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise MinionError(
'Error: Unable to remove {0}: {1}'.format(
dest,
exc.strerror
)
)
os.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv=u'base',
no_cache=False, cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(
url_data.netloc, url_data.path).rstrip(os.sep)
# If dest is a directory, rewrite dest with filename
if dest is not None \
and (os.path.isdir(dest) or dest.endswith((u'/', u'\\'))):
if url_data.query or len(url_data.path) > 1 and not url_data.path.endswith(u'/'):
strpath = url.split(u'/')[-1]
else:
strpath = u'index.html'
if salt.utils.platform.is_windows():
strpath = salt.utils.sanitize_win_path_string(strpath)
dest = os.path.join(dest, strpath)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = u':'.join((url_scheme, url_path))
url_scheme = u'file'
if url_scheme in (u'file', u''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
u'Path \'{0}\' is not absolute'.format(url_path)
)
if dest is None:
with salt.utils.files.fopen(url_path, u'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == u'salt':
result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
if result and dest is None:
with salt.utils.files.fopen(result, u'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return u''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == u's3':
try:
def s3_opt(key, default=None):
u'''Get value of s3.<key> from Minion config or from Pillar'''
if u's3.' + key in self.opts:
return self.opts[u's3.' + key]
try:
return self.opts[u'pillar'][u's3'][key]
except (KeyError, TypeError):
return default
self.utils[u's3.query'](method=u'GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt(u'key'),
keyid=s3_opt(u'keyid'),
service_url=s3_opt(u'service_url'),
verify_ssl=s3_opt(u'verify_ssl', True),
location=s3_opt(u'location'),
path_style=s3_opt(u'path_style', False),
https_enable=s3_opt(u'https_enable', True))
return dest
except Exception as exc:
raise MinionError(
u'Could not fetch from {0}. Exception: {1}'.format(url, exc)
)
if url_data.scheme == u'ftp':
try:
ftp = ftplib.FTP()
ftp.connect(url_data.hostname, url_data.port)
ftp.login(url_data.username, url_data.password)
with salt.utils.files.fopen(dest, u'wb') as fp_:
ftp.retrbinary(u'RETR {0}'.format(url_data.path), fp_.write)
ftp.quit()
return dest
except Exception as exc:
raise MinionError(u'Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == u'swift':
try:
def swift_opt(key, default):
'''
Get value of <key> from Minion config or from Pillar
'''
if key in self.opts:
return self.opts[key]
try:
return self.opts[u'pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt(u'keystone.user', None),
swift_opt(u'keystone.tenant', None),
swift_opt(u'keystone.auth_url', None),
swift_opt(u'keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError(u'Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in (u'http', u'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind(u'@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs[u'auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM
# avail). Here we are working around this by disabling recording
# the body for redirections. The issue is fixed in Tornado 4.3.0
# so on_header callback could be removed when we'll deprecate
# Tornado<4.3.0. See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback.
# Simple bool doesn't work here: on_header creates a new local
# variable instead. This could be avoided in Py3 with 'nonlocal'
# statement. There is no Py2 alternative for this.
#
# write_body[0] is used by the on_chunk callback to tell it whether
# or not we need to write the body of the request to disk. For
# 30x redirects we set this to False because we don't want to
# write the contents to disk, as we will need to wait until we
# get to the redirected URL.
#
# write_body[1] will contain a tornado.httputil.HTTPHeaders
# instance that we will use to parse each header line. We
# initialize this to False, and after we parse the status line we
# will replace it with the HTTPHeaders instance. If/when we have
# found the encoding used in the request, we set this value to
# False to signify that we are done parsing.
#
# write_body[2] is where the encoding will be stored
write_body = [None, False, None]
def on_header(hdr):
if write_body[1] is not False and write_body[2] is None:
if not hdr.strip() and 'Content-Type' not in write_body[1]:
# We've reached the end of the headers and not yet
# found the Content-Type. Reset the values we're
# tracking so that we properly follow the redirect.
write_body[0] = None
write_body[1] = False
return
# Try to find out what content type encoding is used if
# this is a text file
write_body[1].parse_line(hdr) # pylint: disable=no-member
if u'Content-Type' in write_body[1]:
content_type = write_body[1].get(u'Content-Type') # pylint: disable=no-member
if not content_type.startswith(u'text'):
write_body[1] = write_body[2] = False
else:
encoding = u'utf-8'
fields = content_type.split(u';')
for field in fields:
if u'encoding' in field:
encoding = field.split(u'encoding=')[-1]
write_body[2] = encoding
# We have found our encoding. Stop processing headers.
write_body[1] = False
# If write_body[0] is False, this means that this
# header is a 30x redirect, so we need to reset
# write_body[0] to None so that we parse the HTTP
# status code from the redirect target.
if write_body[0] is write_body[1] is False:
write_body[0] = None
# Check the status line of the HTTP request
if write_body[0] is None:
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
write_body[1] = HTTPHeaders()
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
if write_body[2]:
chunk = chunk.decode(write_body[2])
result.append(chunk)
else:
dest_tmp = u"{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.files.fopen(dest_tmp, u'wb') # pylint: disable=resource-leakage
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(
fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs
)
if u'handle' not in query:
raise MinionError(u'Error: {0} reading {1}'.format(query[u'error'], url))
if no_cache:
if write_body[2]:
return u''.join(result)
return six.b(u'').join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError(u'HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError(u'Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'))
return dest
except Exception as exc:
raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP(url_data.hostname)
ftp.login()
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
return dest
except Exception as exc:
raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
query = salt.utils.http.query(
fixed_url,
text=True,
username=url_data.username,
password=url_data.password,
**get_kwargs
)
if 'text' not in query:
raise MinionError('Error: {0}'.format(query['error']))
if no_cache:
return query['body']
else:
dest_tmp = "{0}.part".format(dest)
with salt.utils.fopen(dest_tmp, 'wb') as destfp:
destfp.write(query['body'])
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
env=None,
no_cache=False):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True),
location=self.opts.get('s3.location', None))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
if no_cache:
result = []
def on_chunk(chunk):
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
destfp = salt.utils.fopen(dest_tmp, 'wb')
def on_chunk(chunk):
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
username=url_data.username,
password=url_data.password,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0}'.format(query['error']))
try:
content_length = int(query['handle'].headers['Content-Length'])
except (AttributeError, KeyError, ValueError):
# Shouldn't happen but don't let this raise an exception.
# Instead, just don't do content length checking below.
log.warning(
'No Content-Length header in HTTP response from fetch of '
'{0}, or Content-Length is non-numeric'.format(fixed_url))
content_length = None
if no_cache:
content = ''.join(result)
if content_length is not None \
and len(content) > content_length:
return content[-content_length:]
else:
return content
else:
destfp.close()
destfp = None
dest_tmp_size = os.path.getsize(dest_tmp)
if content_length is not None \
and dest_tmp_size > content_length:
log.warning(
'Size of file downloaded from {0} ({1}) does not '
'match the Content-Length ({2}). This is probably due '
'to an upstream bug in tornado '
'(https://github.com/tornadoweb/tornado/issues/1518). '
'Re-writing the file to correct this.'.format(
fixed_url, dest_tmp_size, content_length))
dest_tmp_bak = dest_tmp + '.bak'
salt.utils.files.rename(dest_tmp, dest_tmp_bak)
with salt.utils.fopen(dest_tmp_bak, 'rb') as fp_bak:
fp_bak.seek(dest_tmp_size - content_length)
with salt.utils.fopen(dest_tmp, 'wb') as fp_new:
while True:
chunk = fp_bak.read(
self.opts['file_buffer_size'])
if not chunk:
break
fp_new.write(chunk)
os.remove(dest_tmp_bak)
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()