def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, url_data.netloc,
url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(None, fixed_url, url_data.username,
url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
env,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
env, url_data.netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(None, fixed_url, url_data.username,
url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path))
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(
method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url', None),
verify_ssl=self.opts.get('s3.verify_ssl', True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(
self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
get_kwargs['prefetch'] = False
else:
get_kwargs['stream'] = True
response = requests.get(fixed_url, **get_kwargs)
response.raise_for_status()
with salt.utils.fopen(dest, 'wb') as destfp:
for chunk in response.iter_content(chunk_size=32 * 1024):
destfp.write(chunk)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_data.path):
raise CommandExecutionError(
'Path {0!r} is not absolute'.format(url_data.path)
)
return url_data.path
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
if salt.utils.is_windows():
netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
else:
netloc = url_data.netloc
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None),
verify_ssl=self.opts.get('s3.verify_ssl',
True))
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.scheme == 'swift':
try:
swift_conn = SaltSwift(self.opts.get('keystone.user', None),
self.opts.get('keystone.tenant', None),
self.opts.get('keystone.auth_url', None),
self.opts.get('keystone.password', None))
swift_conn.get_object(url_data.netloc,
url_data.path[1:],
dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
try:
if requests.__version__[0] == '0':
# 'stream' was called 'prefetch' before 1.0, with flipped meaning
get_kwargs['prefetch'] = False
else:
get_kwargs['stream'] = True
response = requests.get(fixed_url, **get_kwargs)
response.raise_for_status()
with salt.utils.fopen(dest, 'wb') as destfp:
for chunk in response.iter_content(chunk_size=32*1024):
destfp.write(chunk)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
req = requests.get(fixed_url)
with salt.utils.fopen(dest, 'wb') as destfp:
destfp.write(req.content)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
