def _http_request(url, request_timeout=None):
'''
PRIVATE METHOD
Uses json.load to fetch the json results from the solr api.
url : str
a complete url that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None
to use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
'''
try:
request_timeout = __salt__['config.option']('solr.request_timeout')
if request_timeout is None:
data = json.load(url_open(url))
else:
data = json.load(url_open(url, timeout=request_timeout))
return _get_return_dict(True, data, [])
except Exception as e:
return _get_return_dict(False, {}, ["{0} : {1}".format(url, e)])
def _http_request(url, request_timeout=None):
'''
PRIVATE METHOD
Uses json.load to fetch the JSON results from the solr API.
url : str
a complete URL that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None
to use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
'''
_auth(url)
try:
request_timeout = __salt__['config.option']('solr.request_timeout')
if request_timeout is None:
data = json.load(url_open(url))
else:
data = json.load(url_open(url, timeout=request_timeout))
return _get_return_dict(True, data, [])
except Exception as err:
return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)])
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
env, url_data.netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
try:
with contextlib.closing(url_open(url)) as srcfp:
with open(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
return ''
def get_url(self, url, dest, makedirs=False, env="base"):
"""
Get a single file from a URL.
"""
url_data = urlparse(url)
if url_data.scheme == "salt":
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ""
else:
dest = salt.utils.path_join(self.opts["cachedir"], "extrn_files", env, url_data.netloc, url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
try:
with contextlib.closing(url_open(url)) as srcfp:
with salt.utils.fopen(dest, "wb") as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError(
"HTTP error {0} reading {1}: {3}".format(
ex.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]
)
)
except URLError as ex:
raise MinionError("Error reading {0}: {1}".format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.')
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
saltenv, url_data.netloc,
url_data.path)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(None, fixed_url, url_data.username,
url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
env,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
def get_url(self, url, dest, makedirs=False, env='base'):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, env)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = os.path.normpath(
os.sep.join([
self.opts['cachedir'],
'extrn_files',
env,
url_data.netloc,
url_data.path]))
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
try:
with contextlib.closing(url_open(url)) as srcfp:
with open(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
return ''
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
'''
Get a single file from a URL.
'''
if env is not None:
salt.utils.warn_until(
'Boron',
'Passing a salt environment should be done using \'saltenv\' '
'not \'env\'. This functionality will be removed in Salt '
'Boron.'
)
# Backwards compatibility
saltenv = env
url_data = urlparse(url)
if url_data.scheme == 'salt':
return self.get_file(url, dest, makedirs, saltenv)
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
else:
dest = salt.utils.path_join(
self.opts['cachedir'],
'extrn_files',
saltenv,
url_data.netloc,
url_data.path
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path,
url_data.params, url_data.query, url_data.fragment))
passwd_mgr = url_passwd_mgr()
passwd_mgr.add_password(
None, fixed_url, url_data.username, url_data.password)
auth_handler = url_auth_handler(passwd_mgr)
opener = url_build_opener(auth_handler)
url_install_opener(opener)
else:
fixed_url = url
try:
with contextlib.closing(url_open(fixed_url)) as srcfp:
with salt.utils.fopen(dest, 'wb') as destfp:
shutil.copyfileobj(srcfp, destfp)
return dest
except HTTPError as ex:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
ex.code,
url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
except URLError as ex:
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
