def load(self, filename):
"""
_load_
Unpickle data from file
"""
# TODO: currently support both loading from file path or url
# if there are more things to filter may be separate the load function
# urllib2 needs a scheme - assume local file if none given
if not urlparse(filename)[0]:
filename = 'file:' + filename
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = pickle.load(handle)
handle.close()
elif filename.startswith('file:'):
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = pickle.load(handle)
handle.close()
else:
# use own request class so we get authentication if needed
from WMCore.Services.Requests import Requests
request = Requests(filename)
data = request.makeRequest('', incoming_headers={"Accept": "*/*"}, decoder=False)
self.data = pickle.loads(data[0])
return
def post(url, data, timeout=None):
"""Request an URL.
Args:
url:
The web location we want to retrieve.
data:
A dict of (str, unicode) key/value pairs.
timeout:
float. If this value is specified, use it as the definitive timeout (in
seconds) for urlopen() operations. [Optional]
Notes:
If neither `timeout` nor `data['timeout']` is specified. The underlying
defaults are used.
Returns:
A JSON object.
"""
urlopen_kwargs = {}
if timeout is not None:
urlopen_kwargs['timeout'] = timeout
if InputFile.is_inputfile(data):
data = InputFile(data)
request = Request(url, data=data.to_form(), headers=data.headers)
else:
data = json.dumps(data)
request = Request(url,
data=data.encode(),
headers={'Content-Type': 'application/json'})
result = urlopen(request, **urlopen_kwargs).read()
return _parse(result)
def test_make_request(self):
url = 'https://foo.bar'
observed = issue_reporter.make_request(url)
self.assertEqual(
Request(url, headers=fakes.GITHUB_HEADERS).headers,
observed.headers)
self.assertEqual(
Request(url, headers=fakes.GITHUB_HEADERS).get_full_url(),
observed.get_full_url())
def _open_remote_file(fn):
"""
Opens a remote file, trying different locations
fn (str): the filename
return (File): the opened File-like from urllib2
raise HTTPError: in case of failure to find the file
"""
# Sadly our website has to block requests with user-agent that looks too
# much like bots and script-kiddies. That includes the default Python
# user-agent. IOW, it checks that the caller is clever enough to
# change the user-agent. So we have to show that we are "that" clever...
headers = {"User-Agent": "Mozilla/5.0 Odemis"}
for url in VIEWER_ROOT_URLS:
try:
web_url = url + fn
req = Request(web_url, headers=headers)
web_file = urlopen(req, timeout=10)
break
except HTTPError as err:
if err.getcode() == 404 and url != VIEWER_ROOT_URLS[-1]:
logging.info(
"Opening URL %s failed, will try another address",
web_url)
continue
raise
# It should now either have succeeded or raised an exception
return web_file
def send_webhook_request(url, body, user_agent=None):
if url is None:
sys.stderr.write("ERROR No URL provided\n")
return False
sys.stderr.write(
"INFO Sending POST request to url=%s with size=%d bytes payload\n" %
(url, len(body)))
sys.stderr.write("DEBUG Body: %s\n" % body)
try:
if sys.version_info >= (3, 0) and type(body) == str:
body = body.encode()
req = Request(url, body, {
"Content-Type": "application/json",
"User-Agent": user_agent
})
res = urlopen(req)
if 200 <= res.code < 300:
sys.stderr.write(
"INFO Webhook receiver responded with HTTP status=%d\n" %
res.code)
return True
else:
sys.stderr.write(
"ERROR Webhook receiver responded with HTTP status=%d\n" %
res.code)
return False
except HTTPError as e:
sys.stderr.write("ERROR Error sending webhook request: %s\n" % e)
except URLError as e:
sys.stderr.write("ERROR Error sending webhook request: %s\n" % e)
except ValueError as e:
sys.stderr.write("ERROR Invalid URL: %s\n" % e)
return False
def log_event(settings, event, source, sourcetype, host, index):
if event is None:
sys.stderr.write("ERROR No event provided\n")
return False
query = [('source', source), ('sourcetype', sourcetype), ('index', index)]
if host:
query.append(('host', host))
url = '%s/services/receivers/simple?%s' % (settings.get('server_uri'),
urlencode(query))
try:
encoded_body = unicode(event).encode('utf-8')
req = Request(
url, encoded_body,
{'Authorization': 'Splunk %s' % settings.get('session_key')})
res = urlopen(req)
if 200 <= res.code < 300:
sys.stderr.write(
"DEBUG receiver endpoint responded with HTTP status=%d\n" %
res.code)
return True
else:
sys.stderr.write(
"ERROR receiver endpoint responded with HTTP status=%d\n" %
res.code)
return False
except HTTPError as e:
sys.stderr.write("ERROR Error sending receiver request: %s\n" % e)
except URLError as e:
sys.stderr.write("ERROR Error sending receiver request: %s\n" % e)
except Exception as e:
sys.stderr.write("ERROR Error %s\n" % e)
return False
def fetch_dataset(url, sourcefile, destfile, totalsz):
"""
Download the file specified by the given URL.
Args:
url (str): Base URL of the file to be downloaded.
sourcefile (str): Name of the source file.
destfile (str): Path to the destination.
totalsz (int): Size of the file to be downloaded.
"""
req = Request('/'.join([url, sourcefile]),
headers={'User-Agent': 'neon'})
# backport https limitation and workaround per http://python-future.org/imports.html
cloudfile = urlopen(req)
neon_logger.display("Downloading file: {}".format(destfile))
blockchar = u'\u2588' # character to display in progress bar
with open(destfile, 'wb') as f:
data_read = 0
chunksz = 1024**2
while 1:
data = cloudfile.read(chunksz)
if not data:
break
data_read = min(totalsz, data_read + chunksz)
progress_string = u'Download Progress |{:<50}| '.format(
blockchar * int(float(data_read) / totalsz * 50))
sys.stdout.write('\r')
if PY3:
sys.stdout.write(progress_string)
else:
sys.stdout.write(progress_string.encode("utf-8"))
sys.stdout.flush()
f.write(data)
neon_logger.display("Download Complete")
def create_ticket(self, api_key, fields, files=None):
"""
Create ticket on osTicket server.
:arg api_key: (String) API-Key
:arg fields: (String --> String) dictionary containing keys name, email, subject, message
:arg files: (None or list of Strings) pathname of zip files that should be attached
:returns: (int) response code
:raises ValueError: ticket upload failed
:raises urllib.error.HTTPError: key not accepted
:raises urllib.error.URLError: connection problem
"""
if not files:
files = []
fields["attachments"] = []
for fn in files:
with open(fn, "rb") as f:
encoded_data = base64.b64encode(f.read())
att_desc = {
os.path.basename(fn):
"data:application/zip;base64,%s" % encoded_data
}
fields["attachments"].append(att_desc)
description = json.dumps(fields)
req = Request(OS_TICKET_URL,
description,
headers={"X-API-Key": api_key})
f = urlopen(req)
response = f.getcode()
f.close()
if response == RESPONSE_SUCCESS:
return
else:
raise ValueError('Ticket creation failed with error code %s.' %
response)
def exists(self):
req = Request("https://pypi.org/pypi/{}".format(self.name))
req.get_method = lambda: "HEAD"
try:
urlopen(req)
return True
except HTTPError as e:
if e.code == 404:
return False
raise
def _refresh(self, http_request):
refresh_time = datetime.datetime.utcnow()
metadata_root = os.environ.get(
'GCE_METADATA_ROOT', 'metadata.google.internal')
token_url = ('http://{}/computeMetadata/v1/instance/service-accounts/'
'default/token').format(metadata_root)
req = Request(token_url, headers={'Metadata-Flavor': 'Google'})
token_data = json.loads(urlopen(req, timeout=60).read().decode('utf-8'))
self.access_token = token_data['access_token']
self.token_expiry = (refresh_time +
datetime.timedelta(seconds=token_data['expires_in']))
def fetch_dataset(url, sourcefile, destfile, totalsz):
full_url = url + '/' + sourcefile
print(full_url)
req = Request(full_url, headers={'User-Agent': 'Arhat'})
cloudfile = urlopen(req)
with open(destfile, 'wb') as f:
data_read = 0
chunksz = 1024**2
while 1:
data = cloudfile.read(chunksz)
if not data:
break
data_read = min(totalsz, data_read + chunksz)
f.write(data)
def _url_query(self, api_path, authenticate=True): # NOQA
i = 0
while i <= self.retry_limit:
i += 1
try:
query_url = self._server_url + api_path
req = Request(query_url)
starttime = datetime.utcnow()
if self.wif and authenticate:
headers = storjcore.auth.create_headers(
self.btctxstore, self._get_server_address(), self.wif)
req.add_header("Date", headers["Date"])
req.add_header("Authorization", headers["Authorization"])
logger.info("Query: {0} generated in {1}".format(
query_url,
datetime.utcnow() - starttime))
response = urlopen(req, timeout=30)
if 200 <= response.code <= 299:
return response.read()
except HTTPError as e:
if e.code == 409:
raise exceptions.AddressAlreadyRegistered(
self.get_nodeid(), self._server_url)
elif e.code == 404:
raise exceptions.ServerNotFound(self._server_url +
api_path)
elif e.code == 400:
raise exceptions.InvalidAddress(self.get_nodeid())
elif e.code == 401: # auth error (likely clock off)
# log "HTTP Error 401: UNAUTHORIZED"
logger.warning(logmessages.InvalidAuthenticationHeaders())
elif e.code == 500: # pragma: no cover
raise exceptions.ServerError(self._server_url)
else:
raise e # pragma: no cover
except http.client.HTTPException as e:
logger.warning(repr(e))
except URLError as e:
logger.warning(repr(e))
except socket.error as e:
logger.warning(repr(e))
# retry
delay = self.retry_delay
logger.info("Query retry in {0} seconds.".format(delay))
time.sleep(delay)
# retry limit
logger.error("Failed to connect to {0}".format(self._server_url))
raise exceptions.ConnectionError(self._server_url)
def _build_send_req(self, url, method='GET'):
h = {
'Authorization': 'Splunk %s' % self.session_key,
'Content-Type': 'application/json'
}
try:
request = Request(url, headers=h)
request.get_method = lambda: method
response = urlopen(request)
return json.loads(response.read())
except HTTPError as e:
if e.code == http.client.NOT_FOUND:
return None
else:
raise e
def _build_req(self, method, data=None, name=None, query=None):
"""
Build request object
:param method: HTTP Method
:param data: body data
:param name: key,etc.
:param query: query params
:return: request object
"""
h = {'Authorization': 'Splunk %s' % self.session_key}
if h is not None:
h['Content-Type'] = 'application/json'
data_bytes = to_bytes(json.dumps(data))
req = Request(self._build_uri(name, query=query), data_bytes, h)
req.get_method = lambda: method
return req
def urlopen(url, auto_deflate=False):
"""
urlopen wrapper
"""
from future.moves.urllib.request import urlopen, Request
from noc.core.http.proxy import setup_urllib_proxies
setup_urllib_proxies()
if url.startswith("http://") or url.startswith("https://"):
r = Request(url, headers={"User-Agent": "NOC/%s" % version.version.strip()})
else:
r = url
if auto_deflate and url.endswith(".gz"):
u = urlopen(r)
f = six.StringIO(u.read())
return gzip.GzipFile(fileobj=f)
return urlopen(r)
def _get_feed_root(self, url, extra_get_args={}):
"""
Get an Atom feed of application information from the remote provider.
"""
try:
target_url = url
# Forward GET arguments, and add user-agent.
args_dict = {}
headers = {}
args_dict.update(self.request["query"])
if (len(extra_get_args) > 0):
args_dict.update(extra_get_args)
if self._platformInfo:
args_dict.update(self._platformInfo)
args = urlencode(args_dict)
if args != "":
target_url += ("?" + args)
logger.debug("Getting feed from: %s" % target_url)
if self._agent:
headers["User-Agent"] = self._agent
bundle_paths.BundleInstaller().validate_server_cert(
target_url, self._sslpol)
req = Request(target_url, None, headers)
f = urlopen(req, None, URLOPEN_TIMEOUT)
except HTTPError as e:
raise splunk.RESTException(e.code, e.msg)
except URLError as e:
logger.exception(e)
raise splunk.RESTException(
503,
"Splunk is unable to connect to the Internet to find more apps."
)
except Exception as e:
logger.exception(e)
raise splunk.RESTException(404, "Resource not found")
try:
root = safe_lxml.parse(f).getroot()
f.close()
return root
except Exception as e:
raise splunk.InternalServerError(e)
def _build_send_req(self, url, method='GET', data=None):
h = {
'Authorization': 'Splunk %s' % self.session_key,
'Content-Type': 'application/json'
}
try:
encoded_data = urlencode(data) if data else None
# Note: cannot use json.dumps on data because splunk savedsearch REST doesn't like it
request = Request(url,
data=utils.to_bytes(encoded_data),
headers=h)
request.get_method = lambda: method
response = urlopen(request)
return json.loads(response.read())
except HTTPError as e:
if e.code == http.client.NOT_FOUND:
return None
else:
raise SavedsearchInternalException(e)
def download_file(url):
# Create request structure
LOG.debug('Trying to download: {}'.format(url))
try:
req = Request(url)
except ValueError as e:
fatal('Client error. Could not download %r as the URL is invalid. Error: %r' % (url, str(e)))
req.add_header('User-Agent', 'ARM Update Client/%s)' % uc_version)
# Read and return
try:
r = urlopen(req)
content = r.read()
r.close()
return content
except URLError as e:
fatal('Could not download %r. Client error: %r' % (url, str(e)))
except HTTPError as e:
fatal('Could not download %r. Server error: %r' % (url, str(e)))
def create_ticket(self, api_key, fields, files=None):
"""
Create ticket on osTicket server.
:arg api_key: (String) API-Key
:arg fields: (String --> String) dictionary containing keys name, email, subject, message
:arg files: (None or list of Strings) pathname of zip files that should be attached
:returns: (int) response code
:raises ValueError: ticket upload failed
:raises urllib.error.HTTPError: key not accepted
:raises urllib.error.URLError: connection problem
"""
if not files:
files = []
fields["attachments"] = []
for fn in files:
# File is open as bytes and converted to "bytes" with base64.
# We convert it then to a string by "decoding" it from ascii.
# The string is inserted with the rest of the dict fields (all strings).
# The dict is converted to JSON, which is then encoded into bytes using UTF-8 encoding.
# We probably could avoid the bytes -> string -> bytes conversion,
# but it's easier as-is, and doesn't seem to be too costly.
with open(fn, "rb") as f:
encoded_data = base64.b64encode(f.read()).decode("ascii")
att_desc = {
os.path.basename(fn):
"data:application/zip;base64,%s" % encoded_data
}
fields["attachments"].append(att_desc)
description = json.dumps(fields).encode("utf-8")
# data must be bytes, but the headers can be str or bytes
req = Request(OS_TICKET_URL,
data=description,
headers={"X-API-Key": api_key})
f = urlopen(req)
response = f.getcode()
f.close()
if response == RESPONSE_SUCCESS:
return
else:
raise ValueError('Ticket creation failed with error code %s.' %
response)
def shorten(self, url, user_id):
if not (url.startswith('http://') or url.startswith('https://')):
url = 'http://' + url
try:
url2 = self.shorten_template.format(token=str(self.token),
user_ids=str(user_id),
url=str(url))
request = Request(url2,
headers={'Content-Type': 'application/json'})
with urlopen(request) as response:
shortened_url = response.read()
return shortened_url.decode('utf-8')
except HTTPError as error:
self.logger.warn('Botan track error ' + str(error.code) + ':' +
error.read().decode(
'utf-8'))
return None
except URLError as error:
self.logger.warn('Botan track error ' + str(error.reason))
return None
def _generate_cloudwatch_input_request(self, method, data=None, name=None):
base_url = '%s/servicesNS/nobody/Splunk_TA_aws/splunk_ta_aws_aws_cloudwatch/%s?%s'
headers = {
'Authorization': 'Splunk %s' % self.session_key,
'Content-Type': 'application/json'
}
# Handle the query params that are passed in
server_uri = em_common.get_server_uri()
query_params = dict(output_mode='json')
query_params['count'] = 1
query_params['offset'] = 0
# Build the URL and make the request
url = base_url % (server_uri, name or '', urlencode(query_params))
request = Request(url,
to_bytes(urlencode(data)) if data else None,
headers=headers)
request.get_method = lambda: method
return request
def track(self, message, event_name='event'):
try:
uid = message.chat_id
except AttributeError:
self.logger.warn('No chat_id in message')
return False
data = message.to_json()
try:
url = self.url_template.format(
token=str(self.token), uid=str(uid), name=quote(event_name))
request = Request(
url, data=data.encode(), headers={'Content-Type': 'application/json'})
urlopen(request)
return True
except HTTPError as error:
self.logger.warn('Botan track error ' + str(error.code) + ':' + error.read().decode(
'utf-8'))
return False
except URLError as error:
self.logger.warn('Botan track error ' + str(error.reason))
return False
def send_incident(self, incident):
"""
send incident to VictorOps
:param incident: a dict that contains key/value info of the incident
"""
try:
url = "%s/%s/%s" % (INTEGRATION_URL, self.api_key, self.routing_key)
headers = {
'content-type': 'application/json',
}
incident.update({
'monitoring_tool': self.monitoring_tool
})
req = Request(url, utils.to_bytes(json.dumps(incident)), headers)
response = urlopen(req)
message, status_code = response.msg, response.code
if status_code != 200:
raise VictorOpsCouldNotSendAlertException("status_code=%s, message=%s" % (status_code, message))
except Exception as e:
logger.error("Failed to send incident to VictorOps because: %s", e)
raise VictorOpsCouldNotSendAlertException(str(e))
def download(name, url, size, target, step=None):
with tqdm(
bar_format="{l_bar}{bar}| [{remaining}, {rate_fmt}]",
desc="{:<18}".format(name),
leave=False,
total=size,
unit="B",
unit_scale=True,
) as bar:
with target.open("wb", ensure=True) as fh:
url_request = Request(url)
with contextlib.closing(urlopen(url_request)) as socket:
while True:
block = socket.read(4096)
if not block:
break
fh.write(block)
bar.update(len(block))
if step:
ProgressOverall.done_part(step, bar.n / size)
if bar.n != size:
raise RuntimeError(
"Error downloading %s: received %d bytes, expected %d" %
(url, bar.n, size))
def _build_send_req(self, method, stanza_name=None, data=None, query=None):
"""
Build request object
:param method: HTTP Method
:param data: body data
:param stanza_name: stanza in conf file to write to
:param query: query params
:return: response object
"""
headers = {
'Authorization': 'Splunk %s' % self.session_key,
'Content-Type': 'application/json'
}
'''
The REALLY annoying thing about CONF endpoints is that, unlike KV store endpoints,
the format in which they take data arguments is the application/x-www-form-urlencoded.
This means, if you have {a: 'b', c: 'd'}, you would just stringify this for KV store calls
but for CONF endpoints, you need to send it like &a=b&c=d.
:param params: dict of data we want to convert to this other format
:return: a formatted string like a=b&c=d&enough=nonsense
'''
try:
data_string = to_bytes(urlencode(data)) if data else None
req = Request(url=self._build_uri(stanza_name, query=query),
data=data_string,
headers=headers)
req.get_method = lambda: method
res = urlopen(req)
return json.loads(res.read())
except HTTPError as e:
if e.code == http.client.NOT_FOUND:
return None
else:
raise ConfManagerInternalException(e)
def is_valid_response(self, req_args={}, **kwargs):
"""
Submits a reCAPTCHA request for verification and returns its
status.
INPUT:
- ``req_args`` - a dictionary; the arguments of the responding
user's HTTP POST request
- ``kwargs`` - a dictionary of extra keyword arguments
OUTPUT:
- a :class:`ChallengeResponse` instance; whether the user's
response is empty, accepted, or rejected, with an optional
error string
TESTS::
sage: from sagenb.notebook.challenge import reCAPTCHAChallenge
sage: tmp = tmp_dir()
sage: import sagenb.gui.notebook as n
sage: nb = n.Notebook(tmp)
sage: chal = reCAPTCHAChallenge(nb.conf, remote_ip = 'localhost')
sage: req = {}
sage: chal.is_valid_response(req).is_valid
sage: chal.is_valid_response(req).error_code
''
sage: req['recaptcha_response_field'] = ['subplotTimes']
sage: chal.is_valid_response(req).is_valid
False
sage: chal.is_valid_response(req).error_code
'incorrect-captcha-sol'
sage: req['simple_challenge_field'] = ['VBORw0KGgoANSUhEUgAAAB']
sage: chal.is_valid_response(req).is_valid # random
False
sage: chal.is_valid_response(req).error_code # random
'incorrect-captcha-sol'
"""
response_field = req_args.get('recaptcha_response_field', [None])[0]
if not (response_field and len(response_field)):
return ChallengeResponse(None, '')
challenge_field = req_args.get('recaptcha_challenge_field', [None])[0]
if not (challenge_field and len(challenge_field)):
return ChallengeResponse(False, 'incorrect-captcha-sol')
def encode_if_necessary(s):
if isinstance(s, str):
return s.encode('utf-8')
return s
params = urlencode({
'privatekey': encode_if_necessary(self.private_key),
'remoteip': encode_if_necessary(self.remote_ip),
'challenge': encode_if_necessary(challenge_field),
'response': encode_if_necessary(response_field)
})
request = Request(
url="http://%s/verify" % RECAPTCHA_VERIFY_SERVER,
data=params,
headers={
"Content-type": "application/x-www-form-urlencoded",
"User-agent": "reCAPTCHA Python"
}
)
httpresp = urlopen(request)
return_values = httpresp.read().splitlines()
httpresp.close()
return_code = return_values[0]
if (return_code == "true"):
return ChallengeResponse(True)
else:
return ChallengeResponse(False, return_values[1])
def send(self, data):
request = Request(self.endpoint + '?' +
urlencode(self.fixUTF8(data)).encode('utf-8'),
headers={'User-Agent': self.user_agent})
self.open(request)
def handle_POST(self):
"""
Install a remote application in response to an HTTP POST.
"""
self.verifyAllowRemote()
parts = len(self.pathParts)
if parts == self.BASE_DEPTH + 2:
default_version = True
elif parts == self.BASE_DEPTH + 3:
default_version = False
else:
raise splunk.BadRequest
if HTTP_AUTH_TOKEN not in self.args:
raise splunk.BadRequest("Missing argument: %s" % HTTP_AUTH_TOKEN)
if HTTP_ACTION not in self.args:
raise splunk.BadRequest("Missing argument: %s" % HTTP_ACTION)
if self.args[HTTP_ACTION] not in (HTTP_ACTION_INSTALL,
HTTP_ACTION_DOWNLOAD):
raise splunk.BadRequest("Invalid value '%s' for argument '%s'" %
(self.args[HTTP_ACTION], HTTP_ACTION))
# check if this is a cloud stack
if isCloud(self.sessionKey):
app_name = self.pathParts[self.BASE_DEPTH + 1]
# Get all cloud apps and see if the app being installed is vetted for cloud
# i.e install_method == simple
# TODO: Change to just querying for the app in question when BASE-4074
# is finished.
getargs = {'offset': 0, 'limit': 100}
vetted_apps = []
while 1:
serverResponse, serverContent = splunk.rest.simpleRequest(
VETTED_APPS_URI, self.sessionKey, getargs)
if serverResponse.status != 200:
raise splunk.BadRequest(
'Error while querying Splunkbase. Splunkd returned %s'
% serverContent)
vetted_app_data = json.loads(serverContent)
if not vetted_app_data['results']:
break
else:
getargs['offset'] += 100
vetted_apps.extend(vetted_app_data['results'])
for app in vetted_apps:
if app['appid'] == app_name and app[
'install_method'] == VETTED_APP_INSTALL_METHOD:
break
else:
raise splunk.BadRequest(
'App %s is not vetted for Splunk Cloud.' % app_name)
url = self._native_to_foreign_url()
root = self._get_feed_root(url)
if default_version:
root = self._get_latest_version(root)
href = self._parse_link(root)
try:
# Package up a Request with auth information.
req = Request(href)
# XXX: Converting the auth token from a POST arg to a header
# requires us to unquote() it. If the client did not correctly
# quote() the token, login will fail.
req.add_header(HTTP_AUTH_HEADER,
unquote(self.args[HTTP_AUTH_TOKEN]))
# Install using this Request object.
installer = bundle_paths.BundleInstaller()
if self.args[HTTP_ACTION] == HTTP_ACTION_INSTALL:
b, status = installer.install_from_url(req,
sslpol=self._sslpol)
self.response.setStatus(status)
if ((status == bundle_paths.BundleInstaller.STATUS_INSTALLED)
or
(status == bundle_paths.BundleInstaller.STATUS_UPGRADED)):
# Migrate old-style bundles.
logger.debug("Configuring application contents")
try:
b.migrate()
except Exception as e:
logger.exception(e)
self.addMessage("WARN",
"Error during configuration: %s" % e)
# Redirect to local application.
self.response.setHeader("Location",
self._redirect_to_local(b))
# Let splunkd know about newly-installed app.
logger.debug(
"Notifying splunkd that app has been installed")
splunk.rest.simpleRequest('apps/local/_reload',
sessionKey=self.sessionKey)
if status == bundle_paths.BundleInstaller.STATUS_INSTALLED:
self.addMessage("INFO",
"Installed application: %s" % b.name())
elif status == bundle_paths.BundleInstaller.STATUS_UPGRADED:
self.addMessage("INFO",
"Upgraded application: %s" % b.name())
else:
self.addMessage(
"WARN", "Could not install application: %s" % b.name())
else:
assert self.args[HTTP_ACTION] == HTTP_ACTION_DOWNLOAD
downloaded = installer.download_from_url(req,
sslpol=self._sslpol)
self.addMessage("INFO",
"Downloaded application file: %s" % downloaded)
self.response.setHeader('content-type', 'application/json')
response_json = {"downloaded": downloaded}
self.response.write(json.dumps(response_json))
except splunk.ResourceNotFound:
raise
except splunk.AuthorizationFailed:
raise
except splunk.InternalServerError:
raise
except Exception as e:
logger.exception(e)
raise splunk.InternalServerError(e)
def query_server(url, data=None, cert_file=None):
"""
@param str url: URL of server to connect to
@param dict data: optional data to include in the request
@param str cert_file: optional path to PEM certificate file which will be used to validate the identity of the server
Allows for connecting to a server with a provided URL and returns a response dictionary.
"""
# Check if data is not given
if data is None:
data = {}
# Make the url request
headers = {'Content-Type': 'application/json'}
# If no data is provided urllib2 will use GET method. If data is provided urllib2 will use POST method.
if data is not None and len(data) > 0:
request_data = ensure_binary(json.dumps(data))
else:
request_data = None
req = Request(url, request_data, headers)
try:
# Enforce Python version number when urllib2 added support for cafile parameter
if sys.version_info < (2, 7, 9):
if cert_file:
raise RuntimeError(
"Python version 2.7.9 or later is required to establish HTTPS connection with server."
)
response = urlopen(req)
else:
try:
response = urlopen(req, cafile=cert_file)
except TypeError:
# Unsupported cafile param was passed to urlopen so retry without cafile param.
# It is unknown how this condition can arise as the cafile param was added in python 2.7.9 which
# we check above, however, the following issue has been reported:
# TypeError: urlopen() got an unexpected keyword argument 'cafile'
logger.warning(
"Host authentication is not supported. "
"Falling back to connection without host authentication."
)
response = urlopen(req)
except ValueError:
logger.warning(
"Host authentication is not supported because Python is missing SSL. "
"Falling back to connection without host authentication."
)
response = urlopen(req)
except HTTPError as e:
raise RuntimeError('The server could not fulfill the request.'
'\n'
'Error code: ' + str(e.code) + '.'
'\n'
'Please check the followings: \n \
- URL is correct \n \
- server is running \n \
- there is no firewall preventing connections to server'
)
except URLError as e:
raise RuntimeError('Failed to reach the server with URL: ' + "\"" +
url + "\""
'\n'
'Reason: ' + str(e.reason) + '.'
'\n'
'Please check the followings: \n \
- URL is correct \n \
- server is running \n \
- there is no firewall preventing connections to server'
)
return_data = json.loads(response.read())
return return_data
def make_request(url):
"""Make our JSON request to GitHub"""
return Request(url, headers={
"Authorization": "token %s" % GITHUB_API_TOKEN,
"Content-Type": "application/json",
})
