def _build_request(self, query):
if self.method == "GET":
# if '?' in self.endpoint:
# separator = '&'
# else:
# separator = '?'
uri = self.endpoint.strip() + '?' + query
return request.Request(uri.encode('ASCII'))
else:
uri = self.endpoint.strip().encode('ASCII')
return request.Request(uri, data=query)
def scout_server_type(self, host):
"""
Obtain Server header by calling OPTIONS.
:param host: host to check
:returns: Server type, status
"""
try:
url = "http://%s:%s/" % (host[0], host[1])
req = urllib2.Request(url)
req.get_method = lambda: 'OPTIONS'
conn = urllib2.urlopen(req)
header = conn.info().getheader('Server')
server_header = header.split('/')
content = server_header[0]
status = 200
except urllib2.HTTPError as err:
if not self.suppress_errors or self.verbose:
print("-> %s: %s" % (url, err))
content = err
status = err.code
except (urllib2.URLError, socket.timeout) as err:
if not self.suppress_errors or self.verbose:
print("-> %s: %s" % (url, err))
content = err
status = -1
return url, content, status
def generate_request(self, verb, url, headers, post_data=None):
try:
# import pdb; pdb.set_trace()
req = request.Request(url,
data=post_data.encode("utf-8")
if post_data is not None else None,
headers=headers,
method=verb)
if self.proxy:
req.set_proxy(self.proxy_config,
urllib.parse.urlparse(url).scheme)
response = request.urlopen(req,
timeout=60,
context=self.create_ctx())
else:
response = request.urlopen(req,
timeout=60,
context=self.create_ctx())
self.status_code.append(int(response.code))
except error.HTTPError as e:
self.status_code.append(int(e.code))
except error.URLError as e:
self.sns_logger(status_codes={},
exception=str(e.reason),
subject="Grizzly Error")
except Exception:
import traceback
self.sns_logger(status_codes={},
exception=str(traceback.format_exc()),
subject="Grizzly Error")
print(('generic exception: ' + traceback.format_exc()))
def __call__(self, req):
obj = None
try:
(version, account, container, obj) = \
split_path(req.path_info, 4, 4, True)
except ValueError:
# not an object request
pass
if obj and req.method == 'PUT':
# create a POST request with obj name as body
payload = {
"conf": {
"swift_id": obj,
"swift_container": container,
"swift_user": account,
"swift_version": version
}
}
webhook_req = urllib2.Request(URL + ENDPOINT_PATH + "/dags/" +
DAG_TO_TRIGGER + "/dag_runs",
data=payload)
with Timeout(20):
try:
urllib2.urlopen(webhook_req).read()
except (Exception, Timeout):
self.logger.exception('failed POST to webhook %s' %
webhook)
else:
self.logger.info('successfully called webhook %s' %
webhook)
return self.app
def __call__(self, env, start_response):
req = Request(env)
resp = req.get_response(self.app)
self.logger.info("Serverless: available headers: {}".format(
str(dict(req.headers))))
try:
if "X-Function-URL" in req.headers:
version, account, container, obj = split_path(
req.path_info, 4, 4, True)
self.logger.info(
"Serverless: version {}, account {}, container {}, object {}"
.format(version, account, container, obj))
if obj and is_success(resp.status_int) and req.method == 'PUT':
webhook = req.headers.get("X-Function-URL")
data = json.dumps({
"x-auth-token":
req.headers.get("X-Auth-Token"),
"version":
version,
"account":
account,
"container":
container,
"object":
obj,
"project_id":
req.headers.get("X-Project-Id"),
})
self.logger.info(
"Serverless: data to send to a function {}".format(
str(data)))
data_as_bytes = data.encode('utf-8')
webhook_req = urllib2.Request(webhook, data=data_as_bytes)
webhook_req.add_header('Content-Type', 'application/json')
webhook_req.add_header('Content-Length',
len(data_as_bytes))
self.logger.info(
"Serverless: data to send as bytes {}".format(
data_as_bytes))
with Timeout(60):
try:
result = urllib2.urlopen(webhook_req).read()
self.logger.info(
"Serverless: function worked fine. Result {}".
format(str(result)))
except (Exception, Timeout) as ex:
self.logger.error(
'Serverless: failed POST to webhook {}, '
'error {}'.format(webhook, str(ex)))
else:
self.logger.info("Serverless: skipping functions middleware "
"due to absence of function URL")
except ValueError:
# not an object request
pass
return self.app(env, start_response)
def get_auth(url, user, key, auth_version='1.0', **kwargs):
if auth_version != '1.0':
exit('ERROR: swiftclient missing, only auth v1.0 supported')
req = urllib2.Request(url)
req.add_header('X-Auth-User', user)
req.add_header('X-Auth-Key', key)
conn = urllib2.urlopen(req)
headers = conn.info()
return (headers.getheader('X-Storage-Url'),
headers.getheader('X-Auth-Token'))
def _base_method(url, data=None, headers=None,
origin_req_host=None, unverifiable=False,
method=None, decode='utf-8'):
def _caller(stack_number):
try:
info = inspect.stack()[stack_number]
caller_func = info[3]
caller_inst = info[0].f_locals.get('self')
if caller_inst:
return str(caller_inst) + ' ' + caller_func
else:
return str(caller_func)
except:
return str('can not find caller')
if headers is None:
headers = {}
req = request.Request(
url=url, data=data, headers=headers,
origin_req_host=origin_req_host, unverifiable=unverifiable,
method=method)
rval = None
try:
with hub.Timeout(_DEFAULT_TIME_OUT, True):
response = request.urlopen(req)
message = response.read()
msg_dec = message.decode(decode)
rval = json.loads(msg_dec)
except hub.Timeout:
LOG.error({'method': _caller(2),
'caller': _caller(3),
'url': url,
'timeout': str(_DEFAULT_TIME_OUT)})
except error.HTTPError as e:
LOG.error({'method': _caller(2),
'caller': _caller(3),
'url': url,
'error': {
'code': e.code,
'msg': e.msg}})
except:
LOG.error({'method': _caller(2),
'caller': _caller(3),
'url': url,
'error': 'uncovered error'})
return rval
def sanity_check(self, client, computed_requests):
'''
This method checks that the sanity_check_url provides a 200 status code.
If the sanity check fails, the application exists.
'''
req = request.Request(client,
headers=self.computed_requests["headers"][0])
response = request.urlopen(req, timeout=60, context=self.create_ctx())
if response.code != 200:
self.sns_logger(status_codes={},
exception=str(response.code),
subject="Grizzly Sanity Check Failed",
url=client)
raise
else:
self.sns_logger(status_codes={},
exception=str(response.code),
subject="Grizzly Sanity Check Passed",
url=client)
print('Sanity check passed: 200 OK')
return True
def base_request(self,
method,
container=None,
name=None,
prefix=None,
headers=None,
proxy=None,
contents=None,
full_listing=None,
logger=None,
additional_info=None,
timeout=None,
marker=None):
# Common request method
trans_start = time()
url = self.url
if full_listing:
info, body_data = self.base_request(method,
container,
name,
prefix,
headers,
proxy,
timeout=timeout,
marker=marker)
listing = body_data
while listing:
marker = listing[-1]['name']
info, listing = self.base_request(method,
container,
name,
prefix,
headers,
proxy,
timeout=timeout,
marker=marker)
if listing:
body_data.extend(listing)
return [info, body_data]
if headers is None:
headers = {}
if self.token:
headers['X-Auth-Token'] = self.token
if container:
url = '%s/%s' % (url.rstrip('/'), quote(container))
if name:
url = '%s/%s' % (url.rstrip('/'), quote(name))
else:
params = ['format=json']
if prefix:
params.append('prefix=%s' % prefix)
if marker:
params.append('marker=%s' % quote(marker))
url += '?' + '&'.join(params)
req = urllib2.Request(url, headers=headers, data=contents)
if proxy:
proxy = urllib.parse.urlparse(proxy)
req.set_proxy(proxy.netloc, proxy.scheme)
req.get_method = lambda: method
conn = urllib2.urlopen(req, timeout=timeout)
body = conn.read()
info = conn.info()
try:
body_data = json.loads(body)
except ValueError:
body_data = None
trans_stop = time()
if logger:
sent_content_length = 0
for n, v in headers.items():
nl = n.lower()
if nl == 'content-length':
try:
sent_content_length = int(v)
break
except ValueError:
pass
logger.debug("-> " + " ".join(
quote(str(x) if x else "-", ":/")
for x in (strftime('%Y-%m-%dT%H:%M:%S', gmtime(trans_stop)),
method, url, conn.getcode(), sent_content_length,
info['content-length'], trans_start, trans_stop,
trans_stop - trans_start, additional_info)))
return [info, body_data]
