def apiRequest(self, ep, key=None, data=None, method='GET'):
if data is not None or method=='POST':
req = Request("{}{}".format(self.base, ep))
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
if key is not None:
req.add_header('Authorization', "Bearer: {}".format(key))
data = urlencode(data)
req.add_data(data)
elif key is None:
req = Request("{}{}".format(self.base, ep)) # GET without token
else:
if data is None: data = {}
data['access_token'] = key
data = urlencode(data)
req = Request("{}{}?{}".format(self.base, ep, data))
for retries in range(3):
try:
print "requesting {}".format(ep),
res = urlopen(req).read()
print " - done"
return json.loads(res)
except HTTPError as e:
raise
except URLError as e:
pass
def __new__(self, token):
app = grok.getApplication()
users = IOAuthPrincipalSource(app)
url = u"https://graph.facebook.com/me"
print "User token info found: %s" % token.info
req = Request(url)
req.add_header('User-Agent', 'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", "{} {}".format(token.info['token_type'],
token.info['access_token']))
req.add_data(urlencode(dict(access_token=token.info['access_token'],
fields='id,name,first_name,last_name')))
res = urlopen(req).read()
if res: res = json.loads(res)
if res is None:
return None
else:
print "Personal info returned: %s" % res
uid = u"Facebook.{}".format(res['id'])
found = users.find(id=uid)
if len(found)==0:
user = users.new(id=uid)
else:
user = list(found)[0]
user.authInfo = token.info
user.title = unicode(res['name'])
user.description = u'{} {}'.format(res['first_name'], res['last_name'])
user.domain = u'Facebook'
user.login = unicode(res['id'])
user.secret = unicode(token.info['access_token'])
return user
def update(self, code=None, state=None, **args):
''' Either code or error is defined here if this is in response to Authorization '''
super(V2TokenView, self).update(**args)
oauth = self.context.__parent__.__parent__
self.context.__parent__.__parent__.error_msg = None
if self.error is None:
if code is not None:
token = self.context.__parent__
if token.state == state:
self.context.code = code
data = urlencode(self.context.parms)
print "----------------------------------------------------"
print "url=[%s]; data=[%s]" % (self.context.uri, data)
req = Request(self.context.uri)
req.add_header(
'User-Agent',
'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
req.add_header('Content-Type',
'application/x-www-form-urlencoded')
req.add_header('Accept', 'application/json')
req.add_data(data)
try:
res = urlopen(req).read() # should be doing a post
self.context.info = json.loads(res)
try:
# Update session information with auth info
session = ISession(self.request)['OAuth2']
session['info'] = self.context.info
service = self.context.__parent__.service
principal = component.queryAdapter(self.context,
IOAuthPrincipal,
name=service)
session[
'principal'] = principal if principal else None
# If we get here, we can notify subscribers.
grok.notify(
OAuthenticatedEvent(session['principal']))
self.redirect(self.url(grok.getApplication()))
except Exception as e:
print "Problem in adapter for service: {}".format(
str(e))
self.error = str(e)
except HTTPError as e:
print "Error while exchanging tokens: {}".format(
str(e))
self.error = str(e)
else:
self.error = 'State Mismatch'
else:
self.error = 'Invalid code'
if type(self.error) is str and len(self.error):
print "-------------------------------------------------"
print "Error [%s] in token exchange" % self.error
oauth.error_msg = self._render_template()
self.redirect(self.url(grok.getApplication()))
def _verify_cas2_saml(ticket, service):
"""Verifies CAS 3.0+ XML-based authentication ticket and returns extended attributes.
@date: 2011-11-30
@author: Carlos Gonzalez Vila <*****@*****.**>
Returns username and attributes on success and None,None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
# We do the SAML validation
headers = {'soapaction': 'http://www.oasis-open.org/committees/security',
'cache-control': 'no-cache',
'pragma': 'no-cache',
'accept': 'text/xml',
'connection': 'keep-alive',
'content-type': 'text/xml'}
params = {'TARGET': service}
url = Request(urljoin(settings.CAS_SERVER_URL, 'samlValidate') + '?' + urlencode(params), '', headers)
data = get_saml_assertion(ticket)
url.add_data(get_saml_assertion(ticket))
page = urlopen(url)
try:
user = None
attributes = {}
response = page.read()
print(response) # XXX: Is this supposed to be here?
tree = ElementTree.fromstring(response)
# Find the authentication status
success = tree.find('.//' + SAML_1_0_PROTOCOL_NS + 'StatusCode')
if success is not None and success.attrib['Value'] == 'samlp:Success':
# User is validated
attrs = tree.findall('.//' + SAML_1_0_ASSERTION_NS + 'Attribute')
for at in attrs:
if 'uid' in set(at.attrib.values()):
user = at.find(SAML_1_0_ASSERTION_NS + 'AttributeValue').text
attributes['uid'] = user
values = at.findall(SAML_1_0_ASSERTION_NS + 'AttributeValue')
if len(values) > 1:
values_array = []
for v in values:
values_array.append(v.text)
attributes[at.attrib['AttributeName']] = values_array
else:
attributes[at.attrib['AttributeName']] = values[0].text
return user, attributes
finally:
page.close()
def get_system(token,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
LOG.debug("%s cmd:%s hdr:%s payload:%s" %
(method, api_cmd, api_cmd_headers, api_cmd_payload))
response = None
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = urlopen(request_info, timeout=timeout)
response = request.read()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
request.close()
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
if hasattr(e, 'msg') and e.msg:
response = json.loads(e.msg)
else:
response = json.loads("{}")
raise
except URLError:
LOG.error("Cannot access %s" % api_cmd)
raise
finally:
return response
def _get_token(auth_url, auth_project, username, password, user_domain,
project_domain, region_name):
"""
Ask OpenStack Keystone for a token
Returns: token object or None on failure
"""
try:
url = auth_url + "/v3/auth/tokens"
request_info = Request(url)
request_info.add_header("Content-type", "application/json")
request_info.add_header("Accept", "application/json")
payload = json.dumps({
"auth": {
"identity": {
"methods": ["password"],
"password": {
"user": {
"name": username,
"password": password,
"domain": {
"name": user_domain
}
}
}
},
"scope": {
"project": {
"name": auth_project,
"domain": {
"name": project_domain
}
}
}
}
})
request_info.add_data(payload)
request = urlopen(request_info)
# Identity API v3 returns token id in X-Subject-Token
# response header.
token_id = request.info().getheader('X-Subject-Token')
response = json.loads(request.read())
request.close()
# save the region name for service url lookup
return Token(response, token_id, region_name)
except HTTPError as e:
LOG.error("%s, %s" % (e.code, e.read()))
return None
except URLError as e:
LOG.error(e)
return None
def add_data(self, data):
if hasattr(data, "items"):
data = urlencode(data).encode("ascii")
assert isinstance(data, binary_type)
if hasattr(BaseRequest, "add_data"):
BaseRequest.add_data(self, data)
else:
self.data = data
self.add_header("Content-Length", str(len(data)))
def add_data(self, data):
if hasattr(data, "items"):
data = urlencode(data).encode("ascii")
assert isinstance(data, binary_type)
if hasattr(BaseRequest, "add_data"):
BaseRequest.add_data(self, data)
else:
self.data = data
self.add_header("Content-Length", str(len(data)))
def rest_api_request(token, method, api_cmd,
api_cmd_payload=None, timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
api_cmd_headers = dict()
api_cmd_headers['Content-type'] = "application/json"
api_cmd_headers['User-Agent'] = "cert-mon/1.0"
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = None
try:
request = urlopen(request_info, timeout=timeout)
response = request.read()
finally:
if request:
request.close()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
raise
except URLError:
LOG.error("Cannot access %s" % api_cmd)
raise
return response
def soap_req(self, url, http_post=False, skip_auth=False, **kwargs):
soap_action = kwargs.get('soap_action', '')
url = self.MS_TM_API_URL
action = self.MS_TM_SOAP_HEADER + soap_action
headers = {
'SOAPAction': ('"{action}"').format(action=action),
'Content-Type': 'text/xml; charset=utf-8'
}
if soap_action == 'GetLanguages':
payload = {
'xmlns': self.MS_TM_SOAP_XMLNS,
'soap_action': soap_action
}
template = get_template(
'trans/machine/microsoft_terminology_get_langs.jinja')
elif soap_action == 'GetTranslations':
source = kwargs.get('source', '')
language = kwargs.get('language', '')
text = kwargs.get('text', '')
max_result = 5
if soap_action and source and language and text:
payload = {
'action': action,
'url': url,
'xmlns': self.MS_TM_SOAP_XMLNS,
'uuid': uuid4(),
'text': text,
'from_lang': source,
'to_lang': language,
'max_result': max_result
}
template = get_template(
'trans/machine/microsoft_terminology_translate.jinja')
else:
raise MachineTranslationError(
'Wrong SOAP request: "{soap_action}."').format(
soap_action=soap_action)
try:
payload = template.render(payload)
request = Request(url)
request.timeout = 0.5
for header, value in headers.iteritems():
request.add_header(header, value)
request.add_data(payload)
handle = urlopen(request)
except Exception as error:
raise MachineTranslationError('{err}'.format(err=error))
return handle
def request(url, data=None, headers=None, params=None):
u'''Simple HTTP Client'''
if params is not None:
query = urlencode(params)
url = '%s?%s' % (url, query)
req = Request(url, headers=headers)
if data is not None:
req.add_data(data)
try:
logging.debug("%s %s", req.get_method(), url)
res = urlopen(req)
return json.loads(res.read())
except HTTPError as err:
logging.error("%s. Client error GET %s with status %d.", err.reason,
url, err.code)
except URLError as err:
logging.exception(err)
except (ValueError, TypeError) as err:
logging.error(err)
return None
def render(self):
client = self.context.client(self.request)
uri, headers, body = client.sign(self.context.request_token_uri)
req = Request(uri)
for h, v in headers:
req.add_header(h, v)
req.add_data(body)
try:
res = urlopen(req).read() # should be doing a post
access = self.context.process(res)
# Redirect with parameter: oauth_token (optional)
data = dict(oauth_token=access.parms['oauth_token'])
url = "{}?{}".format(self.context.auth_uri, urlencode(data))
self.redirect(url)
return '<a class="autolink" href="%s">Please visit: %s</a>' % (
str(url), str(url))
except HTTPError as e:
print "Error while opening {}: {}".format(str(req), str(e))
self.error = str(e)
def __send_request(self, method, uri, data):
retry_codes = {429: 3}
@request_retry(codes=retry_codes)
def __get_response(_request):
return urlopen(_request).read()
url = self.__url + uri
request = Request(url)
if method == 'POST':
request.add_data(json.dumps(data))
auth = base64.encodestring(
'{0}:{1}'.format(self.user, self.password)).strip()
request.add_header('Authorization', 'Basic {}'.format(auth))
request.add_header('Content-Type', 'application/json')
e = None
try:
response = __get_response(request)
except HTTPError as e:
response = e.read()
if response:
result = json.loads(response)
else:
result = {}
if e is not None:
if result and 'error' in result:
error = '"' + result['error'] + '"'
else:
error = 'No additional error message received'
raise APIError('TestRail API returned HTTP %s (%s)' %
(e.code, error))
return result
class LDSAPI(ABC):
#class LDSAPI(object):
# __metaclass__ = ABCMeta
__sec = 'LDSAPI Wrapper'
sch = None
sch_def = 'https'
url = {
'lds-l': 'data.linz.govt.nz',
'lds-t': 'data-test.linz.govt.nz',
'mfe-t': 'mfe.data-test.linz.govt.nz',
'apiary': 'private-cbd7-koordinates.apiary.io',
'koordinates': 'koordinates.com',
'ambury': 'ambury-ldsl.kx.gd'
}
url_def = 'lds-l'
pxy = {
'linz': 'webproxy1.ad.linz.govt.nz:3128',
'local': '127.0.0.1:3128',
'noproxy': ':'
}
pxy_def = 'noproxy'
ath = {
'key': '.apikey3', #'api_llckey',
'basic': '.credentials'
}
ath_def = 'key'
def __init__(self): #, creds, cfile):
self.cookies = cjar.LWPCookieJar()
#//
self.setProxyRef(self.pxy_def)
# cant set auth because we dont know yet what auth type to use, doesnt make sense to set up on default
# self.setAuthentication(creds, self.ath[self.ath_def], self.ath_def)
self.scheme = self.sch_def
self.auth = None
self.head = {}
@abstractmethod
def setParams(self):
'''abstract host/path setting method'''
pass
def setCommonParams(self,
scheme=None,
host=None,
fmt='json',
sec=None,
pth=None,
url=None):
'''Assigns path/host params or tries to extract them from a url'''
self.fmt = fmt
if url:
p = urlparse(url)
self.scheme = p.scheme
self.host = p.netloc
self.path = p.path
return
if pth:
self.path = self.path_ref[sec][pth] + '?format={0}'.format(
self.fmt)
if host: self.host = super(DataAPI, self).url[host]
self.scheme = scheme or self.sch_def
def setProxyRef(self, pref):
#proxy = ul2.ProxyHandler({'http': self.pxy[pref]})
#self.openerstrs_ntlm = ul2.build_opener(proxy)
self.pref = pref
self.openerstrs_ntlm = self.pxy[pref]
def setAuthentication(self, creds, cfile, auth):
self.ath = {auth: self.ath[auth]}
if auth == 'basic':
self._setBasicAuth(creds, cfile)
elif auth == 'key':
self._setKeyAuth(creds, cfile)
else:
LM.err('Auth error. Need key/basic specifier',
LM._LogExtra('LDSAPI', 'sA'))
raise Exception('Incorrect auth configuration supplied')
def _setBasicAuth(self, creds, cfile):
self.setCredentials(creds(cfile))
self._setRequestAuth("Basic {0}".format(self.b64a))
def _setKeyAuth(self, creds, cfile):
self._setRequestKey(creds(cfile))
self._setRequestAuth("key {0}".format(self.key))
def setCredentials(self, creds):
if type(creds) is dict:
self.usr, self.pwd, self.dom = [creds[i] for i in 'upd']
else:
self.usr = '******'
#self.usr, self.pwd, self.dom = (*creds,None,None)[:3] #only since py3.5
self.usr, self.pwd, self.dom = (creds + (None, None))[:3]
self.b64a = LDSAPI.encode_auth({
'user': self.usr,
'pass': self.pwd,
'domain': self.dom
} if self.dom and self.dom != 'WGRP' else {
'user': self.usr,
'pass': self.pwd
})
#---------------------------------------------------------------------------
def _setRequestKey(self, creds):
self.key = creds['k'] if type(creds) is dict else creds
def _setRequestAuth(self, auth):
self.auth = auth
def setRequest(self, req):
if isinstance(req, string_types):
self.req_str = req
self.req = Request(req)
else:
self.req_str = req.full_url
self.req = req
def _setRequestData(self, data):
self.data = data
def _setRequestInfo(self, info):
self.info = info
def _setRequestHead(self, name, val):
self.head[name] = val
def addRequestHeader(self, name, head):
self.req.add_header(name, head)
def addRequestData(self, data=None):
self.req.add_data(data if data else self.data)
def getRequest(self):
return self.req
def getRequestStr(self, mask=True):
return LDS.kmask(self.req_str) if mask else self.req_str
#---------------------------------------------------------------------------
def setResponse(self, res):
self.res = res
self._setResponseData(res.read())
self._setResponseInfo(res.info())
self._setResponseURL(res.geturl())
self._setResponseHead(
LDSAPI.parseHeaders(self.info._headers
) if hasattr(self.info, '_headers') else None)
def _setResponseData(self, respdata):
self.respdata = respdata
def _setResponseHead(self, head):
self.head = head
def _setResponseInfo(self, info):
self.info = info
def _setResponseURL(self, url):
self.url = url
def getResponse(self):
return {
'info': self.respinfo,
'head': self.resphead,
'data': self.respdata
}
#---------------------------------------------------------------------------
def setExteralLogManager(self, lm):
global LM
LM = lm
@staticmethod
def parseHeaders(head):
# ['Server: nginx\r\n',
# 'Date: Wed, 14 May 2014 20:59:22 GMT\r\n',
# 'Content-Type: application/json\r\n',
# 'Transfer-Encoding: chunked\r\n',
# 'Connection: close\r\n',
# 'Vary: Accept-Encoding\r\n',
# 'Link: <https://data.linz.govt.nz/services/api/v1/layers/?sort=name&kind=raster&format=json>;
# rel="sort-name",
# <https://data.linz.govt.nz/services/api/v1/layers/?sort=-name&kind=raster&format=json>;
# rel="sort-name-desc",
# <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=1&format=json>;
# rel="page-previous",
# <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=3&format=json>;
# rel="page-next",
# <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=4&format=json>;
# rel="page-last"\r\n',
# 'Allow: GET, POST, HEAD, OPTIONS\r\n',
# 'Vary: Accept,Accept-Encoding\r\n',
# 'X-K-gentime: 0.716\r\n']
h = {}
relist = {
'server': 'Server:\s(.*)\r\n',
'date': 'Date:\s(.*)\r\n',
'content-type': 'Content-Type:\s(.*)\r\n',
'transfer-encoding': 'Transfer-Encoding:\s(.*)\r\n',
'connection': 'Connection:\s(.*)\r\n',
'vary': 'Vary:\s(.*)\r\n',
'link': 'Link:\s(.*)\r\n',
'vary-acc': 'Vary:\s(.*?)\r\n',
'x-k-gentime': 'X-K-gentime:\s(.*?)\r\n',
'oauth-scopes': 'OAuth-Scopes:\s(.*?)\r\n'
}
if isinstance(head, HTTPMessage):
for k in relist.keys():
s = [i[1] for i in head._headers if i[0].lower() == k]
if s: h[k] = s[0]
elif isinstance(head, string_types):
for k in relist.keys():
s = re.search(relist[k], '|'.join(head))
if s: h[k] = s.group(1)
elif isinstance(head, list):
for k in relist.keys():
s = [i[1] for i in head if i[0].lower() == k]
if s: h[k] = s[0]
# ---------------------
#Pull apart link string, if its available
lnlist = {
'sort-name': '<(http.*?)>;\s+rel="sort-name"',
'sort-name-desc': '<(http.*?)>;\s+rel="sort-name-desc"',
'page-previous': '<(http.*?)>;\s+rel="page-previous"',
'page-next': '<(http.*?)>;\s+rel="page-next"',
'page-last': '<(http.*?)>;\s+rel="page-last"'
}
link = h['link'].split(',') if 'link' in h else []
for ref in link:
for rex in lnlist.keys():
s = re.search(lnlist[rex], ref)
if s:
if 'page' in rex:
p = re.search('page=(\d+)', s.group(1))
h[rex] = {
'u': s.group(1),
'p': int(p.group(1)) if p else None
}
else:
h[rex] = s.group(1)
continue
return h
def opener(self, purl, puser=None, ppass=None, pscheme=('http', 'https')):
if REDIRECT:
h1, h2 = REDIRECT.BindableHTTPHandler, REDIRECT.BindableHTTPSHandler
else:
h1, h2 = request.HTTPHandler, request.HTTPSHandler
handlers = [h1(), h2(), request.HTTPCookieProcessor(self.cookies)]
if self.pref != 'noproxy' and purl and len(purl) > 1:
#if not noproxy and a proxy url is provided (and its not the placeholder url ie noproxy=':') add a handler
handlers += [
request.ProxyHandler({ps: purl
for ps in pscheme}),
]
#handlers += [request.ProxyHandler({ps:purl}) for ps in pscheme]
if puser and ppass:
#if proxy user/pass provided and a proxy auth handler
pm = request.HTTPPasswordMgrWithDefaultRealm()
pm.add_password(None, purl, puser, ppass)
handlers += [
request.ProxyBasicAuthHandler(pm),
]
return request.build_opener(*handlers)
def connect(self, plus='', head=None, data={}, auth=None):
'''URL connection wrapper, wraps URL strings in request objects, applying selected openers'''
#self.path='/services/api/v1/layers/{id}/versions/{version}/import/'
self.setRequest('{0}://{1}{2}{3}'.format(self.scheme, self.host,
self.path, plus))
# Add user header if provided
if head:
self._setRequestHead(head)
self.addRequestHeader(
shlex.split(head)[0].strip("(),"),
shlex.split(head)[1].strip("(),"))
if auth:
self._setRequestAuth(auth)
self.addRequestHeader("Authorization", auth)
# Add user data if provided
if data: #or true #for testing
#NB. adding a data component in request switches request from GET to POST
data = urlencode(data)
self._setRequestData(data)
self.addRequestData(data)
return self.conn(self.getRequest())
def conn(self, req):
'''URL connection wrappercatching common exceptions and retrying where necessary
param: connreq can be either a url string or a request object
'''
sr = self.__sec, 'Connection Manager'
self.setRequest(req)
req_str = self.getRequestStr()
#if self.auth is set it should have been added to the request header... might be legacy where that hasn't happened
if self.auth:
self.addRequestHeader("Authorization", self.auth)
request.install_opener(self.opener(purl=self.openerstrs_ntlm))
retry = INIT_MAX_RETRY_ATTEMPTS
while retry > 0:
retry -= 1
try:
handle = request.urlopen(self.getRequest()) #,data)
if handle:
if handle.geturl() != req_str:
msg = 'Redirect Warning'
#cannot easily mask redirected url so logging original
LM.info(
msg, LM._LogExtra(*sr,
exc=None,
url=req_str,
rty=0))
return handle
#self.setResponse(handle)
#break
except HTTPError as he:
last_exc = he
if re.search('429', str(he)):
msg = 'RateLimit Error {0}. Sleep awaiting 429 expiry. Attempt {1}'.format(
he, MAX_RETRY_ATTEMPTS - retry)
LM.error(msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
elif retry:
# I'm leaving this code here to test with because LDS was
# somehow throwing exceptions as well as redirecting
#
#if re.search('301',str(he)):
# msg = 'Redirect Error {0}'.format(he)
# #if we have a valid response and its a 301 see if it contains a redirect-to
# if handle and handle.geturl():
# retry = 1
# self.setRequest(handle.geturl()) #TODO reauth?
# msg += '. Attempting alternate connect'
# else:
# retry = 0
# LM.error(msg,LM._LogExtra(*sr,exc=he,url=self.getRequestStr(),rty=0))
# continue
if re.search('401|500', str(he)):
msg = 'HTTP Error {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('403', str(he)):
msg = 'HTTP Error {0} Returns {1}. Attempt {2} (consider proxy)'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('502', str(he)):
msg = 'Proxy Error {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('410', str(he)):
msg = 'Layer removed {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
retry = 0
continue
else:
msg = 'Error with request {0} returns {1}'.format(
req_str, he)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
else:
#Retries have been exhausted, raise the active httpexception
raise HTTPError(he.msg + msg)
except HttpResponseError as rd:
LM.warning('Disconnect. {}'.format(rd),
LM._LogExtra(*sr, exc=rd, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
except URLError as ue:
LM.warning('URL error on connect {}'.format(ue),
LM._LogExtra(*sr, exc=ue, url=req_str, rty=retry))
if re.search('Connection refused|violation of protocol',
str(ue)):
LDSAPI.sleepIncr(retry)
continue
#raise ue
except ConnectionError as ce:
LM.warning('Error on connection. {}'.format(ce),
LM._LogExtra(*sr, exc=ce, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
except ValueError as ve:
LM.error('Value error on connect {}'.format(ve),
LM._LogExtra(*sr, exc=ve, url=req_str, rty=retry))
raise ve
except Exception as xe:
LM.error('Other error on connect {}'.format(xe),
LM._LogExtra(*sr, exc=xe, url=req_str, rty=retry))
raise xe
else:
raise last_exc
#except Exception as e:
# print e
# def _testalt(self,ref='basic'):
# p = os.path.join(os.path.dirname(__file__),cxf or LDSAPI.ath[ref])
# self.setAuthentication(Authentication.creds,p, ref)
@staticmethod
def encode_auth(auth):
'''Build and b64 encode a http authentication string. [Needs to be bytes str, hence en/decode()]'''
if 'domain' in auth:
astr = '{d}\{u}:{p}'.format(u=auth['user'],
p=auth['pass'],
d=auth['domain']).strip().encode()
else:
astr = '{u}:{p}'.format(u=auth['user'],
p=auth['pass']).strip().encode()
return base64.b64encode(astr).decode()
def fetchPages(self, psub=''):
sr = self.__sec, 'Page fetch'
upd = []
page = 0
pagel = None
morepages = True
while morepages:
page = page + 1
pstr = psub + '&page={0}'.format(page)
try:
res = self.connect(plus=pstr)
if res: self.setResponse(res)
else: raise HTTPError('No Response using URL {}'.format(pstr))
#api.dispReq(api.req)
#api.dispRes(api.res)
except HTTPError as he:
LM.error('HTTP Error on page fetch {}'.format(he),
LM._LogExtra(*sr, exc=he, url=pstr))
morepages = False
raise
#continue
except Exception as e:
#Outer catch of unknown errors
LM.error('Error on page fetch {}'.format(he),
LM._LogExtra(*sr, exc=he, url=pstr))
raise
# The logic here is a bit redundant but basically if no last page found then its prob the last page
# otherwise save the last page value and compare to current page. If they're equal get off loop
if 'page-last' in self.head:
pagel = self.head['page-last']['p']
else:
morepages = False
if page == pagel:
morepages = False
jdata = json.loads(self.respdata.decode())
upd += [
jdata,
] if isinstance(jdata, dict) else jdata
return upd
@staticmethod
def sleepIncr(r):
t = (INIT_MAX_RETRY_ATTEMPTS - r) * SLEEP_RETRY_INCR
print('tock' if t % 2 else 'tick', t,
'{}/{}'.format(r, INIT_MAX_RETRY_ATTEMPTS))
time.sleep(t)
@staticmethod
def dispReq(req):
print('Request\n-------\n')
print(LDS.kmask(req.get_full_url()), 'auth',
req.get_header('Authorization'))
@staticmethod
def dispRes(res):
print('Response\n--------\n')
print(res.info())
@staticmethod
def dispJSON(res):
for l in json.loads(res.read()):
print('{0} - {1}\n'.format(l[0], l[1]))
@staticmethod
def _populate(data):
return json.dumps({
"name": data[0],
"type": data[1],
"description": data[2],
"categories": data[3],
"user": data[4],
"options": {
"username": data[5],
"password": data[6]
},
"url_remote": data[7],
"scan_schedule": data[8]
})
def rest_api_request(token,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout=10):
"""
Make a rest-api request
Returns: response as a dictionary
"""
# signal.signal(signal.SIGALRM, _timeout_handler)
# if hasattr(signal, 'SIGALRM'):
# signal.alarm(timeout)
LOG.info("%s cmd:%s hdr:%s payload:%s" %
(method, api_cmd, api_cmd_headers, api_cmd_payload))
response = None
try:
request_info = Request(api_cmd)
request_info.get_method = lambda: method
if token:
request_info.add_header("X-Auth-Token", token.get_id())
request_info.add_header("Accept", "application/json")
if api_cmd_headers is not None:
for header_type, header_value in api_cmd_headers.items():
request_info.add_header(header_type, header_value)
if api_cmd_payload is not None:
request_info.add_data(api_cmd_payload)
request = urlopen(request_info, timeout=timeout)
response = request.read()
if response == "":
response = json.loads("{}")
else:
response = json.loads(response)
request.close()
LOG.info("Response=%s" % response)
except HTTPError as e:
if 401 == e.code:
if token:
token.set_expired()
LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
if hasattr(e, 'msg') and e.msg:
response = json.loads(e.msg)
else:
response = json.loads("{}")
LOG.info("HTTPError response=%s" % (response))
raise OpenStackRestAPIException(e.message, e.code, "%s" % e)
except URLError as e:
LOG.warn("URLError Error e=%s" % (e))
raise OpenStackException(e.message, "%s" % e)
except si_exception.SysInvSignalTimeout as e:
LOG.warn("Timeout Error e=%s" % (e))
raise OpenStackException(e.message, "%s" % e)
finally:
signal.alarm(0)
return response
def add_data(self, data):
if hasattr(data, "iteritems"):
data = urlencode(data)
print(data)
self.add_header("Content-Length", str(len(data)))
BaseRequest.add_data(self, data)
def add_data(self, data):
if hasattr(data, "items"):
data = urlencode(data)
print(data)
self.add_header("Content-Length", str(len(data)))
BaseRequest.add_data(self, data)