Example #1
0
 def apiRequest(self, ep, key=None, data=None, method='GET'):
     if data is not None or method=='POST':
         req = Request("{}{}".format(self.base, ep))
         req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         if key is not None:
             req.add_header('Authorization', "Bearer: {}".format(key))
         data = urlencode(data)
         req.add_data(data)
     elif key is None:            
         req = Request("{}{}".format(self.base, ep)) # GET without token
     else:
         if data is None: data = {}
         data['access_token'] = key
         data = urlencode(data)
         req = Request("{}{}?{}".format(self.base, ep, data))
     for retries in range(3):
         try:
             print "requesting {}".format(ep),
             res = urlopen(req).read()
             print " - done"
             return json.loads(res)
         except HTTPError as e:
             raise
         except URLError as e:
             pass
Example #2
0
    def __new__(self, token):
        app = grok.getApplication()
        users = IOAuthPrincipalSource(app)

        url = u"https://graph.facebook.com/me"
        print "User token info found: %s" % token.info
        req = Request(url)
        
        req.add_header('User-Agent',  'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
        req.add_header("Content-Type", "application/json")
        req.add_header("Authorization", "{} {}".format(token.info['token_type'],
                                                       token.info['access_token']))
        req.add_data(urlencode(dict(access_token=token.info['access_token'],
                                    fields='id,name,first_name,last_name')))
        res = urlopen(req).read()
        if res: res = json.loads(res)
        if res is None:
            return None
        else:
            print "Personal info returned: %s" % res
            uid = u"Facebook.{}".format(res['id'])
            found = users.find(id=uid)
            if len(found)==0:
                user = users.new(id=uid)
            else:
                user = list(found)[0]

            user.authInfo = token.info
            user.title = unicode(res['name'])
            user.description = u'{} {}'.format(res['first_name'], res['last_name'])
            user.domain = u'Facebook'
            user.login = unicode(res['id'])
            user.secret = unicode(token.info['access_token'])
            return user
Example #3
0
    def update(self, code=None, state=None, **args):
        ''' Either code or error is defined here if this is in response to Authorization '''
        super(V2TokenView, self).update(**args)
        oauth = self.context.__parent__.__parent__
        self.context.__parent__.__parent__.error_msg = None

        if self.error is None:
            if code is not None:
                token = self.context.__parent__
                if token.state == state:
                    self.context.code = code
                    data = urlencode(self.context.parms)
                    print "----------------------------------------------------"
                    print "url=[%s]; data=[%s]" % (self.context.uri, data)
                    req = Request(self.context.uri)
                    req.add_header(
                        'User-Agent',
                        'Python urlib2 (grok.zopefoundation.org, Python 2.7)')
                    req.add_header('Content-Type',
                                   'application/x-www-form-urlencoded')
                    req.add_header('Accept', 'application/json')
                    req.add_data(data)
                    try:
                        res = urlopen(req).read()  # should be doing a post
                        self.context.info = json.loads(res)
                        try:
                            # Update session information with auth info
                            session = ISession(self.request)['OAuth2']
                            session['info'] = self.context.info

                            service = self.context.__parent__.service
                            principal = component.queryAdapter(self.context,
                                                               IOAuthPrincipal,
                                                               name=service)
                            session[
                                'principal'] = principal if principal else None
                            # If we get here, we can notify subscribers.
                            grok.notify(
                                OAuthenticatedEvent(session['principal']))
                            self.redirect(self.url(grok.getApplication()))
                        except Exception as e:
                            print "Problem in adapter for service: {}".format(
                                str(e))
                            self.error = str(e)
                    except HTTPError as e:
                        print "Error while exchanging tokens: {}".format(
                            str(e))
                        self.error = str(e)
                else:
                    self.error = 'State Mismatch'
            else:
                self.error = 'Invalid code'
        if type(self.error) is str and len(self.error):
            print "-------------------------------------------------"
            print "Error [%s] in token exchange" % self.error

        oauth.error_msg = self._render_template()
        self.redirect(self.url(grok.getApplication()))
Example #4
0
def _verify_cas2_saml(ticket, service):
    """Verifies CAS 3.0+ XML-based authentication ticket and returns extended attributes.

    @date: 2011-11-30
    @author: Carlos Gonzalez Vila <*****@*****.**>

    Returns username and attributes on success and None,None on failure.
    """

    try:
        from xml.etree import ElementTree
    except ImportError:
        from elementtree import ElementTree

    # We do the SAML validation
    headers = {'soapaction': 'http://www.oasis-open.org/committees/security',
        'cache-control': 'no-cache',
        'pragma': 'no-cache',
        'accept': 'text/xml',
        'connection': 'keep-alive',
        'content-type': 'text/xml'}
    params = {'TARGET': service}
    url = Request(urljoin(settings.CAS_SERVER_URL, 'samlValidate') + '?' + urlencode(params), '', headers)
    data = get_saml_assertion(ticket)
    url.add_data(get_saml_assertion(ticket))

    page = urlopen(url)

    try:
        user = None
        attributes = {}
        response = page.read()
        print(response)  # XXX: Is this supposed to be here?
        tree = ElementTree.fromstring(response)
        # Find the authentication status
        success = tree.find('.//' + SAML_1_0_PROTOCOL_NS + 'StatusCode')
        if success is not None and success.attrib['Value'] == 'samlp:Success':
            # User is validated
            attrs = tree.findall('.//' + SAML_1_0_ASSERTION_NS + 'Attribute')
            for at in attrs:
                if 'uid' in set(at.attrib.values()):
                    user = at.find(SAML_1_0_ASSERTION_NS + 'AttributeValue').text
                    attributes['uid'] = user
                values = at.findall(SAML_1_0_ASSERTION_NS + 'AttributeValue')
                if len(values) > 1:
                    values_array = []
                    for v in values:
                        values_array.append(v.text)
                    attributes[at.attrib['AttributeName']] = values_array
                else:
                   attributes[at.attrib['AttributeName']] = values[0].text
        return user, attributes
    finally:
        page.close()
Example #5
0
def get_system(token,
               method,
               api_cmd,
               api_cmd_headers=None,
               api_cmd_payload=None,
               timeout=10):
    """
    Make a rest-api request
    Returns: response as a dictionary
    """
    LOG.debug("%s cmd:%s hdr:%s payload:%s" %
              (method, api_cmd, api_cmd_headers, api_cmd_payload))

    response = None
    try:
        request_info = Request(api_cmd)
        request_info.get_method = lambda: method
        if token:
            request_info.add_header("X-Auth-Token", token.get_id())
        request_info.add_header("Accept", "application/json")

        if api_cmd_headers is not None:
            for header_type, header_value in api_cmd_headers.items():
                request_info.add_header(header_type, header_value)

        if api_cmd_payload is not None:
            request_info.add_data(api_cmd_payload)

        request = urlopen(request_info, timeout=timeout)
        response = request.read()

        if response == "":
            response = json.loads("{}")
        else:
            response = json.loads(response)
        request.close()

    except HTTPError as e:
        if 401 == e.code:
            if token:
                token.set_expired()
        LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
        if hasattr(e, 'msg') and e.msg:
            response = json.loads(e.msg)
        else:
            response = json.loads("{}")
        raise

    except URLError:
        LOG.error("Cannot access %s" % api_cmd)
        raise

    finally:
        return response
Example #6
0
def _get_token(auth_url, auth_project, username, password, user_domain,
               project_domain, region_name):
    """
    Ask OpenStack Keystone for a token
    Returns: token object or None on failure
    """
    try:
        url = auth_url + "/v3/auth/tokens"
        request_info = Request(url)
        request_info.add_header("Content-type", "application/json")
        request_info.add_header("Accept", "application/json")
        payload = json.dumps({
            "auth": {
                "identity": {
                    "methods": ["password"],
                    "password": {
                        "user": {
                            "name": username,
                            "password": password,
                            "domain": {
                                "name": user_domain
                            }
                        }
                    }
                },
                "scope": {
                    "project": {
                        "name": auth_project,
                        "domain": {
                            "name": project_domain
                        }
                    }
                }
            }
        })

        request_info.add_data(payload)

        request = urlopen(request_info)
        # Identity API v3 returns token id in X-Subject-Token
        # response header.
        token_id = request.info().getheader('X-Subject-Token')
        response = json.loads(request.read())
        request.close()
        # save the region name for service url lookup
        return Token(response, token_id, region_name)

    except HTTPError as e:
        LOG.error("%s, %s" % (e.code, e.read()))
        return None

    except URLError as e:
        LOG.error(e)
        return None
Example #7
0
    def add_data(self, data):
        if hasattr(data, "items"):
            data = urlencode(data).encode("ascii")

        assert isinstance(data, binary_type)

        if hasattr(BaseRequest, "add_data"):
            BaseRequest.add_data(self, data)
        else:
            self.data = data

        self.add_header("Content-Length", str(len(data)))
Example #8
0
    def add_data(self, data):
        if hasattr(data, "items"):
            data = urlencode(data).encode("ascii")

        assert isinstance(data, binary_type)

        if hasattr(BaseRequest, "add_data"):
            BaseRequest.add_data(self, data)
        else:
            self.data = data

        self.add_header("Content-Length", str(len(data)))
Example #9
0
def rest_api_request(token, method, api_cmd,
                     api_cmd_payload=None, timeout=10):
    """
    Make a rest-api request
    Returns: response as a dictionary
    """
    api_cmd_headers = dict()
    api_cmd_headers['Content-type'] = "application/json"
    api_cmd_headers['User-Agent'] = "cert-mon/1.0"

    try:
        request_info = Request(api_cmd)
        request_info.get_method = lambda: method
        if token:
            request_info.add_header("X-Auth-Token", token.get_id())
        request_info.add_header("Accept", "application/json")

        if api_cmd_headers is not None:
            for header_type, header_value in api_cmd_headers.items():
                request_info.add_header(header_type, header_value)

        if api_cmd_payload is not None:
            request_info.add_data(api_cmd_payload)

        request = None
        try:
            request = urlopen(request_info, timeout=timeout)
            response = request.read()
        finally:
            if request:
                request.close()

        if response == "":
            response = json.loads("{}")
        else:
            response = json.loads(response)

    except HTTPError as e:
        if 401 == e.code:
            if token:
                token.set_expired()
        raise

    except URLError:
        LOG.error("Cannot access %s" % api_cmd)
        raise

    return response
Example #10
0
 def soap_req(self, url, http_post=False, skip_auth=False, **kwargs):
     soap_action = kwargs.get('soap_action', '')
     url = self.MS_TM_API_URL
     action = self.MS_TM_SOAP_HEADER + soap_action
     headers = {
         'SOAPAction': ('"{action}"').format(action=action),
         'Content-Type': 'text/xml; charset=utf-8'
     }
     if soap_action == 'GetLanguages':
         payload = {
             'xmlns': self.MS_TM_SOAP_XMLNS,
             'soap_action': soap_action
         }
         template = get_template(
             'trans/machine/microsoft_terminology_get_langs.jinja')
     elif soap_action == 'GetTranslations':
         source = kwargs.get('source', '')
         language = kwargs.get('language', '')
         text = kwargs.get('text', '')
         max_result = 5
         if soap_action and source and language and text:
             payload = {
                 'action': action,
                 'url': url,
                 'xmlns': self.MS_TM_SOAP_XMLNS,
                 'uuid': uuid4(),
                 'text': text,
                 'from_lang': source,
                 'to_lang': language,
                 'max_result': max_result
             }
             template = get_template(
                 'trans/machine/microsoft_terminology_translate.jinja')
     else:
         raise MachineTranslationError(
             'Wrong SOAP request: "{soap_action}."').format(
                 soap_action=soap_action)
     try:
         payload = template.render(payload)
         request = Request(url)
         request.timeout = 0.5
         for header, value in headers.iteritems():
             request.add_header(header, value)
         request.add_data(payload)
         handle = urlopen(request)
     except Exception as error:
         raise MachineTranslationError('{err}'.format(err=error))
     return handle
Example #11
0
def request(url, data=None, headers=None, params=None):
    u'''Simple HTTP Client'''
    if params is not None:
        query = urlencode(params)
        url = '%s?%s' % (url, query)
    req = Request(url, headers=headers)
    if data is not None:
        req.add_data(data)
    try:
        logging.debug("%s %s", req.get_method(), url)
        res = urlopen(req)
        return json.loads(res.read())
    except HTTPError as err:
        logging.error("%s. Client error GET %s with status %d.", err.reason,
                      url, err.code)
    except URLError as err:
        logging.exception(err)
    except (ValueError, TypeError) as err:
        logging.error(err)
    return None
Example #12
0
    def render(self):
        client = self.context.client(self.request)
        uri, headers, body = client.sign(self.context.request_token_uri)

        req = Request(uri)
        for h, v in headers:
            req.add_header(h, v)
        req.add_data(body)
        try:
            res = urlopen(req).read()  # should be doing a post

            access = self.context.process(res)
            #        Redirect with parameter: oauth_token (optional)
            data = dict(oauth_token=access.parms['oauth_token'])
            url = "{}?{}".format(self.context.auth_uri, urlencode(data))
            self.redirect(url)
            return '<a class="autolink" href="%s">Please visit: %s</a>' % (
                str(url), str(url))
        except HTTPError as e:
            print "Error while opening {}: {}".format(str(req), str(e))
            self.error = str(e)
Example #13
0
    def __send_request(self, method, uri, data):
        retry_codes = {429: 3}

        @request_retry(codes=retry_codes)
        def __get_response(_request):
            return urlopen(_request).read()

        url = self.__url + uri
        request = Request(url)
        if method == 'POST':
            request.add_data(json.dumps(data))
        auth = base64.encodestring(
            '{0}:{1}'.format(self.user, self.password)).strip()
        request.add_header('Authorization', 'Basic {}'.format(auth))
        request.add_header('Content-Type', 'application/json')

        e = None
        try:
            response = __get_response(request)
        except HTTPError as e:
            response = e.read()

        if response:
            result = json.loads(response)
        else:
            result = {}

        if e is not None:
            if result and 'error' in result:
                error = '"' + result['error'] + '"'
            else:
                error = 'No additional error message received'
            raise APIError('TestRail API returned HTTP %s (%s)' %
                           (e.code, error))

        return result
Example #14
0
class LDSAPI(ABC):
    #class LDSAPI(object):
    #	__metaclass__ = ABCMeta

    __sec = 'LDSAPI Wrapper'
    sch = None
    sch_def = 'https'

    url = {
        'lds-l': 'data.linz.govt.nz',
        'lds-t': 'data-test.linz.govt.nz',
        'mfe-t': 'mfe.data-test.linz.govt.nz',
        'apiary': 'private-cbd7-koordinates.apiary.io',
        'koordinates': 'koordinates.com',
        'ambury': 'ambury-ldsl.kx.gd'
    }
    url_def = 'lds-l'

    pxy = {
        'linz': 'webproxy1.ad.linz.govt.nz:3128',
        'local': '127.0.0.1:3128',
        'noproxy': ':'
    }
    pxy_def = 'noproxy'

    ath = {
        'key': '.apikey3',  #'api_llckey',
        'basic': '.credentials'
    }
    ath_def = 'key'

    def __init__(self):  #, creds, cfile):

        self.cookies = cjar.LWPCookieJar()
        #//

        self.setProxyRef(self.pxy_def)
        # cant set auth because we dont know yet what auth type to use, doesnt make sense to set up on default
        # self.setAuthentication(creds, self.ath[self.ath_def], self.ath_def)

        self.scheme = self.sch_def
        self.auth = None
        self.head = {}

    @abstractmethod
    def setParams(self):
        '''abstract host/path setting method'''
        pass

    def setCommonParams(self,
                        scheme=None,
                        host=None,
                        fmt='json',
                        sec=None,
                        pth=None,
                        url=None):
        '''Assigns path/host params or tries to extract them from a url'''
        self.fmt = fmt
        if url:
            p = urlparse(url)
            self.scheme = p.scheme
            self.host = p.netloc
            self.path = p.path
            return
        if pth:
            self.path = self.path_ref[sec][pth] + '?format={0}'.format(
                self.fmt)
        if host: self.host = super(DataAPI, self).url[host]
        self.scheme = scheme or self.sch_def

    def setProxyRef(self, pref):
        #proxy = ul2.ProxyHandler({'http': self.pxy[pref]})
        #self.openerstrs_ntlm = ul2.build_opener(proxy)
        self.pref = pref
        self.openerstrs_ntlm = self.pxy[pref]

    def setAuthentication(self, creds, cfile, auth):
        self.ath = {auth: self.ath[auth]}
        if auth == 'basic':
            self._setBasicAuth(creds, cfile)
        elif auth == 'key':
            self._setKeyAuth(creds, cfile)
        else:
            LM.err('Auth error. Need key/basic specifier',
                   LM._LogExtra('LDSAPI', 'sA'))
            raise Exception('Incorrect auth configuration supplied')

    def _setBasicAuth(self, creds, cfile):
        self.setCredentials(creds(cfile))
        self._setRequestAuth("Basic {0}".format(self.b64a))

    def _setKeyAuth(self, creds, cfile):
        self._setRequestKey(creds(cfile))
        self._setRequestAuth("key {0}".format(self.key))

    def setCredentials(self, creds):
        if type(creds) is dict:
            self.usr, self.pwd, self.dom = [creds[i] for i in 'upd']
        else:
            self.usr = '******'
            #self.usr, self.pwd, self.dom = (*creds,None,None)[:3] #only since py3.5
            self.usr, self.pwd, self.dom = (creds + (None, None))[:3]
        self.b64a = LDSAPI.encode_auth({
            'user': self.usr,
            'pass': self.pwd,
            'domain': self.dom
        } if self.dom and self.dom != 'WGRP' else {
            'user': self.usr,
            'pass': self.pwd
        })

        #---------------------------------------------------------------------------

    def _setRequestKey(self, creds):
        self.key = creds['k'] if type(creds) is dict else creds

    def _setRequestAuth(self, auth):
        self.auth = auth

    def setRequest(self, req):
        if isinstance(req, string_types):
            self.req_str = req
            self.req = Request(req)
        else:
            self.req_str = req.full_url
            self.req = req

    def _setRequestData(self, data):
        self.data = data

    def _setRequestInfo(self, info):
        self.info = info

    def _setRequestHead(self, name, val):
        self.head[name] = val

    def addRequestHeader(self, name, head):
        self.req.add_header(name, head)

    def addRequestData(self, data=None):
        self.req.add_data(data if data else self.data)

    def getRequest(self):
        return self.req

    def getRequestStr(self, mask=True):
        return LDS.kmask(self.req_str) if mask else self.req_str

    #---------------------------------------------------------------------------

    def setResponse(self, res):
        self.res = res
        self._setResponseData(res.read())
        self._setResponseInfo(res.info())
        self._setResponseURL(res.geturl())
        self._setResponseHead(
            LDSAPI.parseHeaders(self.info._headers
                                ) if hasattr(self.info, '_headers') else None)

    def _setResponseData(self, respdata):
        self.respdata = respdata

    def _setResponseHead(self, head):
        self.head = head

    def _setResponseInfo(self, info):
        self.info = info

    def _setResponseURL(self, url):
        self.url = url

    def getResponse(self):
        return {
            'info': self.respinfo,
            'head': self.resphead,
            'data': self.respdata
        }

    #---------------------------------------------------------------------------

    def setExteralLogManager(self, lm):
        global LM
        LM = lm

    @staticmethod
    def parseHeaders(head):
        # ['Server: nginx\r\n',
        # 'Date: Wed, 14 May 2014 20:59:22 GMT\r\n',
        # 'Content-Type: application/json\r\n',
        # 'Transfer-Encoding: chunked\r\n',
        # 'Connection: close\r\n',
        # 'Vary: Accept-Encoding\r\n',
        # 'Link: <https://data.linz.govt.nz/services/api/v1/layers/?sort=name&kind=raster&format=json>;
        #	 rel="sort-name",
        #	 <https://data.linz.govt.nz/services/api/v1/layers/?sort=-name&kind=raster&format=json>;
        #	 rel="sort-name-desc",
        #	 <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=1&format=json>;
        #	 rel="page-previous",
        #	 <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=3&format=json>;
        #	 rel="page-next",
        #	 <https://data.linz.govt.nz/services/api/v1/layers/?kind=raster&page=4&format=json>;
        #	 rel="page-last"\r\n',
        # 'Allow: GET, POST, HEAD, OPTIONS\r\n',
        # 'Vary: Accept,Accept-Encoding\r\n',
        # 'X-K-gentime: 0.716\r\n']
        h = {}
        relist = {
            'server': 'Server:\s(.*)\r\n',
            'date': 'Date:\s(.*)\r\n',
            'content-type': 'Content-Type:\s(.*)\r\n',
            'transfer-encoding': 'Transfer-Encoding:\s(.*)\r\n',
            'connection': 'Connection:\s(.*)\r\n',
            'vary': 'Vary:\s(.*)\r\n',
            'link': 'Link:\s(.*)\r\n',
            'vary-acc': 'Vary:\s(.*?)\r\n',
            'x-k-gentime': 'X-K-gentime:\s(.*?)\r\n',
            'oauth-scopes': 'OAuth-Scopes:\s(.*?)\r\n'
        }

        if isinstance(head, HTTPMessage):
            for k in relist.keys():
                s = [i[1] for i in head._headers if i[0].lower() == k]
                if s: h[k] = s[0]
        elif isinstance(head, string_types):
            for k in relist.keys():
                s = re.search(relist[k], '|'.join(head))
                if s: h[k] = s.group(1)
        elif isinstance(head, list):
            for k in relist.keys():
                s = [i[1] for i in head if i[0].lower() == k]
                if s: h[k] = s[0]

        # ---------------------
        #Pull apart link string, if its available
        lnlist = {
            'sort-name': '<(http.*?)>;\s+rel="sort-name"',
            'sort-name-desc': '<(http.*?)>;\s+rel="sort-name-desc"',
            'page-previous': '<(http.*?)>;\s+rel="page-previous"',
            'page-next': '<(http.*?)>;\s+rel="page-next"',
            'page-last': '<(http.*?)>;\s+rel="page-last"'
        }

        link = h['link'].split(',') if 'link' in h else []
        for ref in link:
            for rex in lnlist.keys():
                s = re.search(lnlist[rex], ref)
                if s:
                    if 'page' in rex:
                        p = re.search('page=(\d+)', s.group(1))
                        h[rex] = {
                            'u': s.group(1),
                            'p': int(p.group(1)) if p else None
                        }
                    else:
                        h[rex] = s.group(1)
                    continue

        return h

    def opener(self, purl, puser=None, ppass=None, pscheme=('http', 'https')):
        if REDIRECT:
            h1, h2 = REDIRECT.BindableHTTPHandler, REDIRECT.BindableHTTPSHandler
        else:
            h1, h2 = request.HTTPHandler, request.HTTPSHandler

        handlers = [h1(), h2(), request.HTTPCookieProcessor(self.cookies)]

        if self.pref != 'noproxy' and purl and len(purl) > 1:
            #if not noproxy and a proxy url is provided (and its not the placeholder url ie noproxy=':') add a handler
            handlers += [
                request.ProxyHandler({ps: purl
                                      for ps in pscheme}),
            ]
            #handlers += [request.ProxyHandler({ps:purl}) for ps in pscheme]

            if puser and ppass:
                #if proxy user/pass provided and a proxy auth handler
                pm = request.HTTPPasswordMgrWithDefaultRealm()
                pm.add_password(None, purl, puser, ppass)
                handlers += [
                    request.ProxyBasicAuthHandler(pm),
                ]

        return request.build_opener(*handlers)

    def connect(self, plus='', head=None, data={}, auth=None):
        '''URL connection wrapper, wraps URL strings in request objects, applying selected openers'''

        #self.path='/services/api/v1/layers/{id}/versions/{version}/import/'
        self.setRequest('{0}://{1}{2}{3}'.format(self.scheme, self.host,
                                                 self.path, plus))

        # Add user header if provided
        if head:
            self._setRequestHead(head)
            self.addRequestHeader(
                shlex.split(head)[0].strip("(),"),
                shlex.split(head)[1].strip("(),"))

        if auth:
            self._setRequestAuth(auth)
            self.addRequestHeader("Authorization", auth)

        # Add user data if provided
        if data:  #or true #for testing
            #NB. adding a data component in request switches request from GET to POST
            data = urlencode(data)
            self._setRequestData(data)
            self.addRequestData(data)

        return self.conn(self.getRequest())

    def conn(self, req):
        '''URL connection wrappercatching common exceptions and retrying where necessary
		param: connreq can be either a url string or a request object
		'''
        sr = self.__sec, 'Connection Manager'
        self.setRequest(req)
        req_str = self.getRequestStr()

        #if self.auth is set it should have been added to the request header... might be legacy where that hasn't happened
        if self.auth:
            self.addRequestHeader("Authorization", self.auth)

        request.install_opener(self.opener(purl=self.openerstrs_ntlm))

        retry = INIT_MAX_RETRY_ATTEMPTS
        while retry > 0:
            retry -= 1
            try:
                handle = request.urlopen(self.getRequest())  #,data)
                if handle:
                    if handle.geturl() != req_str:
                        msg = 'Redirect Warning'
                        #cannot easily mask redirected url so logging original
                        LM.info(
                            msg, LM._LogExtra(*sr,
                                              exc=None,
                                              url=req_str,
                                              rty=0))
                    return handle
                #self.setResponse(handle)
                #break
            except HTTPError as he:
                last_exc = he
                if re.search('429', str(he)):
                    msg = 'RateLimit Error {0}. Sleep awaiting 429 expiry. Attempt {1}'.format(
                        he, MAX_RETRY_ATTEMPTS - retry)
                    LM.error(msg,
                             LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                    LDSAPI.sleepIncr(retry)
                    continue
                elif retry:
                    # I'm leaving this code here to test with because LDS was
                    # somehow throwing exceptions as well as redirecting
                    #
                    #if re.search('301',str(he)):
                    #	msg = 'Redirect Error {0}'.format(he)
                    #	#if we have a valid response and its a 301 see if it contains a redirect-to
                    #	if handle and handle.geturl():
                    #		retry = 1
                    #		self.setRequest(handle.geturl()) #TODO reauth?
                    #		msg += '. Attempting alternate connect'
                    #	else:
                    #		retry = 0
                    #	LM.error(msg,LM._LogExtra(*sr,exc=he,url=self.getRequestStr(),rty=0))
                    #	continue
                    if re.search('401|500', str(he)):
                        msg = 'HTTP Error {0} Returns {1}. Attempt {2}'.format(
                            req_str, he, MAX_RETRY_ATTEMPTS - retry)
                        LM.error(
                            msg,
                            LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                        continue
                    elif re.search('403', str(he)):
                        msg = 'HTTP Error {0} Returns {1}. Attempt {2} (consider proxy)'.format(
                            req_str, he, MAX_RETRY_ATTEMPTS - retry)
                        LM.error(
                            msg,
                            LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                        continue
                    elif re.search('502', str(he)):
                        msg = 'Proxy Error {0} Returns {1}. Attempt {2}'.format(
                            req_str, he, MAX_RETRY_ATTEMPTS - retry)
                        LM.error(
                            msg,
                            LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                        continue
                    elif re.search('410', str(he)):
                        msg = 'Layer removed {0} Returns {1}. Attempt {2}'.format(
                            req_str, he, MAX_RETRY_ATTEMPTS - retry)
                        LM.error(
                            msg,
                            LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                        retry = 0
                        continue
                    else:
                        msg = 'Error with request {0} returns {1}'.format(
                            req_str, he)
                        LM.error(
                            msg,
                            LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
                        continue
                else:
                    #Retries have been exhausted, raise the active httpexception
                    raise HTTPError(he.msg + msg)
            except HttpResponseError as rd:
                LM.warning('Disconnect. {}'.format(rd),
                           LM._LogExtra(*sr, exc=rd, url=req_str, rty=retry))
                LDSAPI.sleepIncr(retry)
                continue
            except URLError as ue:
                LM.warning('URL error on connect {}'.format(ue),
                           LM._LogExtra(*sr, exc=ue, url=req_str, rty=retry))
                if re.search('Connection refused|violation of protocol',
                             str(ue)):
                    LDSAPI.sleepIncr(retry)
                continue
                #raise ue
            except ConnectionError as ce:
                LM.warning('Error on connection. {}'.format(ce),
                           LM._LogExtra(*sr, exc=ce, url=req_str, rty=retry))
                LDSAPI.sleepIncr(retry)
                continue
            except ValueError as ve:
                LM.error('Value error on connect {}'.format(ve),
                         LM._LogExtra(*sr, exc=ve, url=req_str, rty=retry))
                raise ve
            except Exception as xe:
                LM.error('Other error on connect {}'.format(xe),
                         LM._LogExtra(*sr, exc=xe, url=req_str, rty=retry))
                raise xe
        else:
            raise last_exc
        #except Exception as e:
        #	print e

# 	def _testalt(self,ref='basic'):
# 		p = os.path.join(os.path.dirname(__file__),cxf or LDSAPI.ath[ref])
# 		self.setAuthentication(Authentication.creds,p, ref)

    @staticmethod
    def encode_auth(auth):
        '''Build and b64 encode a http authentication string. [Needs to be bytes str, hence en/decode()]'''
        if 'domain' in auth:
            astr = '{d}\{u}:{p}'.format(u=auth['user'],
                                        p=auth['pass'],
                                        d=auth['domain']).strip().encode()
        else:
            astr = '{u}:{p}'.format(u=auth['user'],
                                    p=auth['pass']).strip().encode()
        return base64.b64encode(astr).decode()

    def fetchPages(self, psub=''):
        sr = self.__sec, 'Page fetch'
        upd = []
        page = 0
        pagel = None
        morepages = True
        while morepages:
            page = page + 1
            pstr = psub + '&page={0}'.format(page)
            try:
                res = self.connect(plus=pstr)
                if res: self.setResponse(res)
                else: raise HTTPError('No Response using URL {}'.format(pstr))
                #api.dispReq(api.req)
                #api.dispRes(api.res)
            except HTTPError as he:
                LM.error('HTTP Error on page fetch {}'.format(he),
                         LM._LogExtra(*sr, exc=he, url=pstr))
                morepages = False
                raise
                #continue
            except Exception as e:
                #Outer catch of unknown errors
                LM.error('Error on page fetch {}'.format(he),
                         LM._LogExtra(*sr, exc=he, url=pstr))
                raise
            # The logic here is a bit redundant but basically if no last page found then its prob the last page
            # otherwise save the last page value and compare to current page. If they're equal get off loop
            if 'page-last' in self.head:
                pagel = self.head['page-last']['p']
            else:
                morepages = False

            if page == pagel:
                morepages = False

            jdata = json.loads(self.respdata.decode())
            upd += [
                jdata,
            ] if isinstance(jdata, dict) else jdata

        return upd

    @staticmethod
    def sleepIncr(r):
        t = (INIT_MAX_RETRY_ATTEMPTS - r) * SLEEP_RETRY_INCR
        print('tock' if t % 2 else 'tick', t,
              '{}/{}'.format(r, INIT_MAX_RETRY_ATTEMPTS))
        time.sleep(t)

    @staticmethod
    def dispReq(req):
        print('Request\n-------\n')
        print(LDS.kmask(req.get_full_url()), 'auth',
              req.get_header('Authorization'))

    @staticmethod
    def dispRes(res):
        print('Response\n--------\n')
        print(res.info())

    @staticmethod
    def dispJSON(res):
        for l in json.loads(res.read()):
            print('{0} - {1}\n'.format(l[0], l[1]))

    @staticmethod
    def _populate(data):
        return json.dumps({
            "name": data[0],
            "type": data[1],
            "description": data[2],
            "categories": data[3],
            "user": data[4],
            "options": {
                "username": data[5],
                "password": data[6]
            },
            "url_remote": data[7],
            "scan_schedule": data[8]
        })
Example #15
0
def rest_api_request(token,
                     method,
                     api_cmd,
                     api_cmd_headers=None,
                     api_cmd_payload=None,
                     timeout=10):
    """
    Make a rest-api request
    Returns: response as a dictionary
    """

    # signal.signal(signal.SIGALRM, _timeout_handler)
    # if hasattr(signal, 'SIGALRM'):
    #     signal.alarm(timeout)

    LOG.info("%s cmd:%s hdr:%s payload:%s" %
             (method, api_cmd, api_cmd_headers, api_cmd_payload))

    response = None
    try:
        request_info = Request(api_cmd)
        request_info.get_method = lambda: method
        if token:
            request_info.add_header("X-Auth-Token", token.get_id())
        request_info.add_header("Accept", "application/json")

        if api_cmd_headers is not None:
            for header_type, header_value in api_cmd_headers.items():
                request_info.add_header(header_type, header_value)

        if api_cmd_payload is not None:
            request_info.add_data(api_cmd_payload)

        request = urlopen(request_info, timeout=timeout)
        response = request.read()

        if response == "":
            response = json.loads("{}")
        else:
            response = json.loads(response)
        request.close()

        LOG.info("Response=%s" % response)

    except HTTPError as e:
        if 401 == e.code:
            if token:
                token.set_expired()
        LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
        if hasattr(e, 'msg') and e.msg:
            response = json.loads(e.msg)
        else:
            response = json.loads("{}")

        LOG.info("HTTPError response=%s" % (response))
        raise OpenStackRestAPIException(e.message, e.code, "%s" % e)

    except URLError as e:
        LOG.warn("URLError Error e=%s" % (e))
        raise OpenStackException(e.message, "%s" % e)

    except si_exception.SysInvSignalTimeout as e:
        LOG.warn("Timeout Error e=%s" % (e))
        raise OpenStackException(e.message, "%s" % e)

    finally:
        signal.alarm(0)
        return response
Example #16
0
 def add_data(self, data):
     if hasattr(data, "iteritems"):
         data = urlencode(data)
     print(data)
     self.add_header("Content-Length", str(len(data)))
     BaseRequest.add_data(self, data)
Example #17
0
 def add_data(self, data):
     if hasattr(data, "items"):
         data = urlencode(data)
     print(data)
     self.add_header("Content-Length", str(len(data)))
     BaseRequest.add_data(self, data)