Exemplo n.º 1
0
Arquivo: utils_t.py Projeto: ktf/DAS
 def test_httptimestamp(self):
     "Test httptimestamp function"
     day = 'Tue, 28 Aug 2012 16:50:29 GMT'
     self.assertEqual(http_timestamp(day), day)
     gmtime = time.gmtime()
     tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
     self.assertEqual(http_timestamp(gmtime), tstamp)
     ltime  = time.time()
     gmtime = time.gmtime(ltime)
     tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
     self.assertEqual(http_timestamp(gmtime), tstamp)
Exemplo n.º 2
0
 def test_httptimestamp(self):
     "Test httptimestamp function"
     day = 'Tue, 28 Aug 2012 16:50:29 GMT'
     self.assertEqual(http_timestamp(day), day)
     gmtime = time.gmtime()
     tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
     self.assertEqual(http_timestamp(gmtime), tstamp)
     ltime = time.time()
     gmtime = time.gmtime(ltime)
     tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
     self.assertEqual(http_timestamp(gmtime), tstamp)
Exemplo n.º 3
0
def getdata_urllib(url,
                   params,
                   headers=None,
                   expire=3600,
                   post=None,
                   error_expire=300,
                   verbose=0,
                   ckey=None,
                   cert=None,
                   doseq=True,
                   system=None,
                   tstamp=None):
    """
    Invoke URL call and retrieve data from data-service based
    on provided URL and set of parameters. Use post=True to
    invoke POST request.
    """
    contact = 'data-service.'
    if system:
        contact = system + ' ' + contact
    timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
    das_timer(timer_key, verbose)
    encoded_data = urllib.urlencode(params, doseq=doseq)
    if not post:
        url = url + '?' + encoded_data
    if not headers:
        headers = {}
    if tstamp and 'If-Modified-Since' not in headers.keys():
        headers['If-Modified-Since'] = http_timestamp(tstamp)
    if verbose:
        print('+++ getdata, url=%s, headers=%s' % (url, headers))
    req = urllib2.Request(url)
    for key, val in headers.items():
        req.add_header(key, val)
    if verbose > 1:
        handler = urllib2.HTTPHandler(debuglevel=1)
        opener = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    if ckey and cert:
        handler = HTTPSClientAuthHandler(ckey, cert, verbose)
        opener = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    try:
        time0 = time.time()
        if post:
            data = urllib2.urlopen(req, encoded_data)
        else:
            data = urllib2.urlopen(req)
        data_srv_time = time.time() - time0
        info = data.info()
        code = data.getcode()
        if verbose > 1:
            print("+++ response code:", code)
            print("+++ response info\n", info)
        try:  # get HTTP header and look for Expires
            e_time = expire_timestamp(\
                info.__dict__['dict']['expires'])
            if e_time < expire_timestamp(data_srv_time):
                expire = max(e_time, expire_timestamp(expire))
            elif e_time > time.time():
                expire = e_time
        except Exception as _exp:
            pass
    except urllib2.HTTPError as httperror:
        msg  = 'HTTPError, url=%s, args=%s, headers=%s' \
                    % (url, params, headers)
        data = {
            'error': 'Received HTTP error from %s data-service' % contact,
            'reason': msg
        }
        try:
            err = '%s %s' % (contact, extract_http_error(httperror.read()))
            data.update({'error': err})
            msg += '\n' + err
        except Exception as exp:
            data.update({'httperror': None})
            msg += '\n' + str(exp)
        print(msg)
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    except Exception as exp:
        msg  = 'HTTPError, url=%s, args=%s, headers=%s' \
                    % (url, params, headers)
        print(msg + '\n' + str(exp))
        data = {
            'error': 'Received generic error from %s data-service' % contact,
            'reason': msg
        }
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    das_timer(timer_key, verbose)
    return data, expire
Exemplo n.º 4
0
def getdata_urllib(url, params, headers=None, expire=3600, post=None,
    error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None,
    tstamp=None):
    """
    Invoke URL call and retrieve data from data-service based
    on provided URL and set of parameters. Use post=True to
    invoke POST request.
    """
    contact = 'data-service.'
    if  system:
        contact = system + ' ' + contact
    timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
    das_timer(timer_key, verbose)
    encoded_data = urllib.urlencode(params, doseq=doseq)
    if  not post:
        url = url + '?' + encoded_data
    if  not headers:
        headers = {}
    if  tstamp and 'If-Modified-Since' not in headers.keys():
        headers['If-Modified-Since'] = http_timestamp(tstamp)
    if  verbose:
        print('+++ getdata, url=%s, headers=%s' % (url, headers))
    req = urllib2.Request(url)
    for key, val in headers.items():
        req.add_header(key, val)
    if  verbose > 1:
        handler = urllib2.HTTPHandler(debuglevel=1)
        opener  = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    if  ckey and cert:
        handler = HTTPSClientAuthHandler(ckey, cert, verbose)
        opener  = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    try:
        time0 = time.time()
        if  post:
            data = urllib2.urlopen(req, encoded_data)
        else:
            data = urllib2.urlopen(req)
        data_srv_time = time.time()-time0
        info = data.info()
        code = data.getcode()
        if  verbose > 1:
            print("+++ response code:", code)
            print("+++ response info\n", info)
        try: # get HTTP header and look for Expires
            e_time = expire_timestamp(\
                info.__dict__['dict']['expires'])
            if  e_time < expire_timestamp(data_srv_time):
                expire = max(e_time, expire_timestamp(expire))
            elif e_time > time.time():
                expire = e_time
        except Exception as _exp:
            pass
    except urllib2.HTTPError as httperror:
        msg  = 'HTTPError, url=%s, args=%s, headers=%s' \
                    % (url, params, headers)
        data = {'error': 'Received HTTP error from %s data-service' % contact,
                'reason': msg}
        try:
            err  = '%s %s' % (contact, extract_http_error(httperror.read()))
            data.update({'error':err})
            msg += '\n' + err
        except Exception as exp:
            data.update({'httperror': None})
            msg += '\n' + str(exp)
        print(msg)
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    except Exception as exp:
        msg  = 'HTTPError, url=%s, args=%s, headers=%s' \
                    % (url, params, headers)
        print(msg + '\n' + str(exp))
        data = {'error': 'Received generic error from %s data-service' % contact,
                'reason': msg}
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    das_timer(timer_key, verbose)
    return data, expire
Exemplo n.º 5
0
Arquivo: url_utils.py Projeto: ktf/DAS
def getdata_urllib(
    url,
    params,
    headers=None,
    expire=3600,
    post=None,
    error_expire=300,
    verbose=0,
    ckey=None,
    cert=None,
    doseq=True,
    system=None,
    tstamp=None,
):
    """
    Invoke URL call and retrieve data from data-service based
    on provided URL and set of parameters. Use post=True to
    invoke POST request.
    """
    contact = "data-service."
    if system:
        contact = system + " " + contact
    timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True))
    das_timer(timer_key, verbose)
    encoded_data = urllib.urlencode(params, doseq=doseq)
    if not post:
        url = url + "?" + encoded_data
    if not headers:
        headers = {}
    if tstamp and "If-Modified-Since" not in headers.keys():
        headers["If-Modified-Since"] = http_timestamp(tstamp)
    if verbose:
        print "+++ getdata, url=%s, headers=%s" % (url, headers)
    req = urllib2.Request(url)
    for key, val in headers.iteritems():
        req.add_header(key, val)
    if verbose > 1:
        handler = urllib2.HTTPHandler(debuglevel=1)
        opener = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    if ckey and cert:
        handler = HTTPSClientAuthHandler(ckey, cert, verbose)
        opener = urllib2.build_opener(handler)
        urllib2.install_opener(opener)
    try:
        time0 = time.time()
        if post:
            data = urllib2.urlopen(req, encoded_data)
        else:
            data = urllib2.urlopen(req)
        data_srv_time = time.time() - time0
        info = data.info()
        code = data.getcode()
        if verbose > 1:
            print "+++ response code:", code
            print "+++ response info\n", info
        try:  # get HTTP header and look for Expires
            e_time = expire_timestamp(info.__dict__["dict"]["expires"])
            if e_time < expire_timestamp(data_srv_time):
                expire = max(e_time, expire_timestamp(expire))
            elif e_time > time.time():
                expire = e_time
        except Exception as _exp:
            pass
    except urllib2.HTTPError as httperror:
        msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
        data = {"error": "Unable to contact %s" % contact, "reason": msg}
        try:
            err = "%s %s" % (contact, extract_http_error(httperror.read()))
            data.update({"error": err})
            msg += "\n" + err
        except Exception as exp:
            data.update({"httperror": None})
            msg += "\n" + str(exp)
        print msg
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    except Exception as exp:
        msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
        print msg + "\n" + str(exp)
        data = {"error": "Unable to contact %s" % contact, "reason": msg}
        data = json.dumps(data)
        expire = expire_timestamp(error_expire)
    das_timer(timer_key, verbose)
    return data, expire