def test_httptimestamp(self):
"Test httptimestamp function"
day = 'Tue, 28 Aug 2012 16:50:29 GMT'
self.assertEqual(http_timestamp(day), day)
gmtime = time.gmtime()
tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
self.assertEqual(http_timestamp(gmtime), tstamp)
ltime = time.time()
gmtime = time.gmtime(ltime)
tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
self.assertEqual(http_timestamp(gmtime), tstamp)
def test_httptimestamp(self):
"Test httptimestamp function"
day = 'Tue, 28 Aug 2012 16:50:29 GMT'
self.assertEqual(http_timestamp(day), day)
gmtime = time.gmtime()
tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
self.assertEqual(http_timestamp(gmtime), tstamp)
ltime = time.time()
gmtime = time.gmtime(ltime)
tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime)
self.assertEqual(http_timestamp(gmtime), tstamp)
def getdata_urllib(url,
params,
headers=None,
expire=3600,
post=None,
error_expire=300,
verbose=0,
ckey=None,
cert=None,
doseq=True,
system=None,
tstamp=None):
"""
Invoke URL call and retrieve data from data-service based
on provided URL and set of parameters. Use post=True to
invoke POST request.
"""
contact = 'data-service.'
if system:
contact = system + ' ' + contact
timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + '?' + encoded_data
if not headers:
headers = {}
if tstamp and 'If-Modified-Since' not in headers.keys():
headers['If-Modified-Since'] = http_timestamp(tstamp)
if verbose:
print('+++ getdata, url=%s, headers=%s' % (url, headers))
req = urllib2.Request(url)
for key, val in headers.items():
req.add_header(key, val)
if verbose > 1:
handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if ckey and cert:
handler = HTTPSClientAuthHandler(ckey, cert, verbose)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
try:
time0 = time.time()
if post:
data = urllib2.urlopen(req, encoded_data)
else:
data = urllib2.urlopen(req)
data_srv_time = time.time() - time0
info = data.info()
code = data.getcode()
if verbose > 1:
print("+++ response code:", code)
print("+++ response info\n", info)
try: # get HTTP header and look for Expires
e_time = expire_timestamp(\
info.__dict__['dict']['expires'])
if e_time < expire_timestamp(data_srv_time):
expire = max(e_time, expire_timestamp(expire))
elif e_time > time.time():
expire = e_time
except Exception as _exp:
pass
except urllib2.HTTPError as httperror:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
data = {
'error': 'Received HTTP error from %s data-service' % contact,
'reason': msg
}
try:
err = '%s %s' % (contact, extract_http_error(httperror.read()))
data.update({'error': err})
msg += '\n' + err
except Exception as exp:
data.update({'httperror': None})
msg += '\n' + str(exp)
print(msg)
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
print(msg + '\n' + str(exp))
data = {
'error': 'Received generic error from %s data-service' % contact,
'reason': msg
}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
def getdata_urllib(url, params, headers=None, expire=3600, post=None,
error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None,
tstamp=None):
"""
Invoke URL call and retrieve data from data-service based
on provided URL and set of parameters. Use post=True to
invoke POST request.
"""
contact = 'data-service.'
if system:
contact = system + ' ' + contact
timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + '?' + encoded_data
if not headers:
headers = {}
if tstamp and 'If-Modified-Since' not in headers.keys():
headers['If-Modified-Since'] = http_timestamp(tstamp)
if verbose:
print('+++ getdata, url=%s, headers=%s' % (url, headers))
req = urllib2.Request(url)
for key, val in headers.items():
req.add_header(key, val)
if verbose > 1:
handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if ckey and cert:
handler = HTTPSClientAuthHandler(ckey, cert, verbose)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
try:
time0 = time.time()
if post:
data = urllib2.urlopen(req, encoded_data)
else:
data = urllib2.urlopen(req)
data_srv_time = time.time()-time0
info = data.info()
code = data.getcode()
if verbose > 1:
print("+++ response code:", code)
print("+++ response info\n", info)
try: # get HTTP header and look for Expires
e_time = expire_timestamp(\
info.__dict__['dict']['expires'])
if e_time < expire_timestamp(data_srv_time):
expire = max(e_time, expire_timestamp(expire))
elif e_time > time.time():
expire = e_time
except Exception as _exp:
pass
except urllib2.HTTPError as httperror:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
data = {'error': 'Received HTTP error from %s data-service' % contact,
'reason': msg}
try:
err = '%s %s' % (contact, extract_http_error(httperror.read()))
data.update({'error':err})
msg += '\n' + err
except Exception as exp:
data.update({'httperror': None})
msg += '\n' + str(exp)
print(msg)
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
print(msg + '\n' + str(exp))
data = {'error': 'Received generic error from %s data-service' % contact,
'reason': msg}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
def getdata_urllib(
url,
params,
headers=None,
expire=3600,
post=None,
error_expire=300,
verbose=0,
ckey=None,
cert=None,
doseq=True,
system=None,
tstamp=None,
):
"""
Invoke URL call and retrieve data from data-service based
on provided URL and set of parameters. Use post=True to
invoke POST request.
"""
contact = "data-service."
if system:
contact = system + " " + contact
timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + "?" + encoded_data
if not headers:
headers = {}
if tstamp and "If-Modified-Since" not in headers.keys():
headers["If-Modified-Since"] = http_timestamp(tstamp)
if verbose:
print "+++ getdata, url=%s, headers=%s" % (url, headers)
req = urllib2.Request(url)
for key, val in headers.iteritems():
req.add_header(key, val)
if verbose > 1:
handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if ckey and cert:
handler = HTTPSClientAuthHandler(ckey, cert, verbose)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
try:
time0 = time.time()
if post:
data = urllib2.urlopen(req, encoded_data)
else:
data = urllib2.urlopen(req)
data_srv_time = time.time() - time0
info = data.info()
code = data.getcode()
if verbose > 1:
print "+++ response code:", code
print "+++ response info\n", info
try: # get HTTP header and look for Expires
e_time = expire_timestamp(info.__dict__["dict"]["expires"])
if e_time < expire_timestamp(data_srv_time):
expire = max(e_time, expire_timestamp(expire))
elif e_time > time.time():
expire = e_time
except Exception as _exp:
pass
except urllib2.HTTPError as httperror:
msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
data = {"error": "Unable to contact %s" % contact, "reason": msg}
try:
err = "%s %s" % (contact, extract_http_error(httperror.read()))
data.update({"error": err})
msg += "\n" + err
except Exception as exp:
data.update({"httperror": None})
msg += "\n" + str(exp)
print msg
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
print msg + "\n" + str(exp)
data = {"error": "Unable to contact %s" % contact, "reason": msg}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
