def get_data(host, query, idx, limit, debug, threshold=300, ckey=None, cert=None):
"""Contact DAS server and retrieve data for given DAS query"""
if not ckey and not cert:
ckey, cert = get_key_cert()
params = {'input':query, 'idx':idx, 'limit':limit}
path = '/das/cache'
pat = re.compile('http[s]{0,1}://')
if not pat.match(host):
msg = 'Invalid hostname: %s' % host
raise Exception(msg)
url = host + path
headers = {"Accept": "application/json"}
encoded_data = urllib.urlencode(params, doseq=True)
url += '?%s' % encoded_data
req = urllib2.Request(url=url, headers=headers)
if ckey and cert:
ckey = fullpath(ckey)
cert = fullpath(cert)
hdlr = HTTPSClientAuthHandler(ckey, cert)
else:
hdlr = urllib2.HTTPHandler(debuglevel=debug)
opener = urllib2.build_opener(hdlr)
fdesc = opener.open(req)
data = fdesc.read()
fdesc.close()
pat = re.compile(r'^[a-z0-9]{32}')
if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
pid = data
else:
pid = None
sleep = 1 # initial waiting time in seconds
wtime = 30 # final waiting time in seconds
time0 = time.time()
while pid:
params.update({'pid':data})
encoded_data = urllib.urlencode(params, doseq=True)
url = host + path + '?%s' % encoded_data
req = urllib2.Request(url=url, headers=headers)
try:
fdesc = opener.open(req)
data = fdesc.read()
fdesc.close()
except urllib2.HTTPError as err:
return json.dumps({"status":"fail", "reason":str(err)})
if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
pid = data
else:
pid = None
time.sleep(sleep)
if sleep < wtime:
sleep *= 2
else:
sleep = wtime
if (time.time()-time0) > threshold:
reason = "client timeout after %s sec" % int(time.time()-time0)
return json.dumps({"status":"fail", "reason":reason})
return data
def get_data_helper(url,
kwargs=None,
headers=None,
verbose=None,
decoder='json',
post=False,
ckey=None,
cert=None):
"""Retrieve data helper function"""
if url.find('https') != -1:
if not ckey and not cert:
ckey, cert = get_key_cert()
else:
ckey = None
cert = None
if kwargs:
params = kwargs
else:
params = {}
if url.find('/datasets') != -1: # DBS3 use case
params.update({'dataset_access_type': 'PRODUCTION', 'detail': 'True'})
encoded_data = urllib.urlencode(params, doseq=True)
if not post:
url = url + '?' + encoded_data
if verbose:
print "Request:", url, encoded_data, headers, ckey, cert
req = urllib2.Request(url)
if headers:
for key, val in headers.items():
req.add_header(key, val)
else:
headers = {'Accept': 'application/json;text/json'}
if cert:
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if post:
print "POST", req, url, encoded_data, params
res = urllib2.urlopen(req, json.dumps(params))
else:
res = urllib2.urlopen(req)
if decoder == 'json':
data = json.load(res)
else:
data = res.read()
return data
def get_data_and_close(url, ckey=None, cert=None, headers={'Accept': '*/*'}):
"Context Manager to read data from given URL"
if not ckey and not cert:
ckey, cert = get_key_cert()
req = urllib2.Request(url)
if headers:
for key, val in headers.items():
req.add_header(key, val)
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
data = urllib2.urlopen(req)
try:
yield data
finally:
data.close()
def get_data_and_close(url, ckey=None, cert=None, headers={'Accept':'*/*'}):
"Context Manager to read data from given URL"
if not ckey and not cert:
ckey, cert = get_key_cert()
req = urllib2.Request(url)
if headers:
for key, val in headers.items():
req.add_header(key, val)
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
data = urllib2.urlopen(req)
try:
yield data
finally:
data.close()
def get_data_helper(url, kwargs=None, headers=None,
verbose=None, decoder='json', post=False, ckey=None, cert=None):
"""Retrieve data helper function"""
if url.find('https') != -1:
if not ckey and not cert:
ckey, cert = get_key_cert()
else:
ckey = None
cert = None
if kwargs:
params = kwargs
else:
params = {}
if url.find('/datasets') != -1: # DBS3 use case
params.update({'dataset_access_type':'PRODUCTION', 'detail':'True'})
encoded_data = urllib.urlencode(params, doseq=True)
if not post:
url = url + '?' + encoded_data
if verbose:
print "Request:", url, encoded_data, headers, ckey, cert
req = urllib2.Request(url)
if headers:
for key, val in headers.items():
req.add_header(key, val)
else:
headers = {'Accept':'application/json;text/json'}
if cert:
handler = HTTPSClientAuthHandler(ckey, cert)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if post:
print "POST", req, url, encoded_data, params
res = urllib2.urlopen(req, json.dumps(params))
else:
res = urllib2.urlopen(req)
if decoder == 'json':
try:
data = json.load(res)
except Exception as err:
data = {}
print_error(str(err))
else:
data = res.read()
return data
def get_data(host,
query,
idx,
limit,
debug,
threshold=300,
ckey=None,
cert=None):
"""Contact DAS server and retrieve data for given DAS query"""
if not ckey and not cert:
ckey, cert = get_key_cert()
params = {'input': query, 'idx': idx, 'limit': limit}
path = '/das/cache'
pat = re.compile('http[s]{0,1}://')
if not pat.match(host):
msg = 'Invalid hostname: %s' % host
raise Exception(msg)
url = host + path
headers = {"Accept": "application/json"}
encoded_data = urllib.urlencode(params, doseq=True)
url += '?%s' % encoded_data
req = urllib2.Request(url=url, headers=headers)
if ckey and cert:
ckey = fullpath(ckey)
cert = fullpath(cert)
hdlr = HTTPSClientAuthHandler(ckey, cert)
else:
hdlr = urllib2.HTTPHandler(debuglevel=debug)
opener = urllib2.build_opener(hdlr)
fdesc = opener.open(req)
data = fdesc.read()
fdesc.close()
pat = re.compile(r'^[a-z0-9]{32}')
if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
pid = data
else:
pid = None
sleep = 1 # initial waiting time in seconds
wtime = 30 # final waiting time in seconds
time0 = time.time()
while pid:
params.update({'pid': data})
encoded_data = urllib.urlencode(params, doseq=True)
url = host + path + '?%s' % encoded_data
req = urllib2.Request(url=url, headers=headers)
try:
fdesc = opener.open(req)
data = fdesc.read()
fdesc.close()
except urllib2.HTTPError as err:
return json.dumps({"status": "fail", "reason": str(err)})
if data and isinstance(data,
str) and pat.match(data) and len(data) == 32:
pid = data
else:
pid = None
time.sleep(sleep)
if sleep < wtime:
sleep *= 2
else:
sleep = wtime
if (time.time() - time0) > threshold:
reason = "client timeout after %s sec" % int(time.time() - time0)
return json.dumps({"status": "fail", "reason": reason})
return data