def fetch(self, url, params, cache=True):
"Fetch data for given api"
debug = 0
data = "[]"
if cache:
docid = genkey("url=%s params=%s" % (url, params))
res = self.storage.fetch_one('cache', {'_id':docid})
if res and 'data' in res:
if self.verbose:
print("%s::fetch url=%s, params=%s, docid=%s" \
% (self.name, url, params, docid))
return res['data']
if self.verbose:
print("%s::fetch url=%s, params=%s" % (self.name, url, params))
debug = self.verbose-1
try:
data = getdata(url, params, debug=debug)
except Exception as exc:
print(str(exc))
for attempt in xrange(3):
time.sleep(0.1)
print("Attempt %s" % attempt)
try:
data = getdata(url, params, debug=debug)
break
except Exception as err:
print(str(err))
pass
if cache:
self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
return data
def fetch(self, api, params=None):
"Fetch data for given api"
url = '%s/%s/?%s' % (self.url, api, urllib.urlencode(params, doseq=True))
docid = genkey("url=%s params=%s" % (url, params))
res = self.storage.fetch_one('cache', {'_id':docid})
if res and 'data' in res:
if self.verbose:
print("%s::fetch url=%s, params=%s, docid=%s" \
% (self.name, url, params, docid))
data = res['data']
else:
if self.verbose:
print("%s::fetch url=%s, params=%s" % (self.name, url, params))
# NOTE: popularity DB has two different access points, one
# within CERN network and out outside. The former does not require
# authentication, while later passes through CERN SSO.
# The following block reflects this, in a future, when popularity DB
# will move into cmsweb domain we'll no longer need it
if self.url.find('cms-popularity-prod') != -1 or \
self.url.find('cmsweb') != -1:
data = getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
else:
data = sso_getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
data = json.loads(data)
for row in data['DATA']:
yield row
def fetch(self, url, params, cache=True):
"Fetch data for given api"
debug = 0
data = "[]"
if cache:
docid = genkey("url=%s params=%s" % (url, params))
res = self.storage.fetch_one('cache', {'_id': docid})
if res and 'data' in res:
if self.verbose:
print("%s::fetch url=%s, params=%s, docid=%s" \
% (self.name, url, params, docid))
return res['data']
if self.verbose:
print("%s::fetch url=%s, params=%s" % (self.name, url, params))
debug = self.verbose - 1
try:
data = getdata(url, params, debug=debug)
except Exception as exc:
print(str(exc))
for attempt in xrange(3):
time.sleep(0.1)
print("Attempt %s" % attempt)
try:
data = getdata(url, params, debug=debug)
break
except Exception as err:
print(str(err))
pass
if cache:
self.storage.insert('cache', {
'_id': docid,
'data': data,
'url': url,
'params': params
})
return data
def popdb_datasets(tstart, tstop, url):
"Fetch data from popDB for given time frame and print out datasets"
api = "DSStatInTimeWindow"
ckey, cert = get_key_cert()
params = {"tstart": tstart, "tstop": tstop}
url = "%s/%s/?%s" % (url, api, urllib.urlencode(params, doseq=True))
data = getdata(url, ckey=ckey, cert=cert, debug=0)
data = json.loads(data)
headers = []
for row in data["DATA"]:
if not headers:
headers = row.keys()
print(",".join(headers))
out = [str(row[k]) for k in headers]
print(",".join(out))
def popdb_datasets(tstart, tstop, url):
"Fetch data from popDB for given time frame and print out datasets"
api = 'DSStatInTimeWindow'
ckey, cert = get_key_cert()
params = {'tstart': tstart, 'tstop': tstop}
url = '%s/%s/?%s' % (url, api, urllib.urlencode(params, doseq=True))
data = getdata(url, ckey=ckey, cert=cert, debug=0)
data = json.loads(data)
headers = []
for row in data['DATA']:
if not headers:
headers = row.keys()
print(','.join(headers))
out = [str(row[k]) for k in headers]
print(','.join(out))
def fetch(self, api, params=None):
"Fetch data for given api"
url = '%s/%s/?%s' % (self.url, api, urllib.urlencode(params, doseq=True))
docid = genkey("url=%s params=%s" % (url, params))
res = self.storage.fetch_one('cache', {'_id':docid})
if res and 'data' in res:
if self.verbose:
print("%s::fetch url=%s, params=%s, docid=%s" \
% (self.name, url, params, docid))
data = res['data']
else:
if self.verbose:
print("%s::fetch url=%s, params=%s" % (self.name, url, params))
data = getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
data = json.loads(data)
for row in data['DATA']:
yield row
def popdb_datasets(tstart, tstop, url):
"Fetch data from popDB for given time frame and print out datasets"
api = 'DSStatInTimeWindow'
ckey, cert = get_key_cert()
params = {'tstart':tstart, 'tstop':tstop}
url = '%s/%s?%s' % (url, api, urllib.urlencode(params, doseq=True))
# NOTE: popularity DB has two different access points, one
# within CERN network and out outside. The former does not require
# authentication, while later passes through CERN SSO.
# The following block reflects this, in a future, when popularity DB
# will move into cmsweb domain we'll no longer need it
if url.find('cms-popularity-prod.cern.ch') != -1:
data = getdata(url, ckey=ckey, cert=cert, debug=0)
else:
data = sso_getdata(url, ckey=ckey, cert=cert, debug=0)
data = json.loads(data)
headers = []
for row in data['DATA']:
if not headers:
headers = row.keys()
print(','.join(headers))
out = [str(row[k]) for k in headers]
print(','.join(out))