示例#1
0
 def fetch(self, url, params, cache=True):
     "Fetch data for given api"
     debug = 0
     data = "[]"
     if  cache:
         docid = genkey("url=%s params=%s" % (url, params))
         res = self.storage.fetch_one('cache', {'_id':docid})
         if  res and 'data' in res:
             if  self.verbose:
                 print("%s::fetch url=%s, params=%s, docid=%s" \
                         % (self.name, url, params, docid))
             return res['data']
     if  self.verbose:
         print("%s::fetch url=%s, params=%s" % (self.name, url, params))
         debug = self.verbose-1
     try:
         data = getdata(url, params, debug=debug)
     except Exception as exc:
         print(str(exc))
         for attempt in xrange(3):
             time.sleep(0.1)
             print("Attempt %s" % attempt)
             try:
                 data = getdata(url, params, debug=debug)
                 break
             except Exception as err:
                 print(str(err))
                 pass
     if  cache:
         self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
     return data
示例#2
0
 def fetch(self, api, params=None):
     "Fetch data for given api"
     url = '%s/%s/?%s' % (self.url, api, urllib.urlencode(params, doseq=True))
     docid = genkey("url=%s params=%s" % (url, params))
     res = self.storage.fetch_one('cache', {'_id':docid})
     if  res and 'data' in res:
         if  self.verbose:
             print("%s::fetch url=%s, params=%s, docid=%s" \
                     % (self.name, url, params, docid))
         data = res['data']
     else:
         if  self.verbose:
             print("%s::fetch url=%s, params=%s" % (self.name, url, params))
         # NOTE: popularity DB has two different access points, one
         # within CERN network and out outside. The former does not require
         # authentication, while later passes through CERN SSO.
         # The following block reflects this, in a future, when popularity DB
         # will move into cmsweb domain we'll no longer need it
         if  self.url.find('cms-popularity-prod') != -1 or \
                 self.url.find('cmsweb') != -1:
             data = getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
         else:
             data = sso_getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
         self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
     data = json.loads(data)
     for row in data['DATA']:
         yield row
示例#3
0
 def fetch(self, url, params, cache=True):
     "Fetch data for given api"
     debug = 0
     data = "[]"
     if cache:
         docid = genkey("url=%s params=%s" % (url, params))
         res = self.storage.fetch_one('cache', {'_id': docid})
         if res and 'data' in res:
             if self.verbose:
                 print("%s::fetch url=%s, params=%s, docid=%s" \
                         % (self.name, url, params, docid))
             return res['data']
     if self.verbose:
         print("%s::fetch url=%s, params=%s" % (self.name, url, params))
         debug = self.verbose - 1
     try:
         data = getdata(url, params, debug=debug)
     except Exception as exc:
         print(str(exc))
         for attempt in xrange(3):
             time.sleep(0.1)
             print("Attempt %s" % attempt)
             try:
                 data = getdata(url, params, debug=debug)
                 break
             except Exception as err:
                 print(str(err))
                 pass
     if cache:
         self.storage.insert('cache', {
             '_id': docid,
             'data': data,
             'url': url,
             'params': params
         })
     return data
def popdb_datasets(tstart, tstop, url):
    "Fetch data from popDB for given time frame and print out datasets"
    api = "DSStatInTimeWindow"
    ckey, cert = get_key_cert()
    params = {"tstart": tstart, "tstop": tstop}
    url = "%s/%s/?%s" % (url, api, urllib.urlencode(params, doseq=True))
    data = getdata(url, ckey=ckey, cert=cert, debug=0)
    data = json.loads(data)
    headers = []
    for row in data["DATA"]:
        if not headers:
            headers = row.keys()
            print(",".join(headers))
        out = [str(row[k]) for k in headers]
        print(",".join(out))
示例#5
0
def popdb_datasets(tstart, tstop, url):
    "Fetch data from popDB for given time frame and print out datasets"
    api = 'DSStatInTimeWindow'
    ckey, cert = get_key_cert()
    params = {'tstart': tstart, 'tstop': tstop}
    url = '%s/%s/?%s' % (url, api, urllib.urlencode(params, doseq=True))
    data = getdata(url, ckey=ckey, cert=cert, debug=0)
    data = json.loads(data)
    headers = []
    for row in data['DATA']:
        if not headers:
            headers = row.keys()
            print(','.join(headers))
        out = [str(row[k]) for k in headers]
        print(','.join(out))
示例#6
0
 def fetch(self, api, params=None):
     "Fetch data for given api"
     url = '%s/%s/?%s' % (self.url, api, urllib.urlencode(params, doseq=True))
     docid = genkey("url=%s params=%s" % (url, params))
     res = self.storage.fetch_one('cache', {'_id':docid})
     if  res and 'data' in res:
         if  self.verbose:
             print("%s::fetch url=%s, params=%s, docid=%s" \
                     % (self.name, url, params, docid))
         data = res['data']
     else:
         if  self.verbose:
             print("%s::fetch url=%s, params=%s" % (self.name, url, params))
         data = getdata(url, ckey=self.ckey, cert=self.cert, debug=self.verbose)
         self.storage.insert('cache', {'_id':docid, 'data': data, 'url': url, 'params': params})
     data = json.loads(data)
     for row in data['DATA']:
         yield row
示例#7
0
def popdb_datasets(tstart, tstop, url):
    "Fetch data from popDB for given time frame and print out datasets"
    api = 'DSStatInTimeWindow'
    ckey, cert = get_key_cert()
    params = {'tstart':tstart, 'tstop':tstop}
    url = '%s/%s?%s' % (url, api, urllib.urlencode(params, doseq=True))
    # NOTE: popularity DB has two different access points, one
    # within CERN network and out outside. The former does not require
    # authentication, while later passes through CERN SSO.
    # The following block reflects this, in a future, when popularity DB
    # will move into cmsweb domain we'll no longer need it
    if  url.find('cms-popularity-prod.cern.ch') != -1:
        data = getdata(url, ckey=ckey, cert=cert, debug=0)
    else:
        data = sso_getdata(url, ckey=ckey, cert=cert, debug=0)
    data = json.loads(data)
    headers = []
    for row in data['DATA']:
        if  not headers:
            headers = row.keys()
            print(','.join(headers))
        out = [str(row[k]) for k in headers]
        print(','.join(out))