def insert_query_records(self, dasquery): """ Insert DAS query records into DAS cache and return list of services which will answer this query """ services = dasquery.services self.logger.info('Potential services = %s' % services) if not services: msg = 'No data-services for query %s' % dasquery msg += 'mongo_query: %s' % dasquery.mongo_query msg += 'params: %s' % dasquery.params() print dastimestamp('DAS WARNING '), msg # get list of URI which can answer this query ack_services = [] for srv in services: gen = [t for t in getattr(getattr(self, srv), 'apimap')(dasquery)] for url, api, args, iformat, expire in gen: header = dasheader(srv, dasquery, expire, api, url, ctime=0) self.rawcache.insert_query_record(dasquery, header) if srv not in ack_services: ack_services.append(srv) if not ack_services: ack_services = services if dasquery.query.find('records ') != -1: srv_status = True # skip DAS queries w/ records request expire = 2*60 # 2 minutes, it should be overwriten by data-srv header = dasheader("das", dasquery, expire, api='das_core', services=dict(das=ack_services)) header['lookup_keys'] = [] self.rawcache.insert_query_record(dasquery, header) das_timer('das_record', self.verbose) return ack_services
def insert_query_record(self, dasquery, header): """ Insert query record into DAS cache. """ # check presence of API record in a cache dasheader = header['das'] system = dasheader['system'] api = dasheader['api'] collection = 'cache' check_query = True expire = dasheader.get('expire', None) if expire: dasheader['expire'] = adjust_expire(expire) if not self.incache(dasquery, collection, system, api, check_query): msg = "query=%s, header=%s" % (dasquery, header) self.logger.debug(msg) q_record = dict(das=dasheader, query=dasquery.storage_query) q_record['das']['record'] = record_codes('query_record') q_record['das']['status'] = "requested" q_record['qhash'] = dasquery.qhash q_record['das']['ctime'] = [time.time()] res = self.col.insert_one(q_record) if not res: msg = 'unable to insert query record' print(dastimestamp('DAS ERROR '), dasquery, msg, ', will retry') time.sleep(1) res = self.col.insert(q_record) if not res: print(dastimestamp('DAS ERROR '), dasquery, msg)
def insert_query_record(self, dasquery, header): """ Insert query record into DAS cache. """ # check presence of API record in a cache dasheader = header['das'] system = dasheader['system'] api = dasheader['api'] collection = 'cache' check_query = True expire = dasheader.get('expire', None) if expire: dasheader['expire'] = adjust_expire(expire) if not self.incache(dasquery, collection, system, api, check_query): msg = "query=%s, header=%s" % (dasquery, header) self.logger.debug(msg) q_record = dict(das=dasheader, query=dasquery.storage_query) q_record['das']['record'] = record_codes('query_record') q_record['das']['status'] = "requested" q_record['qhash'] = dasquery.qhash q_record['das']['ctime'] = [time.time()] res = self.col.insert_one(q_record) if not res: msg = 'unable to insert query record' print(dastimestamp('DAS ERROR '), dasquery, msg, ', will retry') time.sleep(1) res = self.col.insert(q_record) if not res: print(dastimestamp('DAS ERROR '), dasquery, msg)
def datasets_dbs(self): """ Retrieve a list of DBS datasets (DBS2) """ query = "find dataset,dataset.status" params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} encoded_data = urllib.urlencode(params, doseq=True) url = self.dbs_url + "?" + encoded_data req = urllib2.Request(url) try: stream = urllib2.urlopen(req) except urllib2.HTTPError: msg = "Fail to contact %s" % url print dastimestamp("DAS ERROR"), msg raise Exception(msg) except Exception as exc: print_exc(exc) msg = "Fail to contact %s" % url print dastimestamp("DAS ERROR"), msg raise Exception(msg) gen = qlxml_parser(stream, "dataset") for row in gen: dataset = row["dataset"]["dataset"] rec = {"dataset": dataset} if self.write_hash: storage_query = { "fields": ["dataset"], "spec": [{"key": "dataset.name", "value": '"%s"' % dataset}], "instance": self.dbcoll, } rec.update({"qhash": genkey(storage_query)}) if row["dataset"]["dataset.status"] == "VALID": yield rec stream.close()
def dataset_info(urls, datasetdict, verbose=0): """ Request blockReplicas information from Phedex for a given dataset or a list of dataset (use POST request in later case). Update MongoDB with aggregated information about dataset: site, size, nfiles, nblocks. """ url = urls.get('phedex') + '/blockReplicas' params = {'dataset': [d for d in datasetdict.keys()]} headers = {'Accept':'application/json;text/json'} data, _ = getdata(url, params, headers, post=True, \ ckey=CKEY, cert=CERT, verbose=verbose, system='dbs_phedex') if isinstance(data, basestring): # no response dastimestamp('DBS_PHEDEX ERROR: %s' % data) return jsondict = json.load(data) data.close() for row in jsondict['phedex']['block']: dataset = row['name'].split('#')[0] for rep in row['replica']: rec = dict(dataset=dataset, nfiles=row['files'], size=row['bytes'], site=rep['node'], se=rep['se'], custodial=rep['custodial']) rec.update(datasetdict[dataset]) yield rec data.close()
def check_pid(self, pid, ahash): """ Check status of given pid and return appropriate page content. This is a server callback function for ajaxCheckPid, see js/ajax_utils.js """ cherrypy.response.headers['Cache-Control'] = 'no-cache' cherrypy.response.headers['Pragma'] = 'no-cache' img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base page = '' try: if self.taskmgr.is_alive(pid): page = img + " processing PID=%s" % pid else: kwargs = self.reqmgr.get(pid) if kwargs and kwargs.has_key('dasquery'): del kwargs['dasquery'] # if no kwargs (another request delete it) # use logging DB to look-up user request via ahash if not kwargs: spec = {'ahash':ahash} skey = [('ts', DESCENDING)] res = [r for r in self.logcol.find(spec).sort(skey)] kwargs = res[0]['args'] self.adjust_input(kwargs) self.reqmgr.remove(pid) page = self.get_page_content(kwargs) except Exception as err: msg = 'check_pid fails for pid=%s' % pid print dastimestamp('DAS WEB ERROR '), msg print_exc(err) self.reqmgr.remove(pid) self.taskmgr.remove(pid) return self.error(gen_error_msg({'pid':pid}), wrap=False) return page
def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves DAS records from the cache" msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection) self.logger.info(msg) idx = int(idx) fields, filter_cond = self.get_fields(dasquery) if fields == None: fields = [] if not fields: spec = dasquery.mongo_query.get('spec', {}) elif dasquery.hashes: spec = { 'qhash': { '$in': dasquery.hashes }, 'das.record': spec4data_records() } else: spec = {'qhash': dasquery.qhash, 'das.record': spec4data_records()} if filter_cond: spec.update(filter_cond) if 'records' in dasquery.query: fields = None # retrieve all fields for records DAS query else: # be sure to extract das internal keys fields += das_record_keys() # try to get sort keys all the time to get ordered list of # docs which allow unique_filter to apply afterwards skeys = self.mongo_sort_keys(collection, dasquery) res = self.get_records(collection, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) counter = 0 for row in res: counter += 1 yield row msg = 'qhash %s, found %s record(s) in %s collection' \ % (dasquery.qhash, counter, collection) print(dastimestamp('DAS INFO '), msg) if counter: msg = "yield %s record(s)" % counter self.logger.info(msg) # if no raw records were yield we look-up possible error records # and reset timestamp for record with system:['das'] if not counter: spec = {'qhash': dasquery.qhash} nrec = self.col.find(spec, **PYMONGO_OPTS).count() if nrec: msg = "for query %s, found %s non-result record(s)" \ % (dasquery, nrec) print(dastimestamp('DAS WARNING'), msg) for rec in self.col.find(spec, **PYMONGO_OPTS): if 'query' in rec: print(dastimestamp('DAS das record'), rec) self.update_das_expire(dasquery, etstamp())
def getdata_pycurl(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None): "Fetch data via pycurl library" contact = 'data-service.' if system: contact = system + ' ' + contact if isinstance(params, dict): timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) else: timer_key = '%s?%s' % (url, params) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, \ error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) data = { 'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg, 'ts': time.time() } try: reason = extract_http_error(httperror.read()) data.update({'reason': reason, 'request': msg}) # TODO: err variable did not exit in this function! msg += '\n' + reason except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(dastimestamp('getdata_pycurl'), msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp)) data = { 'error': 'Received generic error from %s data-service' % contact, 'reason': msg, 'ts': time.time() } data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves DAS records from the cache" msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection) self.logger.info(msg) idx = int(idx) fields, filter_cond = self.get_fields(dasquery) if fields == None: fields = [] if not fields: spec = dasquery.mongo_query.get('spec', {}) elif dasquery.hashes: spec = {'qhash':{'$in':dasquery.hashes}, 'das.record': spec4data_records()} else: spec = {'qhash':dasquery.qhash, 'das.record': spec4data_records()} if filter_cond: spec.update(filter_cond) if 'records' in dasquery.query: fields = None # retrieve all fields for records DAS query else: # be sure to extract das internal keys fields += das_record_keys() # try to get sort keys all the time to get ordered list of # docs which allow unique_filter to apply afterwards skeys = self.mongo_sort_keys(collection, dasquery) res = self.get_records(collection, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) counter = 0 for row in res: counter += 1 yield row msg = 'qhash %s, found %s record(s) in %s collection' \ % (dasquery.qhash, counter, collection) print(dastimestamp('DAS INFO '), msg) if counter: msg = "yield %s record(s)" % counter self.logger.info(msg) # if no raw records were yield we look-up possible error records # and reset timestamp for record with system:['das'] if not counter: spec = {'qhash':dasquery.qhash} nrec = self.col.find(spec, **PYMONGO_OPTS).count() if nrec: msg = "for query %s, found %s non-result record(s)" \ % (dasquery, nrec) print(dastimestamp('DAS WARNING'), msg) for rec in self.col.find(spec, **PYMONGO_OPTS): if 'query' in rec: print(dastimestamp('DAS das record'), rec) self.update_das_expire(dasquery, etstamp())
def busy(self): """ Check server load and report busy status if nrequests - nworkers > queue limit """ nrequests = self.reqmgr.size() if (nrequests - self.taskmgr.nworkers()) > self.queue_limit: msg = "#request=%s, queue_limit=%s, #workers=%s" % (nrequests, self.taskmgr.nworkers(), self.queue_limit) print dastimestamp("DAS WEB SERVER IS BUSY "), msg return True return False
def process(gen): "Process generator from getdata" for row in gen: if "error" in row: error = row.get("error") reason = row.get("reason", "") print dastimestamp("DAS ERROR"), error, reason yield row continue if "data" in row: yield json.loads(row["data"])
def getdata_pycurl( url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, ): "Fetch data via pycurl library" contact = "data-service." if system: contact = system + " " + contact timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = "urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} try: reason = extract_http_error(httperror.read()) data.update({"reason": reason, "request": msg}) msg += "\n" + err except Exception as exp: data.update({"httperror": None}) msg += "\n" + str(exp) print dastimestamp("getdata_pycurl"), msg data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = "HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) print dastimestamp("getdata_pycurl"), msg + "\n" + str(exp) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def update(self): """ Update DBS collection with a fresh copy of datasets. Upon first insert of datasets we add dataset:__POPULATED__ record to be used as a flag that cache was populated in this cache. """ if SKIP_UPDATES: return None dbc = self.col if not dbc: print "%s DBSDaemon %s, no connection to DB" % (dastimestamp(), self.dbcoll) return try: time0 = round(time.time()) udict = {"$set": {"ts": time0}} cdict = {"dataset": "__POPULATED__"} gen = self.datasets() msg = "" if not dbc.count(): try: # perform bulk insert operation while True: if not dbc.insert(itertools.islice(gen, self.cache_size)): break except InvalidOperation as err: # please note we need to inspect error message to # distinguish InvalidOperation from generate exhastion if str(err) == "cannot do an empty bulk insert": dbc.insert(cdict) pass except Exception as err: pass # remove records with old ts dbc.remove({"ts": {"$lt": time0 - self.expire}}) msg = "inserted new" else: # we already have records, update their ts for row in gen: spec = dict(dataset=row["dataset"]) dbc.update(spec, udict, upsert=True) msg = "updated old" if find_one(dbc, cdict): dbc.update(cdict, udict) print "%s DBSDaemon %s, %s %s records in %s sec" % ( dastimestamp(), self.dbcoll, msg, dbc.count(), round(time.time() - time0), ) except Exception as exc: print "%s DBSDaemon %s, fail to update, reason %s" % (dastimestamp(), self.dbcoll, str(exc))
def init(self): """ Init db connection and check that it is alive """ try: indexes = [('value', ASCENDING), ('ts', ASCENDING)] create_indexes(self.col, indexes) if not KEEP_EXISTING_RECORDS_ON_RESTART: self.col.remove() except Exception as exc: print dastimestamp(), exc
def update(self): """ Update DBS collection with a fresh copy of datasets. Upon first insert of datasets we add dataset:__POPULATED__ record to be used as a flag that cache was populated in this cache. """ if SKIP_UPDATES: return None dbc = self.col if not dbc: print("%s DBSDaemon %s, no connection to DB" \ % (dastimestamp(), self.dbcoll)) return try: time0 = round(time.time()) udict = {'$set':{'ts':time0}} cdict = {'dataset':'__POPULATED__'} gen = self.datasets() msg = '' if not dbc.count(): try: # perform bulk insert operation res = dbc.insert_many(gen) except InvalidOperation as err: # please note we need to inspect error message to # distinguish InvalidOperation from generate exhastion if str(err) == 'cannot do an empty bulk insert': dbc.insert(cdict) pass except Exception as err: pass # remove records with old ts spec = {'ts':{'$lt':time0-self.expire}} dbc.delete_many(spec) msg = 'inserted' else: # we already have records, update their ts for row in gen: spec = dict(dataset=row['dataset']) dbc.update(spec, udict, upsert=True) msg = 'updated' if find_one(dbc, cdict): dbc.update(cdict, udict) print("%s DBSDaemon %s, %s %s records in %s sec" \ % (dastimestamp(), self.dbcoll, msg, dbc.count(), round(time.time()-time0))) except Exception as exc: print("%s DBSDaemon %s, fail to update, reason %s" \ % (dastimestamp(), self.dbcoll, str(exc)))
def init(self): """ Establish connection to MongoDB back-end and create DB. """ col = None try: conn = db_connection(self.dburi) if conn: dbc = conn[self.dbname] col = dbc[self.colname] # print "### DASMapping:init started successfully" except ConnectionFailure as _err: tstamp = dastimestamp("") thread = threading.current_thread() print "### MongoDB connection failure thread=%s, id=%s, time=%s" % (thread.name, thread.ident, tstamp) except Exception as exc: print_exc(exc) if col: index = [ ("type", DESCENDING), ("system", DESCENDING), ("urn", DESCENDING), ("das_map.das_key", DESCENDING), ("das_map.rec_key", DESCENDING), ("das_map.api_arg", DESCENDING), ] create_indexes(col, index)
def update(self): """ Update some the input values collection for current input field """ if SKIP_UPDATES: return None time0 = time.time() values = self.fetch_values() #print gen if not self.col.count(): try: # perform bulk insert operation self.col.insert(itertools.islice(values, self.cache_size)) # break except InvalidOperation: pass else: # we already have records, update their ts for val in values: spec = dict(value=val['value']) self.col.update(spec, {'$set': {'ts': time0}}, upsert=True) # remove records with old ts self.col.remove({'ts': {'$lt': time0 - self.expire}}) print("%s InputValuesTracker updated" \ " %s collection in %s sec, nrec=%s" \ % (dastimestamp(), self.dbcoll, time.time() - time0, self.col.count()))
def dasmap_reload_handler(self): """ reload KWS after DASMaps reloaded """ print dastimestamp('KWS reloading on DASMaps reload') try: self.dbs_instances = self.dasmgr.mapping.dbs_instances() self.dbs_global = self.dasmgr.mapping.dbs_global_instance() self.kws = KeywordSearchHandler(self.dasmgr) except ConnectionFailure: tstamp = dastimestamp('') mythr = threading.current_thread() print "### MongoDB connection failure thread=%s, id=%s, time=%s" \ % (mythr.name, mythr.ident, tstamp) except Exception as exc: print_exc(exc) self.kws = None
def update(self): """ Update some the input values collection for current input field """ if SKIP_UPDATES: return None time0 = time.time() values = self.fetch_values() #print gen if not self.col.count(): try: # perform bulk insert operation self.col.insert( itertools.islice(values, self.cache_size)) # break except InvalidOperation: pass else: # we already have records, update their ts for val in values: spec = dict(value=val['value']) self.col.update(spec, {'$set': {'ts': time0}}, upsert=True) # remove records with old ts self.col.remove({'ts': {'$lt': time0 - self.expire}}) print("%s InputValuesTracker updated" \ " %s collection in %s sec, nrec=%s" \ % (dastimestamp(), self.dbcoll, time.time() - time0, self.col.count()))
def insert_query_records(self, dasquery): """ Insert DAS query records into DAS cache and return list of services which will answer this query """ services = dasquery.services self.logger.info("Potential services = %s" % services) if not services: msg = "No data-services for query %s" % dasquery msg += "mongo_query: %s" % dasquery.mongo_query msg += "params: %s" % dasquery.params() print(dastimestamp("DAS WARNING "), msg) # get list of URI which can answer this query ack_services = [] for srv in services: gen = [t for t in getattr(getattr(self, srv), "apimap")(dasquery)] for url, api, args, iformat, expire in gen: header = dasheader(srv, dasquery, expire, api, url, ctime=0) self.rawcache.insert_query_record(dasquery, header) if srv not in ack_services: ack_services.append(srv) if not ack_services: ack_services = services if dasquery.query.find("records ") != -1: srv_status = True # skip DAS queries w/ records request # create das record with initial expire tstamp 2 min in a future # it should be sufficient for processing data-srv records expire = time.time() + 2 * 60 header = dasheader("das", dasquery, expire, api="das_core", services=dict(das=ack_services)) header["lookup_keys"] = [] self.rawcache.insert_query_record(dasquery, header) das_timer("das_record", self.verbose) return ack_services
def remove(self, pid): """Remove given pid""" self.clean() attempts = 0 while True: try: self.col.remove(dict(_id=pid), safe=True) break except Exception as err: print_exc(err) time.sleep(0.01) attempts += 1 if attempts > 2: msg = '%s unable to remove pid=%s' % (self.col, pid) print dastimestamp('DAS ERROR '), msg break
def error(query, idx, msg='DAS ERROR: parsing failure'): "Form error message and raise appropriate exception message" out = ' '.join(query) where = '' for jdx in range(0, idx): where += '-' * (len(query[jdx]) + 1) where += '^' msg = dastimestamp(msg) + '\n' + out + '\n' + where raise Exception(msg)
def error(query, idx, msg='DAS ERROR: parsing failure'): "Form error message and raise appropriate exception message" out = ' '.join(query) where = '' for jdx in range(0, idx): where += '-'*(len(query[jdx])+1) where += '^' msg = dastimestamp(msg) + '\n' + out + '\n' + where raise Exception(msg)
def ply_parse_query(query, keys, services, pdir='/tmp', verbose=False): """Get ply object for given query.""" dasply = DASPLY(pdir, keys, services, verbose=verbose) dasply.build() # ply_query = dasply.parser.parse(query) # ply_query = spawn(dasply.parser.parse, query) # return ply_query error = None for trial in xrange(1, 3): try: ply_query = dasply.parser.parse(query) return ply_query except Exception as exc: msg = "Fail to parse query=%s, trial=%s, exception=%s" \ % (query, trial, str(exc)) print dastimestamp('DAS WARNING ') + ' ' + msg error = exc time.sleep(trial/10.) raise error
def process(gen): "Process generator from getdata" for row in gen: if 'error' in row: error = row.get('error') reason = row.get('reason', '') print(dastimestamp('DAS ERROR'), error, reason) yield row continue if 'data' in row: yield json.loads(row['data'])
def status(self): """Return list of all current requests in DAS queue""" requests = [r for r in self.reqmgr.items()] page = self.templatepage('das_status', requests=requests, time=time) sdict = self.dasmgr.status() sdict['web'] = self.taskmgr.status() dasprint(dastimestamp('DAS INFO '), "web TaskManager", sdict['web']) for key, val in sdict.items(): dasprint(dastimestamp('DAS INFO '), "%s TaskManager %s" % (key, val)) page += '<h3>Services</h3>' def dump(idict): "Dump input dict" return ', '.join(['<em>%s:</em> %s' % (k, idict[k]) for k in sorted(idict)]) for key, val in sdict.items(): page += '<div>' stats = ', '.join([dump(v) for v in val.values()]) page += '<b>%s</b>: %s' % (key, stats) page += '</div>' return self.page(page)
def empty_return(self, dasquery, status='busy', reason=None): "Return header/data when DAS server is busy" if not reason: reason = 'DAS server is busy' reason += ', #requests=%s, #workers=%s, queue size=%s' \ % (self.reqmgr.size(), self.taskmgr.nworkers(), self.queue_limit) head = dict(timestamp=time.time()) head.update({'status': status, 'reason': reason, 'ctime':0}) data = [] dasprint(dastimestamp('DAS INFO '), dasquery, 'server status=%s'%status, reason) return self.datastream(dict(head=head, data=data))
def getdata_pycurl(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None): "Fetch data via pycurl library" contact = 'data-service.' if system: contact = system + ' ' + contact if isinstance(params, dict): timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) else: timer_key = '%s?%s' % (url, params) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, \ error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) data = {'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} try: reason = extract_http_error(httperror.read()) data.update({'reason': reason, 'request': msg}) # TODO: err variable did not exit in this function! msg += '\n' + reason except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(dastimestamp('getdata_pycurl'), msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp)) data = {'error': 'Received generic error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def check_pid(self, pid): """ Check status of given pid. This is a server callback function for ajaxCheckPid, see js/ajax_utils.js """ # do not allow caching set_no_cache_flags() img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base page = '' try: if self.taskmgr.is_alive(pid): page = img + " processing PID=%s" % pid else: # at this point we don't know if request arrived to this host # or it was processed. To distinguish the case we'll ask # request manager for that pid if self.reqmgr.has_pid(pid): self.reqmgr.remove(pid) self.taskmgr.remove(pid) page = 'Request PID=%s is completed' % pid page += ', please wait for results to load' else: # there're no request on this server, re-initiate it ref = cherrypy.request.headers.get('Referer', None) if ref: url = urlparse(ref) params = dict(parse_qsl(url.query)) return self.request(**params) else: msg = 'No referer in cherrypy.request.headers' msg += '\nHeaders: %s' % cherrypy.request.headers dasprint(dastimestamp('DAS WEB ERROR '), msg) except Exception as err: msg = 'check_pid fails for pid=%s' % pid dasprint(dastimestamp('DAS WEB ERROR '), msg) print_exc(err) self.reqmgr.remove(pid) self.taskmgr.remove(pid) return self.error(gen_error_msg({'pid':pid}), wrap=False) return page
def init(self): """ Init db connection and check that it is alive """ try: indexes = [('value', ASCENDING), ('ts', ASCENDING)] create_indexes(self.col, indexes) if not KEEP_EXISTING_RECORDS_ON_RESTART: self.col.remove() except Exception as exc: print(dastimestamp(), exc)
def busy(self): """ Check server load and report busy status if nrequests - nworkers > queue limit """ nrequests = self.reqmgr.size() if (nrequests - self.taskmgr.nworkers()) > self.queue_limit: msg = '#request=%s, queue_limit=%s, #workers=%s' \ % (nrequests, self.taskmgr.nworkers(), self.queue_limit) dasprint(dastimestamp('DAS WEB SERVER IS BUSY '), msg) return True return False
def dbs_find(entity, url, kwds, verbose=0): "Find DBS3 entity for given set of parameters" if entity not in ["run", "file", "block"]: msg = "Unsupported entity key=%s" % entity raise Exception(msg) expire = 600 dataset = kwds.get("dataset", None) block = kwds.get("block_name", None) if not block: # TODO: this should go away when DBS will be retired (user in combined srv) block = kwds.get("block", None) lfn = kwds.get("file", None) runs = kwds.get("runs", []) if not (dataset or block or lfn): return url = "%s/%ss" % (url, entity) # DBS3 APIs use plural entity value if dataset: params = {"dataset": dataset} elif block: params = {"block_name": block} elif lfn: params = {"logical_file_name": lfn} if runs: params.update({"run_num": runrange(runs[0], runs[-1], False)}) headers = {"Accept": "application/json;text/json"} source, expire = getdata(url, params, headers, expire, ckey=CKEY, cert=CERT, verbose=verbose) for row in json_parser(source, None): for rec in row: try: if isinstance(rec, basestring): print dastimestamp("DBS3 ERROR:"), row elif entity == "file": yield rec["logical_file_name"] elif entity == "block": yield rec["block_name"] elif entity == "file": yield rec["dataset"] except Exception as exp: msg = 'Fail to parse "%s", exception="%s"' % (rec, exp) print_exc(msg)
def block_run_lumis(url, blocks, runs=None): """ Find block, run, lumi tuple for given set of files and (optional) runs. """ headers = {'Accept': 'text/xml'} urls = [] for blk in blocks: if not blk: continue query = 'find block,run,lumi where block=%s' % blk if runs and isinstance(runs, list): val = ' or '.join(['run=%s' % r for r in runs]) query += ' and (%s)' % val params = {'api':'executeQuery', 'apiversion':'DBS_2_0_9', 'query':query} dbs_url = url + '?' + urllib.urlencode(params) urls.append(dbs_url) if not urls: return gen = urlfetch_getdata(urls, CKEY, CERT, headers) prim_key = 'row' odict = {} # output dict for rec in gen: if 'error' in rec: error = rec.get('error') reason = rec.get('reason', '') print dastimestamp('DAS ERROR'), error, reason yield {'error': error, 'reason': reason} else: source = StringIO.StringIO(rec['data']) lumis = [] for row in qlxml_parser(source, prim_key): run = row['row']['run'] blk = row['row']['block'] lumi = row['row']['lumi'] key = (blk, run) odict.setdefault(key, []).append(lumi) for key, lumis in odict.iteritems(): blk, run = key yield blk, run, lumis
def update_das_query(dasquery, status, reason=None): "Update DAS query record with given status and reason" self.rawcache.update_query_record(dasquery, status, reason=reason) self.rawcache.add_to_record(\ dasquery, {'das.timer': get_das_timer()}, system='das') # make sure that das record is updated, we use 7 iteration which # sum up into 1 minute to cover default syncdelay value of mongo # server (in a future it would be better to find programatically # this syncdelay value, but it seems pymongo driver does not # provide any API for it. for idx in xrange(1, 7): spec = {'qhash':dasquery.qhash, 'das.system':['das']} res = self.rawcache.col.find_one(spec) if res: dbstatus = res.get('das', {}).get('status', None) if dbstatus == status: break msg = 'qhash %s, das.status=%s, status=%s, wait for update' \ % (dasquery.qhash, dbstatus, status) print dastimestamp('DAS WARNING'), msg time.sleep(idx*idx) self.rawcache.update_query_record(dasquery, status, reason=reason)
def init(self): """Init DAS web server, connect to DAS Core""" try: self.reqmgr = RequestManager(lifetime=self.lifetime) self.dasmgr = DASCore(engine=self.engine) self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr) self.daskeys = self.dasmgr.das_keys() self.gfs = db_gridfs(self.dburi) self.daskeys.sort() self.dasmapping = self.dasmgr.mapping self.dbs_url = self.dasmapping.dbs_url() self.dbs_global = self.dasmapping.dbs_global_instance() self.dbs_instances = self.dasmapping.dbs_instances() self.dasmapping.init_presentationcache() self.colors = {"das": gen_color("das")} for system in self.dasmgr.systems: self.colors[system] = gen_color(system) # get SiteDB from global scope self.sitedbmgr = SERVICES.get("sitedb2", None) # Start DBS daemon if self.dataset_daemon: self.dbs_daemon(self.dasconfig["web_server"]) if not self.daskeyslist: keylist = [r for r in self.dasmapping.das_presentation_map()] keylist.sort(key=lambda r: r["das"]) self.daskeyslist = keylist except ConnectionFailure as _err: tstamp = dastimestamp("") mythr = threading.current_thread() print "### MongoDB connection failure thread=%s, id=%s, time=%s" % (mythr.name, mythr.ident, tstamp) except Exception as exc: print_exc(exc) self.dasmgr = None self.reqmgr = None self.dbs_url = None self.dbs_global = None self.dbs_instances = [] self.daskeys = [] self.colors = {} self.q_rewriter = None return # KWS and Query Rewriting failures are not fatal try: # init query rewriter, if needed if self.dasconfig["query_rewrite"]["pk_rewrite_on"]: self.q_rewriter = CMSQueryRewrite(self.repmgr, self.templatepage) except Exception as exc: print_exc(exc) self.q_rewriter = None
def db_monitor(uri, func, sleep, reload_map, reload_time, check_maps, reload_time_bad_maps): """ Check status of MongoDB connection and reload DAS maps once in a while. """ time0 = time.time() valid_maps = False try: valid_maps = check_maps() except Exception as err: print_exc(err) while True: conn = db_connection(uri) if not conn or not is_db_alive(uri): try: conn = db_connection(uri, verbose=False) func() if conn: print "### db_monitor re-established connection %s" % conn valid_maps = check_maps() else: print "### db_monitor, lost connection" except Exception as err: print_exc(err) if conn: # reload invalid more quickly reload_intervl = reload_time if valid_maps else reload_time_bad_maps if time.time() - time0 > reload_intervl: map_state = "INVALID" if not valid_maps else "" msg = "reload %s DAS maps %s" % (map_state, reload_map) print dastimestamp(), msg try: reload_map() valid_maps = check_maps() except Exception as err: print_exc(err) time0 = time.time() time.sleep(sleep)
def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves DAS records from the cache" col = self.mdb[collection] msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection) self.logger.info(msg) idx = int(idx) fields, filter_cond = self.get_fields(dasquery) if not fields: spec = dasquery.mongo_query.get('spec', {}) else: spec = {'qhash':dasquery.qhash, 'das.empty_record':0} if filter_cond: spec.update(filter_cond) if fields: # be sure to extract das internal keys fields += self.das_internal_keys # try to get sort keys all the time to get ordered list of # docs which allow unique_filter to apply afterwards skeys = self.mongo_sort_keys(collection, dasquery) res = self.get_records(col, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) counter = 0 for row in res: counter += 1 yield row if counter: msg = "yield %s record(s)" % counter self.logger.info(msg) # if no raw records were yield we look-up possible error records if not counter: nrec = self.col.find({'qhash':dasquery.qhash}).count() if nrec: msg = "for query %s, found %s non-result record(s)" \ % (dasquery, nrec) prf = 'DAS WARNING, monogocache:get_from_cache ' print dastimestamp(prf), msg
def add(self, pid, kwds): """Add new pid/kwds""" self.clean() if not kwds: return tstamp = time.strftime("%Y%m%d %H:%M:%S", time.localtime()) doc = dict(_id=pid, kwds=json.dumps(kwds), ts=time.time(), timestamp=tstamp) attempts = 0 while True: try: self.col.insert(doc, safe=True) break except DuplicateKeyError as err: break except Exception as err: print_exc(err) time.sleep(0.01) attempts += 1 if attempts > 2: msg = '%s unable to add pid=%s' % (self.col, pid) print dastimestamp('DAS ERROR '), msg break
def init(self): """Init DAS web server, connect to DAS Core""" try: self.reqmgr = RequestManager(lifetime=self.lifetime) self.dasmgr = DASCore(engine=self.engine) self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr) self.daskeys = self.dasmgr.das_keys() self.gfs = db_gridfs(self.dburi) self.daskeys.sort() self.dasmapping = self.dasmgr.mapping self.dbs_url = self.dasmapping.dbs_url() self.dbs_global = self.dasmapping.dbs_global_instance() self.dbs_instances = self.dasmapping.dbs_instances() self.dasmapping.init_presentationcache() self.colors = {'das':gen_color('das')} for system in self.dasmgr.systems: self.colors[system] = gen_color(system) if not self.daskeyslist: keylist = [r for r in self.dasmapping.das_presentation_map()] keylist.sort(key=lambda r: r['das']) self.daskeyslist = keylist except ConnectionFailure as _err: tstamp = dastimestamp('') mythr = threading.current_thread() print("### MongoDB connection failure thread=%s, id=%s, time=%s" \ % (mythr.name, mythr.ident, tstamp)) except Exception as exc: print_exc(exc) self.dasmgr = None self.reqmgr = None self.dbs_url = None self.dbs_global = None self.dbs_instances = [] self.daskeys = [] self.colors = {} self.q_rewriter = None return # KWS and Query Rewriting failures are not fatal try: # init query rewriter, if needed if self.dasconfig['query_rewrite']['pk_rewrite_on']: self.q_rewriter = CMSQueryRewrite(self.repmgr, self.templatepage) except Exception as exc: print_exc(exc) self.q_rewriter = None
def init(self): """Init DAS web server, connect to DAS Core""" try: self.dasmgr = DASCore(multitask=False) self.dbs_instances = self.dasmgr.mapping.dbs_instances() self.dbs_global = self.dasmgr.mapping.dbs_global_instance() if KeywordSearchHandler: self.kws = KeywordSearchHandler(self.dasmgr) except ConnectionFailure: tstamp = dastimestamp('') mythr = threading.current_thread() print("### MongoDB connection failure thread=%s, id=%s, time=%s" \ % (mythr.name, mythr.ident, tstamp)) except Exception as exc: print_exc(exc) self.dasmgr = None self.kws = None
def dbs_find(entity, url, kwds, verbose=0): "Find DBS3 entity for given set of parameters" if entity not in ['run', 'file', 'block']: msg = 'Unsupported entity key=%s' % entity raise Exception(msg) expire = 600 dataset = kwds.get('dataset', None) block = kwds.get('block_name', None) if not block: # TODO: this should go away when DBS will be retired (user in combined srv) block = kwds.get('block', None) lfn = kwds.get('file', None) runs = kwds.get('runs', []) if not (dataset or block or lfn): return url = '%s/%ss' % (url, entity) # DBS3 APIs use plural entity value if dataset: params = {'dataset':dataset} elif block: params = {'block_name': block} elif lfn: params = {'logical_file_name': lfn} if runs: params.update({'run_num': runs}) headers = {'Accept': 'application/json;text/json'} source, expire = \ getdata(url, params, headers, expire, ckey=CKEY, cert=CERT, verbose=verbose) for row in json_parser(source, None): for rec in row: try: if isinstance(rec, basestring): print(dastimestamp('DBS3 ERROR:'), row) elif entity == 'file': yield rec['logical_file_name'] elif entity == 'block': yield rec['block_name'] elif entity == 'file': yield rec['dataset'] except Exception as exp: msg = 'Fail to parse "%s", exception="%s"' % (rec, exp) print_exc(msg)
def get_new_connection(self, uri): "Get new MongoDB connection" key = self.genkey(uri) for idx in range(0, self.retry): try: dbinst = MongoClient(host=uri, **self.mongo_opts) # dbinst = MongoConnection(uri, **self.mongo_opts).client() gfs = dbinst.gridfs fsinst = gridfs.GridFS(gfs) self.conndict[key] = (dbinst, fsinst) self.timedict[key] = time.time() return (dbinst, fsinst) except (ConnectionFailure, AutoReconnect) as exc: tstamp = dastimestamp('') thread = threading.current_thread() print("### MongoDB connection failure thread=%s, id=%s, time=%s" \ % (thread.name, thread.ident, tstamp)) print_exc(exc) except Exception as exc: print_exc(exc) time.sleep(idx) return self.conndict.get(key, (None, None))
def insert_query_records(self, dasquery): """ Insert DAS query records into DAS cache and return list of services which will answer this query """ services = dasquery.services self.logger.info('Potential services = %s' % services) if not services: msg = 'No data-services for query %s' % dasquery msg += 'mongo_query: %s' % dasquery.mongo_query msg += 'params: %s' % dasquery.params() print(dastimestamp('DAS WARNING '), msg) # get list of URI which can answer this query ack_services = [] for srv in services: gen = [t for t in getattr(getattr(self, srv), 'apimap')(dasquery)] for url, api, args, iformat, expire in gen: header = dasheader(srv, dasquery, expire, api, url, ctime=0) self.rawcache.insert_query_record(dasquery, header) if srv not in ack_services: ack_services.append(srv) if not ack_services: ack_services = services if dasquery.query.find('records ') != -1: srv_status = True # skip DAS queries w/ records request # create das record with initial expire tstamp expire = time.time() + self.init_expire header = dasheader("das", dasquery, expire, api='das_core', services=dict(das=ack_services)) header['lookup_keys'] = [] self.rawcache.insert_query_record(dasquery, header) das_timer('das_record', self.verbose) return ack_services
def merge_records(self, dasquery, attempt=0): """ Merge DAS records for provided query. We perform the following steps: 1. get all queries from das.cache by ordering them by primary key 2. run aggregtor function to merge neighbors 3. insert records into das.merge """ ### TMP for asyncio # time.sleep(attempt+3) # pymongo 3.2 don't yet flush in time # remove any entries in merge collection for this query self.merge.delete_many({'qhash': dasquery.qhash}) # proceed self.logger.debug(dasquery) id_list = [] expire = 9999999999 # future # get all API records for given DAS query spec = { 'qhash': dasquery.qhash, 'das.expire': { '$gt': time.time() }, 'das.record': record_codes('query_record') } records = self.col.find(spec, **PYMONGO_OPTS) for row in records: # find smallest expire timestamp to be used by aggregator rexpire = row.get('das', {}).get('expire', expire) if rexpire < expire: expire = rexpire if row['_id'] not in id_list: id_list.append(row['_id']) inserted = 0 lookup_keys = set() fields = dasquery.mongo_query.get('fields') if not fields: # Mongo fields = [] for key in fields: for pkey in self.mapping.mapkeys(key): lookup_keys.add(pkey) for pkey in lookup_keys: skey = [(pkey, DESCENDING)] # lookup all service records spec = {'das_id': {'$in': id_list}, 'das.primary_key': pkey} if self.verbose: nrec = self.col.find(spec, **PYMONGO_OPTS).sort(skey).count() msg = "merging %s records, for %s key" % (nrec, pkey) else: msg = "merging records, for %s key" % pkey self.logger.debug(msg) # use exhaust=False since we process all records in aggregator # and it can be delay in processing records = self.col.find(spec, **PYMONGO_NOEXHAUST).sort(skey) # aggregate all records agen = aggregator(dasquery, records, expire) # diff aggregated records gen = das_diff(agen, self.mapping.diff_keys(pkey.split('.')[0])) # insert all records into das.merge using bulk insert size = self.cache_size try: res = self.merge.insert_many(gen) inserted += len(res.inserted_ids) except InvalidDocument as exp: print(dastimestamp('DAS WARNING'), 'InvalidDocument during merge', str(exp)) msg = "Caught bson error: " + str(exp) self.logger.info(msg) records = self.col.find(spec, **PYMONGO_OPTS).sort(skey) gen = aggregator(dasquery, records, expire) genrows = parse2gridfs(self.gfs, pkey, gen, self.logger) das_dict = { 'das': { 'expire': expire, 'das.record': record_codes('gridfs_record'), 'primary_key': [k for k in lookup_keys], 'system': ['gridfs'] }, 'qhash': dasquery.qhash, 'cache_id': [], 'das_id': id_list } for row in genrows: row.update(das_dict) self.merge.insert(row) except InvalidOperation as exp: pass except DuplicateKeyError as err: print(dastimestamp('DAS WARNING'), 'DuplicateKeyError during merge') if not isinstance(gen, list): raise err status = 'fail' if inserted: status = 'ok' elif not lookup_keys: # we get query w/o fields msg = 'qhash %s, no lookup_keys' % dasquery.qhash print(dastimestamp('DAS WARNING'), msg) status = 'ok' else: # we didn't merge anything, it is DB look-up failure msg = 'qhash %s, did not insert into das.merge, attempt %s' \ % (dasquery.qhash, attempt) print(dastimestamp('DAS WARNING'), msg) empty_expire = etstamp() lkeys = list(lookup_keys) das = dict(expire=empty_expire, primary_key=lkeys[0], condition_keys=lkeys, instance=dasquery.instance, system=['das'], services=dasquery.services, record=record_codes('empty_record'), ts=time.time(), api=[]) empty_record = { 'das': das, 'qhash': dasquery.qhash, 'cache_id': [], 'das_id': id_list } for key in lkeys: empty_record.update({key.split('.')[0]: []}) for key, val in dasquery.mongo_query['spec'].items(): if key.find('.') == -1: empty_record[key] = [] else: # it is compound key, e.g. site.name newkey, newval = convert_dot_notation(key, val) empty_record[newkey] = adjust_mongo_keyvalue(newval) self.merge.insert(empty_record) # update DAS records (both meta and data ones, by using qhash) nval = {'$set': {'das.expire': empty_expire}} spec = {'qhash': dasquery.qhash} self.col.update_many(spec, nval) return status
def get_data(self, kwargs): """ Invoke DAS workflow and get data from the cache. """ head = dict(timestamp=time.time()) head['args'] = kwargs uinput = kwargs.get('input', '') inst = kwargs.get('instance', self.dbs_global) idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 0) # do not impose limit coll = kwargs.get('collection', 'merge') status = kwargs.get('status') error = kwargs.get('error') reason = kwargs.get('reason') dasquery = kwargs.get('dasquery', None) time0 = time.time() if dasquery: dasquery = DASQuery(dasquery, instance=inst) if dasquery.error: return self.page(form + dasquery.error, ctime=time.time()-time0) else: check, content = \ self.generate_dasquery(uinput, inst, html_mode=False) if check: head.update({'status': 'fail', 'reason': content, 'ctime': time.time()-time0, 'input': uinput}) data = [] return head, data dasquery = content # returned content is valid DAS query try: nres = self.dasmgr.nresults(dasquery, coll) data = \ self.dasmgr.get_from_cache(dasquery, idx, limit) # check that we got what we expected data = [r for r in data] if nres and not len(data): for retry in range(1, 3, 5): msg = 'retry in %s sec' % retry dasprint(dastimestamp('DAS WARNING '), msg, dasquery) time.sleep(retry) # retry one more time data = \ self.dasmgr.get_from_cache(dasquery, idx, limit) data = [r for r in data] if len(data): break if nres and not len(data): msg = 'fail to get all data for %s, nres=%s, len(data)=%s' \ % (dasquery, nres, len(data)) dasprint(dastimestamp('DAS WARNING '), msg) status = 'fail' reason = 'Fail to retrieve data from DAS cache, please retry' if dasquery.aggregators: # aggregators split DAS record into sub-system and then # apply aggregator functions, therefore we need to correctly # account for nresults. Resolve generator into list and take # its length as nresults value. data = [r for r in data] nres = len(data) if error: # DAS record contains an error status = 'error' head.update({'status':status, 'nresults':nres, 'ctime': time.time()-time0, 'dasquery': dasquery}) except Exception as exc: status = 'fail' reason = str(exc) print_exc(exc) head.update({'status': status, 'ctime': time.time()-time0, 'dasquery': dasquery}) data = [] head.update({'incache':self.dasmgr.incache(dasquery, coll='cache'), 'apilist':self.dasmgr.apilist(dasquery)}) if reason: head.update({'reason': reason}) if status != 'ok': head.update(self.info()) # check if query had dataset input and returned no results # then run hint functions to find dataset in other DBS instances mquery = dasquery.mongo_query empty = False for item in data: if 'dataset.name' in mquery['spec'] and 'dataset' in mquery['fields'] \ and 'result' not in item: if not item['dataset']: empty = True break if empty: # if no results found add dataset from other DBS instances hints = self.hint_datasets(kwargs) for item in data: item.update({'hints': hints}) return head, data
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) apilist = head.get('apilist') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters aggrtrs = dasquery.aggregators pager = self.pagination(head) main = pager style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: page += '<br/><span class="box_red">%s</span>' % reason for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None pval = None lkey = None if 'das' in row and 'primary_key' in row['das']: pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(dasquery) try: lkey = pkey.split('.')[0] if pkey == 'summary': pval = row[pkey] else: pval = [i for i in DotDict(row).get_values(pkey)] if isinstance(pval, list): if pval and not isinstance(pval[0], list): pval = list(set(pval)) else: pval = list(set(pval)) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': if isinstance(pval, basestring): pval = int(pval) except Exception as exc: msg = "Fail to extract pval for pkey='%s', lkey='%s'" \ % (pkey, lkey) msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval)) print(msg) print_exc(exc) pval = 'N/A' try: if not filters: if pkey == 'summary': page += 'Summary information:' elif pval and pval != 'N/A': page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: args = make_args(lkey, pval, inst) page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (args, pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if 'link' in item: linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = '<a href="/das/download?lfn=%s">Download</a>'\ % lfn if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) links.append(self.templatepage(\ 'xsecdb', primds=path.split('/')[1])) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except Exception as exc: print_exc(exc) pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation if filters and not aggrtrs: page += add_filter_values(row, filters) else: page += adjust_values(func, gen, links, pkey) pad = "" try: if 'das' in row and 'system' in row['das']: systems = self.systems(row['das']['system']) else: systems = "" # no das record print(dastimestamp('DAS ERROR '), \ 'record without DAS key', row) except KeyError as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(dasquery, row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if 'das' in row and 'conflict' in row['das']: conflict = ', '.join(row['das']['conflict']) else: conflict = '' hints = '' for hint in row.get('hints', {}): if hint: hints += self.templatepage('hint', hint=hint, base=self.base, dbs=self.dbs_global) page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict, hints=hints) page += '</div>' old = row main += fltpage if dup and not dasquery.aggregators: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page if total>10: main += '<hr class="line" />' main += pager main += '<hr class="line" />' proc_time = self.processing_time(dasquery) if proc_time: msg = 'processing time: %5.3f sec, ' % proc_time else: msg = '' msg += 'cache server time: %5.3f sec' % head['ctime'] main += '<div align="right">%s</div>' % msg return main
def call(self, query, **kwds): """ Top level DAS api which execute a given query using underlying data-services. It follows the following steps: - parse input query - identify data-sercices based on selection keys and where clause conditions - construct DAS workflow and execute data-service API calls. At this step individual data-services store results into DAS cache. Return status 0/1 depending on success of the calls, can be used by workers on cache server. kwds is provided for compatibility with web layer, e.g. it may invoke this method with additional pid parameter. """ def update_das_query(dasquery, status, reason=None): "Update DAS query record with given status and reason" self.rawcache.update_query_record(dasquery, status, reason=reason) self.rawcache.add_to_record(\ dasquery, {'das.timer': get_das_timer()}, system='das') self.logger.info('input query=%s' % query) das_timer('DASCore::call', self.verbose) if isinstance(query, object) and hasattr(query, '__class__')\ and query.__class__.__name__ == 'DASQuery': dasquery = query else: dasquery = DASQuery(query) for col in ['merge', 'cache']: self.rawcache.remove_expired(dasquery, col) query = dasquery.mongo_query spec = query.get('spec') fields = query.get('fields') if fields == ['records']: msg = 'look-up all records in cache' self.logger.info(msg) return 'in cache' if spec == dict(records='*'): self.logger.info("look-up everything in cache") return 'in cache' for record in self.rawcache.find_specs(dasquery): status = record['das']['status'] msg = 'found query %s in cache, status=%s\n' \ % (record['query'], status) self.logger.info(msg) print(dastimestamp('DAS INFO'), msg) return status self.logger.info(dasquery) das_timer('das_record', self.verbose) services = self.insert_query_records(dasquery) if not services: msg = 'unable to locate data-services to fulfill this request' msg += ', will iterate over all registered services' print(dastimestamp('DAS WARNING '), dasquery, msg) services = dasquery.services if dasquery.services else self.systems try: if self.multitask: jobs = [] for srv in sorted(services): jobs.append(self.taskmgr.spawn(self.worker, srv, dasquery)) self.taskmgr.joinall(jobs) else: for srv in services: self.worker(srv, dasquery) except Exception as exc: print_exc(exc) return 'fail' self.logger.info('\n##### merging ######\n') update_das_query(dasquery, 'merging') das_timer('merge', self.verbose) # check that all query record statuses are ok, i.e. we did insert records # this status is set by self.rawcache.update_cache for idx in range(self.collect_wait_time): records = self.rawcache.find_query_record(dasquery) statuses = [] for row in records: system = row['das']['system'] status = row['das']['status'] self.logger.info("### query record status %s %s %s" % (dasquery.qhash, system, status)) statuses.append(status) all_statuses = sorted(list(set(statuses))) # at this point we're looking that all services will have 'ok' and das status will be 'merging' if len(all_statuses) == 2 and all_statuses == ['merging', 'ok']: break time.sleep(1) # now we can merge records status = self.rawcache.merge_records(dasquery) das_timer('merge', self.verbose) # check if we have service records and properly setup status self.logger.info('\n##### check services ######\n') das_services = self.rawcache.check_services(dasquery) reason = '' status = 'ok' if not das_services: if 'records' in dasquery.query: status = 'ok' # keep status ok for 'records' queries else: reason = 'no data records found in DAS cache' status = 'fail' print(dastimestamp('DAS ERROR '), dasquery, reason) update_das_query(dasquery, status, reason) das_timer('DASCore::call', self.verbose) return status
def helper(self, api, args, expire): """ Class helper function which yields results for given set of input parameters. It yeilds the data record which must contain combined attribute corresponding to systems used to produce record content. """ dbs_url = self.map[api]['services'][self.dbs] phedex_url = self.map[api]['services']['phedex'] # make phedex_api from url, but use xml version for processing phedex_api = phedex_url.replace('/json/', '/xml/') + '/blockReplicas' if api == 'dataset4site_release' or \ api == 'dataset4site_release_parent' or \ api == 'child4site_release_dataset': # DBS part datasets = set() release = args['release'] parent = args.get('parent', None) for row in dbs_dataset4release_parent(dbs_url, release, parent): datasets.add(row) # Phedex part if args['site'].find('.') != -1: # it is SE phedex_args = { 'dataset': list(datasets), 'se': '%s' % args['site'] } else: phedex_args = { 'dataset': list(datasets), 'node': '%s*' % args['site'] } headers = {'Accept': 'text/xml'} source, expire = \ getdata(phedex_api, phedex_args, headers, expire, system='phedex') prim_key = 'block' tags = 'block.replica.node' found = {} for rec in xml_parser(source, prim_key, tags): ddict = DotDict(rec) block = ddict.get('block.name') bbytes = ddict.get('block.bytes') files = ddict.get('block.files') found_dataset = block.split('#')[0] if found_dataset in found: val = found[found_dataset] found[found_dataset] = { 'bytes': val['bytes'] + bbytes, 'files': val['files'] + files } else: found[found_dataset] = {'bytes': bbytes, 'files': files} for name, val in found.items(): record = dict(name=name, size=val['bytes'], files=val['files']) if api == 'child4site_release_dataset': yield {'child': record} else: yield {'dataset': record} del datasets del found if api == 'site4dataset': try: gen = site4dataset(dbs_url, phedex_api, args, expire) for row in gen: sname = row.get('site', {}).get('name', '') skind = self.site_info(phedex_url, sname) row['site'].update({'kind': skind}) yield row except Exception as err: print_exc(err) tstamp = dastimestamp('') msg = tstamp + ' Exception while processing DBS/Phedex info:' msg += str(err) row = { 'site': { 'name': 'Fail to look-up site info', 'error': msg, 'dataset_fraction': 'N/A', 'block_fraction': 'N/A', 'block_completion': 'N/A' }, 'error': msg } yield row if api == 'files4dataset_runs_site' or \ api == 'files4block_runs_site': run_value = args.get('run', []) if isinstance(run_value, dict) and '$in' in run_value: runs = run_value['$in'] elif isinstance(run_value, list): runs = run_value else: if int_number_pattern.match(str(run_value)): runs = [run_value] else: runs = [] args.update({'runs': runs}) files = dbs_find('file', dbs_url, args) site = args.get('site') phedex_api = phedex_url.replace('/json/', '/xml/') + '/fileReplicas' for fname in files4site(phedex_api, files, site): yield {'file': {'name': fname}}