def adjust_params(self, api, kwds, _inst=None): """ Adjust CondDB parameters for specific query requests """ day = 24*60*60 if api == 'get_lumi_info': if kwds.has_key('date') and kwds['date'] != 'optional': value = kwds['date'] if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) elif isinstance(value, dict): value = [kwds['date']['$gte'], kwds['date']['$lte']] elif isinstance(value, int): value = [value, value+day] else: msg = 'Unsupported date format %s' % kwds['date'] raise Exception(msg) kwds['startTime'] = convert_datetime(value[0]) kwds['endTime'] = convert_datetime(value[1]) del kwds['date'] elif kwds.has_key('runList'): val = kwds['runList'] if isinstance(val, dict): # we got a run range kwds['runList'] = '%s-%s' % (val['$gte'], val['$lte']) if api == 'get_run_info': if kwds.has_key('date') and kwds['date'] != 'optional': value = kwds['date'] if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) elif isinstance(value, dict): value = [kwds['date']['$gte'], kwds['date']['$lte']] elif isinstance(value, int): value = [value, value+day] else: msg = 'Unsupported date format %s' % kwds['date'] raise Exception(msg) kwds['startTime'] = convert_datetime(value[0]) kwds['endTime'] = convert_datetime(value[1]) del kwds['date'] if kwds.has_key('Runs') and isinstance(kwds['Runs'], dict): minrun = 0 maxrun = 0 for kkk, vvv in kwds['Runs'].iteritems(): if kkk == '$in': if len(vvv) == 2: minrun, maxrun = vvv else: # in[1, 2, 3] msg = "conddb can not deal with 'in'" self.logger.info(msg) continue elif kkk == '$lte': maxrun = vvv elif kkk == '$gte': minrun = vvv kwds['Runs'] = '%s-%s' % (minrun, maxrun)
def adjust_params(self, api, kwds, _inst=None): """ Adjust CondDB parameters for specific query requests """ day = 24*60*60 if api == 'get_run_info': if 'date' in kwds and kwds['date'] != 'optional': value = kwds['date'] if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) elif isinstance(value, dict): value = [kwds['date']['$gte'], kwds['date']['$lte']] elif isinstance(value, int): value = [value, value+day] else: msg = 'Unsupported date format %s' % kwds['date'] raise Exception(msg) kwds['startTime'] = convert_datetime(value[0]) kwds['endTime'] = convert_datetime(value[1]) del kwds['date'] if 'Runs' in kwds and isinstance(kwds['Runs'], dict): minrun = 0 maxrun = 0 for kkk, vvv in kwds['Runs'].items(): if kkk == '$in': kwds['Runs'] = ','.join([str(r) for r in vvv]) elif kkk == '$lte': maxrun = vvv elif kkk == '$gte': minrun = vvv if minrun and maxrun: kwds['Runs'] = '%s-%s' % (minrun, maxrun) if 'Runs' in kwds: if not kwds['Runs']: del kwds['Runs']
def adjust_params(self, api, kwds, _inst=None): """ Adjust CondDB parameters for specific query requests """ day = 24 * 60 * 60 if api == "get_run_info": if "date" in kwds and kwds["date"] != "optional": value = kwds["date"] if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) elif isinstance(value, dict): value = [kwds["date"]["$gte"], kwds["date"]["$lte"]] elif isinstance(value, int): value = [value, value + day] else: msg = "Unsupported date format %s" % kwds["date"] raise Exception(msg) kwds["startTime"] = convert_datetime(value[0]) kwds["endTime"] = convert_datetime(value[1]) del kwds["date"] if "Runs" in kwds and isinstance(kwds["Runs"], dict): minrun = 0 maxrun = 0 for kkk, vvv in kwds["Runs"].items(): if kkk == "$in": kwds["Runs"] = ",".join([str(r) for r in vvv]) elif kkk == "$lte": maxrun = vvv elif kkk == "$gte": minrun = vvv if minrun and maxrun: kwds["Runs"] = "%s-%s" % (minrun, maxrun) if "Runs" in kwds: if not kwds["Runs"]: del kwds["Runs"]
def test_convert2date(self): """Test convert2date function""" date = '24h' expect = [long(time.time()-24*60*60), long(time.time())] result = convert2date(date) self.assertEqual(expect, result) date = '60m' expect = [long(time.time()-60*60), long(time.time())] result = convert2date(date) self.assertEqual(expect, result) date = '60s' self.assertRaises(Exception, convert2date, date) date = '123' self.assertRaises(Exception, convert2date, date)
def test_convert2date(self): """Test convert2date function""" date = '24h' expect = [long(time.time() - 24 * 60 * 60), long(time.time())] result = convert2date(date) self.assertEqual(expect, result) date = '60m' expect = [long(time.time() - 60 * 60), long(time.time())] result = convert2date(date) self.assertEqual(expect, result) date = '60s' self.assertRaises(Exception, convert2date, date) date = '123' self.assertRaises(Exception, convert2date, date)
def apicall(self, dasquery, url, api, args, dformat, expire): """ An overview data-service worker. """ keys = [ key for key in self.map[api]['keys'] for api in self.map.keys() ] cond = dasquery.mongo_query['spec'] for key, value in cond.items(): if key.find('.') != -1: args['grouping'] = key.split('.')[-1] key, _attr = key.split('.', 1) if key not in keys: continue args['end'] = '%d' % time.time() if not isinstance(value, dict): # we got equal condition if key == 'date': if isinstance(value, list) and len(value) != 2: msg = 'Monitor service requires 2 time stamps.' msg += 'Please use either date last XXh format or' msg += 'date in [YYYYMMDD, YYYYMMDD]' raise Exception(msg) if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) else: value = [value, value + 24 * 60 * 60] args['start'] = value[0] args['end'] = value[1] else: # we got some operator, e.g. key :{'$in' : [1,2,3]} if key == 'date': if '$in' in value: vallist = value['$in'] elif '$lte' in value and '$gte' in value: vallist = (value['$gte'], value['$lte']) else: err = 'Unsupported date value' raise Exception(err) args['start'] = convert_datetime(vallist[0]) args['end'] = convert_datetime(vallist[-1]) else: raise Exception(err) time0 = time.time() res, expire = self.getdata(url, args, expire) dasrows = self.parser(dasquery, dformat, res, args) ctime = time.time() - time0 self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime)
def apicall(self, dasquery, url, api, args, dformat, expire): """ An overview data-service worker. """ keys = [key for key in self.map[api]['keys'] for api in self.map.keys()] cond = dasquery.mongo_query['spec'] for key, value in cond.iteritems(): if key.find('.') != -1: args['grouping'] = key.split('.')[-1] key, _attr = key.split('.', 1) if key not in keys: continue args['end'] = '%d' % time.time() if not isinstance(value, dict): # we got equal condition if key == 'date': if isinstance(value, list) and len(value) != 2: msg = 'Monitor service requires 2 time stamps.' msg += 'Please use either date last XXh format or' msg += 'date in [YYYYMMDD, YYYYMMDD]' raise Exception(msg) if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) else: value = [value, value + 24*60*60] args['start'] = value[0] args['end'] = value[1] else: # we got some operator, e.g. key :{'$in' : [1,2,3]} if key == 'date': if value.has_key('$in'): vallist = value['$in'] elif value.has_key('$lte') and value.has_key('$gte'): vallist = (value['$gte'], value['$lte']) else: err = 'Unsupported date value' raise Exception(err) args['start'] = convert_datetime(vallist[0]) args['end'] = convert_datetime(vallist[-1]) else: raise Exception(err) time0 = time.time() res, expire = self.getdata(url, args, expire) genrows = self.parser(dasquery, dformat, res, args) dasrows = self.set_misses(dasquery, api, genrows) ctime = time.time() - time0 self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime)
def apicall(self, dasquery, url, api, args, dformat, expire): """ A service worker. It parses input query, invoke service API and return results in a list with provided row. """ cond = dasquery.mongo_query['spec'] count = 0 for key, value in cond.items(): err = 'JobSummary does not support key=%s, value=%s' \ % (key, value) if not isinstance(value, dict): # we got equal condition if key == 'date': if isinstance(value, list) and len(value) != 2: msg = 'Dashboard service requires 2 time stamps.' msg += 'Please use either date last XXh format or' msg += 'date in [YYYYMMDD, YYYYMMDD]' raise Exception(msg) if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) else: value = [value, value + 24 * 60 * 60] args['date1'] = convert_datetime(value[0]) args['date2'] = convert_datetime(value[1]) count += 1 else: for param in self.dasmapping.das2api(self.name, api, key): args[param] = value count += 1 else: # we got some operator, e.g. key :{'$in' : [1,2,3]} if key == 'date' or key == 'jobsummary': if '$in' in value: vallist = value['$in'] elif '$lte' in value and '$gte' in value: vallist = (value['$gte'], value['$lte']) else: raise Exception(err) args['date1'] = convert_datetime(vallist[0]) args['date2'] = convert_datetime(vallist[-1]) count += 1 else: raise Exception(err) if not count: # if no parameter are given, don't pass the API msg = 'DashboardService::api\n\n' msg += "--- %s reject API %s, parameters don't match, args=%s" \ % (self.name, api, args) self.logger.info(msg) return else: if not args['date1']: args['date1'] = convert_datetime(time.time() - 24 * 60 * 60) if not args['date2']: args['date2'] = convert_datetime(time.time()) # drop date argument, since it's used by DAS not by dashboard data srv if 'date' in args: args.pop('date') time0 = time.time() res, expire = self.getdata(url, args, expire, headers=self.headers) rawrows = self.parser(res, api, args) dasrows = self.translator(api, rawrows) ctime = time.time() - time0 try: self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime) except Exception as exc: print_exc(exc)
def apicall(self, dasquery, url, api, args, dformat, expire): """ A service worker. It parses input query, invoke service API and return results in a list with provided row. """ cond = dasquery.mongo_query['spec'] count = 0 for key, value in cond.iteritems(): err = 'JobSummary does not support key=%s, value=%s' \ % (key, value) if not isinstance(value, dict): # we got equal condition if key == 'date': if isinstance(value, list) and len(value) != 2: msg = 'Dashboard service requires 2 time stamps.' msg += 'Please use either date last XXh format or' msg += 'date in [YYYYMMDD, YYYYMMDD]' raise Exception(msg) if isinstance(value, str) or isinstance(value, unicode): value = convert2date(value) else: value = [value, value + 24*60*60] args['date1'] = convert_datetime(value[0]) args['date2'] = convert_datetime(value[1]) count += 1 else: for param in self.dasmapping.das2api(self.name, key): args[param] = value count += 1 else: # we got some operator, e.g. key :{'$in' : [1,2,3]} if key == 'date' or key == 'jobsummary': if value.has_key('$in'): vallist = value['$in'] elif value.has_key('$lte') and value.has_key('$gte'): vallist = (value['$gte'], value['$lte']) else: raise Exception(err) args['date1'] = convert_datetime(vallist[0]) args['date2'] = convert_datetime(vallist[-1]) count += 1 else: raise Exception(err) if not count: # if no parameter are given, don't pass the API msg = 'DashboardService::api\n\n' msg += "--- %s reject API %s, parameters don't match, args=%s" \ % (self.name, api, args) self.logger.info(msg) return else: if not args['date1']: args['date1'] = convert_datetime(time.time()-24*60*60) if not args['date2']: args['date2'] = convert_datetime(time.time()) time0 = time.time() res, expire = self.getdata(url, args, expire, headers=self.headers) rawrows = self.parser(res, api, args) genrows = self.translator(api, rawrows) dasrows = self.set_misses(dasquery, api, genrows) ctime = time.time() - time0 try: self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime) except Exception as exc: print_exc(exc)
def apicall(self, dasquery, url, api, args, dformat, expire): """ A service worker. It parses input query, invoke service API and return results in a list with provided row. """ _query = '' _table = 'runsummary' if api == 'rr_xmlrpc_lumis': _table = 'runlumis' for key, val in dasquery.mongo_query['spec'].items(): if key == 'run.run_number': if isinstance(val, int): _query = {'runNumber': '%s' % val} elif isinstance(val, dict): minrun = 0 maxrun = 0 for kkk, vvv in val.items(): if kkk == '$in': runs = ' or '.join([str(r) for r in vvv]) _query = {'runNumber': runs} elif kkk == '$lte': maxrun = vvv elif kkk == '$gte': minrun = vvv if minrun and maxrun: _query = {'runNumber': '>= %s and < %s' % (minrun, maxrun)} elif key == 'date': if isinstance(val, dict): if '$in' in val: value = val['$in'] elif '$lte' in val and '$gte' in val: value = (val['$gte'], val['$lte']) else: msg = 'Unable to get the value from %s=%s' \ % (key, val) raise Exception(msg) try: date1 = convert_datetime(value[0]) date2 = convert_datetime(value[-1]) except: msg = 'Unable to convert to datetime format, %s' \ % value raise Exception(msg) elif isinstance(val, str) or isinstance(val, unicode): date1, date2 = convert2date(val) date1 = rr_date(date1) date2 = rr_date(date2) else: date1 = convert_datetime(val) date2 = convert_datetime(val + 24*60*60) run_time = '>= %s and < %s' % (date1, date2) _query = {'runStartTime': run_time} else: msg = 'RunRegistryService::api\n\n' msg += "--- %s reject API %s, parameters don't match, args=%s" \ % (self.name, api, args) self.logger.info(msg) return if not _query: msg = 'Unable to match input parameters with input query' raise Exception(msg) if 'run' in args and isinstance(args['run'], dict): args['run'] = str(args['run']) msg = "DASAbstractService:RunRegistry, query=%s" % _query self.logger.info(msg) time0 = time.time() rawrows = rr_worker(url, _query, _table) genrows = self.translator(api, rawrows) if _table == 'runsummary': dasrows = run_duration(genrows) else: dasrows = collect_lumis(genrows) ctime = time.time() - time0 try: self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime) except Exception as exc: print_exc(exc)
def apicall(self, dasquery, url, api, args, dformat, expire): """ A service worker. It parses input query, invoke service API and return results in a list with provided row. """ _query = "" for key, val in dasquery.mongo_query['spec'].iteritems(): if key == 'run.run_number': if isinstance(val, int): _query = {'runNumber': '%s' % val} elif isinstance(val, dict): minrun = 0 maxrun = 0 for kkk, vvv in val.iteritems(): if kkk == '$in': if len(vvv) == 2: minrun, maxrun = vvv else: # in[1, 2, 3] msg = "runregistry can not deal with 'in'" self.logger.info(msg) continue elif kkk == '$lte': maxrun = vvv elif kkk == '$gte': minrun = vvv _query = {'runNumber': '>= %s and < %s' % (minrun, maxrun)} elif key == 'date': if isinstance(val, dict): if val.has_key('$in'): value = val['$in'] elif val.has_key('$lte') and val.has_key('$gte'): value = (val['$gte'], val['$lte']) else: msg = 'Unable to get the value from %s=%s' \ % (key, val) raise Exception(msg) try: date1 = convert_datetime(value[0]) date2 = convert_datetime(value[-1]) except: msg = 'Unable to convert to datetime format, %s' \ % value raise Exception(msg) elif isinstance(val, str) or isinstance(val, unicode): date1, date2 = convert2date(val) date1 = rr_date(date1) date2 = rr_date(date2) else: date1 = convert_datetime(val) date2 = convert_datetime(val + 24*60*60) run_time = '>= %s and < %s' % (date1, date2) _query = {'runStartTime': run_time} else: msg = 'RunRegistryService::api\n\n' msg += "--- %s reject API %s, parameters don't match, args=%s" \ % (self.name, api, args) self.logger.info(msg) return if not _query: msg = 'Unable to match input parameters with input query' raise Exception(msg) if args.has_key('run') and isinstance(args['run'], dict): args['run'] = str(args['run']) msg = "DASAbstractService:RunRegistry, query=%s" % _query self.logger.info(msg) time0 = time.time() api_ver = 3 # API version for RunRegistry, v2 is xmlrpc, v3 is REST rawrows = worker(url, _query, api_ver) genrows = self.translator(api, rawrows) dasrows = self.set_misses(dasquery, api, run_duration(genrows, api_ver)) ctime = time.time() - time0 try: self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime) except Exception as exc: print_exc(exc)
def apicall(self, dasquery, url, api, args, dformat, expire): """ A service worker. It parses input query, invoke service API and return results in a list with provided row. """ _query = '' _table = 'runsummary' if api == 'rr_xmlrpc_lumis': _table = 'runlumis' for key, val in dasquery.mongo_query['spec'].items(): if key == 'run.run_number': if isinstance(val, int): _query = {'runNumber': '%s' % val} elif isinstance(val, dict): minrun = 0 maxrun = 0 for kkk, vvv in val.items(): if kkk == '$in': runs = ' or '.join([str(r) for r in vvv]) _query = {'runNumber': runs} elif kkk == '$lte': maxrun = vvv elif kkk == '$gte': minrun = vvv if minrun and maxrun: _query = { 'runNumber': '>= %s and < %s' % (minrun, maxrun) } elif key == 'date': if isinstance(val, dict): if '$in' in val: value = val['$in'] elif '$lte' in val and '$gte' in val: value = (val['$gte'], val['$lte']) else: msg = 'Unable to get the value from %s=%s' \ % (key, val) raise Exception(msg) try: date1 = convert_datetime(value[0]) date2 = convert_datetime(value[-1]) except: msg = 'Unable to convert to datetime format, %s' \ % value raise Exception(msg) elif isinstance(val, str) or isinstance(val, unicode): date1, date2 = convert2date(val) date1 = rr_date(date1) date2 = rr_date(date2) else: date1 = convert_datetime(val) date2 = convert_datetime(val + 24 * 60 * 60) run_time = '>= %s and < %s' % (date1, date2) _query = {'runStartTime': run_time} else: msg = 'RunRegistryService::api\n\n' msg += "--- %s reject API %s, parameters don't match, args=%s" \ % (self.name, api, args) self.logger.info(msg) return if not _query: msg = 'Unable to match input parameters with input query' raise Exception(msg) if 'run' in args and isinstance(args['run'], dict): args['run'] = str(args['run']) msg = "DASAbstractService:RunRegistry, query=%s" % _query self.logger.info(msg) time0 = time.time() rawrows = rr_worker(url, _query, _table) genrows = self.translator(api, rawrows) if _table == 'runsummary': dasrows = run_duration(genrows) else: dasrows = collect_lumis(genrows) ctime = time.time() - time0 try: self.write_to_cache(\ dasquery, expire, url, api, args, dasrows, ctime) except Exception as exc: print_exc(exc)