Example #1
0
 def adjust_params(self, api, kwds, _inst=None):
     """
     Adjust CondDB parameters for specific query requests
     """
     day = 24*60*60
     if  api == 'get_lumi_info':
         if  kwds.has_key('date') and kwds['date'] != 'optional':
             value = kwds['date']
             if  isinstance(value, str) or isinstance(value, unicode):
                 value = convert2date(value)
             elif isinstance(value, dict):
                 value = [kwds['date']['$gte'], kwds['date']['$lte']]
             elif isinstance(value, int):
                 value = [value, value+day]
             else:
                 msg = 'Unsupported date format %s' % kwds['date']
                 raise Exception(msg)
             kwds['startTime'] = convert_datetime(value[0])
             kwds['endTime'] = convert_datetime(value[1])
             del kwds['date']
         elif kwds.has_key('runList'):
             val = kwds['runList']
             if  isinstance(val, dict): # we got a run range
                 kwds['runList'] = '%s-%s' % (val['$gte'], val['$lte'])
     if  api == 'get_run_info':
         if  kwds.has_key('date') and kwds['date'] != 'optional':
             value = kwds['date']
             if  isinstance(value, str) or isinstance(value, unicode):
                 value = convert2date(value)
             elif isinstance(value, dict):
                 value = [kwds['date']['$gte'], kwds['date']['$lte']]
             elif isinstance(value, int):
                 value = [value, value+day]
             else:
                 msg = 'Unsupported date format %s' % kwds['date']
                 raise Exception(msg)
             kwds['startTime'] = convert_datetime(value[0])
             kwds['endTime'] = convert_datetime(value[1])
             del kwds['date']
     if  kwds.has_key('Runs') and isinstance(kwds['Runs'], dict):
         minrun = 0
         maxrun = 0
         for kkk, vvv in kwds['Runs'].iteritems():
             if  kkk == '$in':
                 if len(vvv) == 2:
                     minrun, maxrun = vvv
                 else: # in[1, 2, 3]
                     msg = "conddb can not deal with 'in'"
                     self.logger.info(msg)
                     continue
             elif kkk == '$lte':
                 maxrun = vvv
             elif kkk == '$gte':
                 minrun = vvv
         kwds['Runs'] = '%s-%s' % (minrun, maxrun)
Example #2
0
 def adjust_params(self, api, kwds, _inst=None):
     """
     Adjust CondDB parameters for specific query requests
     """
     day = 24*60*60
     if  api == 'get_run_info':
         if  'date' in kwds and kwds['date'] != 'optional':
             value = kwds['date']
             if  isinstance(value, str) or isinstance(value, unicode):
                 value = convert2date(value)
             elif isinstance(value, dict):
                 value = [kwds['date']['$gte'], kwds['date']['$lte']]
             elif isinstance(value, int):
                 value = [value, value+day]
             else:
                 msg = 'Unsupported date format %s' % kwds['date']
                 raise Exception(msg)
             kwds['startTime'] = convert_datetime(value[0])
             kwds['endTime'] = convert_datetime(value[1])
             del kwds['date']
     if  'Runs' in kwds and isinstance(kwds['Runs'], dict):
         minrun = 0
         maxrun = 0
         for kkk, vvv in kwds['Runs'].items():
             if  kkk == '$in':
                 kwds['Runs'] = ','.join([str(r) for r in vvv])
             elif kkk == '$lte':
                 maxrun = vvv
             elif kkk == '$gte':
                 minrun = vvv
         if  minrun and maxrun:
             kwds['Runs'] = '%s-%s' % (minrun, maxrun)
     if  'Runs' in kwds:
         if  not kwds['Runs']:
             del kwds['Runs']
Example #3
0
 def adjust_params(self, api, kwds, _inst=None):
     """
     Adjust CondDB parameters for specific query requests
     """
     day = 24 * 60 * 60
     if api == "get_run_info":
         if "date" in kwds and kwds["date"] != "optional":
             value = kwds["date"]
             if isinstance(value, str) or isinstance(value, unicode):
                 value = convert2date(value)
             elif isinstance(value, dict):
                 value = [kwds["date"]["$gte"], kwds["date"]["$lte"]]
             elif isinstance(value, int):
                 value = [value, value + day]
             else:
                 msg = "Unsupported date format %s" % kwds["date"]
                 raise Exception(msg)
             kwds["startTime"] = convert_datetime(value[0])
             kwds["endTime"] = convert_datetime(value[1])
             del kwds["date"]
     if "Runs" in kwds and isinstance(kwds["Runs"], dict):
         minrun = 0
         maxrun = 0
         for kkk, vvv in kwds["Runs"].items():
             if kkk == "$in":
                 kwds["Runs"] = ",".join([str(r) for r in vvv])
             elif kkk == "$lte":
                 maxrun = vvv
             elif kkk == "$gte":
                 minrun = vvv
         if minrun and maxrun:
             kwds["Runs"] = "%s-%s" % (minrun, maxrun)
     if "Runs" in kwds:
         if not kwds["Runs"]:
             del kwds["Runs"]
Example #4
0
File: utils_t.py Project: ktf/DAS
    def test_convert2date(self):
        """Test convert2date function"""
        date = '24h'
        expect = [long(time.time()-24*60*60), long(time.time())]
        result = convert2date(date)
        self.assertEqual(expect, result)

        date = '60m'
        expect = [long(time.time()-60*60), long(time.time())]
        result = convert2date(date)
        self.assertEqual(expect, result)

        date = '60s'
        self.assertRaises(Exception, convert2date, date)

        date = '123'
        self.assertRaises(Exception, convert2date, date)
Example #5
0
    def test_convert2date(self):
        """Test convert2date function"""
        date = '24h'
        expect = [long(time.time() - 24 * 60 * 60), long(time.time())]
        result = convert2date(date)
        self.assertEqual(expect, result)

        date = '60m'
        expect = [long(time.time() - 60 * 60), long(time.time())]
        result = convert2date(date)
        self.assertEqual(expect, result)

        date = '60s'
        self.assertRaises(Exception, convert2date, date)

        date = '123'
        self.assertRaises(Exception, convert2date, date)
Example #6
0
    def apicall(self, dasquery, url, api, args, dformat, expire):
        """
        An overview data-service worker.
        """
        keys = [
            key for key in self.map[api]['keys'] for api in self.map.keys()
        ]
        cond = dasquery.mongo_query['spec']
        for key, value in cond.items():
            if key.find('.') != -1:
                args['grouping'] = key.split('.')[-1]
                key, _attr = key.split('.', 1)
            if key not in keys:
                continue
            args['end'] = '%d' % time.time()

            if not isinstance(value, dict):  # we got equal condition
                if key == 'date':
                    if isinstance(value, list) and len(value) != 2:
                        msg = 'Monitor service requires 2 time stamps.'
                        msg += 'Please use either date last XXh format or'
                        msg += 'date in [YYYYMMDD, YYYYMMDD]'
                        raise Exception(msg)
                    if isinstance(value, str) or isinstance(value, unicode):
                        value = convert2date(value)
                    else:
                        value = [value, value + 24 * 60 * 60]
                    args['start'] = value[0]
                    args['end'] = value[1]
            else:  # we got some operator, e.g. key :{'$in' : [1,2,3]}
                if key == 'date':
                    if '$in' in value:
                        vallist = value['$in']
                    elif '$lte' in value and '$gte' in value:
                        vallist = (value['$gte'], value['$lte'])
                    else:
                        err = 'Unsupported date value'
                        raise Exception(err)
                    args['start'] = convert_datetime(vallist[0])
                    args['end'] = convert_datetime(vallist[-1])
                else:
                    raise Exception(err)
            time0 = time.time()
            res, expire = self.getdata(url, args, expire)
            dasrows = self.parser(dasquery, dformat, res, args)
            ctime = time.time() - time0
            self.write_to_cache(\
                dasquery, expire, url, api, args, dasrows, ctime)
Example #7
0
    def apicall(self, dasquery, url, api, args, dformat, expire):
        """
        An overview data-service worker.
        """
        keys = [key for key in self.map[api]['keys'] for api in self.map.keys()]
        cond = dasquery.mongo_query['spec']
        for key, value in cond.iteritems():
            if  key.find('.') != -1:
                args['grouping'] = key.split('.')[-1]
                key, _attr = key.split('.', 1)
            if  key not in keys:
                continue
            args['end'] = '%d' % time.time()

            if  not isinstance(value, dict): # we got equal condition
                if  key == 'date':
                    if  isinstance(value, list) and len(value) != 2:
                        msg  = 'Monitor service requires 2 time stamps.'
                        msg += 'Please use either date last XXh format or'
                        msg += 'date in [YYYYMMDD, YYYYMMDD]'
                        raise Exception(msg)
                    if  isinstance(value, str) or isinstance(value, unicode):
                        value = convert2date(value)
                    else:
                        value = [value, value + 24*60*60]
                    args['start'] = value[0]
                    args['end']   = value[1]
            else: # we got some operator, e.g. key :{'$in' : [1,2,3]}
                if  key == 'date':
                    if  value.has_key('$in'):
                        vallist = value['$in']
                    elif value.has_key('$lte') and value.has_key('$gte'):
                        vallist = (value['$gte'], value['$lte'])
                    else:
                        err = 'Unsupported date value'
                        raise Exception(err)
                    args['start'] = convert_datetime(vallist[0])
                    args['end'] = convert_datetime(vallist[-1])
                else:
                    raise Exception(err)
            time0   = time.time()
            res, expire = self.getdata(url, args, expire)
            genrows = self.parser(dasquery, dformat, res, args)
            dasrows = self.set_misses(dasquery, api, genrows)
            ctime   = time.time() - time0
            self.write_to_cache(\
                dasquery, expire, url, api, args, dasrows, ctime)
Example #8
0
    def apicall(self, dasquery, url, api, args, dformat, expire):
        """
        A service worker. It parses input query, invoke service API 
        and return results in a list with provided row.
        """
        cond = dasquery.mongo_query['spec']
        count = 0
        for key, value in cond.items():
            err = 'JobSummary does not support key=%s, value=%s' \
                    % (key, value)
            if not isinstance(value, dict):  # we got equal condition
                if key == 'date':
                    if isinstance(value, list) and len(value) != 2:
                        msg = 'Dashboard service requires 2 time stamps.'
                        msg += 'Please use either date last XXh format or'
                        msg += 'date in [YYYYMMDD, YYYYMMDD]'
                        raise Exception(msg)
                    if isinstance(value, str) or isinstance(value, unicode):
                        value = convert2date(value)
                    else:
                        value = [value, value + 24 * 60 * 60]
                    args['date1'] = convert_datetime(value[0])
                    args['date2'] = convert_datetime(value[1])
                    count += 1
                else:
                    for param in self.dasmapping.das2api(self.name, api, key):
                        args[param] = value
                        count += 1
            else:  # we got some operator, e.g. key :{'$in' : [1,2,3]}
                if key == 'date' or key == 'jobsummary':
                    if '$in' in value:
                        vallist = value['$in']
                    elif '$lte' in value and '$gte' in value:
                        vallist = (value['$gte'], value['$lte'])
                    else:
                        raise Exception(err)
                    args['date1'] = convert_datetime(vallist[0])
                    args['date2'] = convert_datetime(vallist[-1])
                    count += 1
                else:
                    raise Exception(err)
        if not count:
            # if no parameter are given, don't pass the API
            msg = 'DashboardService::api\n\n'
            msg += "--- %s reject API %s, parameters don't match, args=%s" \
                    % (self.name, api, args)
            self.logger.info(msg)
            return
        else:
            if not args['date1']:
                args['date1'] = convert_datetime(time.time() - 24 * 60 * 60)
            if not args['date2']:
                args['date2'] = convert_datetime(time.time())
        # drop date argument, since it's used by DAS not by dashboard data srv
        if 'date' in args:
            args.pop('date')

        time0 = time.time()
        res, expire = self.getdata(url, args, expire, headers=self.headers)
        rawrows = self.parser(res, api, args)
        dasrows = self.translator(api, rawrows)
        ctime = time.time() - time0
        try:
            self.write_to_cache(\
                dasquery, expire, url, api, args, dasrows, ctime)
        except Exception as exc:
            print_exc(exc)
Example #9
0
    def apicall(self, dasquery, url, api, args, dformat, expire):
        """
        A service worker. It parses input query, invoke service API 
        and return results in a list with provided row.
        """
        cond   = dasquery.mongo_query['spec']
        count  = 0
        for key, value in cond.iteritems():
            err = 'JobSummary does not support key=%s, value=%s' \
                    % (key, value)
            if  not isinstance(value, dict): # we got equal condition
                if  key == 'date':
                    if  isinstance(value, list) and len(value) != 2:
                        msg  = 'Dashboard service requires 2 time stamps.'
                        msg += 'Please use either date last XXh format or'
                        msg += 'date in [YYYYMMDD, YYYYMMDD]'
                        raise Exception(msg)
                    if  isinstance(value, str) or isinstance(value, unicode):
                        value = convert2date(value)
                    else:
                        value = [value, value + 24*60*60]
                    args['date1'] = convert_datetime(value[0])
                    args['date2'] = convert_datetime(value[1])
                    count += 1
                else:
                    for param in self.dasmapping.das2api(self.name, key):
                        args[param] = value
                        count += 1
            else: # we got some operator, e.g. key :{'$in' : [1,2,3]}
                if  key == 'date' or key == 'jobsummary':
                    if  value.has_key('$in'):
                        vallist = value['$in']
                    elif value.has_key('$lte') and value.has_key('$gte'):
                        vallist = (value['$gte'], value['$lte'])
                    else:
                        raise Exception(err)
                    args['date1'] = convert_datetime(vallist[0])
                    args['date2'] = convert_datetime(vallist[-1])
                    count += 1
                else:
                    raise Exception(err)
        if  not count:
            # if no parameter are given, don't pass the API
            msg  = 'DashboardService::api\n\n'
            msg += "--- %s reject API %s, parameters don't match, args=%s" \
                    % (self.name, api, args)
            self.logger.info(msg)
            return
        else:
            if  not args['date1']:
                args['date1'] = convert_datetime(time.time()-24*60*60)
            if  not args['date2']:
                args['date2'] = convert_datetime(time.time())

        time0 = time.time()
        res, expire = self.getdata(url, args, expire, headers=self.headers)
        rawrows = self.parser(res, api, args)
        genrows = self.translator(api, rawrows)
        dasrows = self.set_misses(dasquery, api, genrows)
        ctime = time.time() - time0
        try:
            self.write_to_cache(\
                dasquery, expire, url, api, args, dasrows, ctime)
        except Exception as exc:
            print_exc(exc)
Example #10
0
 def apicall(self, dasquery, url, api, args, dformat, expire):
     """
     A service worker. It parses input query, invoke service API 
     and return results in a list with provided row.
     """
     _query  = ''
     _table  = 'runsummary'
     if  api == 'rr_xmlrpc_lumis':
         _table = 'runlumis'
     for key, val in dasquery.mongo_query['spec'].items():
         if  key == 'run.run_number':
             if  isinstance(val, int):
                 _query = {'runNumber': '%s' % val}
             elif isinstance(val, dict):
                 minrun = 0
                 maxrun = 0
                 for kkk, vvv in val.items():
                     if  kkk == '$in':
                         runs = ' or '.join([str(r) for r in vvv])
                         _query = {'runNumber': runs}
                     elif kkk == '$lte':
                         maxrun = vvv
                     elif kkk == '$gte':
                         minrun = vvv
                 if  minrun and maxrun:
                     _query = {'runNumber': '>= %s and < %s' % (minrun, maxrun)}
         elif key == 'date':
             if  isinstance(val, dict):
                 if  '$in' in val:
                     value = val['$in']
                 elif '$lte' in val and '$gte' in val:
                     value = (val['$gte'], val['$lte'])
                 else:
                     msg = 'Unable to get the value from %s=%s' \
                             % (key, val) 
                     raise Exception(msg)
                 try:
                     date1 = convert_datetime(value[0])
                     date2 = convert_datetime(value[-1])
                 except:
                     msg = 'Unable to convert to datetime format, %s' \
                         % value
                     raise Exception(msg)
             elif  isinstance(val, str) or isinstance(val, unicode):
                 date1, date2 = convert2date(val)
                 date1 = rr_date(date1)
                 date2 = rr_date(date2)
             else:
                 date1 = convert_datetime(val)
                 date2 = convert_datetime(val + 24*60*60)
             run_time = '>= %s and < %s' % (date1, date2)
             _query = {'runStartTime': run_time}
         else:
             msg  = 'RunRegistryService::api\n\n'
             msg += "--- %s reject API %s, parameters don't match, args=%s" \
                     % (self.name, api, args)
             self.logger.info(msg)
             return
     if  not _query:
         msg = 'Unable to match input parameters with input query'
         raise Exception(msg)
     if  'run' in args and isinstance(args['run'], dict):
         args['run'] = str(args['run'])
     msg = "DASAbstractService:RunRegistry, query=%s" % _query
     self.logger.info(msg)
     time0   = time.time()
     rawrows = rr_worker(url, _query, _table)
     genrows = self.translator(api, rawrows)
     if  _table == 'runsummary':
         dasrows = run_duration(genrows)
     else:
         dasrows = collect_lumis(genrows)
     ctime   = time.time() - time0
     try:
         self.write_to_cache(\
             dasquery, expire, url, api, args, dasrows, ctime)
     except Exception as exc:
         print_exc(exc)
Example #11
0
 def apicall(self, dasquery, url, api, args, dformat, expire):
     """
     A service worker. It parses input query, invoke service API 
     and return results in a list with provided row.
     """
     _query  = ""
     for key, val in dasquery.mongo_query['spec'].iteritems():
         if  key == 'run.run_number':
             if  isinstance(val, int):
                 _query = {'runNumber': '%s' % val}
             elif isinstance(val, dict):
                 minrun = 0
                 maxrun = 0
                 for kkk, vvv in val.iteritems():
                     if  kkk == '$in':
                         if len(vvv) == 2:
                             minrun, maxrun = vvv
                         else: # in[1, 2, 3]
                             msg = "runregistry can not deal with 'in'"
                             self.logger.info(msg)
                             continue
                     elif kkk == '$lte':
                         maxrun = vvv
                     elif kkk == '$gte':
                         minrun = vvv
                 _query = {'runNumber': '>= %s and < %s' % (minrun, maxrun)}
         elif key == 'date':
             if  isinstance(val, dict):
                 if  val.has_key('$in'):
                     value = val['$in']
                 elif val.has_key('$lte') and val.has_key('$gte'):
                     value = (val['$gte'], val['$lte'])
                 else:
                     msg = 'Unable to get the value from %s=%s' \
                             % (key, val) 
                     raise Exception(msg)
                 try:
                     date1 = convert_datetime(value[0])
                     date2 = convert_datetime(value[-1])
                 except:
                     msg = 'Unable to convert to datetime format, %s' \
                         % value
                     raise Exception(msg)
             elif  isinstance(val, str) or isinstance(val, unicode):
                 date1, date2 = convert2date(val)
                 date1 = rr_date(date1)
                 date2 = rr_date(date2)
             else:
                 date1 = convert_datetime(val)
                 date2 = convert_datetime(val + 24*60*60)
             run_time = '>= %s and < %s' % (date1, date2)
             _query = {'runStartTime': run_time}
         else:
             msg  = 'RunRegistryService::api\n\n'
             msg += "--- %s reject API %s, parameters don't match, args=%s" \
                     % (self.name, api, args)
             self.logger.info(msg)
             return
     if  not _query:
         msg = 'Unable to match input parameters with input query'
         raise Exception(msg)
     if  args.has_key('run') and isinstance(args['run'], dict):
         args['run'] = str(args['run'])
     msg = "DASAbstractService:RunRegistry, query=%s" % _query
     self.logger.info(msg)
     time0   = time.time()
     api_ver = 3 # API version for RunRegistry, v2 is xmlrpc, v3 is REST
     rawrows = worker(url, _query, api_ver)
     genrows = self.translator(api, rawrows)
     dasrows = self.set_misses(dasquery, api, run_duration(genrows, api_ver))
     ctime   = time.time() - time0
     try:
         self.write_to_cache(\
             dasquery, expire, url, api, args, dasrows, ctime)
     except Exception as exc:
         print_exc(exc)
Example #12
0
 def apicall(self, dasquery, url, api, args, dformat, expire):
     """
     A service worker. It parses input query, invoke service API 
     and return results in a list with provided row.
     """
     _query = ''
     _table = 'runsummary'
     if api == 'rr_xmlrpc_lumis':
         _table = 'runlumis'
     for key, val in dasquery.mongo_query['spec'].items():
         if key == 'run.run_number':
             if isinstance(val, int):
                 _query = {'runNumber': '%s' % val}
             elif isinstance(val, dict):
                 minrun = 0
                 maxrun = 0
                 for kkk, vvv in val.items():
                     if kkk == '$in':
                         runs = ' or '.join([str(r) for r in vvv])
                         _query = {'runNumber': runs}
                     elif kkk == '$lte':
                         maxrun = vvv
                     elif kkk == '$gte':
                         minrun = vvv
                 if minrun and maxrun:
                     _query = {
                         'runNumber': '>= %s and < %s' % (minrun, maxrun)
                     }
         elif key == 'date':
             if isinstance(val, dict):
                 if '$in' in val:
                     value = val['$in']
                 elif '$lte' in val and '$gte' in val:
                     value = (val['$gte'], val['$lte'])
                 else:
                     msg = 'Unable to get the value from %s=%s' \
                             % (key, val)
                     raise Exception(msg)
                 try:
                     date1 = convert_datetime(value[0])
                     date2 = convert_datetime(value[-1])
                 except:
                     msg = 'Unable to convert to datetime format, %s' \
                         % value
                     raise Exception(msg)
             elif isinstance(val, str) or isinstance(val, unicode):
                 date1, date2 = convert2date(val)
                 date1 = rr_date(date1)
                 date2 = rr_date(date2)
             else:
                 date1 = convert_datetime(val)
                 date2 = convert_datetime(val + 24 * 60 * 60)
             run_time = '>= %s and < %s' % (date1, date2)
             _query = {'runStartTime': run_time}
         else:
             msg = 'RunRegistryService::api\n\n'
             msg += "--- %s reject API %s, parameters don't match, args=%s" \
                     % (self.name, api, args)
             self.logger.info(msg)
             return
     if not _query:
         msg = 'Unable to match input parameters with input query'
         raise Exception(msg)
     if 'run' in args and isinstance(args['run'], dict):
         args['run'] = str(args['run'])
     msg = "DASAbstractService:RunRegistry, query=%s" % _query
     self.logger.info(msg)
     time0 = time.time()
     rawrows = rr_worker(url, _query, _table)
     genrows = self.translator(api, rawrows)
     if _table == 'runsummary':
         dasrows = run_duration(genrows)
     else:
         dasrows = collect_lumis(genrows)
     ctime = time.time() - time0
     try:
         self.write_to_cache(\
             dasquery, expire, url, api, args, dasrows, ctime)
     except Exception as exc:
         print_exc(exc)