示例#1
0
    def add_query(self, query, mongoquery):
        """
        Add DAS-QL/MongoDB-QL queries into analytics.
        
        A unique record is contained for each (qhash, dhash) pair.
        For each an array of call-times is contained.
        """
        if  isinstance(mongoquery, dict):
            mongoquery = encode_mongo_query(mongoquery)
        msg = 'query=%s, mongoquery=%s' % (query, mongoquery)
        self.logger.debug(msg)
        dhash = genkey(query)
        qhash = genkey(mongoquery)

        now = time.time()

        existing = self.col.find_one({'qhash': qhash, 'dhash': dhash})
        if existing:
            # check if times contains very old timestamps
            rec = self.col.find({'_id': ObjectId(existing['_id']), 
                                 'times':{'$lt' : now - self.history}})
            if  rec:
                self.col.update({'_id': ObjectId(existing['_id'])},
                    {'$pull': {'times': {'$lt' : now - self.history}}})
            # update times array with new timestamp
            self.col.update({'_id': ObjectId(existing['_id'])},
                            {'$push': {'times': now}})
        else:
            record = dict(query=query, mongoquery=mongoquery,
                        qhash=qhash, dhash=dhash, times=[now])
            self.col.insert(record)

        index = [('qhash', DESCENDING),
                 ('dhash', DESCENDING)]
        create_indexes(self.col, index)
示例#2
0
 def add_api(self, system, query, api, args):
     """
     Add API info to analytics DB. 
     Here args is a dict of API parameters.
     """
     orig_query = query
     if  isinstance(query, dict):
         query = encode_mongo_query(query)
     msg = '(%s, %s, %s, %s)' % (system, query, api, args)
     self.logger.debug(msg)
     # find query record
     qhash = genkey(query)
     record = self.col.find_one({'qhash':qhash}, fields=['dasquery'])
     if  not record:
         self.add_query("", orig_query)
     # find api record
     record = self.col.find_one({'qhash':qhash, 'system':system,
                     'api.name':api, 'api.params':args}) 
     apidict = dict(name=api, params=args)
     if  record:
         self.col.update({'_id':record['_id']}, {'$inc':{'counter':1}})
     else:
         record = dict(system=system, api=apidict, qhash=qhash, counter=1)
         self.col.insert(record)
     index = [('system', DESCENDING), ('dasquery', DESCENDING),
              ('api.name', DESCENDING), ('qhash', DESCENDING) ]
     create_indexes(self.col, index)
示例#3
0
 def test_encode(self):
     "test query encoding"
     q1 = {'fields':None, 'spec':{'a.b.c':'ghi', 'd.e.f': 'jkl'}}
     q2 = {'fields':None, 'spec':{'a.b.c':'ghi', 'd.e.f': 'jkl'}}
     sq1 = DASQuery(q1).storage_query
     sq2 = encode_mongo_query(q2)
     self.assertEqual(json.JSONEncoder(sort_keys=True).encode(sq1),
                      json.JSONEncoder(sort_keys=True).encode(sq2))
示例#4
0
 def test_encode(self):
     "test query encoding"
     q1 = {'fields': None, 'spec': {'a.b.c': 'ghi', 'd.e.f': 'jkl'}}
     q2 = {'fields': None, 'spec': {'a.b.c': 'ghi', 'd.e.f': 'jkl'}}
     sq1 = DASQuery(q1).storage_query
     sq2 = encode_mongo_query(q2)
     self.assertEqual(
         json.JSONEncoder(sort_keys=True).encode(sq1),
         json.JSONEncoder(sort_keys=True).encode(sq2))
示例#5
0
    def test_encode_decode(self):
        """Test encode/decode_query functions"""
        query  = {'fields': None, 'spec': {'block.name':'aaa'}}
        result = encode_mongo_query(query)
        expect = decode_mongo_query(result)
        self.assertEqual(expect, query)

        query  = {'fields': ['block'], 'spec': {'block.size':{'$lt':10}}}
        result = encode_mongo_query(query)
        expect = decode_mongo_query(result)
        self.assertEqual(expect, query)

        pat = re.compile('/test.*')
        query  = {'fields': ['block'], 'spec': {'dataset.name': pat}}
        result = encode_mongo_query(query)
        expect = decode_mongo_query(result)
        self.assertEqual(expect['fields'], query['fields'])
        self.assertEqual(expect['spec']['dataset.name'].pattern,
                         query['spec']['dataset.name'].pattern)
示例#6
0
 def update_apicall(self, query, das_dict):
     """
     Update apicall record with provided DAS dict.
     Moved from AbstractService
     """
     msg = 'DBSAnalytics::update_apicall, query=%s, das_dict=%s'\
             % (query, das_dict)
     self.logger.debug(msg)
     spec = {'apicall.qhash':genkey(encode_mongo_query(query))} 
     record = self.col.find_one(spec)
     self.col.update({'_id':ObjectId(record['_id'])},
         {'$set':{'dasapi':das_dict,
                  'apicall.expire':das_dict['response_expires']}})
示例#7
0
    def _insert_query(self, rawtext, query, error):
        """Internal method to insert a query"""
        if  self.verbose:
            self.logger.debug("DASParserCache: insert %s->%s/%s" %\
	                          (rawtext, query, error))
        # since MongoDB does not support insertion of $ sign in queries
        # we need to encode inserted query
        if  query:
            encquery = encode_mongo_query(query)
        else:
            encquery = ""
        self.col.insert({'raw':rawtext, 'hash':genkey(rawtext),
                         'query':encquery, 'error':str(error)})
示例#8
0
 def update(self, system, query):
     """
     Update records for given system/query.
     """
     if  isinstance(query, dict):
         query = encode_mongo_query(query)
     msg = 'system=%s, query=%s' % (system, query)
     self.logger.debug(msg)
     qhash = genkey(query)
     if  system:
         cond = {'qhash':qhash, 'system':system}
     else:
         cond = {'qhash':qhash}
     self.col.update(cond, {'$inc' : {'counter':1}}, multi=True)
示例#9
0
 def _insert_query(self, rawtext, query, error):
     """Internal method to insert a query"""
     if self.verbose:
         self.logger.debug("DASParserCache: insert %s->%s/%s" %\
                        (rawtext, query, error))
     # since MongoDB does not support insertion of $ sign in queries
     # we need to encode inserted query
     if query:
         encquery = encode_mongo_query(query)
     else:
         encquery = ""
     self.col.insert({
         'raw': rawtext,
         'qhash': genkey(rawtext),
         'query': encquery,
         'error': str(error)
     })
示例#10
0
 def insert_apicall(self, system, query, url, api, api_params, expire):
     """
     Remove obsolete apicall records and
     insert into Analytics DB provided information about API call.
     Moved from AbstractService.
     
     Updated so that we do not have multiple records when performing
     forced updates (ie, the old record is not yet expired) - now
     look for an existing record with the same parameters (I'm hoping
     the fact that some of the variables are indexed will make this
     fast even though not all are), and if it exists just update
     the expiry. Otherwise insert a new record.
     """
     msg = 'query=%s, url=%s,' % (query, url)
     msg += 'api=%s, args=%s, expire=%s' % (api, api_params, expire)
     self.logger.debug(msg)
     expire = expire_timestamp(expire)
     query = encode_mongo_query(query)
     qhash = genkey(query)
     self.remove_expired()
     existing = self.col.find_one({'apicall.system':     system,
                                   'apicall.url':        url,
                                   'apicall.api':        api,
                                   'apicall.api_params': api_params,
                                   'apicall.qhash':      qhash})
     if existing:
         self.logger.debug("updating")
         self.col.update({'_id': existing['_id']},
                         {'$set':{'apicall.expire': expire}})
     else:
         self.col.insert({'apicall':{'api_params':   api_params,
                                     'url':          url,
                                     'api':          api,
                                     'system':       system,
                                     'expire':       expire,
                                     'qhash':        qhash}})
     index_list = [('apicall.url', DESCENDING),
                   ('apicall.api', DESCENDING),
                   ('qhash', DESCENDING)]
     create_indexes(self.col, index_list)