def add_query(self, query, mongoquery):
"""
Add DAS-QL/MongoDB-QL queries into analytics.
A unique record is contained for each (qhash, dhash) pair.
For each an array of call-times is contained.
"""
if isinstance(mongoquery, dict):
mongoquery = encode_mongo_query(mongoquery)
msg = 'query=%s, mongoquery=%s' % (query, mongoquery)
self.logger.debug(msg)
dhash = genkey(query)
qhash = genkey(mongoquery)
now = time.time()
existing = self.col.find_one({'qhash': qhash, 'dhash': dhash})
if existing:
# check if times contains very old timestamps
rec = self.col.find({'_id': ObjectId(existing['_id']),
'times':{'$lt' : now - self.history}})
if rec:
self.col.update({'_id': ObjectId(existing['_id'])},
{'$pull': {'times': {'$lt' : now - self.history}}})
# update times array with new timestamp
self.col.update({'_id': ObjectId(existing['_id'])},
{'$push': {'times': now}})
else:
record = dict(query=query, mongoquery=mongoquery,
qhash=qhash, dhash=dhash, times=[now])
self.col.insert(record)
index = [('qhash', DESCENDING),
('dhash', DESCENDING)]
create_indexes(self.col, index)
def add_api(self, system, query, api, args):
"""
Add API info to analytics DB.
Here args is a dict of API parameters.
"""
orig_query = query
if isinstance(query, dict):
query = encode_mongo_query(query)
msg = '(%s, %s, %s, %s)' % (system, query, api, args)
self.logger.debug(msg)
# find query record
qhash = genkey(query)
record = self.col.find_one({'qhash':qhash}, fields=['dasquery'])
if not record:
self.add_query("", orig_query)
# find api record
record = self.col.find_one({'qhash':qhash, 'system':system,
'api.name':api, 'api.params':args})
apidict = dict(name=api, params=args)
if record:
self.col.update({'_id':record['_id']}, {'$inc':{'counter':1}})
else:
record = dict(system=system, api=apidict, qhash=qhash, counter=1)
self.col.insert(record)
index = [('system', DESCENDING), ('dasquery', DESCENDING),
('api.name', DESCENDING), ('qhash', DESCENDING) ]
create_indexes(self.col, index)
def test_encode(self):
"test query encoding"
q1 = {'fields':None, 'spec':{'a.b.c':'ghi', 'd.e.f': 'jkl'}}
q2 = {'fields':None, 'spec':{'a.b.c':'ghi', 'd.e.f': 'jkl'}}
sq1 = DASQuery(q1).storage_query
sq2 = encode_mongo_query(q2)
self.assertEqual(json.JSONEncoder(sort_keys=True).encode(sq1),
json.JSONEncoder(sort_keys=True).encode(sq2))
def test_encode(self):
"test query encoding"
q1 = {'fields': None, 'spec': {'a.b.c': 'ghi', 'd.e.f': 'jkl'}}
q2 = {'fields': None, 'spec': {'a.b.c': 'ghi', 'd.e.f': 'jkl'}}
sq1 = DASQuery(q1).storage_query
sq2 = encode_mongo_query(q2)
self.assertEqual(
json.JSONEncoder(sort_keys=True).encode(sq1),
json.JSONEncoder(sort_keys=True).encode(sq2))
def test_encode_decode(self):
"""Test encode/decode_query functions"""
query = {'fields': None, 'spec': {'block.name':'aaa'}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect, query)
query = {'fields': ['block'], 'spec': {'block.size':{'$lt':10}}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect, query)
pat = re.compile('/test.*')
query = {'fields': ['block'], 'spec': {'dataset.name': pat}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect['fields'], query['fields'])
self.assertEqual(expect['spec']['dataset.name'].pattern,
query['spec']['dataset.name'].pattern)
def update_apicall(self, query, das_dict):
"""
Update apicall record with provided DAS dict.
Moved from AbstractService
"""
msg = 'DBSAnalytics::update_apicall, query=%s, das_dict=%s'\
% (query, das_dict)
self.logger.debug(msg)
spec = {'apicall.qhash':genkey(encode_mongo_query(query))}
record = self.col.find_one(spec)
self.col.update({'_id':ObjectId(record['_id'])},
{'$set':{'dasapi':das_dict,
'apicall.expire':das_dict['response_expires']}})
def _insert_query(self, rawtext, query, error):
"""Internal method to insert a query"""
if self.verbose:
self.logger.debug("DASParserCache: insert %s->%s/%s" %\
(rawtext, query, error))
# since MongoDB does not support insertion of $ sign in queries
# we need to encode inserted query
if query:
encquery = encode_mongo_query(query)
else:
encquery = ""
self.col.insert({'raw':rawtext, 'hash':genkey(rawtext),
'query':encquery, 'error':str(error)})
def update(self, system, query):
"""
Update records for given system/query.
"""
if isinstance(query, dict):
query = encode_mongo_query(query)
msg = 'system=%s, query=%s' % (system, query)
self.logger.debug(msg)
qhash = genkey(query)
if system:
cond = {'qhash':qhash, 'system':system}
else:
cond = {'qhash':qhash}
self.col.update(cond, {'$inc' : {'counter':1}}, multi=True)
def _insert_query(self, rawtext, query, error):
"""Internal method to insert a query"""
if self.verbose:
self.logger.debug("DASParserCache: insert %s->%s/%s" %\
(rawtext, query, error))
# since MongoDB does not support insertion of $ sign in queries
# we need to encode inserted query
if query:
encquery = encode_mongo_query(query)
else:
encquery = ""
self.col.insert({
'raw': rawtext,
'qhash': genkey(rawtext),
'query': encquery,
'error': str(error)
})
def insert_apicall(self, system, query, url, api, api_params, expire):
"""
Remove obsolete apicall records and
insert into Analytics DB provided information about API call.
Moved from AbstractService.
Updated so that we do not have multiple records when performing
forced updates (ie, the old record is not yet expired) - now
look for an existing record with the same parameters (I'm hoping
the fact that some of the variables are indexed will make this
fast even though not all are), and if it exists just update
the expiry. Otherwise insert a new record.
"""
msg = 'query=%s, url=%s,' % (query, url)
msg += 'api=%s, args=%s, expire=%s' % (api, api_params, expire)
self.logger.debug(msg)
expire = expire_timestamp(expire)
query = encode_mongo_query(query)
qhash = genkey(query)
self.remove_expired()
existing = self.col.find_one({'apicall.system': system,
'apicall.url': url,
'apicall.api': api,
'apicall.api_params': api_params,
'apicall.qhash': qhash})
if existing:
self.logger.debug("updating")
self.col.update({'_id': existing['_id']},
{'$set':{'apicall.expire': expire}})
else:
self.col.insert({'apicall':{'api_params': api_params,
'url': url,
'api': api,
'system': system,
'expire': expire,
'qhash': qhash}})
index_list = [('apicall.url', DESCENDING),
('apicall.api', DESCENDING),
('qhash', DESCENDING)]
create_indexes(self.col, index_list)