def lookup_query(self, rawtext):
"""
Check the parser cache for a given rawtext query.
Search is done with the hash of this string.
Returns a tuple (status, value) for the cases
(PARSERCACHE_VALID, mongo_query) - valid query found
(PARSERCACHE_INVALID, error) - error message for invalid query
(PARSERCACHE_NOTFOUND, None) - not in the cache
"""
result = self.col.find_one({'hash':genkey(rawtext)},
fields=['query', 'error'])
if result and result['query']:
if self.verbose:
self.logger.debug("DASParserCache: found valid %s->%s" %\
(rawtext, result['query']))
query = decode_mongo_query(result['query'])
return (PARSERCACHE_VALID, query)
elif result and result['error']:
if self.verbose:
self.logger.debug("DASParserCache: found invalid %s->%s" %\
(rawtext, result['error']))
return (PARSERCACHE_INVALID, result['error'])
else:
if self.verbose:
self.logger.debug("DASParserCache: not found %s" %\
(rawtext))
return (PARSERCACHE_NOTFOUND, None)
def lookup_query(self, rawtext):
"""
Check the parser cache for a given rawtext query.
Search is done with the qhash of this string.
Returns a tuple (status, value) for the cases
(PARSERCACHE_VALID, mongo_query) - valid query found
(PARSERCACHE_INVALID, error) - error message for invalid query
(PARSERCACHE_NOTFOUND, None) - not in the cache
"""
result = find_one(self.col, {'qhash':genkey(rawtext)}, \
fields=['query', 'error'])
if result and result['query']:
if self.verbose:
self.logger.debug("DASParserCache: found valid %s->%s" %\
(rawtext, result['query']))
query = decode_mongo_query(result['query'])
return (PARSERCACHE_VALID, query)
elif result and result['error']:
if self.verbose:
self.logger.debug("DASParserCache: found invalid %s->%s" %\
(rawtext, result['error']))
return (PARSERCACHE_INVALID, result['error'])
else:
if self.verbose:
self.logger.debug("DASParserCache: not found %s" %\
(rawtext))
return (PARSERCACHE_NOTFOUND, None)
def test_decode(self):
"test query decoding"
sq1 = {
'fields':
None,
'spec': [{
'key': 'a.b.c',
'value': '"ghi"'
}, {
'key': 'd.e.f',
'value': '"jkl"'
}]
}
sq2 = {
'fields':
None,
'spec': [{
'key': 'a.b.c',
'value': '"ghi"'
}, {
'key': 'd.e.f',
'value': '"jkl"'
}]
}
q1 = DASQuery(sq1).mongo_query
q2 = decode_mongo_query(sq2)
self.assertEqual(
json.JSONEncoder(sort_keys=True).encode(q1),
json.JSONEncoder(sort_keys=True).encode(q2))
def test_encode_decode(self):
"""Test encode/decode_query functions"""
query = {'fields': None, 'spec': {'block.name':'aaa'}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect, query)
query = {'fields': ['block'], 'spec': {'block.size':{'$lt':10}}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect, query)
pat = re.compile('/test.*')
query = {'fields': ['block'], 'spec': {'dataset.name': pat}}
result = encode_mongo_query(query)
expect = decode_mongo_query(result)
self.assertEqual(expect['fields'], query['fields'])
self.assertEqual(expect['spec']['dataset.name'].pattern,
query['spec']['dataset.name'].pattern)
def test_decode(self):
"test query decoding"
sq1 = {'fields':None,
'spec':[{'key':'a.b.c', 'value': '"ghi"'},
{'key':'d.e.f', 'value':'"jkl"'}]}
sq2 = {'fields':None,
'spec':[{'key':'a.b.c', 'value': '"ghi"'},
{'key':'d.e.f', 'value':'"jkl"'}]}
q1 = DASQuery(sq1).mongo_query
q2 = decode_mongo_query(sq2)
self.assertEqual(json.JSONEncoder(sort_keys=True).encode(q1),
json.JSONEncoder(sort_keys=True).encode(q2))
def get_superset_keys(self, key, value):
"""
This is a special-case version of similar_keys,
intended for analysers that want to quickly
find possible superset queries of a simple
query of the form key=value.
"""
msg = "%s=%s" % (key, value)
self.logger.debug(msg)
cond = {'query.spec.key': key}
for row in self.col.find(cond):
mongo_query = decode_mongo_query(row['query'])
for thiskey, thisvalue in mongo_query.iteritems():
if thiskey == key:
if fnmatch.fnmatch(value, thisvalue):
yield thisvalue
def get_superset_keys(self, key, value):
"""
This is a special-case version of similar_keys,
intended for analysers that want to quickly
find possible superset queries of a simple
query of the form key=value.
"""
msg = "%s=%s" % (key, value)
self.logger.debug(msg)
cond = {'query.spec.key': key}
for row in self.col.find(cond, **PYMONGO_OPTS):
mongo_query = decode_mongo_query(row['query'])
for thiskey, thisvalue in mongo_query.items():
if thiskey == key:
if fnmatch.fnmatch(value, thisvalue):
yield thisvalue