def pass_apicall(self, dasquery, url, api, api_params):
"""
Filter provided apicall wrt existing apicall records in Analytics DB.
"""
self.analytics.remove_expired()
msg = 'API=%s, args=%s' % (api, api_params)
for row in self.analytics.list_apicalls(url=url, api=api):
input_query = {'spec':api_params}
exist_query = {'spec':row['apicall']['api_params']}
if compare_specs(input_query, exist_query):
msg += '\nwill re-use existing api call with args=%s, query=%s'\
% (row['apicall']['api_params'], exist_query)
self.logger.info(msg)
try:
# update DAS cache with empty result set
args = self.inspect_params(api, api_params)
cond = {'das.qhash': row['apicall']['qhash']}
record = self.localcache.col.find_one(cond)
if record and record.has_key('das') and \
record['das'].has_key('expire'):
expire = record['das']['expire']
self.write_to_cache(\
dasquery, expire, url, api, args, [], 0)
except Exception as exc:
print_exc(exc)
msg = 'failed api %s\n' % api
msg += 'input query %s\n' % input_query
msg += 'existing query %s\n' % exist_query
msg += 'Unable to look-up existing query and extract '
msg += 'expire timestamp'
raise Exception(msg)
return False
return True
def is_subset_of(self, other):
"""
Return true if we are not a superset of the other.
This is perhaps not the same thing as "is subset of"
"""
return compare_specs(self.mongo_query, other.mongo_query)
def is_superset_of(self, other):
"""
Return true if we are a superset of other
"""
return compare_specs(other.mongo_query, self.mongo_query)
def is_subset_of(self, other):
"""
Return true if we are not a superset of the other.
This is perhaps not the same thing as "is subset of"
"""
return compare_specs(self.mongo_query, other.mongo_query)
def is_superset_of(self, other):
"""
Return true if we are a superset of other
"""
return compare_specs(other.mongo_query, self.mongo_query)
def test_compare_specs(self):
"""
Test compare_specs funtion.
"""
input = {'spec': {u'node': u'*', u'block': '*', u'se': u'*'}}
exist = {'spec': {u'node': u'T1_CH_CERN', u'se': u'T1_CH_CERN', u'block': u'*'}}
result = compare_specs(input, exist)
self.assertEqual(False, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':'T1_CH_CERN'})
exist_query = dict(fields=['site'], spec={})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':'T1_CH_CERN'})
exist_query = dict(fields=None, spec={'test':'T1_*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':'site'})
exist_query = dict(fields=None, spec={'test':'site_ch'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # no pattern
input_query = dict(fields=None, spec={'test':'site_ch'})
exist_query = dict(fields=None, spec={'test':'site'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # no pattern
input_query = dict(fields=None, spec={'test':'site*'})
exist_query = dict(fields=None, spec={'test':'site_ch'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=None, spec={'test':'site_ch'})
exist_query = dict(fields=None, spec={'test':'site*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=['site','block'], spec={'test':'site*'})
exist_query = dict(fields=['site'], spec={'test':'site_ch'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=['site','block'], spec={'test':'site_ch'})
exist_query = dict(fields=['site'], spec={'test':'site*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is a superset, but different fields
input_query = dict(fields=['block'], spec={'test':'site*'})
exist_query = dict(fields=['site'], spec={'test':'site*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # different fields
input_query = dict(fields=None, spec={'test':'T1_CH_*'})
exist_query = dict(fields=None, spec={'test':'T1_CH_CERN'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=None, spec={'test':'T1_CH_CERN'})
exist_query = dict(fields=None, spec={'test':'T1_CH_*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':{'$gt':10}})
exist_query = dict(fields=None, spec={'test':{'$gt':11}})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=None, spec={'test':{'$gt':10}})
exist_query = dict(fields=None, spec={'test':{'$gt':9}})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':{'$lt':10}})
exist_query = dict(fields=None, spec={'test':{'$lt':9}})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=None, spec={'test':{'$lt':10}})
exist_query = dict(fields=None, spec={'test':{'$lt':11}})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':{'$in':[1,2,3]}})
exist_query = dict(fields=None, spec={'test':{'$in':[2,3]}})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # exist_query is not a superset
input_query = dict(fields=None, spec={'test':{'$in':[2,3]}})
exist_query = dict(fields=None, spec={'test':{'$in':[1,2,3]}})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result) # exist_query is a superset
input_query = dict(fields=None, spec={'test':'aaa'})
exist_query = dict(fields=None, spec={'test':'aaa', 'test2':'bbb'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result) # different set of spec keys
input_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'})
exist_query = dict(fields=['conf'], spec={'dataset.name':'/a/b/c'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result)
input_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'})
exist_query = dict(fields=None, spec={'dataset.name':'/a/b/c'})
result = compare_specs(input_query, exist_query)
self.assertEqual(False, result)
input_query = dict(fields=None, spec={'dataset.name':'/a/b/c'})
exist_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'})
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result)
input_query = {'fields':None, 'spec':{'block.name':'ABCDEFG'}}
exist_query = {'fields':None, 'spec':{'block.name':'ABCDE*'}}
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result)
input_query = {'spec': {u'api':u'a', u'lfn':'test'}}
exist_query = {'spec': {u'api':u'a', u'lfn':u'test'}}
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result)
def test_compare_specs_with_patterns(self):
"""Test compare_specs with str/unicode dicts"""
query1 = {'spec':{'dataset.name':'*Run201*RECO'}}
query2 = {'spec':{'dataset.name':'*Run2011*RECO'}}
query3 = {'spec':{'dataset.name':'*Run20*RECO'}}
query4 = {'spec':{'dataset.name':'*Cosmics*Run20*RECO'}}
query5 = {'spec':{'dataset.name':'*Run2011*PromptReco*RECO'}}
result = compare_specs(query2, query1)
self.assertEqual(True, result)
result = compare_specs(query1, query3)
self.assertEqual(True, result)
result = compare_specs(query2, query3)
self.assertEqual(True, result)
result = compare_specs(query4, query3)
self.assertEqual(True, result)
result = compare_specs(query4, query1)
self.assertEqual(False, result)
result = compare_specs(query5, query2)
self.assertEqual(True, result)
query2 = {'spec':{'dataset.name':'*Run2011*RECO'}}
query3 = {'spec':{'dataset.name':'*Run20*RECO*'}}
query5 = {'spec':{'dataset.name':'*Run2011*PromptReco*RECO*'}}
result = compare_specs(query5, query2)
self.assertEqual(False, result)
result = compare_specs(query5, query3)
self.assertEqual(True, result)
query6 = {'spec':{'dataset.name':'Run*RECO*RECO'}}
query7 = {'spec':{'dataset.name':'Run*RECO1*RECO2'}}
result = compare_specs(query7, query6)
self.assertEqual(False, result)
query8 = {'spec':{'dataset.name':'Run*0RECO*0RECO'}}
query9 = {'spec':{'dataset.name':'Run*RECO*RECO*End'}}
query10 = {'spec':{'dataset.name':'Run*1RECO*2RECO*3End'}}
result = compare_specs(query8, query6)
self.assertEqual(True, result)
result = compare_specs(query10, query9)
self.assertEqual(True, result)
def test_str_vs_unicode_compare_specs(self):
"""Test compare_specs with str/unicode dicts"""
input_query = {u'fields': [u'zip'], 'spec': {u'zip.code': 10000}, u'filters': [u'zip.Placemark.address']}
exist_query = {'fields': ['zip'], 'spec': {'zip.code': 10000}, 'filters': ['zip.Placemark.address']}
result = compare_specs(input_query, exist_query)
self.assertEqual(True, result)
