def pass_apicall(self, dasquery, url, api, api_params): """ Filter provided apicall wrt existing apicall records in Analytics DB. """ self.analytics.remove_expired() msg = 'API=%s, args=%s' % (api, api_params) for row in self.analytics.list_apicalls(url=url, api=api): input_query = {'spec':api_params} exist_query = {'spec':row['apicall']['api_params']} if compare_specs(input_query, exist_query): msg += '\nwill re-use existing api call with args=%s, query=%s'\ % (row['apicall']['api_params'], exist_query) self.logger.info(msg) try: # update DAS cache with empty result set args = self.inspect_params(api, api_params) cond = {'das.qhash': row['apicall']['qhash']} record = self.localcache.col.find_one(cond) if record and record.has_key('das') and \ record['das'].has_key('expire'): expire = record['das']['expire'] self.write_to_cache(\ dasquery, expire, url, api, args, [], 0) except Exception as exc: print_exc(exc) msg = 'failed api %s\n' % api msg += 'input query %s\n' % input_query msg += 'existing query %s\n' % exist_query msg += 'Unable to look-up existing query and extract ' msg += 'expire timestamp' raise Exception(msg) return False return True
def is_subset_of(self, other): """ Return true if we are not a superset of the other. This is perhaps not the same thing as "is subset of" """ return compare_specs(self.mongo_query, other.mongo_query)
def is_superset_of(self, other): """ Return true if we are a superset of other """ return compare_specs(other.mongo_query, self.mongo_query)
def test_compare_specs(self): """ Test compare_specs funtion. """ input = {'spec': {u'node': u'*', u'block': '*', u'se': u'*'}} exist = {'spec': {u'node': u'T1_CH_CERN', u'se': u'T1_CH_CERN', u'block': u'*'}} result = compare_specs(input, exist) self.assertEqual(False, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':'T1_CH_CERN'}) exist_query = dict(fields=['site'], spec={}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':'T1_CH_CERN'}) exist_query = dict(fields=None, spec={'test':'T1_*'}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':'site'}) exist_query = dict(fields=None, spec={'test':'site_ch'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # no pattern input_query = dict(fields=None, spec={'test':'site_ch'}) exist_query = dict(fields=None, spec={'test':'site'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # no pattern input_query = dict(fields=None, spec={'test':'site*'}) exist_query = dict(fields=None, spec={'test':'site_ch'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=None, spec={'test':'site_ch'}) exist_query = dict(fields=None, spec={'test':'site*'}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=['site','block'], spec={'test':'site*'}) exist_query = dict(fields=['site'], spec={'test':'site_ch'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=['site','block'], spec={'test':'site_ch'}) exist_query = dict(fields=['site'], spec={'test':'site*'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is a superset, but different fields input_query = dict(fields=['block'], spec={'test':'site*'}) exist_query = dict(fields=['site'], spec={'test':'site*'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # different fields input_query = dict(fields=None, spec={'test':'T1_CH_*'}) exist_query = dict(fields=None, spec={'test':'T1_CH_CERN'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=None, spec={'test':'T1_CH_CERN'}) exist_query = dict(fields=None, spec={'test':'T1_CH_*'}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':{'$gt':10}}) exist_query = dict(fields=None, spec={'test':{'$gt':11}}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=None, spec={'test':{'$gt':10}}) exist_query = dict(fields=None, spec={'test':{'$gt':9}}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':{'$lt':10}}) exist_query = dict(fields=None, spec={'test':{'$lt':9}}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=None, spec={'test':{'$lt':10}}) exist_query = dict(fields=None, spec={'test':{'$lt':11}}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':{'$in':[1,2,3]}}) exist_query = dict(fields=None, spec={'test':{'$in':[2,3]}}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # exist_query is not a superset input_query = dict(fields=None, spec={'test':{'$in':[2,3]}}) exist_query = dict(fields=None, spec={'test':{'$in':[1,2,3]}}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) # exist_query is a superset input_query = dict(fields=None, spec={'test':'aaa'}) exist_query = dict(fields=None, spec={'test':'aaa', 'test2':'bbb'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) # different set of spec keys input_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'}) exist_query = dict(fields=['conf'], spec={'dataset.name':'/a/b/c'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) input_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'}) exist_query = dict(fields=None, spec={'dataset.name':'/a/b/c'}) result = compare_specs(input_query, exist_query) self.assertEqual(False, result) input_query = dict(fields=None, spec={'dataset.name':'/a/b/c'}) exist_query = dict(fields=None, spec={'dataset.name':'/a/b/c*'}) result = compare_specs(input_query, exist_query) self.assertEqual(True, result) input_query = {'fields':None, 'spec':{'block.name':'ABCDEFG'}} exist_query = {'fields':None, 'spec':{'block.name':'ABCDE*'}} result = compare_specs(input_query, exist_query) self.assertEqual(True, result) input_query = {'spec': {u'api':u'a', u'lfn':'test'}} exist_query = {'spec': {u'api':u'a', u'lfn':u'test'}} result = compare_specs(input_query, exist_query) self.assertEqual(True, result)
def test_compare_specs_with_patterns(self): """Test compare_specs with str/unicode dicts""" query1 = {'spec':{'dataset.name':'*Run201*RECO'}} query2 = {'spec':{'dataset.name':'*Run2011*RECO'}} query3 = {'spec':{'dataset.name':'*Run20*RECO'}} query4 = {'spec':{'dataset.name':'*Cosmics*Run20*RECO'}} query5 = {'spec':{'dataset.name':'*Run2011*PromptReco*RECO'}} result = compare_specs(query2, query1) self.assertEqual(True, result) result = compare_specs(query1, query3) self.assertEqual(True, result) result = compare_specs(query2, query3) self.assertEqual(True, result) result = compare_specs(query4, query3) self.assertEqual(True, result) result = compare_specs(query4, query1) self.assertEqual(False, result) result = compare_specs(query5, query2) self.assertEqual(True, result) query2 = {'spec':{'dataset.name':'*Run2011*RECO'}} query3 = {'spec':{'dataset.name':'*Run20*RECO*'}} query5 = {'spec':{'dataset.name':'*Run2011*PromptReco*RECO*'}} result = compare_specs(query5, query2) self.assertEqual(False, result) result = compare_specs(query5, query3) self.assertEqual(True, result) query6 = {'spec':{'dataset.name':'Run*RECO*RECO'}} query7 = {'spec':{'dataset.name':'Run*RECO1*RECO2'}} result = compare_specs(query7, query6) self.assertEqual(False, result) query8 = {'spec':{'dataset.name':'Run*0RECO*0RECO'}} query9 = {'spec':{'dataset.name':'Run*RECO*RECO*End'}} query10 = {'spec':{'dataset.name':'Run*1RECO*2RECO*3End'}} result = compare_specs(query8, query6) self.assertEqual(True, result) result = compare_specs(query10, query9) self.assertEqual(True, result)
def test_str_vs_unicode_compare_specs(self): """Test compare_specs with str/unicode dicts""" input_query = {u'fields': [u'zip'], 'spec': {u'zip.code': 10000}, u'filters': [u'zip.Placemark.address']} exist_query = {'fields': ['zip'], 'spec': {'zip.code': 10000}, 'filters': ['zip.Placemark.address']} result = compare_specs(input_query, exist_query) self.assertEqual(True, result)