def test_AndOrGroups(self): self.assertTrue( FilterEngine('1 > 2, 4 > 3 > 2; True=True', strict_coerce=False).evaluate()) self.assertFalse( FilterEngine('1 > 2, 4 > 3 > 2; True=False', strict_coerce=False).evaluate())
def test_CompoundInequality(self): self.assertTrue(FilterEngine('3 > 2 > 1', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2 > 3', strict_coerce=False).evaluate()) with self.assertRaises(DuplicateCriteriaInDIDFilter): FilterEngine('1 < 2 > 3', strict_coerce=False) with self.assertRaises(DuplicateCriteriaInDIDFilter): FilterEngine('1 < 2 > 3', strict_coerce=False)
def test_BackwardsCompatibilityCreatedBefore(self, session=None): did_name = self._create_tmp_DID() after = datetime.strftime(datetime.now() + timedelta(seconds=1), "%Y-%m-%dT%H:%M:%S.%fZ") # w/ +1s buffer dids = [] q = FilterEngine('created_before={}'.format(after), model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, ignore_dids=None, session=None): if not ignore_dids: ignore_dids = set() # backwards compatability for filters as single {}. if isinstance(filters, dict): filters = [filters] # instantiate fe and create mongo query fe = FilterEngine(filters, model_class=None, strict_coerce=False) mongo_query_str = fe.create_mongo_query( additional_filters=[('scope', operator.eq, scope.internal), ('vo', operator.eq, scope.vo)]) if recursive: # TODO: possible, but requires retrieving the results of a concurrent sqla query to call list_content on for datasets and containers raise exception.UnsupportedOperation( "'{}' metadata module does not currently support recursive searches" .format(self.plugin_name.lower())) if long: query_result = self.col.find(mongo_query_str) if limit: query_result = query_result.limit(limit) for did in query_result: did_full = did_full = "{}:{}".format(did['scope'], did['name']) if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs ignore_dids.add(did_full) yield { 'scope': InternalScope(did['scope']), 'name': did['name'], 'did_type': "N/A", 'bytes': "N/A", 'length': "N/A" } else: query_result = self.col.find(mongo_query_str) if limit: query_result = query_result.limit(limit) for did in query_result: did_full = did_full = "{}:{}".format(did['scope'], did['name']) if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs ignore_dids.add(did_full) yield did['name']
def test_OrGroups(self): self.assertTrue(FilterEngine('True = True; True = True', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('True = True; True = False', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('True = False; False = True', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('3 > 2; 2 > 1', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('1 > 2; 2 > 1', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2; 2 > 3', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('1 > 2; 4 > 3 > 2', strict_coerce=False).evaluate())
def test_InputSanitisation(self): filters = FilterEngine(' TestKeyword1 = True , TestKeyword2 = 0; 1 < TestKeyword4 <= 2', strict_coerce=False).filters filters_expected = [[('TestKeyword1', operator.eq, 1), ('TestKeyword2', operator.eq, 0)], [('TestKeyword4', operator.gt, 1), ('TestKeyword4', operator.le, 2)]] self.assertEqual(filters, filters_expected) with self.assertRaises(ValueError): FilterEngine('did_type >= 1', strict_coerce=False) with self.assertRaises(ValueError): FilterEngine('name >= 1', strict_coerce=False) with self.assertRaises(ValueError): FilterEngine('length >= test', strict_coerce=False) with self.assertRaises(ValueError): FilterEngine('name >= *', strict_coerce=False)
def test_BackwardsCompatibilityCreatedAfter(self): test_expressions = { "created_after=1900-01-01 00:00:00": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]], "created_after=1900-01-01T00:00:00": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]], "created_after=1900-01-01 00:00:00.000Z": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]], "created_after=1900-01-01T00:00:00.000Z": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]] } for input_datetime_expression, filters_expected in test_expressions.items(): filters = FilterEngine(input_datetime_expression, strict_coerce=False).filters self.assertEqual(filters, filters_expected)
def test_BackwardsCompatibilityLength(self): test_expressions = { 'length > 0': [[('length', operator.gt, 0)]], 'length < 0': [[('length', operator.lt, 0)]], 'length >= 0': [[('length', operator.ge, 0)]], 'length <= 0': [[('length', operator.le, 0)]], 'length == 0': [[('length', operator.eq, 0)]] } for input_length_expression, filters_expected in test_expressions.items(): filters = FilterEngine(input_length_expression, strict_coerce=False).filters self.assertEqual(filters, filters_expected)
def test_AndGroups(self): self.assertTrue(FilterEngine('True = True, False = False', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('True = True, False = True', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('3 > 2, 2 > 1', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2, 2 > 1', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2, 2 > 3', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2, 4 > 3 > 2', strict_coerce=False).evaluate())
def test_CompoundInequality(self, session=None): did_name = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name, key='run_number', value=1) dids = [] q = FilterEngine( '0 < run_number < 2', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine( '0 < run_number <= 1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine( '0 <= run_number < 1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertNotEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_typecastString(self): test_expressions = { 'testkeyint1 = 0': int, 'testkeyfloat1 = 0.5': float, 'testkeystr1 = test': str, 'testbool1 = false': bool, 'testbool2 = False': bool, 'testbool3 = FALSE': bool, 'testbool4 = true': bool, 'testbool5 = True': bool, 'testbool6 = TRUE': bool, 'testkeydate1 = 1900-01-01 00:00:00': datetime, 'testkeydate2 = 1900-01-01 00:00:00.000Z': datetime, 'testkeydate3 = 1900-01-01T00:00:00': datetime, 'testkeydate4 = 1900-01-01T00:00:00.000Z': datetime } for input_length_expression, type_expected in test_expressions.items(): filters = FilterEngine(input_length_expression, strict_coerce=False).filters self.assertIsInstance(filters[0][0][2], type_expected)
def test_AndOrGroups(self, session=None): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value='1') set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test") set_metadata(scope=self.tmp_scope, name=did_name3, key='run_number', value='1') set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="test") dids = [] q = FilterEngine( 'run_number = 1, project != test; project = test', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3) # 1, 2, 3 dids = [] q = FilterEngine( 'run_number = 1, project = test; run_number != 1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 2, 3
def test_OperatorsEqualNotEqual(self, session=None): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value=1) set_metadata(scope=self.tmp_scope, name=did_name2, key='run_number', value=2) dids = [] q = FilterEngine( 'run_number=1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1) dids = [] q = FilterEngine( 'run_number!=1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 1, 3 (NULL counted in not equals)
def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, ignore_dids=None, session=None): """ Search data identifiers. :param scope: the scope name. :param filters: dictionary of attributes by which the results should be filtered. :param did_type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file. :param ignore_case: ignore case distinctions. :param limit: limit number. :param offset: offset number. :param long: Long format option to display more information for each DID. :param session: The database session in use. :param recursive: Recursively list DIDs content. :param ignore_dids: List of DIDs to refrain from yielding. """ if not ignore_dids: ignore_dids = set() # mapping for semantic <type> to a (set of) recognised DIDType(s). type_to_did_type_mapping = { 'all': [DIDType.CONTAINER, DIDType.DATASET, DIDType.FILE], 'collection': [DIDType.CONTAINER, DIDType.DATASET], 'container': [DIDType.CONTAINER], 'dataset': [DIDType.DATASET], 'file': [DIDType.FILE] } # backwards compatability for filters as single {}. if isinstance(filters, dict): filters = [filters] # for each or_group, make sure there is a mapped "did_type" filter. # if type maps to many DIDTypes, the corresponding or_group will be copied the required number of times to satisfy all the logical possibilities. filters_tmp = [] for or_group in filters: if 'type' not in or_group: or_group_type = did_type.lower() else: or_group_type = or_group.pop('type').lower() if or_group_type not in type_to_did_type_mapping.keys(): raise exception.UnsupportedOperation('{} is not a valid type. Valid types are {}'.format(or_group_type, type_to_did_type_mapping.keys())) for mapped_did_type in type_to_did_type_mapping[or_group_type]: or_group['did_type'] = mapped_did_type filters_tmp.append(or_group.copy()) filters = filters_tmp # instantiate fe and create sqla query fe = FilterEngine(filters, model_class=models.DataIdentifier) query = fe.create_sqla_query( additional_model_attributes=[ models.DataIdentifier.scope, models.DataIdentifier.name, models.DataIdentifier.did_type, models.DataIdentifier.bytes, models.DataIdentifier.length ], additional_filters=[ (models.DataIdentifier.scope, operator.eq, scope), (models.DataIdentifier.suppressed, operator.ne, true()) ] ) if limit: query = query.limit(limit) if recursive: from rucio.core.did import list_content # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion collections_content = [] for did in query.yield_per(100): if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET): collections_content += [d for d in list_content(scope=did.scope, name=did.name)] # Replace any name filtering with recursed DID names. for did in collections_content: for or_group in filters: or_group['name'] = did['name'] for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, did_type=did_type, limit=limit, offset=offset, long=long, ignore_dids=ignore_dids, session=session): yield result for did in query.yield_per(5): # don't unpack this as it makes it dependent on query return order! if long: did_full = "{}:{}".format(did.scope, did.name) if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive. ignore_dids.add(did_full) yield { 'scope': did.scope, 'name': did.name, 'did_type': str(did.did_type), 'bytes': did.bytes, 'length': did.length } else: did_full = "{}:{}".format(did.scope, did.name) if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive. ignore_dids.add(did_full) yield did.name
def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, ignore_dids=None, session=None): if not json_implemented(session=session): raise NotImplementedError if not ignore_dids: ignore_dids = set() # backwards compatability for filters as single {}. if isinstance(filters, dict): filters = [filters] # instantiate fe and create sqla query, note that coercion to a model keyword # is not appropriate here as the filter words are stored in a single json column. fe = FilterEngine(filters, model_class=models.DidMeta, strict_coerce=False) query = fe.create_sqla_query(additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], additional_filters=[(models.DidMeta.scope, operator.eq, scope)], json_column=models.DidMeta.meta) if limit: query = query.limit(limit) if recursive: from rucio.core.did import list_content # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion collections_content = [] for did in query.yield_per(100): if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET): collections_content += [ d for d in list_content(scope=did.scope, name=did.name) ] # Replace any name filtering with recursed DID names. for did in collections_content: for or_group in filters: or_group['name'] = did['name'] for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, did_type=did_type, limit=limit, offset=offset, long=long, ignore_dids=ignore_dids, session=session): yield result try: for did in query.yield_per( 5 ): # don't unpack this as it makes it dependent on query return order! if long: did_full = "{}:{}".format(did.scope, did.name) if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive. ignore_dids.add(did_full) yield { 'scope': did.scope, 'name': did.name, 'did_type': None, # not available with JSON plugin 'bytes': None, # not available with JSON plugin 'length': None # not available with JSON plugin } else: did_full = "{}:{}".format(did.scope, did.name) if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive. ignore_dids.add(did_full) yield did.name except DataError as e: raise exception.InvalidMetadata( "Database query failed: {}. This can be raised when the datatype of a key is inconsistent between dids." .format(e))
def test_OperatorsEqualNotEqual(self, session=None): # Plugin: DID # did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value=1) set_metadata(scope=self.tmp_scope, name=did_name2, key='run_number', value=2) dids = [] q = FilterEngine('run_number=1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1) dids = [] q = FilterEngine('run_number!=1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 1, 3 (NULL counted in not equals) # Plugin: JSON # if json_implemented(session=session): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeyint1', value=1) set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeyint2', value=2) set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeyint3', value=2) dids = [] q = FilterEngine('testkeyint1=1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1) if session.bind.dialect.name != 'oracle' and json_implemented(session=session): dids = [] q = FilterEngine('testkeyint1!=1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2)
def test_OneSidedInequality(self): self.assertTrue(FilterEngine('1 < 2', strict_coerce=False).evaluate()) self.assertFalse(FilterEngine('1 > 2', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('1 <= 1', strict_coerce=False).evaluate()) self.assertTrue(FilterEngine('1 >= 1', strict_coerce=False).evaluate())
def test_OperatorsEqualNotEqual(self): self.assertTrue( FilterEngine('True = True', strict_coerce=False).evaluate()) self.assertTrue( FilterEngine('True != False', strict_coerce=False).evaluate())
def test_Wildcards(self, session=None): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() did_name4 = self._create_tmp_DID() did_name5 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='project', value="test1") set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test2") set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="anothertest1") set_metadata(scope=self.tmp_scope, name=did_name4, key='project', value="anothertest2") dids = [] q = FilterEngine( 'project = test*', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map( lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 1, 2 dids = [] q = FilterEngine( 'project = *test*', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map( lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4) # 1, 2, 3, 4 dids = [] q = FilterEngine( 'project != *anothertest*', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map( lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 3) # 3, 4, 5 (NULL counted in not equals) dids = [] q = FilterEngine( 'project != *test*', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list( map( lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 1) # 5 (NULL counted in not equals)
def test_BackwardsCompatibilityLength(self, session=None): did_name = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name, key='length', value='10') dids = [] q = FilterEngine( 'length >= 10', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine( 'length > 9', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine( 'length <= 10', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine( 'length < 11', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual( list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_CompoundInequality(self, session=None): # Plugin: DID # did_name = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name, key='run_number', value=1) dids = [] q = FilterEngine('0 < run_number < 2', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine('0 < run_number <= 1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine('0 <= run_number < 1', model_class=models.DataIdentifier).create_sqla_query( additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertNotEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1) # Plugin: JSON # if session.bind.dialect.name != 'oracle' and json_implemented(session=session): did_name = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name, key='testkeyint1', value=1) dids = [] q = FilterEngine('0 < testkeyint1 < 2', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine('0 < testkeyint1 <= 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1) dids = [] q = FilterEngine('0 <= testkeyint1 < 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertNotEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_AndOrGroups(self, session=None): # Plugin: DID # did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value='1') set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test") set_metadata(scope=self.tmp_scope, name=did_name3, key='run_number', value='1') set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="test") dids = [] q = FilterEngine('run_number = 1, project != test; project = test', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3) # 1, 2, 3 dids = [] q = FilterEngine('run_number = 1, project = test; run_number != 1', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 2, 3 # Plugin: JSON # if session.bind.dialect.name != 'oracle' and json_implemented(session=session): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeyint1', value='1') set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeystr1', value="test") set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeyint1', value='1') set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeystr1', value="test") dids = [] q = FilterEngine('testkeyint1 = 1, testkeystr1 != test; testkeystr1 = test', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3) # 1, 2, 3 dids = [] q = FilterEngine('testkeyint1 = 1, testkeystr1 = test; testkeyint1 != 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 2, 3
def test_Wildcards(self, session=None): # Plugin: DID # did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() did_name4 = self._create_tmp_DID() did_name5 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='project', value="test1") set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test2") set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="anothertest1") set_metadata(scope=self.tmp_scope, name=did_name4, key='project', value="anothertest2") dids = [] q = FilterEngine('project = test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 1, 2 dids = [] q = FilterEngine('project = *test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4) # 1, 2, 3, 4 dids = [] q = FilterEngine('project != *anothertest*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 3) # 3, 4, 5 (NULL counted in not equals) dids = [] q = FilterEngine('project != *test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name]) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 1) # 5 (NULL counted in not equals) # Plugin: JSON # if session.bind.dialect.name != 'oracle' and json_implemented(session=session): did_name1 = self._create_tmp_DID() did_name2 = self._create_tmp_DID() did_name3 = self._create_tmp_DID() did_name4 = self._create_tmp_DID() did_name5 = self._create_tmp_DID() set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeystr1', value="test1") set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeystr1', value="test2") set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeystr1', value="anothertest1") set_metadata(scope=self.tmp_scope, name=did_name4, key='testkeystr1', value="anothertest2") dids = [] q = FilterEngine('testkeystr1 = test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 1, 2 dids = [] q = FilterEngine('testkeystr1 = *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4) # 1, 2, 3, 4 dids = [] q = FilterEngine('testkeystr1 != *anothertest*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 3, 4 dids = [] q = FilterEngine('testkeystr1 != *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query( additional_model_attributes=[ models.DidMeta.scope, models.DidMeta.name ], json_column=models.DidMeta.meta) dids += [did for did in q.yield_per(5)] dids = set(dids) self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 0)