def test_AndOrGroups(self):
self.assertTrue(
FilterEngine('1 > 2, 4 > 3 > 2; True=True',
strict_coerce=False).evaluate())
self.assertFalse(
FilterEngine('1 > 2, 4 > 3 > 2; True=False',
strict_coerce=False).evaluate())
def test_CompoundInequality(self):
self.assertTrue(FilterEngine('3 > 2 > 1', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2 > 3', strict_coerce=False).evaluate())
with self.assertRaises(DuplicateCriteriaInDIDFilter):
FilterEngine('1 < 2 > 3', strict_coerce=False)
with self.assertRaises(DuplicateCriteriaInDIDFilter):
FilterEngine('1 < 2 > 3', strict_coerce=False)
def test_BackwardsCompatibilityCreatedBefore(self, session=None):
did_name = self._create_tmp_DID()
after = datetime.strftime(datetime.now() + timedelta(seconds=1), "%Y-%m-%dT%H:%M:%S.%fZ") # w/ +1s buffer
dids = []
q = FilterEngine('created_before={}'.format(after), model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def list_dids(self,
scope,
filters,
did_type='collection',
ignore_case=False,
limit=None,
offset=None,
long=False,
recursive=False,
ignore_dids=None,
session=None):
if not ignore_dids:
ignore_dids = set()
# backwards compatability for filters as single {}.
if isinstance(filters, dict):
filters = [filters]
# instantiate fe and create mongo query
fe = FilterEngine(filters, model_class=None, strict_coerce=False)
mongo_query_str = fe.create_mongo_query(
additional_filters=[('scope', operator.eq,
scope.internal), ('vo', operator.eq,
scope.vo)])
if recursive:
# TODO: possible, but requires retrieving the results of a concurrent sqla query to call list_content on for datasets and containers
raise exception.UnsupportedOperation(
"'{}' metadata module does not currently support recursive searches"
.format(self.plugin_name.lower()))
if long:
query_result = self.col.find(mongo_query_str)
if limit:
query_result = query_result.limit(limit)
for did in query_result:
did_full = did_full = "{}:{}".format(did['scope'], did['name'])
if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs
ignore_dids.add(did_full)
yield {
'scope': InternalScope(did['scope']),
'name': did['name'],
'did_type': "N/A",
'bytes': "N/A",
'length': "N/A"
}
else:
query_result = self.col.find(mongo_query_str)
if limit:
query_result = query_result.limit(limit)
for did in query_result:
did_full = did_full = "{}:{}".format(did['scope'], did['name'])
if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs
ignore_dids.add(did_full)
yield did['name']
def test_OrGroups(self):
self.assertTrue(FilterEngine('True = True; True = True', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('True = True; True = False', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('True = False; False = True', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('3 > 2; 2 > 1', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('1 > 2; 2 > 1', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2; 2 > 3', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('1 > 2; 4 > 3 > 2', strict_coerce=False).evaluate())
def test_InputSanitisation(self):
filters = FilterEngine(' TestKeyword1 = True , TestKeyword2 = 0; 1 < TestKeyword4 <= 2', strict_coerce=False).filters
filters_expected = [[('TestKeyword1', operator.eq, 1),
('TestKeyword2', operator.eq, 0)],
[('TestKeyword4', operator.gt, 1),
('TestKeyword4', operator.le, 2)]]
self.assertEqual(filters, filters_expected)
with self.assertRaises(ValueError):
FilterEngine('did_type >= 1', strict_coerce=False)
with self.assertRaises(ValueError):
FilterEngine('name >= 1', strict_coerce=False)
with self.assertRaises(ValueError):
FilterEngine('length >= test', strict_coerce=False)
with self.assertRaises(ValueError):
FilterEngine('name >= *', strict_coerce=False)
def test_BackwardsCompatibilityCreatedAfter(self):
test_expressions = {
"created_after=1900-01-01 00:00:00": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]],
"created_after=1900-01-01T00:00:00": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]],
"created_after=1900-01-01 00:00:00.000Z": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]],
"created_after=1900-01-01T00:00:00.000Z": [[('created_at', operator.ge, datetime(1900, 1, 1, 0, 0))]]
}
for input_datetime_expression, filters_expected in test_expressions.items():
filters = FilterEngine(input_datetime_expression, strict_coerce=False).filters
self.assertEqual(filters, filters_expected)
def test_BackwardsCompatibilityLength(self):
test_expressions = {
'length > 0': [[('length', operator.gt, 0)]],
'length < 0': [[('length', operator.lt, 0)]],
'length >= 0': [[('length', operator.ge, 0)]],
'length <= 0': [[('length', operator.le, 0)]],
'length == 0': [[('length', operator.eq, 0)]]
}
for input_length_expression, filters_expected in test_expressions.items():
filters = FilterEngine(input_length_expression, strict_coerce=False).filters
self.assertEqual(filters, filters_expected)
def test_AndGroups(self):
self.assertTrue(FilterEngine('True = True, False = False', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('True = True, False = True', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('3 > 2, 2 > 1', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2, 2 > 1', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2, 2 > 3', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2, 4 > 3 > 2', strict_coerce=False).evaluate())
def test_CompoundInequality(self, session=None):
did_name = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope,
name=did_name,
key='run_number',
value=1)
dids = []
q = FilterEngine(
'0 < run_number < 2',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine(
'0 < run_number <= 1',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine(
'0 <= run_number < 1',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertNotEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_typecastString(self):
test_expressions = {
'testkeyint1 = 0': int,
'testkeyfloat1 = 0.5': float,
'testkeystr1 = test': str,
'testbool1 = false': bool,
'testbool2 = False': bool,
'testbool3 = FALSE': bool,
'testbool4 = true': bool,
'testbool5 = True': bool,
'testbool6 = TRUE': bool,
'testkeydate1 = 1900-01-01 00:00:00': datetime,
'testkeydate2 = 1900-01-01 00:00:00.000Z': datetime,
'testkeydate3 = 1900-01-01T00:00:00': datetime,
'testkeydate4 = 1900-01-01T00:00:00.000Z': datetime
}
for input_length_expression, type_expected in test_expressions.items():
filters = FilterEngine(input_length_expression, strict_coerce=False).filters
self.assertIsInstance(filters[0][0][2], type_expected)
def test_AndOrGroups(self, session=None):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope,
name=did_name1,
key='run_number',
value='1')
set_metadata(scope=self.tmp_scope,
name=did_name2,
key='project',
value="test")
set_metadata(scope=self.tmp_scope,
name=did_name3,
key='run_number',
value='1')
set_metadata(scope=self.tmp_scope,
name=did_name3,
key='project',
value="test")
dids = []
q = FilterEngine(
'run_number = 1, project != test; project = test',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(lambda did: did.name in (did_name1, did_name2, did_name3),
dids)).count(True), 3) # 1, 2, 3
dids = []
q = FilterEngine(
'run_number = 1, project = test; run_number != 1',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(lambda did: did.name in (did_name1, did_name2, did_name3),
dids)).count(True), 2) # 2, 3
def test_OperatorsEqualNotEqual(self, session=None):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope,
name=did_name1,
key='run_number',
value=1)
set_metadata(scope=self.tmp_scope,
name=did_name2,
key='run_number',
value=2)
dids = []
q = FilterEngine(
'run_number=1',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(lambda did: did.name in (did_name1, did_name2, did_name3),
dids)).count(True), 1)
dids = []
q = FilterEngine(
'run_number!=1',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(lambda did: did.name in (did_name1, did_name2, did_name3),
dids)).count(True), 2) # 1, 3 (NULL counted in not equals)
def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None,
offset=None, long=False, recursive=False, ignore_dids=None, session=None):
"""
Search data identifiers.
:param scope: the scope name.
:param filters: dictionary of attributes by which the results should be filtered.
:param did_type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file.
:param ignore_case: ignore case distinctions.
:param limit: limit number.
:param offset: offset number.
:param long: Long format option to display more information for each DID.
:param session: The database session in use.
:param recursive: Recursively list DIDs content.
:param ignore_dids: List of DIDs to refrain from yielding.
"""
if not ignore_dids:
ignore_dids = set()
# mapping for semantic <type> to a (set of) recognised DIDType(s).
type_to_did_type_mapping = {
'all': [DIDType.CONTAINER, DIDType.DATASET, DIDType.FILE],
'collection': [DIDType.CONTAINER, DIDType.DATASET],
'container': [DIDType.CONTAINER],
'dataset': [DIDType.DATASET],
'file': [DIDType.FILE]
}
# backwards compatability for filters as single {}.
if isinstance(filters, dict):
filters = [filters]
# for each or_group, make sure there is a mapped "did_type" filter.
# if type maps to many DIDTypes, the corresponding or_group will be copied the required number of times to satisfy all the logical possibilities.
filters_tmp = []
for or_group in filters:
if 'type' not in or_group:
or_group_type = did_type.lower()
else:
or_group_type = or_group.pop('type').lower()
if or_group_type not in type_to_did_type_mapping.keys():
raise exception.UnsupportedOperation('{} is not a valid type. Valid types are {}'.format(or_group_type, type_to_did_type_mapping.keys()))
for mapped_did_type in type_to_did_type_mapping[or_group_type]:
or_group['did_type'] = mapped_did_type
filters_tmp.append(or_group.copy())
filters = filters_tmp
# instantiate fe and create sqla query
fe = FilterEngine(filters, model_class=models.DataIdentifier)
query = fe.create_sqla_query(
additional_model_attributes=[
models.DataIdentifier.scope,
models.DataIdentifier.name,
models.DataIdentifier.did_type,
models.DataIdentifier.bytes,
models.DataIdentifier.length
], additional_filters=[
(models.DataIdentifier.scope, operator.eq, scope),
(models.DataIdentifier.suppressed, operator.ne, true())
]
)
if limit:
query = query.limit(limit)
if recursive:
from rucio.core.did import list_content
# Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
collections_content = []
for did in query.yield_per(100):
if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
# Replace any name filtering with recursed DID names.
for did in collections_content:
for or_group in filters:
or_group['name'] = did['name']
for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, did_type=did_type, limit=limit, offset=offset,
long=long, ignore_dids=ignore_dids, session=session):
yield result
for did in query.yield_per(5): # don't unpack this as it makes it dependent on query return order!
if long:
did_full = "{}:{}".format(did.scope, did.name)
if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
ignore_dids.add(did_full)
yield {
'scope': did.scope,
'name': did.name,
'did_type': str(did.did_type),
'bytes': did.bytes,
'length': did.length
}
else:
did_full = "{}:{}".format(did.scope, did.name)
if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
ignore_dids.add(did_full)
yield did.name
def list_dids(self,
scope,
filters,
did_type='collection',
ignore_case=False,
limit=None,
offset=None,
long=False,
recursive=False,
ignore_dids=None,
session=None):
if not json_implemented(session=session):
raise NotImplementedError
if not ignore_dids:
ignore_dids = set()
# backwards compatability for filters as single {}.
if isinstance(filters, dict):
filters = [filters]
# instantiate fe and create sqla query, note that coercion to a model keyword
# is not appropriate here as the filter words are stored in a single json column.
fe = FilterEngine(filters,
model_class=models.DidMeta,
strict_coerce=False)
query = fe.create_sqla_query(additional_model_attributes=[
models.DidMeta.scope, models.DidMeta.name
],
additional_filters=[(models.DidMeta.scope,
operator.eq, scope)],
json_column=models.DidMeta.meta)
if limit:
query = query.limit(limit)
if recursive:
from rucio.core.did import list_content
# Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
collections_content = []
for did in query.yield_per(100):
if (did.did_type == DIDType.CONTAINER
or did.did_type == DIDType.DATASET):
collections_content += [
d for d in list_content(scope=did.scope, name=did.name)
]
# Replace any name filtering with recursed DID names.
for did in collections_content:
for or_group in filters:
or_group['name'] = did['name']
for result in self.list_dids(scope=did['scope'],
filters=filters,
recursive=True,
did_type=did_type,
limit=limit,
offset=offset,
long=long,
ignore_dids=ignore_dids,
session=session):
yield result
try:
for did in query.yield_per(
5
): # don't unpack this as it makes it dependent on query return order!
if long:
did_full = "{}:{}".format(did.scope, did.name)
if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
ignore_dids.add(did_full)
yield {
'scope': did.scope,
'name': did.name,
'did_type': None, # not available with JSON plugin
'bytes': None, # not available with JSON plugin
'length': None # not available with JSON plugin
}
else:
did_full = "{}:{}".format(did.scope, did.name)
if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
ignore_dids.add(did_full)
yield did.name
except DataError as e:
raise exception.InvalidMetadata(
"Database query failed: {}. This can be raised when the datatype of a key is inconsistent between dids."
.format(e))
def test_OperatorsEqualNotEqual(self, session=None):
# Plugin: DID
#
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value=1)
set_metadata(scope=self.tmp_scope, name=did_name2, key='run_number', value=2)
dids = []
q = FilterEngine('run_number=1', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1)
dids = []
q = FilterEngine('run_number!=1', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 1, 3 (NULL counted in not equals)
# Plugin: JSON
#
if json_implemented(session=session):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeyint1', value=1)
set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeyint2', value=2)
set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeyint3', value=2)
dids = []
q = FilterEngine('testkeyint1=1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 1)
if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
dids = []
q = FilterEngine('testkeyint1!=1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2)
def test_OneSidedInequality(self):
self.assertTrue(FilterEngine('1 < 2', strict_coerce=False).evaluate())
self.assertFalse(FilterEngine('1 > 2', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('1 <= 1', strict_coerce=False).evaluate())
self.assertTrue(FilterEngine('1 >= 1', strict_coerce=False).evaluate())
def test_OperatorsEqualNotEqual(self):
self.assertTrue(
FilterEngine('True = True', strict_coerce=False).evaluate())
self.assertTrue(
FilterEngine('True != False', strict_coerce=False).evaluate())
def test_Wildcards(self, session=None):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
did_name4 = self._create_tmp_DID()
did_name5 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope,
name=did_name1,
key='project',
value="test1")
set_metadata(scope=self.tmp_scope,
name=did_name2,
key='project',
value="test2")
set_metadata(scope=self.tmp_scope,
name=did_name3,
key='project',
value="anothertest1")
set_metadata(scope=self.tmp_scope,
name=did_name4,
key='project',
value="anothertest2")
dids = []
q = FilterEngine(
'project = test*',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(
lambda did: did.name in
(did_name1, did_name2, did_name3, did_name4, did_name5),
dids)).count(True), 2) # 1, 2
dids = []
q = FilterEngine(
'project = *test*',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(
lambda did: did.name in
(did_name1, did_name2, did_name3, did_name4, did_name5),
dids)).count(True), 4) # 1, 2, 3, 4
dids = []
q = FilterEngine(
'project != *anothertest*',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(
lambda did: did.name in
(did_name1, did_name2, did_name3, did_name4, did_name5),
dids)).count(True),
3) # 3, 4, 5 (NULL counted in not equals)
dids = []
q = FilterEngine(
'project != *test*',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(
map(
lambda did: did.name in
(did_name1, did_name2, did_name3, did_name4, did_name5),
dids)).count(True), 1) # 5 (NULL counted in not equals)
def test_BackwardsCompatibilityLength(self, session=None):
did_name = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope,
name=did_name,
key='length',
value='10')
dids = []
q = FilterEngine(
'length >= 10',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine(
'length > 9', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine(
'length <= 10',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine(
'length < 11',
model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(
list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_CompoundInequality(self, session=None):
# Plugin: DID
#
did_name = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name, key='run_number', value=1)
dids = []
q = FilterEngine('0 < run_number < 2', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine('0 < run_number <= 1', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine('0 <= run_number < 1', model_class=models.DataIdentifier).create_sqla_query(
additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertNotEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
# Plugin: JSON
#
if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
did_name = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name, key='testkeyint1', value=1)
dids = []
q = FilterEngine('0 < testkeyint1 < 2', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine('0 < testkeyint1 <= 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
dids = []
q = FilterEngine('0 <= testkeyint1 < 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertNotEqual(list(map(lambda did: did.name == did_name, dids)).count(True), 1)
def test_AndOrGroups(self, session=None):
# Plugin: DID
#
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='run_number', value='1')
set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test")
set_metadata(scope=self.tmp_scope, name=did_name3, key='run_number', value='1')
set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="test")
dids = []
q = FilterEngine('run_number = 1, project != test; project = test', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3) # 1, 2, 3
dids = []
q = FilterEngine('run_number = 1, project = test; run_number != 1', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 2, 3
# Plugin: JSON
#
if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeyint1', value='1')
set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeystr1', value="test")
set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeyint1', value='1')
set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeystr1', value="test")
dids = []
q = FilterEngine('testkeyint1 = 1, testkeystr1 != test; testkeystr1 = test', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 3) # 1, 2, 3
dids = []
q = FilterEngine('testkeyint1 = 1, testkeystr1 = test; testkeyint1 != 1', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3), dids)).count(True), 2) # 2, 3
def test_Wildcards(self, session=None):
# Plugin: DID
#
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
did_name4 = self._create_tmp_DID()
did_name5 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='project', value="test1")
set_metadata(scope=self.tmp_scope, name=did_name2, key='project', value="test2")
set_metadata(scope=self.tmp_scope, name=did_name3, key='project', value="anothertest1")
set_metadata(scope=self.tmp_scope, name=did_name4, key='project', value="anothertest2")
dids = []
q = FilterEngine('project = test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 1, 2
dids = []
q = FilterEngine('project = *test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4) # 1, 2, 3, 4
dids = []
q = FilterEngine('project != *anothertest*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 3) # 3, 4, 5 (NULL counted in not equals)
dids = []
q = FilterEngine('project != *test*', model_class=models.DataIdentifier).create_sqla_query(additional_model_attributes=[models.DataIdentifier.name])
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 1) # 5 (NULL counted in not equals)
# Plugin: JSON
#
if session.bind.dialect.name != 'oracle' and json_implemented(session=session):
did_name1 = self._create_tmp_DID()
did_name2 = self._create_tmp_DID()
did_name3 = self._create_tmp_DID()
did_name4 = self._create_tmp_DID()
did_name5 = self._create_tmp_DID()
set_metadata(scope=self.tmp_scope, name=did_name1, key='testkeystr1', value="test1")
set_metadata(scope=self.tmp_scope, name=did_name2, key='testkeystr1', value="test2")
set_metadata(scope=self.tmp_scope, name=did_name3, key='testkeystr1', value="anothertest1")
set_metadata(scope=self.tmp_scope, name=did_name4, key='testkeystr1', value="anothertest2")
dids = []
q = FilterEngine('testkeystr1 = test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 1, 2
dids = []
q = FilterEngine('testkeystr1 = *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 4) # 1, 2, 3, 4
dids = []
q = FilterEngine('testkeystr1 != *anothertest*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 2) # 3, 4
dids = []
q = FilterEngine('testkeystr1 != *test*', model_class=models.DidMeta, strict_coerce=False).create_sqla_query(
additional_model_attributes=[
models.DidMeta.scope,
models.DidMeta.name
],
json_column=models.DidMeta.meta)
dids += [did for did in q.yield_per(5)]
dids = set(dids)
self.assertEqual(list(map(lambda did: did.name in (did_name1, did_name2, did_name3, did_name4, did_name5), dids)).count(True), 0)