def __init__(
self,
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
transformer: Transformer,
):
"""Initialize the collection for the given parameters.
Parameters:
resource_cls (EntryResource): The `EntryResource` model
that is stored by the collection.
resource_mapper (BaseResourceMapper): A resource mapper
object that handles aliases and format changes between
deserialization and response.
transformer (Transformer): The Lark `Transformer` used to
interpret the filter.
"""
self.parser = LarkParser()
self.resource_cls = resource_cls
self.resource_mapper = resource_mapper
self.transformer = transformer
self.provider_prefix = CONFIG.provider.prefix
self.provider_fields = [
field if isinstance(field, str) else field["name"] for field in
CONFIG.provider_fields.get(resource_mapper.ENDPOINT, [])
]
self._all_fields: Set[str] = None
def set_up(self):
from optimade.filtertransformers.elasticsearch import Transformer, Quantity
self.parser = LarkParser(version=(0, 10, 0), variant="elastic")
nelements = Quantity(name="nelements")
elements_only = Quantity(name="elements_only")
elements_ratios = Quantity(name="elements_ratios")
elements_ratios.nested_quantity = elements_ratios
elements = Quantity(
name="elements",
length_quantity=nelements,
has_only_quantity=elements_only,
nested_quantity=elements_ratios,
)
dimension_types = Quantity(name="dimension_types")
dimension_types.length_quantity = dimension_types
quantities = [
nelements,
elements_only,
elements_ratios,
elements,
dimension_types,
Quantity(name="chemical_formula_reduced"),
]
self.transformer = Transformer(quantities=quantities)
def test_list_length_aliases(self, mapper):
from optimade.filtertransformers.mongo import MongoTransformer
transformer = MongoTransformer(mapper=mapper("StructureMapper")())
parser = LarkParser(version=self.version, variant=self.variant)
assert transformer.transform(parser.parse("elements LENGTH 3")) == {
"nelements": 3
}
assert transformer.transform(
parser.parse('elements HAS "Li" AND elements LENGTH = 3')
) == {"$and": [{"elements": {"$in": ["Li"]}}, {"nelements": 3}]}
assert transformer.transform(parser.parse("elements LENGTH > 3")) == {
"nelements": {"$gt": 3}
}
assert transformer.transform(parser.parse("elements LENGTH < 3")) == {
"nelements": {"$lt": 3}
}
assert transformer.transform(parser.parse("elements LENGTH = 3")) == {
"nelements": 3
}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH <= 3")
) == {"nsites": {"$lte": 3}}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH >= 3")
) == {"nsites": {"$gte": 3}}
def __init__(
self,
collection: Union[pymongo.collection.Collection,
mongomock.collection.Collection],
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
):
"""Initialize the MongoCollection for the given parameters.
Parameters:
collection (Union[pymongo.collection.Collection, mongomock.collection.Collection]):
The backend-specific collection.
resource_cls (EntryResource): The `EntryResource` model
that is stored by the collection.
resource_mapper (BaseResourceMapper): A resource mapper
object that handles aliases and format changes between
deserialization and response.
"""
super().__init__(
collection,
resource_cls,
resource_mapper,
MongoTransformer(mapper=resource_mapper),
)
self.parser = LarkParser(
version=(0, 10, 1), variant="default"
) # The MongoTransformer only supports v0.10.1 as the latest grammar
# check aliases do not clash with mongo operators
self._check_aliases(self.resource_mapper.all_aliases())
self._check_aliases(self.resource_mapper.all_length_aliases())
def __init__(
self,
name: str,
resource_cls: "EntryResource",
resource_mapper: "BaseResourceMapper",
):
"""Initialize the AsyncMongoCollection for the given parameters.
Parameters:
name: The name of the collection.
resource_cls: The `EntryResource` model that is stored by the collection.
resource_mapper: A resource mapper object that handles aliases and format
changes between deserialization and response.
"""
from optimade_gateway.mongo.database import ( # pylint: disable=import-outside-toplevel
MONGO_DB, )
super().__init__(
resource_cls=resource_cls,
resource_mapper=resource_mapper,
transformer=MongoTransformer(mapper=resource_mapper),
)
self.parser = LarkParser(version=(1, 0, 0), variant="default")
self.collection: MongoCollection = MONGO_DB[name]
# Check aliases do not clash with mongo operators
self._check_aliases(self.resource_mapper.all_aliases())
self._check_aliases(self.resource_mapper.all_length_aliases())
def __init__(
self,
name: str,
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
database: str = CONFIG.mongo_database,
):
"""Initialize the MongoCollection for the given parameters.
Parameters:
name: The name of the collection.
resource_cls: The type of entry resource that is stored by the collection.
resource_mapper: A resource mapper object that handles aliases and
format changes between deserialization and response.
database: The name of the underlying MongoDB database to connect to.
"""
super().__init__(
resource_cls,
resource_mapper,
MongoTransformer(mapper=resource_mapper),
)
self.parser = LarkParser(version=(1, 0, 0), variant="default")
self.collection = CLIENT[database][name]
# check aliases do not clash with mongo operators
self._check_aliases(self.resource_mapper.all_aliases())
self._check_aliases(self.resource_mapper.all_length_aliases())
def __init__(
self, collection, resource_cls: EntryResource, resource_mapper: ResourceMapper
):
self.collection = collection
self.parser = LarkParser()
self.resource_cls = resource_cls
self.resource_mapper = resource_mapper
def __init__(
self,
collection,
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
transformer: Transformer,
):
"""Initialize the collection for the given parameters.
Parameters:
collection: The backend-specific collection.
resource_cls (EntryResource): The `EntryResource` model
that is stored by the collection.
resource_mapper (BaseResourceMapper): A resource mapper
object that handles aliases and format changes between
deserialization and response.
transformer (Transformer): The Lark `Transformer` used to
interpret the filter.
"""
self.collection = collection
self.parser = LarkParser()
self.resource_cls = resource_cls
self.resource_mapper = resource_mapper
self.transformer = transformer
self.provider_prefix = CONFIG.provider.prefix
self.provider_fields = CONFIG.provider_fields.get(
resource_mapper.ENDPOINT, [])
def test_mongo_special_id(self, mapper):
from optimade.filtertransformers.mongo import MongoTransformer
from bson import ObjectId
class MyMapper(mapper("StructureMapper")):
ALIASES = (("immutable_id", "_id"), )
transformer = MongoTransformer(mapper=MyMapper())
parser = LarkParser(version=self.version, variant=self.variant)
assert transformer.transform(
parser.parse('immutable_id = "5cfb441f053b174410700d02"')) == {
"_id": {
"$eq": ObjectId("5cfb441f053b174410700d02")
}
}
assert transformer.transform(
parser.parse('immutable_id != "5cfb441f053b174410700d02"')) == {
"_id": {
"$ne": ObjectId("5cfb441f053b174410700d02")
}
}
for op in ("CONTAINS", "STARTS WITH", "ENDS WITH", "HAS"):
with pytest.raises(
BadRequest,
match=
r".*not supported for query on field 'immutable_id', can only test for equality.*",
):
transformer.transform(
parser.parse(f'immutable_id {op} "abcdef"'))
def __init__(
self,
entity: Entity,
resource_cls: EntryResource,
resource_mapper: ResourceMapper,
):
self.entity = entity
self.parser = LarkParser()
self.resource_cls = resource_cls
self.resource_mapper = resource_mapper
self.transformer = AiidaTransformer()
self.provider = CONFIG.provider.prefix
self.provider_fields = CONFIG.provider_fields[resource_mapper.ENDPOINT]
self.parser = LarkParser()
# "Cache"
self._data_available: int = None
self._data_returned: int = None
self._filter_fields: set = None
self._latest_filter: dict = None
def __init__(self):
self.opers = {
"=": self.eq,
">": self.gt,
">=": self.ge,
"<": self.lt,
"<=": self.le,
"!=": self.ne,
"OR": self.or_,
"AND": self.and_,
"NOT": self.not_,
}
self.parser = LarkParser(version=(0, 9, 7))
def test_list_length_aliases(self):
from optimade.server.mappers import StructureMapper
transformer = MongoTransformer(mapper=StructureMapper())
parser = LarkParser(version=self.version, variant=self.variant)
self.assertEqual(
transformer.transform(parser.parse("elements LENGTH 3")),
{"nelements": 3})
self.assertEqual(
transformer.transform(
parser.parse('elements HAS "Li" AND elements LENGTH = 3')),
{"$and": [{
"elements": {
"$in": ["Li"]
}
}, {
"nelements": 3
}]},
)
self.assertEqual(
transformer.transform(parser.parse("elements LENGTH > 3")),
{"nelements": {
"$gt": 3
}},
)
self.assertEqual(
transformer.transform(parser.parse("elements LENGTH < 3")),
{"nelements": {
"$lt": 3
}},
)
self.assertEqual(
transformer.transform(parser.parse("elements LENGTH = 3")),
{"nelements": 3})
self.assertEqual(
transformer.transform(
parser.parse("cartesian_site_positions LENGTH <= 3")),
{"nsites": {
"$lte": 3
}},
)
self.assertEqual(
transformer.transform(
parser.parse("cartesian_site_positions LENGTH >= 3")),
{"nsites": {
"$gte": 3
}},
)
def __init__(
self,
collection: orm.entities.Collection,
resource_cls: EntryResource,
resource_mapper: ResourceMapper,
):
self.collection = collection
self.parser = LarkParser()
self.resource_cls = resource_cls
self.resource_mapper = resource_mapper
self.transformer = AiidaTransformerV0_10_1()
self.provider = CONFIG.provider["prefix"]
self.provider_fields = CONFIG.provider_fields[resource_mapper.ENDPOINT]
self.page_limit = CONFIG.page_limit
self.db_page_limit = CONFIG.db_page_limit
self.parser = LarkParser(version=(0, 10, 0))
# "Cache"
self._data_available: int = None
self._data_returned: int = None
self._filter_fields: set = None
self._latest_filter: dict = None
def test_other_provider_fields(self, mapper):
"""Test that fields from other providers generate
queries that treat the value of the field as `null`.
"""
from optimade.filtertransformers.mongo import MongoTransformer
t = MongoTransformer(mapper=mapper("StructureMapper"))
p = LarkParser(version=self.version, variant=self.variant)
assert t.transform(p.parse("_other_provider_field > 1")) == {
"_other_provider_field": {
"$gt": 1
}
}
def __init__(self):
p = LarkParser(version=(1, 0, 0), variant="default")
t = MongoTransformer()
self.transform = lambda inp: t.transform(p.parse(inp))
client = MongoClient('mongodb://{}:{}@{}:{}/?authSource={}'.format(
"admin", "admin", "localhost", "27017", "admin"))
db = client["MaterialsDB"]
self.cl = db["Data.Calculation.StaticCalculation"]
self.lu = Lower2Upper()
self.data = info1
self.info = info2
def __init__(
self,
collection: Union[pymongo.collection.Collection,
mongomock.collection.Collection],
resource_cls: EntryResource,
resource_mapper: ResourceMapper,
):
super().__init__(collection, resource_cls, resource_mapper)
self.transformer = MongoTransformer()
self.provider = CONFIG.provider["prefix"]
self.provider_fields = CONFIG.provider_fields.get(
resource_mapper.ENDPOINT, [])
self.parser = LarkParser(
version=(0, 10, 1), variant="default"
) # The MongoTransformer only supports v0.10.1 as the latest grammar
def setUpClass(cls):
parser = LarkParser(version=(0, 9, 7))
transformer = MongoTransformer()
def convert(_, q):
parsed = parser.parse(q)
return transformer.transform(parsed)
cls.convert = convert
cls.client = MongoClient()
cls.db = cls.client[f"test_db_{uuid.uuid4()}"]
cls.coll = cls.db.data
cls.coll.insert_many([{
"a": a,
"b": b
} for a, b in itertools.product(range(10), range(10))])
def test_list_length_aliases():
"""Check LENGTH aliases for lists"""
from optimade.server.mappers import StructureMapper
transformer = AiidaTransformer(mapper=StructureMapper())
parser = LarkParser(version=VERSION, variant=VARIANT)
assert transformer.transform(parser.parse("elements LENGTH 3")) == {
"nelements": 3
}
assert transformer.transform(
parser.parse('elements HAS "Li" AND elements LENGTH = 3')) == {
"and": [{
"elements": {
"contains": ["Li"]
}
}, {
"nelements": 3
}]
}
assert transformer.transform(parser.parse("elements LENGTH > 3")) == ({
"nelements": {
">": 3
}
})
assert transformer.transform(parser.parse("elements LENGTH < 3")) == ({
"nelements": {
"<": 3
}
})
assert transformer.transform(parser.parse("elements LENGTH = 3")) == {
"nelements": 3
}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH <= 3")) == {
"nsites": {
"<=": 3
}
}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH >= 3")) == {
"nsites": {
">=": 3
}
}
def test_aliased_length_operator(self, mapper):
from optimade.filtertransformers.mongo import MongoTransformer
class MyMapper(mapper("StructureMapper")):
ALIASES = (("elements", "my_elements"), ("nelements", "nelem"))
LENGTH_ALIASES = (
("chemsys", "nelements"),
("cartesian_site_positions", "nsites"),
("elements", "nelements"),
)
PROVIDER_FIELDS = ("chemsys",)
transformer = MongoTransformer(mapper=MyMapper())
parser = LarkParser(version=self.version, variant=self.variant)
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH <= 3")
) == {"nsites": {"$lte": 3}}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH < 3")
) == {"nsites": {"$lt": 3}}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH 3")
) == {"nsites": 3}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH 3")
) == {"nsites": 3}
assert transformer.transform(
parser.parse("cartesian_site_positions LENGTH >= 10")
) == {"nsites": {"$gte": 10}}
assert transformer.transform(
parser.parse("structure_features LENGTH > 10")
) == {"structure_features.11": {"$exists": True}}
assert transformer.transform(parser.parse("nsites LENGTH > 10")) == {
"nsites.11": {"$exists": True}
}
assert transformer.transform(parser.parse("elements LENGTH 3")) == {"nelem": 3}
assert transformer.transform(parser.parse('elements HAS "Ag"')) == {
"my_elements": {"$in": ["Ag"]}
}
assert transformer.transform(parser.parse("chemsys LENGTH 3")) == {"nelem": 3}
def __init__(
self,
collection: Union[pymongo.collection.Collection,
mongomock.collection.Collection],
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
):
super().__init__(collection, resource_cls, resource_mapper)
self.transformer = MongoTransformer(mapper=resource_mapper)
self.provider_prefix = CONFIG.provider.prefix
self.provider_fields = CONFIG.provider_fields.get(
resource_mapper.ENDPOINT, [])
self.parser = LarkParser(
version=(0, 10, 1), variant="default"
) # The MongoTransformer only supports v0.10.1 as the latest grammar
# check aliases do not clash with mongo operators
self._check_aliases(self.resource_mapper.all_aliases())
self._check_aliases(self.resource_mapper.all_length_aliases())
def parser():
return LarkParser()
def test_aliases(self, mapper):
"""Test that valid aliases are allowed, but do not affect r-values."""
from optimade.filtertransformers.mongo import MongoTransformer
class MyStructureMapper(mapper("StructureMapper")):
ALIASES = (
("elements", "my_elements"),
("A", "D"),
("property_a", "D"),
("B", "E"),
("C", "F"),
("_exmpl_nested_field", "nested_field"),
)
PROVIDER_FIELDS = ("D", "E", "F", "nested_field")
mapper = MyStructureMapper
t = MongoTransformer(mapper=mapper)
p = LarkParser(version=self.version, variant=self.variant)
assert mapper.get_backend_field("elements") == "my_elements"
test_filter = 'elements HAS "A"'
assert t.transform(p.parse(test_filter)) == {
"my_elements": {
"$in": ["A"]
}
}
test_filter = 'elements HAS ANY "A","B","C" AND elements HAS "D"'
assert t.transform(p.parse(test_filter)) == {
"$and": [
{
"my_elements": {
"$in": ["A", "B", "C"]
}
},
{
"my_elements": {
"$in": ["D"]
}
},
]
}
test_filter = 'elements = "A"'
assert t.transform(p.parse(test_filter)) == {
"my_elements": {
"$eq": "A"
}
}
test_filter = 'property_a HAS "B"'
assert t.transform(p.parse(test_filter)) == {"D": {"$in": ["B"]}}
test_filter = "_exmpl_nested_field.sub_property > 1234.5"
assert t.transform(p.parse(test_filter)) == {
"nested_field.sub_property": {
"$gt": 1234.5
}
}
test_filter = "_exmpl_nested_field.sub_property.x IS UNKNOWN"
assert t.transform(p.parse(test_filter)) == {
"$or": [
{
"nested_field.sub_property.x": {
"$exists": False
}
},
{
"nested_field.sub_property.x": {
"$eq": None
}
},
]
}
def test_suspected_timestamp_fields(self, mapper):
import datetime
import bson.tz_util
from optimade.filtertransformers.mongo import MongoTransformer
from optimade.server.warnings import TimestampNotRFCCompliant
example_RFC3339_date = "2019-06-08T04:13:37Z"
example_RFC3339_date_2 = "2019-06-08T04:13:37"
example_non_RFC3339_date = "2019-06-08T04:13:37.123Z"
expected_datetime = datetime.datetime(
year=2019,
month=6,
day=8,
hour=4,
minute=13,
second=37,
microsecond=0,
tzinfo=bson.tz_util.utc,
)
assert self.transform(f'last_modified > "{example_RFC3339_date}"') == {
"last_modified": {
"$gt": expected_datetime
}
}
assert self.transform(
f'last_modified > "{example_RFC3339_date_2}"') == {
"last_modified": {
"$gt": expected_datetime
}
}
non_rfc_datetime = expected_datetime.replace(microsecond=123000)
with pytest.warns(TimestampNotRFCCompliant):
assert self.transform(
f'last_modified > "{example_non_RFC3339_date}"') == {
"last_modified": {
"$gt": non_rfc_datetime
}
}
class MyMapper(mapper("StructureMapper")):
ALIASES = (("last_modified", "ctime"), )
transformer = MongoTransformer(mapper=MyMapper)
parser = LarkParser(version=self.version, variant=self.variant)
assert transformer.transform(
parser.parse(f'last_modified > "{example_RFC3339_date}"')) == {
"ctime": {
"$gt": expected_datetime
}
}
assert transformer.transform(
parser.parse(f'last_modified > "{example_RFC3339_date_2}"')) == {
"ctime": {
"$gt": expected_datetime
}
}
def test_filtering_on_relationships(self, mapper):
"""Test the nested properties with special names
like "structures", "references" etc. are applied
to the relationships field.
"""
from optimade.filtertransformers.mongo import MongoTransformer
t = MongoTransformer(mapper=mapper("StructureMapper"))
p = LarkParser(version=self.version, variant=self.variant)
assert t.transform(p.parse('references.id HAS "dummy/2019"')) == {
"relationships.references.data.id": {
"$in": ["dummy/2019"]
}
}
assert t.transform(
p.parse('structures.id HAS ANY "dummy/2019", "dijkstra1968"')) == {
"relationships.structures.data.id": {
"$in": ["dummy/2019", "dijkstra1968"]
}
}
assert t.transform(
p.parse('structures.id HAS ALL "dummy/2019", "dijkstra1968"')) == {
"relationships.structures.data.id": {
"$all": ["dummy/2019", "dijkstra1968"]
}
}
assert t.transform(p.parse('structures.id HAS ONLY "dummy/2019"')) == {
"$and": [
{
"relationships.structures.data": {
"$size": 1
}
},
{
"relationships.structures.data.id": {
"$all": ["dummy/2019"]
}
},
]
}
assert t.transform(
p.parse(
'structures.id HAS ONLY "dummy/2019" AND structures.id HAS "dummy/2019"'
)) == {
"$and": [
{
"$and": [
{
"relationships.structures.data": {
"$size": 1,
}
},
{
"relationships.structures.data.id": {
"$all": ["dummy/2019"]
}
},
]
},
{
"relationships.structures.data.id": {
"$in": ["dummy/2019"]
}
},
],
}
def set_up(self):
from optimade.filtertransformers.mongo import MongoTransformer
p = LarkParser(version=self.version, variant=self.variant)
t = MongoTransformer()
self.transform = lambda inp: t.transform(p.parse(inp))
def setUp(self):
p = LarkParser(version=self.version, variant=self.variant)
t = MongoTransformer()
self.transform = lambda inp: t.transform(p.parse(inp))
def set_up(self):
self.parser = LarkParser(version=self.version, variant=self.variant)
def test_parser_version(self):
v = (0, 9, 5)
p = LarkParser(version=v)
assert isinstance(p.parse(self.test_filters[0]), Tree)
assert p.version == v
return "string"
def number(self, arg):
# number: SIGNED_INT | SIGNED_FLOAT
print("Node: ", "number", arg)
return "number"
def __default__(self, data, children, meta):
print("Node: ", data, children)
return data
if __name__ == "__main__": # pragma: no cover
from optimade.filterparser import LarkParser
p = LarkParser(version=(0, 10, 0))
# t = DebugTransformer()
t = TransformerSkeleton()
# f = 'a.a = "text" OR a<a AND NOT b>=8'
# single list
f = "list HAS < 3"
f = "list HAS < 3, > 4" # -> error
f = "list HAS ALL < 3, > 4"
# multiple lists
f = "list1:list2 HAS < 3 : > 4"
def set_up(self):
self.test_filters = []
for fn in sorted(glob(os.path.join(testfile_dir, "*.inp"))):
with open(fn) as f:
self.test_filters.append(f.read().strip())
self.parser = LarkParser(version=(0, 9, 5))