def get_providers() -> list:
"""Retrieve Materials-Consortia providers (from https://providers.optimade.org/v1/links).
Fallback order if providers.optimade.org is not available:
1. Try Materials-Consortia/providers on GitHub.
2. Try submodule `providers`' list of providers.
3. Log warning that providers list from Materials-Consortia is not included in the
`/links`-endpoint.
Returns:
List of raw JSON-decoded providers including MongoDB object IDs.
"""
import requests
try:
import simplejson as json
except ImportError:
import json
provider_list_urls = [
"https://providers.optimade.org/v1/links",
"https://raw.githubusercontent.com/Materials-Consortia/providers",
"/master/src/links/v1/providers.json",
]
for provider_list_url in provider_list_urls:
try:
providers = requests.get(provider_list_url).json()
except (
requests.exceptions.ConnectionError,
requests.exceptions.ConnectTimeout,
json.JSONDecodeError,
):
pass
else:
break
else:
try:
from optimade.server.data import providers
except ImportError:
from optimade.server.logger import LOGGER
LOGGER.warning("""Could not retrieve a list of providers!
Tried the following resources:
{}
The list of providers will not be included in the `/links`-endpoint.
""".format("".join([f" * {_}\n" for _ in provider_list_urls])))
return []
providers_list = []
for provider in providers.get("data", []):
# Remove/skip "exmpl"
if provider["id"] == "exmpl":
continue
provider.update(provider.pop("attributes", {}))
# Add MongoDB ObjectId
provider["_id"] = {
"$oid": mongo_id_for_database(provider["id"], provider["type"])
}
providers_list.append(provider)
return providers_list
LOGGER.debug(
"Adding Materials-Consortia providers to links from optimade.org..."
)
providers = get_providers()
for doc in providers:
links_coll.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting index links!")
else:
LOGGER.warning(
"Not inserting test data for index meta-database for backend %s",
CONFIG.database_backend.value,
)
# Add CORS middleware first
app.add_middleware(CORSMiddleware, allow_origins=["*"])
# Then add required OPTIMADE middleware
for middleware in OPTIMADE_MIDDLEWARE:
app.add_middleware(middleware)
# Add exception handlers
for exception, handler in OPTIMADE_EXCEPTIONS:
app.add_exception_handler(exception, handler)
# Add all endpoints to unversioned URL
for endpoint in (index_info, links, versions):
from typing import Dict, Tuple, List, Any
from optimade.filterparser import LarkParser
from optimade.filtertransformers.mongo import MongoTransformer
from optimade.models import EntryResource
from optimade.server.config import CONFIG
from optimade.server.entry_collections import EntryCollection
from optimade.server.logger import LOGGER
from optimade.server.mappers import BaseResourceMapper
if CONFIG.database_backend.value == "mongodb":
from pymongo import MongoClient, version_tuple
if version_tuple[0] < 4:
LOGGER.warning(
"Support for pymongo<=3 (and thus MongoDB v3) is deprecated and will be "
"removed in the next minor release.")
LOGGER.info("Using: Real MongoDB (pymongo)")
elif CONFIG.database_backend.value == "mongomock":
from mongomock import MongoClient
LOGGER.info("Using: Mock MongoDB (mongomock)")
if CONFIG.database_backend.value in ("mongomock", "mongodb"):
CLIENT = MongoClient(CONFIG.mongo_uri)
class MongoCollection(EntryCollection):
"""Class for querying MongoDB collections (implemented by either pymongo or mongomock)