def general_exception(
request: Request,
exc: Exception,
status_code: int = 500, # A status_code in `exc` will take precedence
errors: List[OptimadeError] = None,
) -> JSONResponse:
debug_info = {}
if CONFIG.debug:
tb = "".join(
traceback.format_exception(etype=type(exc),
value=exc,
tb=exc.__traceback__))
LOGGER.error("Traceback:\n%s", tb)
debug_info[f"_{CONFIG.provider.prefix}_traceback"] = tb
try:
http_response_code = int(exc.status_code)
except AttributeError:
http_response_code = int(status_code)
try:
title = str(exc.title)
except AttributeError:
title = str(exc.__class__.__name__)
try:
detail = str(exc.detail)
except AttributeError:
detail = str(exc)
if errors is None:
errors = [
OptimadeError(detail=detail,
status=http_response_code,
title=title)
]
response = ErrorResponse(
meta=meta_values(
url=request.url,
data_returned=0,
data_available=0,
more_data_available=False,
**debug_info,
),
errors=errors,
)
return JSONResponse(
status_code=http_response_code,
content=jsonable_encoder(response, exclude_unset=True),
)
def create_optimade_index(self) -> None:
"""Load or create an index that can handle aliased OPTIMADE fields and attach it
to the current client.
"""
body = self.predefined_index.get(self.name)
if body is None:
body = self.create_elastic_index_from_mapper(
self.resource_mapper, self.all_fields
)
properties = {}
for field in list(body["mappings"]["doc"]["properties"].keys()):
properties[self.resource_mapper.get_backend_field(field)] = body[
"mappings"
]["doc"]["properties"].pop(field)
body["mappings"]["doc"]["properties"] = properties
self.client.indices.create(index=self.name, body=body, ignore=400)
LOGGER.debug(f"Created Elastic index for {self.name!r} with body {body}")
def load_entries(endpoint_name: str, endpoint_collection: MongoCollection):
LOGGER.debug("Loading test %s...", endpoint_name)
endpoint_collection.collection.insert_many(getattr(data, endpoint_name, []))
if endpoint_name == "links":
LOGGER.debug(
" Adding Materials-Consortia providers to links from optimade.org"
)
providers = get_providers()
for doc in providers:
endpoint_collection.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting test %s!", endpoint_name)
def load_entries(endpoint_name: str, endpoint_collection: EntryCollection):
LOGGER.debug("Loading test %s...", endpoint_name)
endpoint_collection.insert(getattr(data, endpoint_name, []))
if (CONFIG.database_backend.value in ("mongomock", "mongodb")
and endpoint_name == "links"):
LOGGER.debug(
"Adding Materials-Consortia providers to links from optimade.org"
)
providers = get_providers(add_mongo_id=True)
for doc in providers:
endpoint_collection.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(
bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting test %s!", endpoint_name)
from optimade.server.exception_handlers import OPTIMADE_EXCEPTIONS
from optimade.server.middleware import OPTIMADE_MIDDLEWARE
from optimade.server.routers import (
info,
landing,
links,
references,
structures,
versions,
)
from optimade.server.routers.utils import BASE_URL_PREFIXES, JSONAPIResponse
if config_warnings:
LOGGER.warn(
f"Invalid config file or no config file provided, running server with default settings. Errors: "
f"{[warnings.formatwarning(w.message, w.category, w.filename, w.lineno, '') for w in config_warnings]}"
)
else:
LOGGER.info(
f"Loaded settings from {os.getenv('OPTIMADE_CONFIG_FILE', DEFAULT_CONFIG_FILE_PATH)}."
)
if CONFIG.debug: # pragma: no cover
LOGGER.info("DEBUG MODE")
app = FastAPI(
root_path=CONFIG.root_path,
title="OPTIMADE API",
description=
(f"""The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.
from optimade import __api_version__, __version__
import optimade.server.exception_handlers as exc_handlers
from optimade.server.logger import LOGGER
from optimade.server.middleware import (
AddWarnings,
CheckWronglyVersionedBaseUrls,
EnsureQueryParamIntegrity,
HandleApiHint,
)
from optimade.server.routers import index_info, links, versions
from optimade.server.routers.utils import BASE_URL_PREFIXES
if CONFIG.config_file is None:
LOGGER.warn(
f"Invalid config file or no config file provided, running server with default settings. Errors: "
f"{[warnings.formatwarning(w.message, w.category, w.filename, w.lineno, '') for w in config_warnings]}"
)
else:
LOGGER.info(f"Loaded settings from {CONFIG.config_file}.")
if CONFIG.debug: # pragma: no cover
LOGGER.info("DEBUG MODE")
app = FastAPI(
title="OPTIMADE API - Index meta-database",
description=
(f"""The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.
This is the "special" index meta-database.
This specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v{__version__}) v{__version__}."""
),
from typing import Dict, Tuple, List, Any
from optimade.filterparser import LarkParser
from optimade.filtertransformers.mongo import MongoTransformer
from optimade.models import EntryResource
from optimade.server.config import CONFIG
from optimade.server.entry_collections import EntryCollection
from optimade.server.logger import LOGGER
from optimade.server.mappers import BaseResourceMapper
if CONFIG.database_backend.value == "mongodb":
from pymongo import MongoClient
LOGGER.info("Using: Real MongoDB (pymongo)")
elif CONFIG.database_backend.value == "mongomock":
from mongomock import MongoClient
LOGGER.info("Using: Mock MongoDB (mongomock)")
if CONFIG.database_backend.value in ("mongomock", "mongodb"):
CLIENT = MongoClient(CONFIG.mongo_uri)
class MongoCollection(EntryCollection):
"""Class for querying MongoDB collections (implemented by either pymongo or mongomock)
containing serialized [`EntryResource`][optimade.models.entries.EntryResource]s objects.
"""
def __init__(
self,
HandleApiHint,
)
from optimade.server.routers import (
info,
landing,
links,
references,
structures,
versions,
)
from optimade.server.routers.utils import BASE_URL_PREFIXES
if CONFIG.config_file is None:
LOGGER.warn(
f"Invalid config file or no config file provided, running server with default settings. Errors: "
f"{[warnings.formatwarning(w.message, w.category, w.filename, w.lineno, '') for w in config_warnings]}"
)
else:
LOGGER.info(f"Loaded settings from {CONFIG.config_file}.")
if CONFIG.debug: # pragma: no cover
LOGGER.info("DEBUG MODE")
app = FastAPI(
title="OPTIMADE API",
description=(
f"""The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.
This specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v{__version__}) v{__version__}."""
),
version=__api_version__,
with warnings.catch_warnings(record=True) as w:
from optimade.server.config import CONFIG
config_warnings = w
from optimade import __api_version__, __version__
from optimade.server.logger import LOGGER
from optimade.server.exception_handlers import OPTIMADE_EXCEPTIONS
from optimade.server.middleware import OPTIMADE_MIDDLEWARE
from optimade.server.routers import index_info, links, versions
from optimade.server.routers.utils import BASE_URL_PREFIXES
if os.getenv("OPTIMADE_CONFIG_FILE") is None:
LOGGER.warn(
f"Invalid config file or no config file provided, running server with default settings. Errors: "
f"{[warnings.formatwarning(w.message, w.category, w.filename, w.lineno, '') for w in config_warnings]}"
)
else:
LOGGER.info(f"Loaded settings from {os.getenv('OPTIMADE_CONFIG_FILE')}.")
if CONFIG.debug: # pragma: no cover
LOGGER.info("DEBUG MODE")
app = FastAPI(
root_path=CONFIG.root_path,
title="OPTIMADE API - Index meta-database",
description=
(f"""The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.
This is the "special" index meta-database.
This specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v{__version__}) v{__version__}."""
def general_exception(
request: Request,
exc: Exception,
status_code: int = 500, # A status_code in `exc` will take precedence
errors: List[OptimadeError] = None,
) -> JSONAPIResponse:
"""Handle an exception
Parameters:
request: The HTTP request resulting in the exception being raised.
exc: The exception being raised.
status_code: The returned HTTP status code for the error response.
errors: List of error resources as defined in
[the OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#json-response-schema-common-fields).
Returns:
A JSON HTTP response based on [`ErrorResponse`][optimade.models.responses.ErrorResponse].
"""
debug_info = {}
if CONFIG.debug:
tb = "".join(
traceback.format_exception(type(exc),
value=exc,
tb=exc.__traceback__))
LOGGER.error("Traceback:\n%s", tb)
debug_info[f"_{CONFIG.provider.prefix}_traceback"] = tb
try:
http_response_code = int(exc.status_code)
except AttributeError:
http_response_code = int(status_code)
try:
title = str(exc.title)
except AttributeError:
title = str(exc.__class__.__name__)
try:
detail = str(exc.detail)
except AttributeError:
detail = str(exc)
if errors is None:
errors = [
OptimadeError(detail=detail,
status=http_response_code,
title=title)
]
response = ErrorResponse(
meta=meta_values(
url=request.url,
data_returned=0,
data_available=0,
more_data_available=False,
**debug_info,
),
errors=errors,
)
return JSONAPIResponse(
status_code=http_response_code,
content=jsonable_encoder(response, exclude_unset=True),
)
def get_providers() -> list:
"""Retrieve Materials-Consortia providers (from https://providers.optimade.org/v1/links).
Fallback order if providers.optimade.org is not available:
1. Try Materials-Consortia/providers on GitHub.
2. Try submodule `providers`' list of providers.
3. Log warning that providers list from Materials-Consortia is not included in the
`/links`-endpoint.
Returns:
List of raw JSON-decoded providers including MongoDB object IDs.
"""
import requests
try:
import simplejson as json
except ImportError:
import json
provider_list_urls = [
"https://providers.optimade.org/v1/links",
"https://raw.githubusercontent.com/Materials-Consortia/providers",
"/master/src/links/v1/providers.json",
]
for provider_list_url in provider_list_urls:
try:
providers = requests.get(provider_list_url).json()
except (
requests.exceptions.ConnectionError,
requests.exceptions.ConnectTimeout,
json.JSONDecodeError,
):
pass
else:
break
else:
try:
from optimade.server.data import providers
except ImportError:
from optimade.server.logger import LOGGER
LOGGER.warning("""Could not retrieve a list of providers!
Tried the following resources:
{}
The list of providers will not be included in the `/links`-endpoint.
""".format("".join([f" * {_}\n" for _ in provider_list_urls])))
return []
providers_list = []
for provider in providers.get("data", []):
# Remove/skip "exmpl"
if provider["id"] == "exmpl":
continue
provider.update(provider.pop("attributes", {}))
# Add MongoDB ObjectId
provider["_id"] = {
"$oid": mongo_id_for_database(provider["id"], provider["type"])
}
providers_list.append(provider)
return providers_list
from typing import Dict, Tuple, List, Any
from optimade.filterparser import LarkParser
from optimade.filtertransformers.mongo import MongoTransformer
from optimade.models import EntryResource
from optimade.server.config import CONFIG
from optimade.server.entry_collections import EntryCollection
from optimade.server.logger import LOGGER
from optimade.server.mappers import BaseResourceMapper
if CONFIG.database_backend.value == "mongodb":
from pymongo import MongoClient, version_tuple
if version_tuple[0] < 4:
LOGGER.warning(
"Support for pymongo<=3 (and thus MongoDB v3) is deprecated and will be "
"removed in the next minor release.")
LOGGER.info("Using: Real MongoDB (pymongo)")
elif CONFIG.database_backend.value == "mongomock":
from mongomock import MongoClient
LOGGER.info("Using: Mock MongoDB (mongomock)")
if CONFIG.database_backend.value in ("mongomock", "mongodb"):
CLIENT = MongoClient(CONFIG.mongo_uri)
class MongoCollection(EntryCollection):
"""Class for querying MongoDB collections (implemented by either pymongo or mongomock)
from optimade.filtertransformers.elasticsearch import ElasticTransformer
from optimade.server.config import CONFIG
from optimade.server.logger import LOGGER
from optimade.models import EntryResource
from optimade.server.mappers import BaseResourceMapper
from optimade.server.entry_collections import EntryCollection
if CONFIG.database_backend.value == "elastic":
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch_dsl import Search
CLIENT = Elasticsearch(hosts=CONFIG.elastic_hosts)
LOGGER.info("Using: Elasticsearch backend at %s", CONFIG.elastic_hosts)
class ElasticCollection(EntryCollection):
def __init__(
self,
name: str,
resource_cls: EntryResource,
resource_mapper: BaseResourceMapper,
client: Optional["Elasticsearch"] = None,
):
"""Initialize the ElasticCollection for the given parameters.
Parameters:
name: The name of the collection.
resource_cls: The type of entry resource that is stored by the collection.