Beispiel #1
0
def get_entries(
    collection: AiidaCollection,
    response: EntryResponseMany,
    request: Request,
    params: EntryListingQueryParams,
) -> EntryResponseMany:
    """Generalized /{entry} endpoint getter"""
    (
        results,
        data_returned,
        more_data_available,
        data_available,
        fields,
    ) = collection.find(params)

    pagination = handle_pagination(request=request,
                                   more_data_available=more_data_available,
                                   nresults=len(results))

    if fields:
        results = handle_response_fields(results, fields, collection)

    return response(
        links=ToplevelLinks(**pagination),
        data=results,
        meta=meta_values(str(request.url), data_returned, data_available,
                         more_data_available),
    )
def get_info(request: Request) -> IndexInfoResponse:
    return IndexInfoResponse(
        meta=meta_values(request.url, 1, 1, more_data_available=False),
        data=IndexInfoResource(
            id=IndexInfoResource.schema()["properties"]["id"]["const"],
            type=IndexInfoResource.schema()["properties"]["type"]["const"],
            attributes=IndexInfoAttributes(
                api_version=f"{__api_version__}",
                available_api_versions=[{
                    "url":
                    f"{get_base_url(request.url)}/v{__api_version__.split('.')[0]}/",
                    "version": f"{__api_version__}",
                }],
                formats=["json"],
                available_endpoints=["info", "links"],
                entry_types_by_format={"json": []},
                is_index=True,
            ),
            relationships={
                "default":
                IndexRelationship(
                    data={
                        "type":
                        RelatedLinksResource.schema()["properties"]["type"]
                        ["const"],
                        "id":
                        CONFIG.default_db,
                    })
            },
        ),
    )
Beispiel #3
0
async def post_queries(
    request: Request,
    query: QueryCreate,
) -> QueriesResponseSingle:
    """`POST /queries`

    Create or return existing gateway query according to `query`.
    """
    await validate_resource(
        await collection_factory(CONFIG.gateways_collection), query.gateway_id)

    result, created = await resource_factory(query)

    if created:
        asyncio.create_task(perform_query(url=request.url, query=result))

    collection = await collection_factory(CONFIG.queries_collection)

    return QueriesResponseSingle(
        links=ToplevelLinks(next=None),
        data=result,
        meta=meta_values(
            url=request.url,
            data_returned=1,
            data_available=await collection.acount(),
            more_data_available=False,
            **{f"_{CONFIG.provider.prefix}_created": created},
        ),
    )
Beispiel #4
0
def get_info(request: Request):
    from optimade.models import BaseInfoResource, BaseInfoAttributes

    parse_result = urllib.parse.urlparse(str(request.url))
    base_url = get_base_url(parse_result)

    return InfoResponse(
        meta=meta_values(str(request.url), 1, 1, more_data_available=False),
        data=BaseInfoResource(
            id=BaseInfoResource.schema()["properties"]["id"]["const"],
            type=BaseInfoResource.schema()["properties"]["type"]["const"],
            attributes=BaseInfoAttributes(
                api_version=__api_version__,
                available_api_versions=[
                    {
                        "url": f"{base_url}/v{__api_version__.split('.')[0]}",
                        "version": __api_version__,
                    }
                ],
                formats=["json"],
                available_endpoints=["info", "links"] + list(ENTRY_INFO_SCHEMAS.keys()),
                entry_types_by_format={"json": list(ENTRY_INFO_SCHEMAS.keys())},
                is_index=False,
            ),
        ),
    )
Beispiel #5
0
def get_entry_info(request: Request, entry: str):
    from optimade.models import EntryInfoResource

    valid_entry_info_endpoints = ENTRY_INFO_SCHEMAS.keys()
    if entry not in valid_entry_info_endpoints:
        raise StarletteHTTPException(
            status_code=404,
            detail=f"Entry info not found for {entry}, valid entry info endpoints are: {', '.join(valid_entry_info_endpoints)}",
        )

    schema = ENTRY_INFO_SCHEMAS[entry]()
    queryable_properties = {"id", "type", "attributes"}
    properties = retrieve_queryable_properties(schema, queryable_properties)

    output_fields_by_format = {"json": list(properties.keys())}

    return EntryInfoResponse(
        meta=meta_values(str(request.url), 1, 1, more_data_available=False),
        data=EntryInfoResource(
            formats=list(output_fields_by_format.keys()),
            description=schema.get("description", "Entry Resources"),
            properties=properties,
            output_fields_by_format=output_fields_by_format,
        ),
    )
Beispiel #6
0
def general_exception(
    request: Request,
    exc: Exception,
    status_code: int = 500,  # A status_code in `exc` will take precedence
    errors: List[OptimadeError] = None,
) -> JSONResponse:
    debug_info = {}
    if CONFIG.debug:
        tb = "".join(
            traceback.format_exception(etype=type(exc),
                                       value=exc,
                                       tb=exc.__traceback__))
        LOGGER.error("Traceback:\n%s", tb)
        debug_info[f"_{CONFIG.provider.prefix}_traceback"] = tb

    try:
        http_response_code = int(exc.status_code)
    except AttributeError:
        http_response_code = int(status_code)

    try:
        title = str(exc.title)
    except AttributeError:
        title = str(exc.__class__.__name__)

    try:
        detail = str(exc.detail)
    except AttributeError:
        detail = str(exc)

    if errors is None:
        errors = [
            OptimadeError(detail=detail,
                          status=http_response_code,
                          title=title)
        ]

    response = ErrorResponse(
        meta=meta_values(
            url=request.url,
            data_returned=0,
            data_available=0,
            more_data_available=False,
            **debug_info,
        ),
        errors=errors,
    )

    return JSONResponse(
        status_code=http_response_code,
        content=jsonable_encoder(response, exclude_unset=True),
    )
async def landing(request: Request):
    """Show a human-readable landing page when the base URL is accessed."""

    meta = meta_values(request.url, 1, 1, more_data_available=False)
    major_version = __api_version__.split(".")[0]
    versioned_url = f"{get_base_url(request.url)}/v{major_version}/"

    context = {
        "request": request,
        "request_url": request.url,
        "api_version": __api_version__,
        "implementation": meta.implementation,
        "versioned_url": versioned_url,
        "provider": meta.provider,
        "index_base_url": CONFIG.index_base_url,
        "endpoints": list(ENTRY_COLLECTIONS.keys()) + ["info"],
    }

    return TEMPLATES.TemplateResponse("landing_page.html", context)
Beispiel #8
0
async def get_gateway(request: Request,
                      gateway_id: str) -> GatewaysResponseSingle:
    """`GET /gateways/{gateway ID}`

    Return a single [`GatewayResource`][optimade_gateway.models.gateways.GatewayResource].
    """
    collection = await collection_factory(CONFIG.gateways_collection)
    result = await get_valid_resource(collection, gateway_id)

    return GatewaysResponseSingle(
        links=ToplevelLinks(next=None),
        data=result,
        meta=meta_values(
            url=request.url,
            data_returned=1,
            data_available=await collection.acount(),
            more_data_available=False,
        ),
    )
Beispiel #9
0
async def get_entries(
    collection: AsyncMongoCollection,
    response_cls: "EntryResponseMany",
    request: "Request",
    params: "EntryListingQueryParams",
) -> "EntryResponseMany":
    """Generalized `/{entries}` endpoint getter"""
    (
        results,
        data_returned,
        more_data_available,
        fields,
        include_fields,
    ) = await collection.afind(params=params)

    if more_data_available:
        # Deduce the `next` link from the current request
        query = urllib.parse.parse_qs(request.url.query)
        query["page_offset"] = [int(query.get("page_offset", [0])[0]) + len(results)]  # type: ignore[list-item, arg-type]
        urlencoded = urllib.parse.urlencode(query, doseq=True)
        base_url = get_base_url(request.url)

        links = ToplevelLinks(next=f"{base_url}{request.url.path}?{urlencoded}")
    else:
        links = ToplevelLinks(next=None)

    if fields or include_fields:
        results = handle_response_fields(results, fields, include_fields)

    return response_cls(
        links=links,
        data=results,
        meta=meta_values(
            url=request.url,
            data_returned=data_returned,
            data_available=await collection.acount(),
            more_data_available=more_data_available,
        ),
    )
Beispiel #10
0
async def post_gateways(request: Request,
                        gateway: GatewayCreate) -> GatewaysResponseSingle:
    """`POST /gateways`

    Create or return existing gateway according to `gateway`.
    """
    if gateway.database_ids:
        databases_collection = await collection_factory(
            CONFIG.databases_collection)

        databases = await databases_collection.get_multiple(filter={
            "id": {
                "$in": await clean_python_types(gateway.database_ids)
            }
        })

        if not isinstance(gateway.databases, list):
            gateway.databases = []

        current_database_ids = [_.id for _ in gateway.databases]
        gateway.databases.extend(
            (_ for _ in databases if _.id not in current_database_ids))

    result, created = await resource_factory(gateway)
    collection = await collection_factory(CONFIG.gateways_collection)

    return GatewaysResponseSingle(
        links=ToplevelLinks(next=None),
        data=result,
        meta=meta_values(
            url=request.url,
            data_returned=1,
            data_available=await collection.acount(),
            more_data_available=False,
            **{f"_{CONFIG.provider.prefix}_created": created},
        ),
    )
Beispiel #11
0
async def get_query(
    request: Request,
    query_id: str,
    response: Response,
) -> QueriesResponseSingle:
    """`GET /queries/{query_id}`

    Return a single [`QueryResource`][optimade_gateway.models.queries.QueryResource].
    """
    collection = await collection_factory(CONFIG.queries_collection)
    query: QueryResource = await get_valid_resource(collection, query_id)

    if query.attributes.response and query.attributes.response.errors:
        for error in query.attributes.response.errors:
            if error.status:
                for part in error.status.split(" "):
                    try:
                        response.status_code = int(part)
                        break
                    except ValueError:
                        pass
                if response.status_code and response.status_code >= 300:
                    break
        else:
            response.status_code = 500

    return QueriesResponseSingle(
        links=ToplevelLinks(next=None),
        data=query,
        meta=meta_values(
            url=request.url,
            data_returned=1,
            data_available=await collection.acount(),
            more_data_available=False,
        ),
    )
Beispiel #12
0
def get_single_entry(
    collection: AiidaCollection,
    entry_id: str,
    response: EntryResponseOne,
    request: Request,
    params: SingleEntryQueryParams,
) -> EntryResponseOne:
    """Generalized /{entry}/{entry_id} endpoint getter"""
    params.filter = f"id={entry_id}"
    (
        results,
        data_returned,
        more_data_available,
        data_available,
        fields,
    ) = collection.find(params)

    if more_data_available:
        raise HTTPException(
            status_code=500,
            detail=
            "more_data_available MUST be False for single entry response, "
            f"however it is {more_data_available}",
        )

    links = ToplevelLinks(next=None)

    if fields and results is not None:
        results = handle_response_fields(results, fields, collection)[0]

    return response(
        links=links,
        data=results,
        meta=meta_values(str(request.url), data_returned, data_available,
                         more_data_available),
    )
Beispiel #13
0
async def post_search(request: Request,
                      search: Search) -> QueriesResponseSingle:
    """`POST /search`

    Coordinate a new OPTIMADE query in multiple databases through a gateway:

    1. Search for gateway in DB using `optimade_urls` and `database_ids`
    1. Create [`GatewayCreate`][optimade_gateway.models.gateways.GatewayCreate] model
    1. `POST` gateway resource to get ID - using functionality of `POST /gateways`
    1. Create new [Query][optimade_gateway.models.queries.QueryCreate] resource
    1. `POST` Query resource - using functionality of `POST /queries`
    1. Return `POST /queries` response -
        [`QueriesResponseSingle`][optimade_gateway.models.responses.QueriesResponseSingle]

    """
    databases_collection = await collection_factory(CONFIG.databases_collection
                                                    )
    # NOTE: It may be that the final list of base URLs (`base_urls`) contains the same
    # provider(s), but with differring base URLS, if, for example, a versioned base URL
    # is supplied.
    base_urls = set()

    if search.database_ids:
        databases = await databases_collection.get_multiple(filter={
            "id": {
                "$in": await clean_python_types(search.database_ids)
            }
        })
        base_urls |= {
            get_resource_attribute(database, "attributes.base_url")
            for database in databases if get_resource_attribute(
                database, "attributes.base_url") is not None
        }

    if search.optimade_urls:
        base_urls |= {_ for _ in search.optimade_urls if _ is not None}

    if not base_urls:
        msg = "No (valid) OPTIMADE URLs with:"
        if search.database_ids:
            msg += (
                f"\n  Database IDs: {search.database_ids} and corresponding found URLs: "
                f"{[get_resource_attribute(database, 'attributes.base_url') for database in databases]}"
            )
        if search.optimade_urls:
            msg += f"\n  Passed OPTIMADE URLs: {search.optimade_urls}"
        raise BadRequest(detail=msg)

    # Ensure all URLs are `pydantic.AnyUrl`s
    if not all(isinstance(_, AnyUrl) for _ in base_urls):
        raise InternalServerError(
            "Could unexpectedly not validate all base URLs as proper URLs.")

    databases = await databases_collection.get_multiple(
        filter={"base_url": {
            "$in": await clean_python_types(base_urls)
        }})
    if len(databases) == len(base_urls):
        # At this point it is expected that the list of databases in `databases`
        # is a complete set of databases requested.
        gateway = GatewayCreate(databases=databases)
    elif len(databases) < len(base_urls):
        # There are unregistered databases
        current_base_urls = {
            get_resource_attribute(database, "attributes.base_url")
            for database in databases
        }
        databases.extend([
            LinksResource(
                id=(f"{url.user + '@' if url.user else ''}{url.host}"
                    f"{':' + url.port if url.port else ''}"
                    f"{url.path.rstrip('/') if url.path else ''}").replace(
                        ".", "__"),
                type="links",
                attributes=LinksResourceAttributes(
                    name=(f"{url.user + '@' if url.user else ''}{url.host}"
                          f"{':' + url.port if url.port else ''}"
                          f"{url.path.rstrip('/') if url.path else ''}"),
                    description="",
                    base_url=url,
                    link_type=LinkType.CHILD,
                    homepage=None,
                ),
            ) for url in base_urls - current_base_urls
        ])
    else:
        LOGGER.error(
            "Found more database entries in MongoDB than then number of passed base URLs."
            " This suggests ambiguity in the base URLs of databases stored in MongoDB.\n"
            "  base_urls: %s\n  databases %s",
            base_urls,
            databases,
        )
        raise InternalServerError(
            "Unambiguous base URLs. See logs for more details.")

    gateway = GatewayCreate(databases=databases)
    gateway, created = await resource_factory(gateway)

    if created:
        LOGGER.debug("A new gateway was created for a query (id=%r)",
                     gateway.id)
    else:
        LOGGER.debug("A gateway was found and reused for a query (id=%r)",
                     gateway.id)

    query = QueryCreate(
        endpoint=search.endpoint,
        gateway_id=gateway.id,
        query_parameters=search.query_parameters,
    )
    query, created = await resource_factory(query)

    if created:
        asyncio.create_task(perform_query(url=request.url, query=query))

    collection = await collection_factory(CONFIG.queries_collection)

    return QueriesResponseSingle(
        links=ToplevelLinks(next=None),
        data=query,
        meta=meta_values(
            url=request.url,
            data_returned=1,
            data_available=await collection.acount(),
            more_data_available=False,
            **{f"_{CONFIG.provider.prefix}_created": created},
        ),
    )
Beispiel #14
0
def render_landing_page(url: str) -> HTMLResponse:
    """Render and cache the landing page.

    This function uses the template file `./static/landing_page.html`, adapted
    from the original Jinja template. Instead of Jinja, some basic string
    replacement is used to fill out the fields from the server configuration.

    !!! warning "Careful"
        The removal of Jinja means that the fields are no longer validated as
        web safe before inclusion in the template.

    """
    meta = meta_values(url, 1, 1, more_data_available=False)
    major_version = __api_version__.split(".")[0]
    versioned_url = f"{get_base_url(url)}/v{major_version}/"

    template_dir = Path(__file__).parent.joinpath("static").resolve()

    html = (template_dir / "landing_page.html").read_text()

    # Build a dictionary that maps the old Jinja keys to the new simplified replacements
    replacements = {
        "api_version": __api_version__,
    }

    if meta.provider:
        replacements.update({
            "provider.name":
            meta.provider.name,
            "provider.prefix":
            meta.provider.prefix,
            "provider.description":
            meta.provider.description,
            "provider.homepage":
            str(meta.provider.homepage) or "",
        })

    if meta.implementation:
        replacements.update({
            "implementation.name":
            meta.implementation.name or "",
            "implementation.version":
            meta.implementation.version or "",
            "implementation.source_url":
            str(meta.implementation.source_url or ""),
        })

    for replacement in replacements:
        html = html.replace(f"{{{{ {replacement} }}}}",
                            replacements[replacement])

    # Build the list of endpoints. The template already opens and closes the `<ul>` tag.
    endpoints_list = [
        f'<li><a href="{versioned_url}{endp}">{versioned_url}{endp}</a></li>'
        for endp in list(ENTRY_COLLECTIONS.keys()) + ["info"]
    ]
    html = html.replace("{% ENDPOINTS %}", "\n".join(endpoints_list))

    # If the index base URL has been configured, also list it
    index_base_url_html = ""
    if CONFIG.index_base_url:
        index_base_url_html = f"""<h3>Index base URL:</h3>
<p><a href={CONFIG.index_base_url}>{CONFIG.index_base_url}</a></p>
"""
    html = html.replace("{% INDEX_BASE_URL %}", index_base_url_html)

    return HTMLResponse(html)
Beispiel #15
0
async def get_search(
    request: Request,
    response: Response,
    search_params: SearchQueryParams = Depends(),
    entry_params: EntryListingQueryParams = Depends(),
) -> Union[QueriesResponseSingle, EntryResponseMany, ErrorResponse,
           RedirectResponse]:
    """`GET /search`

    Coordinate a new OPTIMADE query in multiple databases through a gateway:

    1. Create a [`Search`][optimade_gateway.models.search.Search] `POST` data - calling
        `POST /search`.
    1. Wait [`search_params.timeout`][optimade_gateway.queries.params.SearchQueryParams]
        seconds before returning the query, if it has not finished before.
    1. Return query - similar to `GET /queries/{query_id}`.

    This endpoint works similarly to `GET /queries/{query_id}`, where one passes the query
    parameters directly in the URL, instead of first POSTing a query and then going to its
    URL. Hence, a
    [`QueryResponseSingle`][optimade_gateway.models.responses.QueriesResponseSingle] is
    the standard response model for this endpoint.

    If the timeout time is reached and the query has not yet finished, the user is
    redirected to the specific URL for the query.

    If the `as_optimade` query parameter is `True`, the response will be parseable as a
    standard OPTIMADE entry listing endpoint like, e.g., `/structures`.
    For more information see the
    [OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/master/optimade.rst#entry-listing-endpoints).

    """
    try:
        search = Search(
            query_parameters=OptimadeQueryParameters(
                **{
                    field: getattr(entry_params, field)
                    for field in OptimadeQueryParameters.__fields__
                    if getattr(entry_params, field)
                }),
            optimade_urls=search_params.optimade_urls,
            endpoint=search_params.endpoint,
            database_ids=search_params.database_ids,
        )
    except ValidationError as exc:
        raise BadRequest(detail=(
            "A Search object could not be created from the given URL query "
            f"parameters. Error(s): {exc.errors}")) from exc

    queries_response = await post_search(request, search=search)

    if not queries_response.data:
        LOGGER.error("QueryResource not found in POST /search response:\n%s",
                     queries_response)
        raise RuntimeError(
            "Expected the response from POST /search to return a QueryResource, it did "
            "not")

    once = True
    start_time = time()
    while (  # pylint: disable=too-many-nested-blocks
            time() < (start_time + search_params.timeout) or once):
        # Make sure to run this at least once (e.g., if timeout=0)
        once = False

        collection = await collection_factory(CONFIG.queries_collection)

        query: QueryResource = await collection.get_one(
            **{"filter": {
                "id": queries_response.data.id
            }})

        if query.attributes.state == QueryState.FINISHED:
            if query.attributes.response and query.attributes.response.errors:
                for error in query.attributes.response.errors:
                    if error.status:
                        for part in error.status.split(" "):
                            try:
                                response.status_code = int(part)
                                break
                            except ValueError:
                                pass
                        if response.status_code and response.status_code >= 300:
                            break
                else:
                    response.status_code = 500

            if search_params.as_optimade:
                return await query.response_as_optimade(url=request.url)

            return QueriesResponseSingle(
                links=ToplevelLinks(next=None),
                data=query,
                meta=meta_values(
                    url=request.url,
                    data_returned=1,
                    data_available=await collection.acount(),
                    more_data_available=False,
                ),
            )

        await asyncio.sleep(0.1)

    # The query has not yet succeeded and we're past the timeout time -> Redirect to
    # /queries/<id>
    return RedirectResponse(query.links.self)
Beispiel #16
0
    async def response_as_optimade(
        self,
        url: Optional[Union[urllib.parse.ParseResult, urllib.parse.SplitResult,
                            StarletteURL, str]] = None,
    ) -> Union[EntryResponseMany, ErrorResponse]:
        """Return `attributes.response` as a valid OPTIMADE entry listing response.

        Note, this method disregards the state of the query and will simply return the
        query results as they currently are (if there are any at all).

        Parameters:
            url: Optionally, update the `meta.query.representation` value with this.

        Returns:
            A valid OPTIMADE entry-listing response according to the
            [OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/master/optimade.rst#entry-listing-endpoints)
            or an error response, if errors were returned or occurred during the query.

        """
        from optimade.server.routers.utils import (  # pylint: disable=import-outside-toplevel
            meta_values, )

        async def _update_id(
                entry_: Union[EntryResource,
                              Dict[str, Any]], database_provider_: str
        ) -> Union[EntryResource, Dict[str, Any]]:
            """Internal utility function to prepend the entries' `id` with
            `provider/database/`.

            Parameters:
                entry_: The entry as a model or a dictionary.
                database_provider_: `provider/database` string.

            Returns:
                The entry with an updated `id` value.

            """
            if isinstance(entry_, dict):
                _entry = deepcopy(entry_)
                _entry["id"] = f"{database_provider_}/{entry_['id']}"
            else:
                _entry = entry_.copy(deep=True)
                _entry.id = f"{database_provider_}/{entry_.id}"  # type: ignore[union-attr]
            return _entry

        if not self.attributes.response:
            # The query has not yet been initiated
            return ErrorResponse(
                errors=[{
                    "detail":
                    ("Can not return as a valid OPTIMADE response as the query has"
                     " not yet been initialized."),
                    "id":
                    "OPTIMADE_GATEWAY_QUERY_NOT_INITIALIZED",
                }],
                meta=meta_values(
                    url=url or f"/queries/{self.id}?",
                    data_returned=0,
                    data_available=0,
                    more_data_available=False,
                ),
            )

        meta_ = self.attributes.response.meta
        if url:
            meta_ = meta_.dict(exclude_unset=True)
            for repeated_key in (
                    "query",
                    "api_version",
                    "time_stamp",
                    "provider",
                    "implementation",
            ):
                meta_.pop(repeated_key, None)
            meta_ = meta_values(url=url, **meta_)

        # Error response
        if self.attributes.response.errors:
            return ErrorResponse(
                errors=self.attributes.response.errors,
                meta=meta_,
            )

        # Data response
        results = []
        for database_provider, entries in self.attributes.response.data.items(
        ):
            results.extend([
                await _update_id(entry, database_provider) for entry in entries
            ])

        return self.attributes.endpoint.get_response_model()(
            data=results,
            meta=meta_,
            links=self.attributes.response.links,
        )
async def perform_query(
    url: "URL",
    query: "QueryResource",
) -> "Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse]":
    """Perform OPTIMADE query with gateway.

    Parameters:
        url: Original request URL.
        query: The query to be performed.

    Returns:
        This function returns the final response; a
        [`GatewayQueryResponse`][optimade_gateway.models.queries.GatewayQueryResponse].

    """
    await update_query(query, "state", QueryState.STARTED)

    gateway: GatewayResource = await get_valid_resource(
        await collection_factory(CONFIG.gateways_collection),
        query.attributes.gateway_id,
    )

    filter_queries = await prepare_query_filter(
        database_ids=[_.id for _ in gateway.attributes.databases],
        filter_query=query.attributes.query_parameters.filter,
    )

    url = url.replace(path=f"{url.path.rstrip('/')}/{query.id}")
    await update_query(
        query,
        "response",
        GatewayQueryResponse(
            data={},
            links=ToplevelLinks(next=None),
            meta=meta_values(
                url=url,
                data_available=0,
                data_returned=0,
                more_data_available=False,
            ),
        ),
        operator=None,
        **{"$set": {
            "state": QueryState.IN_PROGRESS
        }},
    )

    loop = asyncio.get_running_loop()
    with ThreadPoolExecutor(
            max_workers=min(32, (os.cpu_count() or 0) +
                            4, len(gateway.attributes.databases))) as executor:
        # Run OPTIMADE DB queries in a thread pool, i.e., not using the main OS thread,
        # where the asyncio event loop is running.
        query_tasks = []
        for database in gateway.attributes.databases:
            query_params = await get_query_params(
                query_parameters=query.attributes.query_parameters,
                database_id=database.id,
                filter_mapping=filter_queries,
            )
            query_tasks.append(
                loop.run_in_executor(
                    executor=executor,
                    func=functools.partial(
                        db_find,
                        database=database,
                        endpoint=query.attributes.endpoint.value,
                        response_model=query.attributes.endpoint.
                        get_response_model(),
                        query_params=query_params,
                    ),
                ))

        for query_task in query_tasks:
            (db_response, db_id) = await query_task

            await process_db_response(
                response=db_response,
                database_id=db_id,
                query=query,
                gateway=gateway,
            )

    # Pagination
    #
    # if isinstance(results, list) and get_resource_attribute(
    #     query,
    #     "attributes.response.meta.more_data_available",
    #     False,
    #     disambiguate=False,  # Extremely minor speed-up
    # ):
    #     # Deduce the `next` link from the current request
    #     query_string = urllib.parse.parse_qs(url.query)
    #     query_string["page_offset"] = [
    #         int(query_string.get("page_offset", [0])[0])  # type: ignore[list-item]
    #         + len(results[: query.attributes.query_parameters.page_limit])
    #     ]
    #     urlencoded = urllib.parse.urlencode(query_string, doseq=True)
    #     base_url = get_base_url(url)

    #     links = ToplevelLinks(next=f"{base_url}{url.path}?{urlencoded}")

    #     await update_query(query, "response.links", links)

    await update_query(query, "state", QueryState.FINISHED)
    return query.attributes.response
Beispiel #18
0
def general_exception(
    request: Request,
    exc: Exception,
    status_code: int = 500,  # A status_code in `exc` will take precedence
    errors: List[OptimadeError] = None,
) -> JSONAPIResponse:
    """Handle an exception

    Parameters:
        request: The HTTP request resulting in the exception being raised.
        exc: The exception being raised.
        status_code: The returned HTTP status code for the error response.
        errors: List of error resources as defined in
            [the OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#json-response-schema-common-fields).

    Returns:
        A JSON HTTP response based on [`ErrorResponse`][optimade.models.responses.ErrorResponse].

    """
    debug_info = {}
    if CONFIG.debug:
        tb = "".join(
            traceback.format_exception(type(exc),
                                       value=exc,
                                       tb=exc.__traceback__))
        LOGGER.error("Traceback:\n%s", tb)
        debug_info[f"_{CONFIG.provider.prefix}_traceback"] = tb

    try:
        http_response_code = int(exc.status_code)
    except AttributeError:
        http_response_code = int(status_code)

    try:
        title = str(exc.title)
    except AttributeError:
        title = str(exc.__class__.__name__)

    try:
        detail = str(exc.detail)
    except AttributeError:
        detail = str(exc)

    if errors is None:
        errors = [
            OptimadeError(detail=detail,
                          status=http_response_code,
                          title=title)
        ]

    response = ErrorResponse(
        meta=meta_values(
            url=request.url,
            data_returned=0,
            data_available=0,
            more_data_available=False,
            **debug_info,
        ),
        errors=errors,
    )

    return JSONAPIResponse(
        status_code=http_response_code,
        content=jsonable_encoder(response, exclude_unset=True),
    )