def uri_is_valid(cls, v):
if isinstance(v, AnyUrl):
return v
if settings.MONGO_HOST in ("localhost", "127.0.0.1"):
try:
return AnyUrl.build(
scheme=settings.MONGO_SCHEME,
host=settings.MONGO_HOST,
port=settings.MONGO_PORT,
user=settings.MONGO_USER,
password=settings.MONGO_PASSWORD,
path=f"/{settings.MONGO_DB}",
query="retryWrites=true&w=majority",
)
except Exception as e:
raise AttributeError(v) from e
try:
return AnyUrl.build(
scheme=settings.MONGO_SCHEME,
host=settings.MONGO_HOST,
user=settings.MONGO_USER,
password=settings.MONGO_PASSWORD,
path=f"/{settings.MONGO_DB}",
query="retryWrites=true&w=majority",
)
except Exception as exc:
logger.error(exc)
raise AttributeError(v) from exc
async def test_post_gateways(client: (
'Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
), ):
"""Test POST /gateways"""
from bson.objectid import ObjectId
from optimade.models import LinksResource
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.config import CONFIG
from optimade_gateway.models.responses import GatewaysResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
data = {
"databases": [{
"id": "test_post_gateways",
"type": "links",
"attributes": {
"name": "PyTest test_post_gateways",
"description": "This is a valid test database",
"base_url": "https://example.org/test",
"homepage": "https://example.org",
"link_type": "child",
},
}]
}
response = await client("/gateways", method="post", json=data)
assert response.status_code == 200, f"Request failed: {response.json()}"
url = response.url
response = GatewaysResponseSingle(**response.json())
assert response
assert getattr(response.meta,
f"_{CONFIG.provider.prefix}_created"), response.meta.dict()
datum = response.data
assert datum, response
for response_db, test_db in zip(datum.attributes.databases,
data["databases"]):
assert (
response_db.dict() == LinksResource(**test_db).dict()
), f"Response: {response_db!r}\n\nTest data: {LinksResource(**test_db)!r}"
assert datum.links.dict() == {
"self":
AnyUrl(
url=
f"{'/'.join(str(url).split('/')[:-1])}{BASE_URL_PREFIXES['major']}/gateways/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
mongo_filter = {"_id": ObjectId(datum.id)}
assert await MONGO_DB["gateways"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["gateways"].find_one(mongo_filter)
assert db_datum["databases"] == data["databases"]
def test_any_url_parts():
url = validate_url('http://example.org')
assert str(url) == 'http://example.org'
assert repr(url) == "AnyUrl('http://example.org', scheme='http', host='example.org', tld='org', host_type='domain')"
assert url.scheme == 'http'
assert url.host == 'example.org'
assert url.tld == 'org'
assert url.host_type == 'domain'
assert url.port is None
assert url == AnyUrl('http://example.org', scheme='https', host='example.org')
def data_must_be_url_or_base64(cls, v, values):
if values['type'] == 'url':
return AnyUrl.validate(v)
elif values['type'] == 'b64':
match = re.match(
"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$",
v)
if match is not None:
return match.group(0)
else:
raise ValueError()
def _assemble_db_connection(cls, v: Optional[str],
values: Dict[str, str]) -> str:
if isinstance(v, str):
return v
return AnyUrl.build(
scheme="postgres",
user=values.get("POSTGRES_USER"),
password=values.get("POSTGRES_PASSWORD"),
host=values.get("POSTGRES_SERVER") or "localhost",
port=values.get("POSTGRES_PORT"),
path=f"/{values.get('POSTGRES_DB')}",
)
def assemble_celery_dsn(cls, v: Optional[str], values: Dict[str,
Any]) -> Any:
if isinstance(v, str):
return v
return AnyUrl.build(
scheme="amqp",
user=values.get("CELERY_USER"),
password=values.get("CELERY_PASSWORD"),
host=values.get("CELERY_SERVER"),
path=f"/{values.get('CELERY_VHOST')}",
)
def test_any_url_obj():
class Model(BaseModel):
v: AnyUrl
url = Model(v='http://example.org').v
assert str(url) == 'http://example.org'
assert repr(url) == ("<AnyUrl('http://example.org' scheme='http' host='example.org' tld='org' host_type='domain')>")
assert url.scheme == 'http'
assert url.host == 'example.org'
assert url.tld == 'org'
assert url.host_type == 'domain'
assert url.port is None
assert url == AnyUrl('http://example.org', scheme='https', host='example.org')
url2 = Model(v='http://*****:*****@example.org:1234/the/path/?query=here#fragment=is;this=bit').v
assert str(url2) == 'http://*****:*****@example.org:1234/the/path/?query=here#fragment=is;this=bit'
assert repr(url2) == (
"<AnyUrl('http://*****:*****@example.org:1234/the/path/?query=here#fragment=is;this=bit' "
"scheme='http' user='******' host='example.org' tld='org' host_type='domain' port='1234' "
"path='/the/path/' query='query=here' fragment='fragment=is;this=bit')>"
)
assert url2.scheme == 'http'
assert url2.user == 'user:password'
assert url2.host == 'example.org'
assert url.host_type == 'domain'
assert url2.port == '1234'
assert url2.path == '/the/path/'
assert url2.query == 'query=here'
assert url2.fragment == 'fragment=is;this=bit'
url3 = Model(v='ftp://123.45.67.8:8329/').v
assert url3.scheme == 'ftp'
assert url3.host == '123.45.67.8'
assert url3.host_type == 'ipv4'
assert url3.port == '8329'
url4 = Model(v='wss://[2001:db8::ff00:42]:8329').v
assert url4.scheme == 'wss'
assert url4.host == '[2001:db8::ff00:42]'
assert url4.host_type == 'ipv6'
assert url4.port == '8329'
url5 = Model(v='https://£££.org').v
assert url5.host == 'xn--9aaa.org'
assert url5.host_type == 'int_domain'
assert str(url5) == 'https://xn--9aaa.org'
url = Model(v='http://example.co.uk').v
assert str(url) == 'http://example.co.uk'
assert url.scheme == 'http'
assert url.host == 'example.co.uk'
assert url.tld == 'uk' # wrong but no better solution
assert url.host_type == 'domain'
async def test_post_databases(client: (
'Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
), ):
"""Test POST /databases"""
from bson.objectid import ObjectId
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.config import CONFIG
from optimade_gateway.models.responses import DatabasesResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
data = {
"name": "PyTest test_post_databases",
"base_url": "https://example.org/test",
}
response = await client("/databases", method="post", json=data)
assert response.status_code == 200, f"Request failed: {response.json()}"
url = response.url
response = DatabasesResponseSingle(**response.json())
assert response
assert getattr(response.meta,
f"_{CONFIG.provider.prefix}_created"), response.meta.dict()
datum = response.data
assert datum, response
for field in data:
assert (
getattr(response.data.attributes, field) == data[field]
), f"Response: {response.data.attributes.dict()!r}\n\nTest data: {data!r}"
assert datum.links.dict() == {
"self":
AnyUrl(
url=
f"{'/'.join(str(url).split('/')[:-1])}{BASE_URL_PREFIXES['major']}/databases/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
mongo_filter = {"_id": ObjectId(datum.id)}
assert await MONGO_DB["databases"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["databases"].find_one(mongo_filter)
for field in data:
assert db_datum[field] == data[field]
async def test_post_gateways_database_ids(client: (
'Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
), ):
"""Test POST /gateways with `database_ids` specified"""
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.config import CONFIG
from optimade_gateway.models.responses import GatewaysResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
# Databases for gateway "twodbs"
data = {"database_ids": ["mcloud/2dstructures", "mcloud/optimade-sample"]}
response = await client("/gateways", method="post", json=data)
assert response.status_code == 200, f"Request failed: {response.json()}"
url = response.url
response = GatewaysResponseSingle(**response.json())
assert response
assert not getattr(
response.meta,
f"_{CONFIG.provider.prefix}_created"), response.meta.dict()
datum = response.data
assert datum, response
assert datum.id == "twodbs"
for database in datum.attributes.databases:
assert database.id in data["database_ids"]
assert datum.links.dict() == {
"self":
AnyUrl(
url=
f"{'/'.join(str(url).split('/')[:-1])}{BASE_URL_PREFIXES['major']}/gateways/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
mongo_filter = {"id": datum.id}
assert await MONGO_DB["gateways"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["gateways"].find_one(mongo_filter)
for db in db_datum["databases"]:
assert db["id"] in data["database_ids"]
def map_back(cls, doc: dict) -> dict:
from optimade.server.routers.utils import BASE_URL_PREFIXES
if "_id" in doc:
_id = str(doc.pop("_id"))
if "id" not in doc:
doc["id"] = _id
doc["links"] = {
"self": AnyUrl(
url=(
f"{CONFIG.base_url.strip('/')}{BASE_URL_PREFIXES['major']}"
f"/{cls.ENDPOINT}/{doc['id']}"
),
scheme=CONFIG.base_url.split("://", maxsplit=1)[0],
host=CONFIG.base_url.split("://", maxsplit=2)[1].split("/")[0],
)
}
return super().map_back(doc)
def validate( # type: ignore
cls, value: str, field: "ModelField", config: "BaseConfig"
) -> Union["AnyUrl", str]:
""" """
if value.startswith("mailto:"):
schema = value[0:7]
email = value[7:]
realname = parseaddr(email)[0]
name, email = validate_email(email)
if realname:
email = formataddr((name, email))
return schema + email
elif value.startswith("mllp:") or value.startswith("llp:"):
# xxx: find validation
return value
elif value in FHIR_PRIMITIVES:
# Extensions may contain a valueUrl for a primitive FHIR type
return value
return AnyUrl.validate(value, field, config)
def build_dsn(
driver: str,
host: str,
port: int,
db: Optional[str] = None,
user: Optional[str] = None,
password: Optional[str] = None,
**params: Any,
):
if params:
params = urlencode(params)
return AnyUrl.build(
scheme=driver,
host=host,
port=str(port),
path=f"/{db}",
user=user,
password=password,
query=params,
)
class TrafficLightProtocol(BaseModel):
"""
Provides details about the TLP classification of the document.
"""
label: Annotated[
Label,
Field(description='Provides the TLP label of the document.', title='Label of TLP'),
]
url: Annotated[
Optional[AnyUrl],
Field(
description='Provides a URL where to find the textual description of the TLP version which is used in this'
' document. Default is the URL to the definition by FIRST.',
examples=[
'https://www.us-cert.gov/tlp',
'https://www.bsi.bund.de/SharedDocs/Downloads/DE/BSI/Kritis/Merkblatt_TLP.pdf',
],
title='URL of TLP version',
),
] = AnyUrl(url='https://www.first.org/tlp/', host='www.first.org', scheme='https')
class test_parser_dbf(TestCase):
filename = 'S0041283.dbf'
url = AnyUrl(None,
scheme="ftp",
user="******",
password="******",
host="santens.ru",
port="21",
path="/path/to/folder/")
file = UploadFile(filename=filename,
content_type='application/octet-stream',
file=io.BytesIO(open(filename, mode='rb').read()))
@pytest.mark.asyncio
async def test_parser_dbf_source(self):
filename = 'S0041283.dbf'
assert isinstance(await parse_source(**self), exch.Packet)
@pytest.mark.asyncio
async def test_parser_dbf_packet(self):
assert isinstance(await parse_format(request=None, **self),
exch.FormatPacket)
async def test_post_queries(
client: (
'Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
),
mock_gateway_responses: "Callable[[dict], None]",
get_gateway: "Callable[[str], Awaitable[dict]]",
):
"""Test POST /queries"""
import asyncio
from bson.objectid import ObjectId
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.config import CONFIG
from optimade_gateway.models.queries import OptimadeQueryParameters, QueryState
from optimade_gateway.models.responses import QueriesResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
data = {
"gateway_id": "singledb",
"query_parameters": {"filter": 'elements HAS "Cu"', "page_limit": 15},
}
mock_gateway_responses(await get_gateway(data["gateway_id"]))
response = await client("/queries", method="post", json=data)
assert response.status_code == 202, f"Request failed: {response.json()}"
url = response.url
response = QueriesResponseSingle(**response.json())
assert response
assert getattr(
response.meta, f"_{CONFIG.provider.prefix}_created"
), response.meta.dict()
datum = response.data
assert datum, response
assert (
datum.attributes.query_parameters.dict()
== OptimadeQueryParameters(**data["query_parameters"]).dict()
), f"Response: {datum.attributes.query_parameters!r}\n\nTest data: {OptimadeQueryParameters(**data['query_parameters'])!r}"
assert datum.links.dict() == {
"self": AnyUrl(
url=f"{'/'.join(str(url).split('/')[:-1])}{BASE_URL_PREFIXES['major']}/queries/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
assert datum.attributes.state == QueryState.CREATED
assert datum.attributes.response is None
mongo_filter = {"_id": ObjectId(datum.id)}
assert await MONGO_DB["queries"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["queries"].find_one(mongo_filter)
for key in data:
assert db_datum[key] == data[key]
await asyncio.sleep(1) # Ensure mock URL is queried
def test_build_url(kwargs, expected):
assert AnyUrl(None, **kwargs) == expected
'http://info.example.com?fred',
'http://info.example.com/?fred',
'http://xn--mgbh0fb.xn--kgbechtv/',
'http://example.com/blue/red%3Fand+green',
'http://www.example.com/?array%5Bkey%5D=value',
'http://xn--rsum-bpad.example.org/',
'http://123.45.67.8/',
'http://123.45.67.8:8329/',
'http://[2001:db8::ff00:42]:8329',
'http://[2001::1]:8329',
'http://[2001:db8::1]/',
'http://www.example.com:8000/foo',
'http://www.cwi.nl:80/%7Eguido/Python.html',
'https://www.python.org/путь',
'http://андрей@example.com',
AnyUrl('https://example.com', scheme='https', host='example.com'),
'https://exam_ple.com/',
'http://twitter.com/@handle/',
],
)
def test_any_url_success(value):
class Model(BaseModel):
v: AnyUrl
assert Model(v=value).v, value
@pytest.mark.parametrize(
'value,err_type,err_msg,err_ctx',
[
('http:///example.com/', 'value_error.url.host', 'URL host invalid',
async def test_get_single_database(
client:
('Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
),
top_dir: "Path",
):
"""Test GET /databases/{id}"""
import json
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.utils import clean_python_types
from optimade_gateway.models.responses import DatabasesResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
database_id = "mcloud/optimade-sample"
response = await client(f"/databases/{database_id}")
assert response.status_code == 200, f"Request failed: {response.json()}"
url = response.url
response = DatabasesResponseSingle(**response.json())
assert response
datum = response.data
assert datum, response
with open(top_dir / "tests/static/test_databases.json") as handle:
all_test_data = json.load(handle)
for data in all_test_data:
if data["id"] == database_id:
test_data = data
break
else:
pytest.fail(
f"Could not find expected test database in test data. ID: {database_id!r}"
)
for field in test_data:
if field in ("id", "type", "links", "relationships", "meta"):
continue
assert (
await clean_python_types(response.data.attributes.dict()[field]
) == data[field]
), f"Field: {field!r}\n\nResponse: {response.data.attributes.dict()!r}\n\nTest data: {data!r}"
test_links = {
"self":
AnyUrl(
url=
f"{'/'.join(str(url).split('/')[:-3])}{BASE_URL_PREFIXES['major']}/databases/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
assert (datum.links.dict() == test_links
), f"Response: {datum.links.dict()}\n\nTest data: {test_links}"
mongo_filter = {"id": datum.id}
assert await MONGO_DB["databases"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["databases"].find_one(mongo_filter)
for field in test_data:
assert db_datum[field] == data[field]
async def test_post_gateways_create_with_db_ids(client: (
'Callable[[str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], Awaitable[Response]]'
), ):
"""Test POST /gateways with `database_ids`, while creating gateway"""
from optimade.server.routers.utils import BASE_URL_PREFIXES
from pydantic import AnyUrl
from optimade_gateway.common.config import CONFIG
from optimade_gateway.models.responses import GatewaysResponseSingle
from optimade_gateway.mongo.database import MONGO_DB
data = {
"databases": [{
"id": "test_post_gateways",
"type": "links",
"attributes": {
"name": "PyTest test_post_gateways",
"description": "This is a valid test database",
"base_url": "https://example.org/test",
"homepage": "https://example.org",
"link_type": "child",
},
}],
"database_ids": ["mcloud/2dstructures"],
}
response = await client("/gateways", method="post", json=data)
assert response.status_code == 200, f"Request failed: {response.json()}"
url = response.url
response = GatewaysResponseSingle(**response.json())
assert response
assert getattr(response.meta,
f"_{CONFIG.provider.prefix}_created"), response.meta.dict()
datum = response.data
assert datum, response
for database in datum.attributes.databases:
assert database.id in [
data["databases"][0]["id"], data["database_ids"][0]
]
assert datum.links.dict() == {
"self":
AnyUrl(
url=
f"{'/'.join(str(url).split('/')[:-1])}{BASE_URL_PREFIXES['major']}/gateways/{datum.id}",
scheme=url.scheme,
host=url.host,
)
}
mongo_filter = {"id": datum.id}
assert await MONGO_DB["gateways"].count_documents(mongo_filter) == 1
db_datum = await MONGO_DB["gateways"].find_one(mongo_filter)
for db in db_datum["databases"]:
assert db["id"] in [
data["databases"][0]["id"], data["database_ids"][0]
]
