def test_get_providers():
"""Make sure valid responses are handled as expected."""
try:
from optimade.server.data import providers
except ImportError:
pytest.fail("Cannot import providers from optimade.server.data, "
"please initialize the `providers` submodule!")
from optimade.server.routers.utils import get_providers, mongo_id_for_database
side_effects = [
mocked_providers_list_response,
ConnectionError,
]
for side_effect in side_effects:
with mock.patch("requests.get", side_effect=side_effect):
providers_list = [
_ for _ in providers.get("data", []) if _["id"] != "exmpl"
]
for provider in providers_list:
provider.update({
"_id": {
"$oid":
mongo_id_for_database(provider["id"], provider["type"])
}
})
assert get_providers() == providers_list
def load_entries(endpoint_name: str, endpoint_collection: MongoCollection):
print(f"loading test {endpoint_name}...")
if endpoint_name != "links":
endpoint_collection.collection.insert_many(
data.get(endpoint_name, []))
if endpoint_name == "links":
print(
"adding Materials-Consortia providers to links from optimade.org"
)
endpoint_collection.collection.insert_many(
bson.json_util.loads(bson.json_util.dumps(get_providers())))
print(f"done inserting test {endpoint_name}...")
def load_entries(endpoint_name: str, endpoint_collection: MongoCollection):
LOGGER.debug("Loading test %s...", endpoint_name)
endpoint_collection.collection.insert_many(getattr(data, endpoint_name, []))
if endpoint_name == "links":
LOGGER.debug(
" Adding Materials-Consortia providers to links from optimade.org"
)
providers = get_providers()
for doc in providers:
endpoint_collection.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting test %s!", endpoint_name)
def load_entries(endpoint_name: str, endpoint_collection: EntryCollection):
LOGGER.debug("Loading test %s...", endpoint_name)
endpoint_collection.insert(getattr(data, endpoint_name, []))
if (CONFIG.database_backend.value in ("mongomock", "mongodb")
and endpoint_name == "links"):
LOGGER.debug(
"Adding Materials-Consortia providers to links from optimade.org"
)
providers = get_providers(add_mongo_id=True)
for doc in providers:
endpoint_collection.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(
bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting test %s!", endpoint_name)
def test_get_providers_warning(caplog, top_dir):
"""Make sure a warning is logged as a last resort."""
import copy
from optimade.server.routers.utils import get_providers
provider_list_urls = [
"https://providers.optimade.org/v1/links",
"https://raw.githubusercontent.com/Materials-Consortia/providers",
"/master/src/links/v1/providers.json",
]
providers_cache = False
try:
from optimade.server import data
if getattr(data, "providers", None) is not None:
providers_cache = copy.deepcopy(data.providers)
caplog.clear()
with mock.patch("requests.get", side_effect=ConnectionError):
del data.providers # pylint: disable=no-member
assert get_providers() == []
warning_message = """Could not retrieve a list of providers!
Tried the following resources:
{}
The list of providers will not be included in the `/links`-endpoint.
""".format("".join([f" * {_}\n" for _ in provider_list_urls]))
assert warning_message in caplog.messages
finally:
if providers_cache:
from optimade.server import data
data.providers = providers_cache
# Trying to import providers to make sure it's there again now
from optimade.server.data import providers
assert providers == providers_cache
def test_get_providers_warning(caplog, top_dir):
"""Make sure a warning is logged as a last resort."""
import copy
from optimade.server.routers.utils import get_providers, PROVIDER_LIST_URLS
providers_cache = False
try:
from optimade.server import data
if getattr(data, "providers", None) is not None:
providers_cache = copy.deepcopy(data.providers)
caplog.clear()
with mock.patch("requests.get", side_effect=ConnectionError):
del data.providers # pylint: disable=no-member
assert get_providers() == []
warning_message = """Could not retrieve a list of providers!
Tried the following resources:
{}
The list of providers will not be included in the `/links`-endpoint.
""".format("".join([f" * {_}\n" for _ in PROVIDER_LIST_URLS]))
assert warning_message in caplog.messages
finally:
if providers_cache:
from optimade.server import data
data.providers = providers_cache
# Trying to import providers to make sure it's there again now
from optimade.server.data import providers
assert providers == providers_cache
with open(CONFIG.index_links_path) as f:
data = json.load(f)
processed = []
for db in data:
db["_id"] = {"$oid": mongo_id_for_database(db["id"], db["type"])}
processed.append(db)
LOGGER.debug(" Inserting index links into collection from %s...",
CONFIG.index_links_path)
links_coll.collection.insert_many(
bson.json_util.loads(bson.json_util.dumps(processed)))
LOGGER.debug(
" Adding Materials-Consortia providers to links from optimade.org...")
providers = get_providers()
for doc in providers:
links_coll.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting index links!")
# Add various middleware
app.add_middleware(CORSMiddleware, allow_origins=["*"])
app.add_middleware(EnsureQueryParamIntegrity)
app.add_middleware(CheckWronglyVersionedBaseUrls)
app.add_middleware(HandleApiHint)
app.add_middleware(AddWarnings)
processed = []
for db in data:
db["_id"] = {"$oid": mongo_id_for_database(db["id"], db["type"])}
processed.append(db)
LOGGER.debug("Inserting index links into collection from %s...",
CONFIG.index_links_path)
links_coll.insert(bson.json_util.loads(bson.json_util.dumps(processed)))
if CONFIG.database_backend.value in ("mongodb", "mongomock"):
LOGGER.debug(
"Adding Materials-Consortia providers to links from optimade.org..."
)
providers = get_providers(add_mongo_id=True)
for doc in providers:
links_coll.collection.replace_one(
filter={"_id": ObjectId(doc["_id"]["$oid"])},
replacement=bson.json_util.loads(bson.json_util.dumps(doc)),
upsert=True,
)
LOGGER.debug("Done inserting index links!")
else:
LOGGER.warning(
"Not inserting test data for index meta-database for backend %s",
CONFIG.database_backend.value,
)
def make_pages():
"""Create the rendered pages (index, and per-provider detail page)."""
# Create output folder, copy static files
if os.path.exists(OUT_FOLDER):
shutil.rmtree(OUT_FOLDER)
os.mkdir(OUT_FOLDER)
os.mkdir(os.path.join(OUT_FOLDER, HTML_FOLDER))
shutil.copytree(STATIC_FOLDER_ABS, os.path.join(OUT_FOLDER, STATIC_FOLDER))
env = Environment(
loader=PackageLoader("mod"),
autoescape=select_autoescape(["html", "xml"]),
)
env.filters["extract_url"] = extract_url
providers = get_providers()
if not providers:
raise RuntimeError("Unable to retrieve providers list.")
last_check_time = datetime.datetime.utcnow().strftime(
"%A %B %d, %Y at %H:%M UTC")
all_provider_data = []
# Create HTML view for each provider
for provider in providers:
provider_data = {
"id": provider["id"],
"last_check_time": last_check_time
}
print(" - {}".format(provider["id"]))
subpage = os.path.join(HTML_FOLDER,
get_html_provider_fname(provider["id"]))
subpage_abspath = os.path.join(OUT_FOLDER, subpage)
provider_data["subpage"] = subpage
provider_data["attributes"] = provider
base_url = provider.get("base_url")
if base_url is None:
provider_data["index_metadb"] = {
"state":
"unspecified",
"tooltip_lines": [
"The provider did not specify a base URL for the Index Meta-Database"
],
"color":
"dark-gray",
}
else:
provider_data["index_metadb"] = {}
try:
index_metadb_data = get_index_metadb_data(base_url)
provider_data["index_metadb"] = index_metadb_data
except Exception:
provider_data["index_metadb"] = {
"state":
"unknown",
"tooltip_lines":
"Generic error while fetching the data:\n{}".format(
traceback.format_exc()).splitlines(),
"color":
"orange",
}
provider_data[
"title"] = f'{provider_data["attributes"].get("name")}: OPTIMADE provider dashboard'
# Write provider html
provider_html = env.get_template("singlepage.html").render(
**provider_data)
with open(subpage_abspath, "w") as f:
f.write(provider_html)
all_provider_data.append(provider_data)
print(" - Page {} generated.".format(subpage))
all_data = {}
all_data["providers"] = sorted(all_provider_data,
key=lambda provider: provider["id"])
all_data["globalsummary"] = {
"with_base_url":
len(all_data["providers"]),
"num_sub_databases":
sum([
provider_data.get("index_metadb", {}).get("num_non_null_subdbs", 0)
for provider_data in all_provider_data
]),
}
# Write main overview index
print("[main index]")
rendered = env.get_template("main_index.html").render(**all_data)
outfile = os.path.join(OUT_FOLDER, "index.html")
with open(outfile, "w") as f:
f.write(rendered)
print(" - index.html generated")
