def test_get_providers(): """Make sure valid responses are handled as expected.""" try: from optimade.server.data import providers except ImportError: pytest.fail("Cannot import providers from optimade.server.data, " "please initialize the `providers` submodule!") from optimade.server.routers.utils import get_providers, mongo_id_for_database side_effects = [ mocked_providers_list_response, ConnectionError, ] for side_effect in side_effects: with mock.patch("requests.get", side_effect=side_effect): providers_list = [ _ for _ in providers.get("data", []) if _["id"] != "exmpl" ] for provider in providers_list: provider.update({ "_id": { "$oid": mongo_id_for_database(provider["id"], provider["type"]) } }) assert get_providers() == providers_list
def load_entries(endpoint_name: str, endpoint_collection: MongoCollection): print(f"loading test {endpoint_name}...") if endpoint_name != "links": endpoint_collection.collection.insert_many( data.get(endpoint_name, [])) if endpoint_name == "links": print( "adding Materials-Consortia providers to links from optimade.org" ) endpoint_collection.collection.insert_many( bson.json_util.loads(bson.json_util.dumps(get_providers()))) print(f"done inserting test {endpoint_name}...")
def load_entries(endpoint_name: str, endpoint_collection: MongoCollection): LOGGER.debug("Loading test %s...", endpoint_name) endpoint_collection.collection.insert_many(getattr(data, endpoint_name, [])) if endpoint_name == "links": LOGGER.debug( " Adding Materials-Consortia providers to links from optimade.org" ) providers = get_providers() for doc in providers: endpoint_collection.collection.replace_one( filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads(bson.json_util.dumps(doc)), upsert=True, ) LOGGER.debug("Done inserting test %s!", endpoint_name)
def load_entries(endpoint_name: str, endpoint_collection: EntryCollection): LOGGER.debug("Loading test %s...", endpoint_name) endpoint_collection.insert(getattr(data, endpoint_name, [])) if (CONFIG.database_backend.value in ("mongomock", "mongodb") and endpoint_name == "links"): LOGGER.debug( "Adding Materials-Consortia providers to links from optimade.org" ) providers = get_providers(add_mongo_id=True) for doc in providers: endpoint_collection.collection.replace_one( filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads( bson.json_util.dumps(doc)), upsert=True, ) LOGGER.debug("Done inserting test %s!", endpoint_name)
def test_get_providers_warning(caplog, top_dir): """Make sure a warning is logged as a last resort.""" import copy from optimade.server.routers.utils import get_providers provider_list_urls = [ "https://providers.optimade.org/v1/links", "https://raw.githubusercontent.com/Materials-Consortia/providers", "/master/src/links/v1/providers.json", ] providers_cache = False try: from optimade.server import data if getattr(data, "providers", None) is not None: providers_cache = copy.deepcopy(data.providers) caplog.clear() with mock.patch("requests.get", side_effect=ConnectionError): del data.providers # pylint: disable=no-member assert get_providers() == [] warning_message = """Could not retrieve a list of providers! Tried the following resources: {} The list of providers will not be included in the `/links`-endpoint. """.format("".join([f" * {_}\n" for _ in provider_list_urls])) assert warning_message in caplog.messages finally: if providers_cache: from optimade.server import data data.providers = providers_cache # Trying to import providers to make sure it's there again now from optimade.server.data import providers assert providers == providers_cache
def test_get_providers_warning(caplog, top_dir): """Make sure a warning is logged as a last resort.""" import copy from optimade.server.routers.utils import get_providers, PROVIDER_LIST_URLS providers_cache = False try: from optimade.server import data if getattr(data, "providers", None) is not None: providers_cache = copy.deepcopy(data.providers) caplog.clear() with mock.patch("requests.get", side_effect=ConnectionError): del data.providers # pylint: disable=no-member assert get_providers() == [] warning_message = """Could not retrieve a list of providers! Tried the following resources: {} The list of providers will not be included in the `/links`-endpoint. """.format("".join([f" * {_}\n" for _ in PROVIDER_LIST_URLS])) assert warning_message in caplog.messages finally: if providers_cache: from optimade.server import data data.providers = providers_cache # Trying to import providers to make sure it's there again now from optimade.server.data import providers assert providers == providers_cache
with open(CONFIG.index_links_path) as f: data = json.load(f) processed = [] for db in data: db["_id"] = {"$oid": mongo_id_for_database(db["id"], db["type"])} processed.append(db) LOGGER.debug(" Inserting index links into collection from %s...", CONFIG.index_links_path) links_coll.collection.insert_many( bson.json_util.loads(bson.json_util.dumps(processed))) LOGGER.debug( " Adding Materials-Consortia providers to links from optimade.org...") providers = get_providers() for doc in providers: links_coll.collection.replace_one( filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads(bson.json_util.dumps(doc)), upsert=True, ) LOGGER.debug("Done inserting index links!") # Add various middleware app.add_middleware(CORSMiddleware, allow_origins=["*"]) app.add_middleware(EnsureQueryParamIntegrity) app.add_middleware(CheckWronglyVersionedBaseUrls) app.add_middleware(HandleApiHint) app.add_middleware(AddWarnings)
processed = [] for db in data: db["_id"] = {"$oid": mongo_id_for_database(db["id"], db["type"])} processed.append(db) LOGGER.debug("Inserting index links into collection from %s...", CONFIG.index_links_path) links_coll.insert(bson.json_util.loads(bson.json_util.dumps(processed))) if CONFIG.database_backend.value in ("mongodb", "mongomock"): LOGGER.debug( "Adding Materials-Consortia providers to links from optimade.org..." ) providers = get_providers(add_mongo_id=True) for doc in providers: links_coll.collection.replace_one( filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads(bson.json_util.dumps(doc)), upsert=True, ) LOGGER.debug("Done inserting index links!") else: LOGGER.warning( "Not inserting test data for index meta-database for backend %s", CONFIG.database_backend.value, )
def make_pages(): """Create the rendered pages (index, and per-provider detail page).""" # Create output folder, copy static files if os.path.exists(OUT_FOLDER): shutil.rmtree(OUT_FOLDER) os.mkdir(OUT_FOLDER) os.mkdir(os.path.join(OUT_FOLDER, HTML_FOLDER)) shutil.copytree(STATIC_FOLDER_ABS, os.path.join(OUT_FOLDER, STATIC_FOLDER)) env = Environment( loader=PackageLoader("mod"), autoescape=select_autoescape(["html", "xml"]), ) env.filters["extract_url"] = extract_url providers = get_providers() if not providers: raise RuntimeError("Unable to retrieve providers list.") last_check_time = datetime.datetime.utcnow().strftime( "%A %B %d, %Y at %H:%M UTC") all_provider_data = [] # Create HTML view for each provider for provider in providers: provider_data = { "id": provider["id"], "last_check_time": last_check_time } print(" - {}".format(provider["id"])) subpage = os.path.join(HTML_FOLDER, get_html_provider_fname(provider["id"])) subpage_abspath = os.path.join(OUT_FOLDER, subpage) provider_data["subpage"] = subpage provider_data["attributes"] = provider base_url = provider.get("base_url") if base_url is None: provider_data["index_metadb"] = { "state": "unspecified", "tooltip_lines": [ "The provider did not specify a base URL for the Index Meta-Database" ], "color": "dark-gray", } else: provider_data["index_metadb"] = {} try: index_metadb_data = get_index_metadb_data(base_url) provider_data["index_metadb"] = index_metadb_data except Exception: provider_data["index_metadb"] = { "state": "unknown", "tooltip_lines": "Generic error while fetching the data:\n{}".format( traceback.format_exc()).splitlines(), "color": "orange", } provider_data[ "title"] = f'{provider_data["attributes"].get("name")}: OPTIMADE provider dashboard' # Write provider html provider_html = env.get_template("singlepage.html").render( **provider_data) with open(subpage_abspath, "w") as f: f.write(provider_html) all_provider_data.append(provider_data) print(" - Page {} generated.".format(subpage)) all_data = {} all_data["providers"] = sorted(all_provider_data, key=lambda provider: provider["id"]) all_data["globalsummary"] = { "with_base_url": len(all_data["providers"]), "num_sub_databases": sum([ provider_data.get("index_metadb", {}).get("num_non_null_subdbs", 0) for provider_data in all_provider_data ]), } # Write main overview index print("[main index]") rendered = env.get_template("main_index.html").render(**all_data) outfile = os.path.join(OUT_FOLDER, "index.html") with open(outfile, "w") as f: f.write(rendered) print(" - index.html generated")