def test_mock(): host = "localhost:10000" with Client(host=host, headers={"a": "b"}) as client: spec = client.swagger_spec assert spec.http_client.headers == { "Content-Type": "application/json", "a": "b" } assert spec.origin_url == f"http://{host}/apispec.json" assert spec.spec_dict["host"] == host assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format host = "contribs-apis:10000" with Client(host=host) as client: spec = client.swagger_spec assert spec.http_client.headers == {"Content-Type": "application/json"} assert spec.origin_url == f"http://{host}/apispec.json" assert spec.spec_dict["host"] == host assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format host = "192.168.0.40:10000" with Client(host=host) as client: spec = client.swagger_spec assert spec.http_client.headers == {"Content-Type": "application/json"} assert spec.origin_url == f"http://{host}/apispec.json" assert spec.spec_dict["host"] == host assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format with pytest.raises(ValueError): with Client(host="not.valid.org") as client: spec = client.swagger_spec
def show_component(request, oid): ckwargs = client_kwargs(request) headers = ckwargs.get("headers", {}) resp = None if headers.get("X-Anonymous-Consumer", False): ctx = get_context(request) msg = f""" Please <a href=\"{ctx['OAUTH_URL']}\">log in</a> to show contribution component. """.strip() return HttpResponse(msg, status=403) client = Client(**ckwargs) try: resp = client.get_structure(oid) except HTTPNotFound: try: resp = client.get_table(oid) except HTTPNotFound: try: resp = client.get_attachment(oid) except HTTPNotFound: return HttpResponse( f"Component with ObjectId {oid} not found.", status=404) if resp is not None: return HttpResponse(resp.info().display()) return HttpResponse(status=404)
def download_contribution(request, cid): ckwargs = client_kwargs(request) headers = ckwargs.get("headers", {}) if headers.get("X-Anonymous-Consumer", False): ctx = get_context(request) msg = f""" Please <a href=\"{ctx['OAUTH_URL']}\">log in</a> to download contribution. """.strip() return HttpResponse(msg, status=403) tmpdir = Path("/tmp") outdir = tmpdir / "download" client = Client(**ckwargs) client.download_contributions(query={"id": cid}, outdir=outdir, include=list(COMPONENTS)) zipfile = Path(make_archive(tmpdir / cid, "zip", outdir)) resp = zipfile.read_bytes() rmtree(outdir) os.remove(zipfile) response = HttpResponse(resp, content_type="application/zip") response["Content-Disposition"] = f"attachment; filename={cid}.zip" return response
def index(request): ctx = get_context(request) cname = os.environ["PORTAL_CNAME"] template_dir = get_app_template_dirs("templates/notebooks")[0] htmls = os.path.join(template_dir, cname, "*.html") ctx["notebooks"] = [ p.split("/" + cname + "/")[-1].replace(".html", "") for p in glob(htmls) ] ctx["PORTAL_CNAME"] = cname ctx["landing_pages"] = [] mask = [ "name", "title", "authors", "is_public", "description", "references" ] client = Client(**client_kwargs(request)) entries = client.projects.get_entries(_fields=mask).result()["data"] for entry in entries: authors = entry["authors"].strip().split(",", 1) if len(authors) > 1: authors[1] = authors[1].strip() entry["authors"] = authors entry["description"] = entry["description"].split(".", 1)[0] + "." # visibility governed by is_public flag and X-Consumer-Groups header ctx["landing_pages"].append(entry) return render(request, "home.html", ctx.flatten())
def download(request): if not request.GET: return HttpResponse(status=404) if "project" not in request.GET: return HttpResponse(status=404) client = Client(**client_kwargs(request)) params = deepcopy(request.GET) fmt = params.pop("format")[0] fields = params.pop("_fields")[0].split(",") kwargs = {"fields": ["id"]} for k, v in params.items(): kwargs[k] = v resp = client.contributions.get_entries(**kwargs).result() if resp["total_count"] > 1000: return HttpResponse(status=404) kwargs = {"_fields": fields, "format": fmt, "short_mime": "gz"} for k, v in params.items(): kwargs[k] = v resp = client.contributions.download_entries(**kwargs).result() filename = f'{kwargs["project"]}.{fmt}.gz' response = HttpResponse(resp, content_type="application/gzip") response["Content-Disposition"] = f"attachment; filename={filename}" return response
def download_contribution(request, cid): client = Client(**client_kwargs(request)) data = client.contributions.download_entries( short_mime="gz", format="json", _fields=["_all"], id=cid ).result() filename = request.path[1:] response = HttpResponse(data, content_type="application/gzip") response["Content-Disposition"] = f"attachment; filename={filename}" return response
def contribution(request, cid): ctx = get_context(request) client = Client(**client_kwargs(request)) contrib = client.contributions.get_entry( pk=cid, _fields=["identifier", "notebook"]).result() nb = client.notebooks.get_entry(pk=contrib["notebook"]["id"], _fields=["_all"]).result() ctx["identifier"], ctx["cid"] = contrib["identifier"], cid ctx["nb"], _ = export_notebook(nb, cid) return render(request, "contribution.html", ctx.flatten())
def test_live(): with Client(apikey="1234") as client: assert client.url == f"https://{DEFAULT_HOST}" spec = client.swagger_spec assert spec.http_client.headers == { "Content-Type": "application/json", "x-api-key": "1234" } assert spec.origin_url == f"https://{DEFAULT_HOST}/apispec.json" assert spec.spec_dict["host"] == DEFAULT_HOST assert spec.spec_dict["schemes"] == ["https"] assert spec.user_defined_formats["email"] == email_format host = "ml-api.materialsproject.org" with Client(host=host) as client: spec = client.swagger_spec assert spec.http_client.headers == {"Content-Type": "application/json"} assert spec.origin_url == f"https://{host}/apispec.json" assert spec.spec_dict["host"] == host assert spec.spec_dict["schemes"] == ["https"] assert spec.user_defined_formats["email"] == email_format
def _reconcile_include(request, project: str, fields: list): ckwargs = client_kwargs(request) avail_components = set() client = Client(**ckwargs) info = client.projects.get_entry(pk=project, _fields=["columns"]).result() for column in info["columns"]: path = column["path"] if not path.startswith("data.") and path in COMPONENTS: avail_components.add(path) return list(set(fields) & avail_components)
def test_Client(): kwargs = {"host": "127.0.0.1"} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {} assert spec.origin_url == "http://127.0.0.1/apispec.json" assert spec.spec_dict["host"] == "127.0.0.1" assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format kwargs = {"apikey": "1234"} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {"x-api-key": "1234"} assert spec.origin_url == f"https://{DEFAULT_HOST}/apispec.json" assert spec.spec_dict["host"] == DEFAULT_HOST assert spec.spec_dict["schemes"] == ["https"] assert spec.user_defined_formats["email"] == email_format kwargs = {"headers": {"a": "b"}, "host": "localhost:5000"} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {"a": "b"} assert spec.origin_url == "http://localhost:5000/apispec.json" assert spec.spec_dict["host"] == "localhost:5000" assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format
def test_Client(): kwargs = {} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {} assert spec.origin_url == f"http://{DEFAULT_HOST}/apispec.json" assert spec.spec_dict["host"] == DEFAULT_HOST assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format kwargs = {"apikey": "1234", "headers": {"a": "b"}, "host": "api.example.com"} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {"x-api-key": "1234"} assert spec.origin_url == "https://api.example.com/apispec.json" assert spec.spec_dict["host"] == "api.example.com" assert spec.spec_dict["schemes"] == ["https"] assert spec.user_defined_formats["email"] == email_format kwargs = {"headers": {"a": "b"}} spec = Client(**kwargs).swagger_spec assert spec.http_client.headers == {"a": "b"} assert spec.origin_url == f"http://{DEFAULT_HOST}/apispec.json" assert spec.spec_dict["host"] == DEFAULT_HOST assert spec.spec_dict["schemes"] == ["http"] assert spec.user_defined_formats["email"] == email_format
def landingpage(request, project): ctx = get_context(request) try: client = Client(**client_kwargs(request)) prov = client.projects.get_entry(pk=project, _fields=["_all"]).result() ctx["name"] = project long_title = prov.get("long_title") ctx["title"] = long_title if long_title else prov["title"] ctx["descriptions"] = prov["description"].strip().split(".", 1) authors = prov["authors"].strip().split(",", 1) ctx["authors"] = {"main": authors[0].strip()} if len(authors) > 1: ctx["authors"]["etal"] = authors[1].strip() ctx["references"] = prov["references"][:5] ctx["more_references"] = prov["references"][5:] other = prov.get("other", "") if other: ctx["other"] = j2h.convert( json=remap(other, visit=visit), table_attributes='class="table is-narrow is-fullwidth has-background-light"', ) if prov["columns"]: ctx["columns"] = ["identifier", "id", "formula"] + [ col["path"] if col["unit"] == "NaN" else f'{col["path"]} [{col["unit"]}]' for col in prov["columns"] ] ctx["search_columns"] = ["identifier", "formula"] + [ col["path"] for col in prov["columns"] if col["unit"] == "NaN" and col["path"] not in ["structures", "tables"] ] ctx["ranges"] = json.dumps( { f'{col["path"]} [{col["unit"]}]': [col["min"], col["max"]] for col in prov["columns"] if col["unit"] != "NaN" } ) except Exception as ex: ctx["alert"] = str(ex) templates = [f"{project}_index.html", "landingpage.html"] template = select_template(templates) return HttpResponse(template.render(ctx.flatten(), request))
def browse(request): ctx = get_context(request) ctx["landing_pages"] = [] mask = [ "name", "title", "authors", "is_public", "description", "references" ] client = Client(**client_kwargs(request)) entries = client.projects.get_entries(_fields=mask).result()["data"] for entry in entries: authors = entry["authors"].strip().split(",", 1) if len(authors) > 1: authors[1] = authors[1].strip() entry["authors"] = authors entry["description"] = entry["description"].split(".", 1)[0] + "." # visibility governed by is_public flag and X-Authenticated-Groups header ctx["landing_pages"].append(entry) return render(request, "browse.html", ctx.flatten())
def download_component(request, oid): ckwargs = client_kwargs(request) headers = ckwargs.get("headers", {}) content = None if headers.get("X-Anonymous-Consumer", False): ctx = get_context(request) msg = f""" Please <a href=\"{ctx['OAUTH_URL']}\">log in</a> to download contribution component. """.strip() return HttpResponse(msg, status=403) client = Client(**ckwargs) try: resp = client.structures.get_entry(pk=oid, _fields=["name", "cif"]).result() name = resp["name"] content = gzip.compress(bytes(resp["cif"], "utf-8")) content_type = "application/gzip" filename = f"{oid}_{name}.cif.gz" except HTTPNotFound: try: resp = client.get_table(oid) content = gzip.compress(bytes(resp.to_csv(), "utf-8")) resp = client.tables.get_entry(pk=oid, _fields=["name"]).result() name = resp["name"] content_type = "application/gzip" filename = f"{oid}_{name}.csv.gz" except HTTPNotFound: try: resp = client.get_attachment(oid) name = resp["name"] content = resp.decode() content_type = resp["mime"] filename = f"{oid}_{name}" except HTTPNotFound: return HttpResponse( f"Component with ObjectId {oid} not found.", status=404) if content: response = HttpResponse(content, content_type=content_type) response["Content-Disposition"] = f"attachment; filename={filename}" return response return HttpResponse(status=404)
def download_contribution(request, cid): client = Client(**client_kwargs(request)) resp = client.contributions.get_entry(pk=cid, _fields=["project"]).result() resp = client.projects.get_entry(pk=resp["project"], _fields=["columns"]).result() fields = ["project", "identifier", "formula", "is_public", "last_modified"] fields += [ column["path"] + ".display" for column in resp["columns"] if column["path"].startswith("data.") ] fields += ["structures", "tables"] resp = client.contributions.download_entries( id=cid, short_mime="gz", format="json", _fields=fields ).result() filename = request.path[1:] response = HttpResponse(resp, content_type="application/gzip") response["Content-Disposition"] = f"attachment; filename={filename}" return response
def contribution(request, cid): ckwargs = client_kwargs(request) headers = ckwargs.get("headers", {}) ctx = get_context(request) if headers.get("X-Anonymous-Consumer", False): ctx["alert"] = f""" Please <a href=\"{ctx['OAUTH_URL']}\">log in</a> to view contribution. """.strip() return render(request, "contribution.html", ctx.flatten()) client = Client(**ckwargs) try: contrib = client.contributions.get_entry( pk=cid, _fields=["identifier", "needs_build", "notebook"]).result() except HTTPNotFound: return HttpResponse(f"Contribution {cid} not found.", status=404) if "notebook" not in contrib or contrib.get("needs_build", True): url = f"{client.url}/notebooks/build" r = requests.get(url, params={"cids": cid, "force": True}) if r.status_code == requests.codes.ok: status = r.json().get("result", {}).get("status") if status != "COMPLETED": ctx["alert"] = f"Notebook build failed with status {status}" return render(request, "contribution.html", ctx.flatten()) contrib = client.contributions.get_entry( pk=cid, _fields=["identifier", "notebook"]).result() else: ctx["alert"] = f"Notebook build failed with status {r.status_code}" return render(request, "contribution.html", ctx.flatten()) nid = contrib["notebook"]["id"] try: nb = client.notebooks.get_entry(pk=nid, _fields=["_all"]).result() except HTTPNotFound: return HttpResponse(f"Notebook {nid} not found.", status=404) ctx["identifier"], ctx["cid"] = contrib["identifier"], cid ctx["nb"], _ = export_notebook(nb, cid) return render(request, "contribution.html", ctx.flatten())
def contribution(request, cid): ctx = get_context(request) client = Client(**client_kwargs(request)) contrib = client.contributions.get_entry( pk=cid, _fields=["identifier", "notebook"]).result() if "notebook" not in contrib: url = f"{client.url}/notebooks/build" r = requests.get(url, params={"cids": cid}) if r.status_code == requests.codes.ok: contrib = client.contributions.get_entry( pk=cid, _fields=["identifier", "notebook"]).result() else: ctx["alert"] = f"Notebook build failed with status {r.status_code}" return render(request, "contribution.html", ctx.flatten()) nid = contrib["notebook"] nb = client.notebooks.get_entry(pk=nid, _fields=["_all"]).result() ctx["identifier"], ctx["cid"] = contrib["identifier"], cid ctx["nb"], _ = export_notebook(nb, cid) return render(request, "contribution.html", ctx.flatten())
def download_component(request, oid): client = Client(**client_kwargs(request)) try: resp = client.structures.get_entry(pk=oid, _fields=["cif"]).result() content = resp["cif"] ext = "cif" except HTTPNotFound: try: resp = client.get_table(oid) content = resp.to_csv() ext = "csv" except HTTPNotFound: return HttpResponse(status=404) if content: content = gzip.compress(bytes(content, "utf-8")) response = HttpResponse(content, content_type="application/gzip") response["Content-Disposition"] = f"attachment; filename={oid}.{ext}.gz" return response return HttpResponse(status=404)
def generate_downloads(names=None): q = {"name__in": names} if names else {} client = Client(host=os.environ["MPCONTRIBS_API_HOST"], headers=HEADERS) projects = client.projects.get_entries( _fields=["name", "stats"], **q ).result().get("data", []) skip = {"columns", "contributions"} print("PROJECTS", len(projects)) for project in projects: name = project["name"] include = [k for k, v in project["stats"].items() if k not in skip and v] for fmt in FORMATS: query = {"project": name, "format": fmt} resp = make_download(client, query, []) print(name, json.loads(resp.content)) if include: for r in range(1, len(include)+1): for combo in combinations(include, r): resp = make_download(client, query, combo) print(name, combo, json.loads(resp.content))
def landingpage(request, project): ckwargs = client_kwargs(request) headers = ckwargs.get("headers", {}) ctx = get_context(request) not_logged_in = headers.get("X-Anonymous-Consumer", False) if not_logged_in: ctx["alert"] = f""" Please <a href=\"{ctx['OAUTH_URL']}\">log in</a> to browse and filter contributions. """.strip() try: client = Client(**ckwargs) prov = client.projects.get_entry(pk=project, _fields=["_all"]).result() except HTTPNotFound: msg = f"Project '{project}' not found or access denied!" if not_logged_in: ctx["alert"] += f" {msg}" else: ctx["alert"] = msg else: ctx["name"] = project ctx["owner"] = prov["owner"].split(":", 1)[-1] long_title = prov.get("long_title") ctx["title"] = long_title if long_title else prov["title"] ctx["descriptions"] = prov["description"].strip().split(".", 1) authors = prov["authors"].strip().split(",", 1) ctx["authors"] = {"main": authors[0].strip()} if len(authors) > 1: ctx["authors"]["etal"] = authors[1].strip() ctx["references"] = prov["references"][:5] ctx["more_references"] = prov["references"][5:] other = prov.get("other", "") if other: ctx["other"] = j2h.convert( json=remap(other, visit=visit), table_attributes= 'class="table is-narrow is-fullwidth has-background-light"', ) if prov["columns"]: ctx["columns"] = ["identifier", "id", "formula"] + [ col["path"] if col["unit"] == "NaN" else f'{col["path"]} [{col["unit"]}]' for col in prov["columns"] ] ctx["search_columns"] = ["identifier", "formula"] + [ col["path"] for col in prov["columns"] if col["unit"] == "NaN" and col["path"] not in COMPONENTS ] ctx["components"] = [ col["path"] for col in prov["columns"] if col["path"] in COMPONENTS ] ctx["ranges"] = json.dumps({ f'{col["path"]} [{col["unit"]}]': [col["min"], col["max"]] for col in prov["columns"] if col["unit"] != "NaN" }) templates = [f"{project}_index.html", "landingpage.html"] template = select_template(templates) return HttpResponse(template.render(ctx.flatten(), request))
def make_download(headers, query, include=None): client = Client(headers=headers) include = include or [] key = _get_download_key(query, include) total_count, total_pages = client.get_totals(query=query, op="download") if total_count < 1: return JsonResponse({"error": "No results for query."}) kwargs = { k: v for k, v in query.items() if k not in {"format", "_sort", "_fields", "_limit", "per_page"} } last_modified = client.contributions.get_entries( _sort="-last_modified", _fields=["last_modified"], _limit=1, **kwargs).result()["data"][0]["last_modified"] json_resp = {"status": "UNDEFINED"} try: s3_client.head_object(Bucket=BUCKET, Key=key, IfModifiedSince=last_modified) json_resp["status"] = "READY" # latest version already generated except ClientError: try: s3_resp = s3_client.head_object(Bucket=BUCKET, Key=key) next_version = int(s3_resp["Metadata"].get("version", 1)) + 1 except ClientError: next_version = 1 # about to generate first version filename = _get_filename(query, include) fmt = query.get("format", "json") redis_key = f"{BUCKET}:{filename}:{fmt}:{next_version}" json_resp["redis_key"] = redis_key status = redis_store.get(redis_key) if status is None: payload = { "redis_key": redis_key, "host": os.environ["MPCONTRIBS_CLIENT_HOST"], "headers": headers, "query": query, "include": include } try: response = lambda_client.invoke( FunctionName="mpcontribs-make-download", InvocationType='Event', Payload=json.dumps(payload)) if response["StatusCode"] == 202: status = "SUBMITTED" json_resp["status"] = status redis_store.set(redis_key, status) else: status = "ERROR" json_resp["status"] = status json_resp["error"] = "Failed to queue download request" redis_store.set(redis_key, status) except Exception as e: status = "ERROR" json_resp["status"] = status json_resp["error"] = str(e) redis_store.set(redis_key, status) else: json_resp["status"] = status return JsonResponse(json_resp)
LOG_GVRH, DIELECTRIC, JDFT2D, MP_GAP, MP_IS_METAL, MP_E_FORM, PEROVSKITES, GLASS, EXPT_IS_METAL, EXPT_GAP, STEELS, PHONONS, ) api_key = os.environ["MPCONTRIBS_API_KEY"] client = Client(api_key, host="ml-api.materialsproject.cloud") mpr = MPRester() def chunks(data, SIZE=500): it = iter(data) for i in range(0, len(data), SIZE): if isinstance(data, dict): yield {k: data[k] for k in islice(it, SIZE)} else: yield data[i:i + SIZE] def pretty_column_map(columns_old): colmap = {} for col in columns_old:
def __init__( self, api_key=DEFAULT_API_KEY, endpoint=DEFAULT_ENDPOINT, notify_db_version=False, include_user_agent=True, monty_decode: bool = True, use_document_model: bool = True, ): """ Args: api_key (str): A String API key for accessing the MaterialsProject REST interface. Please obtain your API key at https://next-gen.materialsproject.org/api. If this is None, the code will check if there is a "MP_API_KEY" setting. If so, it will use that environment variable. This makes easier for heavy users to simply add this environment variable to their setups and MPRester can then be called without any arguments. endpoint (str): Url of endpoint to access the MaterialsProject REST interface. Defaults to the standard Materials Project REST address at "https://api.materialsproject.org", but can be changed to other urls implementing a similar interface. notify_db_version (bool): If True, the current MP database version will be retrieved and logged locally in the ~/.pmgrc.yaml. If the database version changes, you will be notified. The current database version is also printed on instantiation. These local logs are not sent to materialsproject.org and are not associated with your API key, so be aware that a notification may not be presented if you run MPRester from multiple computing environments. include_user_agent (bool): If True, will include a user agent with the HTTP request including information on pymatgen and system version making the API request. This helps MP support pymatgen users, and is similar to what most web browsers send with each page request. Set to False to disable the user agent. monty_decode: Decode the data using monty into python objects use_document_model: If False, skip the creating the document model and return data as a dictionary. This can be simpler to work with but bypasses data validation and will not give auto-complete for available fields. """ if api_key and len(api_key) == 16: raise ValueError( "Please use a new API key from https://next-gen.materialsproject.org/api " "Keys for the new API are 32 characters, whereas keys for the legacy " "API are 16 characters.") self.api_key = api_key self.endpoint = endpoint self.session = BaseRester._create_session( api_key=api_key, include_user_agent=include_user_agent) try: self.contribs = Client(api_key) except Exception as error: self.contribs = None warnings.warn(f"Problem loading MPContribs client: {error}") self._all_resters = [] if notify_db_version: raise NotImplementedError("This has not yet been implemented.") if not self.endpoint.endswith("/"): self.endpoint += "/" for cls in BaseRester.__subclasses__(): rester = cls( api_key=api_key, endpoint=endpoint, include_user_agent=include_user_agent, session=self.session, monty_decode=monty_decode, use_document_model=use_document_model, ) # type: BaseRester self._all_resters.append(rester) setattr( self, cls.suffix.replace("/", "_"), # type: ignore rester, )
def test_Client_Live(): Client()