def test_load_client():
kwargs = {}
spec = load_client(**kwargs).swagger_spec
assert spec.http_client.headers == {}
assert spec.origin_url == f"http://{DEFAULT_HOST}/apispec.json"
assert spec.spec_dict["host"] == DEFAULT_HOST
assert spec.spec_dict["schemes"] == ["http"]
assert spec.user_defined_formats["email"] == email_format
kwargs = {
"apikey": "1234",
"headers": {
"a": "b"
},
"host": "api.example.com"
}
spec = load_client(**kwargs).swagger_spec
assert spec.http_client.headers == {"x-api-key": "1234"}
assert spec.origin_url == f"https://api.example.com/apispec.json"
assert spec.spec_dict["host"] == "api.example.com"
assert spec.spec_dict["schemes"] == ["https"]
assert spec.user_defined_formats["email"] == email_format
kwargs = {"headers": {"a": "b"}}
spec = load_client(**kwargs).swagger_spec
assert spec.http_client.headers == {"a": "b"}
assert spec.origin_url == f"http://{DEFAULT_HOST}/apispec.json"
assert spec.spec_dict["host"] == DEFAULT_HOST
assert spec.spec_dict["schemes"] == ["http"]
assert spec.user_defined_formats["email"] == email_format
def index(request):
ctx = get_context(request)
cname = os.environ["PORTAL_CNAME"]
template_dir = get_app_template_dirs("templates/notebooks")[0]
htmls = os.path.join(template_dir, cname, "*.html")
ctx["notebooks"] = [
p.split("/" + cname + "/")[-1].replace(".html", "")
for p in glob(htmls)
]
ctx["PORTAL_CNAME"] = cname
ctx["landing_pages"] = []
mask = ["project", "title", "authors", "is_public", "description", "urls"]
client = load_client(
headers=get_consumer(request)) # sets/returns global variable
entries = client.projects.get_entries(_fields=mask).result()["data"]
for entry in entries:
authors = entry["authors"].strip().split(",", 1)
if len(authors) > 1:
authors[1] = authors[1].strip()
entry["authors"] = authors
entry["description"] = entry["description"].split(".", 1)[0] + "."
ctx["landing_pages"].append(
entry
) # visibility governed by is_public flag and X-Consumer-Groups header
return render(request, "home.html", ctx.flatten())
def csv(request, project):
from pandas import DataFrame
from pandas.io.json._normalize import nested_to_record
client = load_client(
headers=get_consumer(request)) # sets/returns global variable
contribs = client.contributions.get_entries(
project=project, _fields=["identifier", "id", "formula",
"data"]).result()["data"] # first 20 only
data = []
for contrib in contribs:
data.append({})
for k, v in nested_to_record(contrib, sep=".").items():
if v is not None and not k.endswith(".value") and not k.endswith(
".unit"):
vs = v.split(" ")
if k.endswith(".display") and len(vs) > 1:
key = k.replace("data.", "").replace(".display",
"") + f" [{vs[1]}]"
data[-1][key] = vs[0]
else:
data[-1][k] = v
df = DataFrame(data)
response = HttpResponse(df.to_csv(), content_type="text/csv")
response["Content-Disposition"] = "attachment; filename={}.csv".format(
project)
return response
def contribution(request, cid):
ctx = RequestContext(request)
client = load_client()
nb = client.notebooks.get_entry(cid=cid).response().result
ctx['nb'], ctx['js'] = export_notebook(nb, cid)
return render(request, "mpcontribs_explorer_contribution.html",
ctx.flatten())
def index(request):
ctx = RequestContext(request)
ctx['landing_pages'] = []
mask = ['project', 'title', 'authors']
client = load_client()
provenances = client.projects.get_entries(mask=mask).response().result
for provenance in provenances:
entry = {'project': provenance['project']}
try:
entry['url'] = reverse(provenance['project'] + ':index')
except:
entry['contribs'] = client.contributions.get_entries(
projects=[provenance['project']],
mask=['identifier'],
per_page=2).response().result
entry['title'] = provenance['title']
authors = provenance['authors'].split(',', 1)
style = 'font-size: 13px;'
if '/' in authors[0]:
style += 'margin-top: -7px;'
prov_display = f'<span class="pull-right" style="{style}">{authors[0].replace("/", "/<br>")}'
if len(authors) > 1:
prov_display += '''<button class="btn btn-sm btn-link" data-html="true"
data-toggle="tooltip" data-placement="bottom" data-container="body"
title="{}" style="padding: 0px 0px 2px 5px;">et al.</a>'''.format(
authors[1].strip().replace(', ', '<br/>'))
prov_display += '</span>'
entry['provenance'] = prov_display
ctx['landing_pages'].append(
entry) # consider everything in DB released
return render(request, "mpcontribs_portal_index.html", ctx.flatten())
def index(request):
ctx = RequestContext(request)
try:
client = load_client()
resp = client.projects.get_entries(mask=['project']).response()
ctx['projects'] = [r['project'] for r in resp.result]
except Exception as ex:
ctx['alert'] = f'{ex}'
return render(request, "mpcontribs_explorer_index.html", ctx.flatten())
def cif(request, sid):
client = load_client(
headers=get_consumer(request)) # sets/returns global variable
cif = client.structures.get_entry(pk=sid, _fields=["cif"]).result()["cif"]
if cif:
response = HttpResponse(cif, content_type="text/plain")
response["Content-Disposition"] = "attachment; filename={}.cif".format(
sid)
return response
return HttpResponse(status=404)
def download_json(request, cid):
client = load_client()
contrib = client.contributions.get_entry(cid=cid).response().result
if contrib:
jcontrib = json.dumps(contrib)
response = HttpResponse(jcontrib, content_type='application/json')
response[
'Content-Disposition'] = 'attachment; filename={}.json'.format(cid)
return response
return HttpResponse(status=404)
def download_json(request, cid):
client = load_client(
headers=get_consumer(request)) # sets/returns global variable
contrib = client.contributions.get_entry(pk=cid, fields=["_all"]).result()
if contrib:
jcontrib = json.dumps(contrib)
response = HttpResponse(jcontrib, content_type="application/json")
response[
"Content-Disposition"] = "attachment; filename={}.json".format(cid)
return response
return HttpResponse(status=404)
def get_context(project, columns=None):
ctx = {'project': project}
client = load_client()
prov = client.projects.get_entry(project=project).response().result
for k in ['id', 'project', 'other']:
prov.pop(k)
ctx['title'] = prov.pop('title')
ctx['provenance'] = RecursiveDict(prov).render()
data = client.projects.get_table(project=project,
columns=columns,
per_page=3).response().result
if data['items']:
columns = list(data['items'][0].keys())
table = Table(data['items'], columns=columns)
ctx['table'] = table.render(project=project)
return ctx
def landingpage(request):
ctx = get_context(request)
try:
project = request.path.replace("/", "")
client = load_client(headers=get_consumer(request))
prov = client.projects.get_entry(pk=project, _fields=["_all"]).result()
ctx["project"] = project
long_title = prov.get("long_title")
ctx["title"] = long_title if long_title else prov["title"]
ctx["descriptions"] = prov["description"].strip().split(".", 1)
authors = prov["authors"].strip().split(",", 1)
ctx["authors"] = {"main": authors[0].strip()}
if len(authors) > 1:
ctx["authors"]["etal"] = authors[1].strip()
ctx["urls"] = prov["urls"]
other = prov.get("other", "")
if other:
ctx["other"] = json.dumps(HierarchicalData(other))
if prov["columns"]:
ctx["columns"] = ["identifier", "id", "formula"] + list(
prov["columns"].keys())
ctx["search_columns"] = ["identifier", "formula"] + [
col for col in prov["columns"].keys()
if not col.endswith("]") and not col.startswith("structures")
]
ctx["ranges"] = json.dumps(prov["columns"])
# TODO contribs key is only used in dilute_diffusion and should go through the table
# from mpcontribs.io.core.utils import get_short_object_id
# ctx['contribs'] = []
# for contrib in client.contributions.get_entries(
# project=project, _fields=['id', 'identifier', 'data.formula']
# ).result()['data']:
# formula = contrib.get('data', {}).get('formula')
# if formula:
# contrib['formula'] = formula
# contrib['short_cid'] = get_short_object_id(contrib['id'])
# ctx['contribs'].append(contrib)
except Exception as ex:
ctx["alert"] = str(ex)
templates = [f"{project}_index.html", "landingpage.html"]
template = select_template(templates)
return HttpResponse(template.render(ctx.flatten(), request))
def contribution(request, cid):
ctx = get_context(request)
client = load_client(
headers=get_consumer(request)) # sets/returns global variable
contrib = client.contributions.get_entry(pk=cid,
_fields=["id",
"identifier"]).result()
ctx["identifier"], ctx["cid"] = contrib["identifier"], contrib["id"]
nb = client.notebooks.get_entry(
pk=cid).result() # generate notebook with cells
ctx["ncells"] = len(nb["cells"])
if not nb["cells"][-1]["outputs"]:
try:
nb = client.notebooks.get_entry(pk=cid).result(
timeout=1) # trigger cell execution
except HTTPTimeoutError as e:
dots = '<span class="loader__dot">.</span><span class="loader__dot">.</span><span class="loader__dot">.</span>'
ctx["alert"] = f"Detail page is building in the background {dots}"
ctx["nb"], ctx["js"] = export_notebook(nb, cid)
return render(request, "contribution.html", ctx.flatten())
# -
# # MPContribs
# +
import yaml
path_i = os.path.join(os.environ["PROJ_irox"], "config", "config.yml")
with open(path_i) as file:
config_dict = yaml.load(file, Loader=yaml.FullLoader)
api_key = config_dict["mpcontrib"]["api_key"]
# +
project = 'active_learned_irox_polymorphs'
client = load_client(api_key)
# print(dir(client))
# print(dir(client.projects))
# -
# # Deleting all data to start over
# +
total_count = 1234
while total_count != 0:
deleted = client.contributions.delete_entries(project=project).result()
total_count = deleted["total_count"]
num_deleted = deleted["count"]
def cif(request, sid):
client = load_client()
cif = client.structures.get_cif(sid=sid).response().result
if cif:
return HttpResponse(cif, content_type='text/plain')
return HttpResponse(status=404)
from matbench_config import BENCHMARK_DEBUG_01, LOG_KVRH, \
LOG_GVRH, \
DIELECTRIC, \
JDFT2D, \
MP_GAP, \
MP_IS_METAL, \
MP_E_FORM, \
PEROVSKITES, \
GLASS, \
EXPT_IS_METAL, \
EXPT_GAP, \
STEELS, \
PHONONS
api_key = os.environ["MPCONTRIBS_API_KEY"]
client = load_client(api_key, host='ml-api.materialsproject.cloud')
mpr = MPRester()
def chunks(data, SIZE=500):
it = iter(data)
for i in range(0, len(data), SIZE):
if isinstance(data, dict):
yield {k: data[k] for k in islice(it, SIZE)}
else:
yield data[i:i + SIZE]
def pretty_column_map(columns_old):
colmap = {}
for col in columns_old: