class Tables(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private table"
)
name = StringField(required=True, help_text="table name")
columns = ListField(StringField(), required=True, help_text="column names")
data = ListField(ListField(StringField()), required=True, help_text="table rows")
config = DictField(help_text="graph config")
meta = {
"collection": "tables",
"indexes": [
"contribution",
"is_public",
"name",
"columns",
{"fields": ("contribution", "name"), "unique": True},
],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
Notebooks.objects(pk=document.contribution.id).delete()
class Structures(DynamicDocument):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this structure belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private structure"
)
name = StringField(required=True, help_text="structure name")
label = StringField(required=True, help_text="structure label")
lattice = DictField(required=True, help_text="lattice")
sites = ListField(DictField(), required=True, help_text="sites")
charge = FloatField(null=True, help_text="charge")
klass = StringField(help_text="@class")
module = StringField(help_text="@module")
meta = {
"collection": "structures",
"indexes": ["contribution", "is_public", "name", "label"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
set_root_keys = set(k.split(".", 1)[0] for k in document._delta()[0].keys())
cid = document.contribution.id
nbs = Notebooks.objects(pk=cid)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
else:
nbs.delete()
document.update(unset__cif=True)
Contributions.objects(pk=cid).update(unset__structures=True)
class Tables(DynamicDocument):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private table"
)
name = StringField(required=True, help_text="table name")
label = StringField(required=True, help_text="table label")
columns = ListField(StringField(), required=True, help_text="column names")
data = ListField(ListField(StringField()), required=True, help_text="table rows")
config = DictField(help_text="graph config")
meta = {
"collection": "tables",
"indexes": ["contribution", "is_public", "name", "label", "columns"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
set_root_keys = set(k.split(".", 1)[0] for k in document._delta()[0].keys())
cid = document.contribution.id
nbs = Notebooks.objects(pk=cid)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
else:
nbs.delete()
if "data" in set_root_keys:
document.update(unset__total_data_rows=True)
Contributions.objects(pk=cid).update(unset__tables=True)
class Structures(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this structure belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private structure"
)
name = StringField(required=True, help_text="structure name")
label = StringField(required=True, help_text="structure label")
lattice = DictField(required=True, help_text="lattice")
sites = ListField(DictField(), required=True, help_text="sites")
charge = FloatField(null=True, help_text="charge")
klass = StringField(help_text="@class")
module = StringField(help_text="@module")
meta = {
"collection": "structures",
"indexes": ["contribution", "is_public", "label"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
Notebooks.objects(pk=document.contribution.id).delete()
class Review(Document):
# Review documents are saved in the collection 'reviews'
meta = {'collection': 'reviews'}
# The review references the user who created the review and the restaurant the review was created for.
# Both references include a rule that deletes the review should either of the references be deleted themselves.
user = LazyReferenceField('Patron',
required=True,
reverse_delete_rule=CASCADE)
restaurant = LazyReferenceField('Restaurant',
required=True,
reverse_delete_rule=CASCADE)
# General content values of the review
rating = IntField(required=True, validation=_validate_rating)
date = DateTimeField(required=True)
content = StringField(required=True, max_length=1500)
# Images contains a list of urls used to access images from the s3 bucket
images = ListField(StringField())
class Contributions(DynamicDocument):
project = LazyReferenceField(Projects,
required=True,
passthrough=True,
reverse_delete_rule=CASCADE)
identifier = StringField(required=True,
help_text="material/composition identifier")
formula = StringField(help_text="formula (set dynamically)")
is_public = BooleanField(required=True,
default=False,
help_text="public/private contribution")
data = DictField(
help_text="free-form data to be shown in Contribution Card")
last_modified = DateTimeField(required=True,
default=datetime.utcnow,
help_text="time of last modification")
meta = {
"collection":
"contributions",
"indexes":
["project", "identifier", "formula", "is_public", "last_modified"],
}
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
document.data = validate_data(document.data,
sender=sender,
project=document.project)
if hasattr(document, "formula"):
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier,
document.identifier)
document.last_modified = datetime.utcnow()
@classmethod
def post_save(cls, sender, document, **kwargs):
# avoid circular import
from mpcontribs.api.notebooks.document import Notebooks
from mpcontribs.api.cards.document import Cards
# TODO unset and rebuild columns key in Project for updated (nested) keys only
set_root_keys = set(
k.split(".", 1)[0] for k in document._delta()[0].keys())
nbs = Notebooks.objects(pk=document.id)
cards = Cards.objects(pk=document.id)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
cards.update(set__is_public=document.is_public)
else:
nbs.delete()
cards.delete()
if "data" in set_root_keys:
Projects.objects(pk=document.project.id).update(
unset__columns=True)
class Editor(Document):
"""
An Editor of a publication.
"""
meta = {'collection': 'test_editor'}
id = StringField(primary_key=True)
first_name = StringField(required=True, help_text="Editor's first name.", db_field='fname')
last_name = StringField(required=True, help_text="Editor's last name.")
metadata = MapField(field=StringField(), help_text="Arbitrary metadata.")
company = LazyReferenceField(Publisher)
class Post(Document):
thread = LazyReferenceField(Thread, required=True)
content = StringField(required=True)
author = LazyReferenceField(User)
date_created = DateTimeField()
date_updated = DateTimeField()
deleted = BooleanField(default=False)
replyto = LazyReferenceField(User)
# Need 39 characters to store an IPV6 address
# ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
# @see https://en.wikipedia.org/wiki/IPv6_address
ip = StringField(max_length=39)
# Automatically set the date_created field to now when the Post is first created.
# Automatically set the date_updated field to now every time the Post is saved.
def save(self, *args, **kwargs):
if not self.date_created:
self.date_created = datetime.datetime.now()
self.date_updated = datetime.datetime.now()
return super(Post, self).save(*args, **kwargs)
class Restaurant(Document):
# Restaurant documents are saved in the collection 'restaurants'
meta = {'collection': 'restaurants'}
# General required values for a restaurant document
restaurantName = StringField(required=True)
restaurantTags = ListField(StringField(max_length=50), default=[])
description = StringField(required=True, max_length=2500)
dateOpen = DateTimeField(required=True)
# Referenced values for a restaurant document
# If the owner of the restaurant is deleted, then the restaurant is also deleted. The
# restaurant being deleted deletes all associated reviews, whose are then pulled from their
# associated users.
ownerid = LazyReferenceField('Owner',
required=True,
reverse_delete_rule=CASCADE)
reviews = ListField(LazyReferenceField('Review'), default=list)
# Location values for a restaurant document
address = StringField(required=True)
address2 = StringField()
city = StringField(required=True)
zipcode = StringField(required=True)
state = StringField(required=True)
location = GeoPointField(required=True)
# Details about the restaurant, such as hours open and other miscellaneous details
hours = EmbeddedDocumentField(Hours, required=True, default=DEFAULT_HOURS)
details = EmbeddedDocumentField(Details)
# Website link, list of menus, and list of images
# Menu and images contain a list of urls used to access images via S3 Bucket
website = StringField()
menu = ListField(StringField())
images = ListField(StringField())
# Limelight condition to display specific restaurant discount
limelightCondition = StringField(default="")
class Cards(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
primary_key=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public or private card"
)
html = StringField(required=True, default="", help_text="embeddable html code")
meta = {"collection": "cards", "indexes": ["is_public"]}
class Notebooks(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
primary_key=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(required=True,
default=False,
help_text="public or private notebook")
nbformat = IntField(default=5, help_text="nbformat version")
nbformat_minor = IntField(default=0, help_text="nbformat minor version")
metadata = DictField(Metadata(), help_text="notebook metadata")
cells = ListField(Cell(), max_length=30, help_text="cells")
meta = {"collection": "notebooks", "indexes": ["is_public"]}
problem_key = "application/vnd.plotly.v1+json"
escaped_key = problem_key.replace(".", "~dot~")
def transform(self, incoming=True):
if incoming:
old_key = self.problem_key
new_key = self.escaped_key
else:
old_key = self.escaped_key
new_key = self.problem_key
for cell in self.cells:
for output in cell.get("outputs", []):
if old_key in output.get("data", {}):
output["data"][new_key] = output["data"].pop(old_key)
def clean(self):
self.transform()
def restore(self):
self.transform(incoming=False)
class Contributions(Document):
project = LazyReferenceField(Projects,
required=True,
passthrough=True,
reverse_delete_rule=CASCADE)
identifier = StringField(required=True,
help_text="material/composition identifier")
formula = StringField(help_text="formula (set dynamically)")
is_public = BooleanField(required=True,
default=False,
help_text="public/private contribution")
data = DictField(
help_text="free-form data to be shown in Contribution Card")
meta = {
"collection": "contributions",
"indexes": ["project", "identifier", "formula", "is_public"],
}
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
document.data = validate_data(document.data,
sender=sender,
project=document.project)
if hasattr(document, "formula"):
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier,
document.identifier)
@classmethod
def post_save(cls, sender, document, **kwargs):
# avoid circular import
from mpcontribs.api.notebooks.document import Notebooks
from mpcontribs.api.cards.document import Cards
Notebooks.objects(pk=document.id).delete()
Cards.objects(pk=document.id).delete()
class Owner(Client):
# The owner contains a list of their restaurants.
restaurants = ListField(LazyReferenceField('Restaurant'))
class Patron(Client):
# Reviews is a list of references to all reviews created by the user. The patron also contains special
# profiling values.
reviews = ListField(LazyReferenceField('Review'))
about = StringField()
tags = ListField(StringField(max_length=50), default=[])
class Contributions(DynamicDocument):
project = LazyReferenceField("Projects",
required=True,
passthrough=True,
reverse_delete_rule=CASCADE)
identifier = StringField(required=True,
help_text="material/composition identifier")
formula = StringField(
help_text="formula (set dynamically if not provided)")
is_public = BooleanField(required=True,
default=True,
help_text="public/private contribution")
last_modified = DateTimeField(required=True,
default=datetime.utcnow,
help_text="time of last modification")
needs_build = BooleanField(default=True, help_text="needs notebook build?")
data = DictField(
default=dict,
validation=valid_dict,
pullout_key="display",
help_text="simple free-form data",
)
structures = ListField(ReferenceField("Structures", null=True),
default=list,
max_length=10)
tables = ListField(ReferenceField("Tables", null=True),
default=list,
max_length=10)
attachments = ListField(ReferenceField("Attachments", null=True),
default=list,
max_length=10)
notebook = ReferenceField("Notebooks")
meta = {
"collection":
"contributions",
"indexes": [
"project",
"identifier",
"formula",
"is_public",
"last_modified",
"needs_build",
"notebook",
{
"fields": [(r"data.$**", 1)]
},
# can only use wildcardProjection option with wildcard index on all document fields
{
"fields": [(r"$**", 1)],
"wildcardProjection": {
"project": 1
}
},
] + list(COMPONENTS.keys()),
}
@queryset_manager
def objects(doc_cls, queryset):
return queryset.no_dereference().only("project", "identifier",
"formula", "is_public",
"last_modified", "needs_build")
@classmethod
def post_init(cls, sender, document, **kwargs):
# replace existing components with according ObjectIds
for component, fields in COMPONENTS.items():
lst = document._data.get(component)
if lst and lst[0].id is None: # id is None for incoming POST
resource = get_resource(component)
for i, o in enumerate(lst):
digest = get_md5(resource, o, fields)
objs = resource.document.objects(md5=digest)
exclude = list(resource.document._fields.keys())
obj = objs.exclude(*exclude).only("id").first()
if obj:
lst[i] = obj.to_dbref()
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
# set formula field
if hasattr(document, "formula") and not document.formula:
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier,
document.identifier)
# project is LazyReferenceField & load columns due to custom queryset manager
project = document.project.fetch().reload("columns")
columns = {col.path: col for col in project.columns}
# run data through Pint Quantities and save as dicts
def make_quantities(path, key, value):
key = key.strip()
if key in quantity_keys or not isinstance(value,
(str, int, float)):
return key, value
# can't be a quantity if contains 2+ spaces
str_value = str(value).strip()
if str_value.count(" ") > 1:
return key, value
# don't parse if column.unit indicates string type
field = delimiter.join(["data"] + list(path) + [key])
if field in columns:
if columns[field].unit == "NaN":
return key, str_value
# parse as quantity
q = get_quantity(str_value)
if not q:
return key, value
# silently ignore "nan"
if isnan(q.nominal_value):
return False
## try compact representation
#qq = q.value.to_compact()
#q = new_error_units(q, qq)
## reduce dimensionality if possible
#if not q.check(0):
# qq = q.value.to_reduced_units()
# q = new_error_units(q, qq)
# ensure that the same units are used across contributions
if field in columns:
column = columns[field]
if column.unit != str(q.value.units):
try:
qq = q.value.to(column.unit)
q = new_error_units(q, qq)
except DimensionalityError:
raise ValueError(
f"Can't convert [{q.units}] to [{column.unit}] for {field}!"
)
# significant digits
q = truncate_digits(q)
# return new value dict
display = str(q.value) if isnan(q.std_dev) else str(q)
value = {
"display": display,
"value": q.nominal_value,
"error": q.std_dev,
"unit": str(q.units),
}
return key, value
document.data = remap(document.data,
visit=make_quantities,
enter=enter)
document.last_modified = datetime.utcnow()
document.needs_build = True
@classmethod
def pre_delete(cls, sender, document, **kwargs):
args = list(COMPONENTS.keys())
document.reload(*args)
for component in COMPONENTS.keys():
# check if other contributions exist before deletion!
for idx, obj in enumerate(getattr(document, component)):
q = {component: obj.id}
if sender.objects(**q).count() < 2:
obj.delete()
class Contributions(DynamicDocument):
project = LazyReferenceField(
"Projects", required=True, passthrough=True, reverse_delete_rule=CASCADE
)
identifier = StringField(required=True, help_text="material/composition identifier")
formula = StringField(help_text="formula (set dynamically)")
is_public = BooleanField(
required=True, default=False, help_text="public/private contribution"
)
last_modified = DateTimeField(
required=True, default=datetime.utcnow, help_text="time of last modification"
)
data = DictField(
default={}, validation=valid_dict, help_text="simple free-form data"
)
# TODO in flask-mongorest: also get all ReferenceFields when download requested
structures = ListField(ReferenceField("Structures"), default=list, max_length=10)
tables = ListField(ReferenceField("Tables"), default=list, max_length=10)
notebook = ReferenceField("Notebooks")
meta = {
"collection": "contributions",
"indexes": [
"project",
"identifier",
"formula",
"is_public",
"last_modified",
{"fields": [(r"data.$**", 1)]},
],
}
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# set formula field
if hasattr(document, "formula"):
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier, document.identifier)
# project is LazyReferenceField
project = document.project.fetch()
# run data through Pint Quantities and save as dicts
def make_quantities(path, key, value):
if key not in quantity_keys and isinstance(value, (str, int, float)):
str_value = str(value)
words = str_value.split()
try_quantity = bool(len(words) == 2 and is_float(words[0]))
if try_quantity or is_float(value):
field = delimiter.join(["data"] + list(path) + [key])
q = Q_(str_value).to_compact()
if not q.check(0):
q.ito_reduced_units()
# ensure that the same units are used across contributions
try:
column = project.columns.get(path=field)
if column.unit != str(q.units):
q.ito(column.unit)
except DoesNotExist:
pass # column doesn't exist yet (generated in post_save)
except DimensionalityError:
raise ValueError(
f"Can't convert [{q.units}] to [{column.unit}]!"
)
v = Decimal(str(q.magnitude))
vt = v.as_tuple()
if vt.exponent < 0:
dgts = len(vt.digits)
dgts = max_dgts if dgts > max_dgts else dgts
v = f"{v:.{dgts}g}"
if try_quantity:
q = Q_(f"{v} {q.units}")
value = {
"display": str(q),
"value": q.magnitude,
"unit": str(q.units),
}
return key, value
document.data = remap(document.data, visit=make_quantities, enter=enter)
@classmethod
def post_save(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# avoid circular imports
from mpcontribs.api.projects.document import Column
from mpcontribs.api.notebooks.document import Notebooks
# project is LazyReferenceField
project = document.project.fetch()
# set columns field for project
def update_columns(path, key, value):
path = delimiter.join(["data"] + list(path) + [key])
is_quantity = isinstance(value, dict) and quantity_keys == set(value.keys())
is_text = bool(
not is_quantity and isinstance(value, str) and key not in quantity_keys
)
if is_quantity or is_text:
try:
column = project.columns.get(path=path)
if is_quantity:
v = value["value"]
if v > column.max:
column.max = v
project.save().reload("columns")
elif v < column.min:
column.min = v
project.save().reload("columns")
except DoesNotExist:
column = Column(path=path)
if is_quantity:
column.unit = value["unit"]
column.min = column.max = value["value"]
project.modify(push__columns=column)
ncolumns = len(project.columns)
if ncolumns > 50:
raise ValueError("Reached maximum number of columns (50)!")
return True
# run update_columns over document data
remap(document.data, visit=update_columns, enter=enter)
# add/remove columns for other components
for path in ["structures", "tables"]:
try:
project.columns.get(path=path)
except DoesNotExist:
if getattr(document, path):
project.update(push__columns=Column(path=path))
# generate notebook for this contribution
if document.notebook is not None:
document.notebook.delete()
cells = [
nbf.new_code_cell(
"client = Client(\n"
'\theaders={"X-Consumer-Groups": "admin"},\n'
f'\thost="{MPCONTRIBS_API_HOST}"\n'
")"
),
nbf.new_code_cell(f'client.get_contribution("{document.id}").pretty()'),
]
if document.tables:
cells.append(nbf.new_markdown_cell("## Tables"))
for table in document.tables:
cells.append(
nbf.new_code_cell(f'client.get_table("{table.id}").plot()')
)
if document.structures:
cells.append(nbf.new_markdown_cell("## Structures"))
for structure in document.structures:
cells.append(
nbf.new_code_cell(f'client.get_structure("{structure.id}")')
)
ws = connect_kernel()
for cell in cells:
if cell["cell_type"] == "code":
cell["outputs"] = execute(ws, str(document.id), cell["source"])
ws.close()
cells[0] = nbf.new_code_cell("client = Client('<your-api-key-here>')")
doc = deepcopy(seed_nb)
doc["cells"] += cells
document.notebook = Notebooks(**doc).save()
document.last_modified = datetime.utcnow()
document.save(signal_kwargs={"skip": True})
@classmethod
def pre_delete(cls, sender, document, **kwargs):
document.reload("notebook", "structures", "tables")
# remove reference documents
if document.notebook is not None:
document.notebook.delete()
for structure in document.structures:
structure.delete()
for table in document.tables:
table.delete()
@classmethod
def post_delete(cls, sender, document, **kwargs):
# reset columns field for project
project = document.project.fetch()
for column in list(project.columns):
if not isnan(column.min) and not isnan(column.max):
column.min, column.max = get_min_max(sender, column.path)
if isnan(column.min) and isnan(column.max):
# just deleted last contribution with this column
project.update(pull__columns__path=column.path)
else:
# use wildcard index if available -> single field query
field = column.path.replace(delimiter, "__") + "__type"
qs = sender.objects(**{field: "string"}).only(column.path)
if qs.count() < 1 or qs.filter(project__name=project.name).count() < 1:
project.update(pull__columns__path=column.path)
class Contributions(DynamicDocument):
project = LazyReferenceField("Projects",
required=True,
passthrough=True,
reverse_delete_rule=CASCADE)
identifier = StringField(required=True,
help_text="material/composition identifier")
formula = StringField(
help_text="formula (set dynamically if not provided)")
is_public = BooleanField(required=True,
default=False,
help_text="public/private contribution")
last_modified = DateTimeField(required=True,
default=datetime.utcnow,
help_text="time of last modification")
data = DictField(default={},
validation=valid_dict,
help_text="simple free-form data")
structures = ListField(ReferenceField("Structures"),
default=list,
max_length=10)
tables = ListField(ReferenceField("Tables"), default=list, max_length=10)
notebook = ReferenceField("Notebooks")
meta = {
"collection":
"contributions",
"indexes": [
"project",
"identifier",
"formula",
"is_public",
"last_modified",
{
"fields": [(r"data.$**", 1)]
},
],
}
@classmethod
def post_init(cls, sender, document, **kwargs):
# replace existing structures/tables with according ObjectIds
for component in ["structures", "tables"]:
lst = getattr(document, component)
if lst and lst[0].id is None: # id is None for incoming POST
dmodule = import_module(f"mpcontribs.api.{component}.document")
klass = component.capitalize()
Docs = getattr(dmodule, klass)
vmodule = import_module(f"mpcontribs.api.{component}.views")
Resource = getattr(vmodule, f"{klass}Resource")
resource = Resource()
for i, o in enumerate(lst):
d = resource.serialize(
o, fields=["lattice", "sites", "charge"])
s = json.dumps(d, sort_keys=True).encode("utf-8")
digest = md5(s).hexdigest()
obj = Docs.objects(md5=digest).only("id").first()
if obj:
obj.reload()
lst[i] = obj
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# set formula field
if hasattr(document, "formula") and not document.formula:
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier,
document.identifier)
# project is LazyReferenceField
project = document.project.fetch()
# run data through Pint Quantities and save as dicts
def make_quantities(path, key, value):
if key in quantity_keys or not isinstance(value,
(str, int, float)):
return key, value
str_value = str(value)
if str_value.count(" ") > 1:
return key, value
q = get_quantity(str_value)
if not q:
return key, value
# silently ignore "nan"
if isnan(q.nominal_value):
return False
# try compact representation
qq = q.value.to_compact()
q = new_error_units(q, qq)
# reduce dimensionality if possible
if not q.check(0):
qq = q.value.to_reduced_units()
q = new_error_units(q, qq)
# ensure that the same units are used across contributions
field = delimiter.join(["data"] + list(path) + [key])
try:
column = project.columns.get(path=field)
if column.unit != str(q.value.units):
qq = q.value.to(column.unit)
q = new_error_units(q, qq)
except DoesNotExist:
pass # column doesn't exist yet (generated in post_save)
except DimensionalityError:
raise ValueError(
f"Can't convert [{q.units}] to [{column.unit}]!")
v = Decimal(str(q.nominal_value))
vt = v.as_tuple()
if vt.exponent < 0:
dgts = len(vt.digits)
dgts = max_dgts if dgts > max_dgts else dgts
s = f"{v:.{dgts}g}"
if not isnan(q.std_dev):
s += f"+/-{q.std_dev:.{dgts}g}"
s += f" {q.units}"
q = get_quantity(s)
# return new value dict
display = str(q.value) if isnan(q.std_dev) else str(q)
value = {
"display": display,
"value": q.nominal_value,
"error": q.std_dev,
"unit": str(q.units),
}
return key, value
document.data = remap(document.data,
visit=make_quantities,
enter=enter)
@classmethod
def post_save(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# project is LazyReferenceField
project = document.project.fetch()
# set columns field for project
def update_columns(path, key, value):
path = delimiter.join(["data"] + list(path) + [key])
is_quantity = isinstance(value, dict) and quantity_keys.issubset(
value.keys())
is_text = bool(not is_quantity and isinstance(value, str)
and key not in quantity_keys)
if is_quantity or is_text:
project.reload("columns")
try:
column = project.columns.get(path=path)
if is_quantity:
v = value["value"]
if isnan(column.max) or v > column.max:
column.max = v
if isnan(column.min) or v < column.min:
column.min = v
except DoesNotExist:
column = {"path": path}
if is_quantity:
column["unit"] = value["unit"]
column["min"] = column["max"] = value["value"]
project.columns.create(**column)
project.save().reload("columns")
ncolumns = len(project.columns)
if ncolumns > 50:
raise ValueError("Reached maximum number of columns (50)!")
return True
# run update_columns over document data
remap(document.data, visit=update_columns, enter=enter)
# add/remove columns for other components
for path in ["structures", "tables"]:
try:
project.columns.get(path=path)
except DoesNotExist:
if getattr(document, path):
project.columns.create(path=path)
project.save().reload("columns")
# generate notebook for this contribution
if document.notebook is not None:
document.notebook.delete()
cells = [
nbf.new_code_cell("client = Client(\n"
'\theaders={"X-Consumer-Groups": "admin"},\n'
f'\thost="{MPCONTRIBS_API_HOST}"\n'
")"),
nbf.new_code_cell(
f'client.get_contribution("{document.id}").pretty()'),
]
if document.tables:
cells.append(nbf.new_markdown_cell("## Tables"))
for table in document.tables:
cells.append(
nbf.new_code_cell(
f'client.get_table("{table.id}").plot()'))
if document.structures:
cells.append(nbf.new_markdown_cell("## Structures"))
for structure in document.structures:
cells.append(
nbf.new_code_cell(
f'client.get_structure("{structure.id}")'))
loop = asyncio.new_event_loop()
task = loop.create_task(
execute_cells(str(document.id), cells, loop=loop))
outputs = loop.run_until_complete(task)
for task in asyncio.all_tasks(loop=loop):
print(f"Cancelling {task}")
task.cancel()
outputs = loop.run_until_complete(task)
loop.close()
for idx, output in outputs.items():
cells[idx]["outputs"] = output
cells[0] = nbf.new_code_cell("client = Client()")
doc = deepcopy(seed_nb)
doc["cells"] += cells
# avoid circular imports
from mpcontribs.api.notebooks.document import Notebooks
document.notebook = Notebooks(**doc).save()
document.last_modified = datetime.utcnow()
document.save(signal_kwargs={"skip": True})
@classmethod
def pre_delete(cls, sender, document, **kwargs):
document.reload("notebook", "structures", "tables")
# remove reference documents
if document.notebook is not None:
document.notebook.delete()
for component in ["structures", "tables"]:
# check if other contributions exist before deletion!
for obj in getattr(document, component):
q = {component: obj.id}
if sender.objects(**q).count() < 2:
obj.delete()
@classmethod
def post_delete(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# reset columns field for project
project = document.project.fetch()
for column in list(project.columns):
if not isnan(column.min) and not isnan(column.max):
column.min, column.max = get_min_max(sender, column.path)
if isnan(column.min) and isnan(column.max):
# just deleted last contribution with this column
project.update(pull__columns__path=column.path)
else:
# use wildcard index if available -> single field query
field = column.path.replace(delimiter, "__") + "__type"
qs = sender.objects(**{field: "string"}).only(column.path)
if qs.count() < 1 or qs.filter(
project__name=project.name).count() < 1:
project.update(pull__columns__path=column.path)
class Contributions(DynamicDocument):
project = LazyReferenceField(
"Projects", required=True, passthrough=True, reverse_delete_rule=CASCADE
)
identifier = StringField(required=True, help_text="material/composition identifier")
formula = StringField(help_text="formula (set dynamically if not provided)")
is_public = BooleanField(
required=True, default=False, help_text="public/private contribution"
)
last_modified = DateTimeField(
required=True, default=datetime.utcnow, help_text="time of last modification"
)
data = DictField(
default={}, validation=valid_dict, help_text="simple free-form data"
)
structures = ListField(ReferenceField("Structures"), default=list, max_length=10)
tables = ListField(ReferenceField("Tables"), default=list, max_length=10)
notebook = LazyReferenceField("Notebooks", passthrough=True)
meta = {
"collection": "contributions",
"indexes": [
"project",
"identifier",
"formula",
"is_public",
"last_modified",
{"fields": [(r"data.$**", 1)]},
"notebook",
]
+ list(COMPONENTS.keys()),
}
@classmethod
def post_init(cls, sender, document, **kwargs):
# replace existing structures/tables with according ObjectIds
for component, fields in COMPONENTS.items():
lst = getattr(document, component)
if lst and lst[0].id is None: # id is None for incoming POST
dmodule = import_module(f"mpcontribs.api.{component}.document")
klass = component.capitalize()
Docs = getattr(dmodule, klass)
vmodule = import_module(f"mpcontribs.api.{component}.views")
Resource = getattr(vmodule, f"{klass}Resource")
resource = Resource()
for i, o in enumerate(lst):
d = resource.serialize(o, fields=fields)
s = json.dumps(d, sort_keys=True).encode("utf-8")
digest = md5(s).hexdigest()
obj = Docs.objects(md5=digest).only("id").first()
if obj:
obj.reload()
lst[i] = obj
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# set formula field
if hasattr(document, "formula") and not document.formula:
formulae = current_app.config["FORMULAE"]
document.formula = formulae.get(document.identifier, document.identifier)
# project is LazyReferenceField
project = document.project.fetch()
# run data through Pint Quantities and save as dicts
def make_quantities(path, key, value):
if key in quantity_keys or not isinstance(value, (str, int, float)):
return key, value
str_value = str(value)
if str_value.count(" ") > 1:
return key, value
q = get_quantity(str_value)
if not q:
return key, value
# silently ignore "nan"
if isnan(q.nominal_value):
return False
# try compact representation
qq = q.value.to_compact()
q = new_error_units(q, qq)
# reduce dimensionality if possible
if not q.check(0):
qq = q.value.to_reduced_units()
q = new_error_units(q, qq)
# ensure that the same units are used across contributions
field = delimiter.join(["data"] + list(path) + [key])
try:
column = project.columns.get(path=field)
if column.unit != str(q.value.units):
qq = q.value.to(column.unit)
q = new_error_units(q, qq)
except DoesNotExist:
pass # column doesn't exist yet (generated in post_save)
except DimensionalityError:
raise ValueError(f"Can't convert [{q.units}] to [{column.unit}]!")
# significant digits
q = truncate_digits(q)
# return new value dict
display = str(q.value) if isnan(q.std_dev) else str(q)
value = {
"display": display,
"value": q.nominal_value,
"error": q.std_dev,
"unit": str(q.units),
}
return key, value
document.data = remap(document.data, visit=make_quantities, enter=enter)
@classmethod
def post_save(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# project is LazyReferenceField
project = document.project.fetch()
# set columns field for project
def update_columns(path, key, value):
path = delimiter.join(["data"] + list(path) + [key])
is_quantity = isinstance(value, dict) and quantity_keys.issubset(
value.keys()
)
is_text = bool(
not is_quantity and isinstance(value, str) and key not in quantity_keys
)
if is_quantity or is_text:
project.reload("columns")
try:
column = project.columns.get(path=path)
if is_quantity:
v = value["value"]
if isnan(column.max) or v > column.max:
column.max = v
if isnan(column.min) or v < column.min:
column.min = v
except DoesNotExist:
column = {"path": path}
if is_quantity:
column["unit"] = value["unit"]
column["min"] = column["max"] = value["value"]
project.columns.create(**column)
project.save().reload("columns")
ncolumns = len(project.columns)
if ncolumns > 50:
raise ValueError("Reached maximum number of columns (50)!")
return True
# run update_columns over document data
remap(document.data, visit=update_columns, enter=enter)
# add/remove columns for other components
for path in COMPONENTS.keys():
try:
project.columns.get(path=path)
except DoesNotExist:
if getattr(document, path):
project.columns.create(path=path)
project.save().reload("columns")
document.last_modified = datetime.utcnow()
@classmethod
def pre_delete(cls, sender, document, **kwargs):
args = ["notebook"] + list(COMPONENTS.keys())
document.reload(*args)
# remove reference documents
if document.notebook is not None:
from mpcontribs.api.notebooks.document import Notebooks
Notebooks.objects(id=document.notebook.id).delete()
for component in COMPONENTS.keys():
# check if other contributions exist before deletion!
for obj in getattr(document, component):
q = {component: obj.id}
if sender.objects(**q).count() < 2:
obj.delete()
@classmethod
def post_delete(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return
# reset columns field for project
project = document.project.fetch()
for column in list(project.columns):
if not isnan(column.min) and not isnan(column.max):
column.min, column.max = get_min_max(sender, column.path)
if isnan(column.min) and isnan(column.max):
# just deleted last contribution with this column
project.update(pull__columns__path=column.path)
else:
# use wildcard index if available -> single field query
field = column.path.replace(delimiter, "__") + "__type"
qs = sender.objects(**{field: "string"}).only(column.path)
if qs.count() < 1 or qs.filter(project__name=project.name).count() < 1:
project.update(pull__columns__path=column.path)