def post_save(cls, sender, document, **kwargs):
# avoid circular import
from mpcontribs.api.notebooks.document import Notebooks
from mpcontribs.api.cards.document import Cards
Notebooks.objects(pk=document.id).delete()
Cards.objects(pk=document.id).delete()
def post_save(cls, sender, document, **kwargs):
admin_email = current_app.config["MAIL_DEFAULT_SENDER"]
admin_topic = current_app.config["MAIL_TOPIC"]
if kwargs.get("created"):
ts = current_app.config["USTS"]
email_project = [document.owner, document.project]
token = ts.dumps(email_project)
scheme = "http" if current_app.config["DEBUG"] else "https"
link = url_for("projects.applications",
token=token,
_scheme=scheme,
_external=True)
subject = f'New project "{document.project}"'
hours = int(current_app.config["USTS_MAX_AGE"] / 3600)
html = render_template("admin_email.html",
doc=document,
link=link,
hours=hours)
send_email(admin_topic, subject, html)
resp = sns_client.create_topic(
Name=f"mpcontribs_{document.project}",
Attributes={"DisplayName": f"MPContribs {document.title}"},
)
sns_client.subscribe(TopicArn=resp["TopicArn"],
Protocol="email",
Endpoint=document.owner)
else:
set_keys = document._delta()[0].keys()
if "is_approved" in set_keys and document.is_approved:
subject = f'Your project "{document.project}" has been approved'
if current_app.config["DEBUG"]:
portal = "http://localhost:" + os.environ["PORTAL_PORT"]
else:
portal = "https://" + os.environ["PORTAL_CNAME"]
html = render_template(
"owner_email.html",
approved=True,
admin_email=admin_email,
host=portal,
)
topic_arn = ":".join(
admin_topic.split(":")[:-1] +
["mpcontribs_" + document.project])
send_email(topic_arn, subject, html)
if set_keys:
# import here to avoid circular
from mpcontribs.api.contributions.document import Contributions
from mpcontribs.api.notebooks.document import Notebooks
from mpcontribs.api.cards.document import Cards
contributions = Contributions.objects.only("pk").filter(
project=document.project)
Notebooks.objects(contribution__in=contributions).delete()
Cards.objects(contribution__in=contributions).delete()
def pre_delete(cls, sender, document, **kwargs):
args = ["notebook"] + list(COMPONENTS.keys())
document.reload(*args)
# remove reference documents
if document.notebook is not None:
from mpcontribs.api.notebooks.document import Notebooks
Notebooks.objects(id=document.notebook.id).delete()
for component in COMPONENTS.keys():
# check if other contributions exist before deletion!
for obj in getattr(document, component):
q = {component: obj.id}
if sender.objects(**q).count() < 2:
obj.delete()
def post_save(cls, sender, document, **kwargs):
set_root_keys = set(k.split(".", 1)[0] for k in document._delta()[0].keys())
cid = document.contribution.id
nbs = Notebooks.objects(pk=cid)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
else:
nbs.delete()
document.update(unset__cif=True)
Contributions.objects(pk=cid).update(unset__structures=True)
def post_save(cls, sender, document, **kwargs):
# avoid circular import
from mpcontribs.api.notebooks.document import Notebooks
from mpcontribs.api.cards.document import Cards
# TODO unset and rebuild columns key in Project for updated (nested) keys only
set_root_keys = set(
k.split(".", 1)[0] for k in document._delta()[0].keys())
nbs = Notebooks.objects(pk=document.id)
cards = Cards.objects(pk=document.id)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
cards.update(set__is_public=document.is_public)
else:
nbs.delete()
cards.delete()
if "data" in set_root_keys:
Projects.objects(pk=document.project.id).update(
unset__columns=True)
def post_save(cls, sender, document, **kwargs):
Notebooks.objects(pk=document.contribution.id).delete()
def build():
with no_dereference(Contributions) as Contribs:
# TODO get a random max_docs slice to avoid collisions in parallel Fargate tasks
# remove dangling and unset missing notebooks
nbs_total, nbs_count = -1, -1
ctrbs_cnt = Contribs.objects._cursor.collection.estimated_document_count(
)
nbs_cnt = Notebooks.objects._cursor.collection.estimated_document_count(
)
if ctrbs_cnt != nbs_cnt:
contribs = Contribs.objects(notebook__exists=True).only("notebook")
nids = [contrib.notebook.id for contrib in contribs]
if len(nids) < nbs_cnt:
nbs = Notebooks.objects(id__nin=nids).only("id")
nbs_total = nbs.count()
max_docs = 2500
nbs[:max_docs].delete()
nbs_count = nbs_total if nbs_total < max_docs else max_docs
else:
missing_nids = set(nids) - set(
Notebooks.objects.distinct("id"))
if missing_nids:
upd_contribs = Contribs.objects(
notebook__in=list(missing_nids))
nupd_total = upd_contribs.count()
nupd = upd_contribs.update(unset__notebook="")
print(
f"unset notebooks for {nupd}/{nupd_total} contributions"
)
# build missing notebooks
max_docs = NotebooksResource.max_limit
cids = request.args.get("cids", "").split(",")[:max_docs]
if cids[0]:
documents = Contribs.objects(id__in=cids)
else:
documents = Contribs.objects(notebook__exists=False)[:max_docs]
total = documents.count()
count = 0
for document in documents:
if document.notebook is not None:
# NOTE document.notebook.delete() doesn't trigger pre_delete signal?
nb = Notebooks.objects.get(id=document.notebook.id)
nb.delete()
cells = [
# define client only once in kernel
# avoids API calls for regex expansion for query parameters
nbf.new_code_cell("\n".join([
"if 'client' not in locals():",
"\tclient = Client(",
'\t\theaders={"X-Authenticated-Groups": "admin"},',
f'\t\thost="{MPCONTRIBS_API_HOST}"',
"\t)",
])),
nbf.new_code_cell(
f'client.get_contribution("{document.id}").pretty()'),
]
if document.tables:
cells.append(nbf.new_markdown_cell("## Tables"))
for table in document.tables:
cells.append(
nbf.new_code_cell("\n".join([
f'df = client.get_table("{table.id}")',
"df.plot(**df.attrs)",
])))
if document.structures:
cells.append(nbf.new_markdown_cell("## Structures"))
for structure in document.structures:
cells.append(
nbf.new_code_cell(
f'client.get_structure("{structure.id}")'))
cid = str(document.id)
outputs = execute_cells(cid, cells)
if not outputs:
raise ValueError(f"notebook generation for {cid} failed!")
for idx, output in outputs.items():
cells[idx]["outputs"] = output
doc = deepcopy(seed_nb)
doc["cells"] += cells[1:] # skip localhost Client
document.notebook = Notebooks(**doc).save()
document.save(signal_kwargs={"skip": True})
count += 1
return f"{count}/{total} notebooks built & {nbs_count}/{nbs_total} notebooks deleted"