def insertMany(self, table, uid, eppn, records):
"""Insert several records at once.
Typically used for inserting criteriaEntry en reviewEntry records.
Parameters
----------
table: string
The table in which the record will be inserted.
uid: ObjectId
The user that creates the record, typically the logged in user.
eppn: string
The `eppn` of that same user. This is the unique identifier that comes from
the DARIAH authentication service.
records: iterable of dict
The records (as dicts) to insert.
"""
justNow = now()
newRecords = [
{
N.dateCreated: justNow,
N.creator: uid,
N.modified: [MOD_FMT.format(eppn, justNow)],
**record,
}
for record in records
]
self.mongoCmd(N.insertMany, table, N.insert_many, newRecords)
def insertUser(self, record):
"""Insert a user record, i.e. a record corresponding to a user.
NB: the creator of this record is the system, by name of the
`creatorId` attribute.
Parameters
----------
record: dict
The user information to be stored, as a dictionary.
Returns
-------
None
But note that the new _id and the generated field values are added to the
record.
"""
creatorId = self.creatorId
justNow = now()
record.update(
{
N.dateLastLogin: justNow,
N.statusLastLogin: N.Approved,
N.mayLogin: True,
N.creator: creatorId,
N.dateCreated: justNow,
N.modified: [MOD_FMT.format(CREATOR, justNow)],
}
)
result = self.mongoCmd(N.insertUser, N.user, N.insert_one, record)
self.recollect(N.user)
record[N._id] = result.inserted_id
def updateUser(self, record):
"""Updates user information.
When users log in, or when they are assigned an other status,
some of their attributes will change.
Parameters
----------
record: dict
The new user information as a dict.
"""
if N.isPristine in record:
del record[N.isPristine]
justNow = now()
record.update(
{
N.dateLastLogin: justNow,
N.statusLastLogin: N.Approved,
N.modified: [MOD_FMT.format(CREATOR, justNow)],
}
)
criterion = {N._id: G(record, N._id)}
updates = {k: v for (k, v) in record.items() if k != N._id}
instructions = {M_SET: updates, M_UNSET: {N.isPristine: E}}
self.mongoCmd(N.updateUser, N.user, N.update_one, criterion, instructions)
self.recollect(N.user)
def collectActualItems(self, tables=None):
"""Determines which items are "actual".
Actual items are those types and criteria that are specified in a
package record that is itself actual.
A package record is actual if the current data is between its start
and end days.
!!! caution
If only value table needs to be collected that are not
involved in the concept of "actual", nothing will be done.
Parameters
----------
tables: set of string, optional `None`
"""
if tables is not None and not (tables & ACTUAL_TABLES):
return
justNow = now()
packageActual = {
G(record, N._id)
for record in self.mongoCmd(
N.collectActualItems,
N.package,
N.find,
{N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}},
)
}
for record in self.package.values():
record[N.actual] = G(record, N._id) in packageActual
typeActual = set(
chain.from_iterable(
G(record, N.typeContribution) or []
for (_id, record) in self.package.items()
if _id in packageActual
)
)
for record in self.typeContribution.values():
record[N.actual] = G(record, N._id) in typeActual
criteriaActual = {
_id
for (_id, record) in self.criteria.items()
if G(record, N.package) in packageActual
}
for record in self.criteria.values():
record[N.actual] = G(record, N._id) in criteriaActual
self.typeCriteria = {}
for (_id, record) in self.criteria.items():
if _id in criteriaActual:
for tp in G(record, N.typeContribution) or []:
self.typeCriteria.setdefault(tp, set()).add(_id)
if DEBUG_SYNCH:
serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""")
def getDefaultDate():
today = now()
return (
today.year,
today.month,
today.day,
today.hour,
today.minute,
today.second,
)
def fromStr(self, editVal):
if not editVal:
return None
if editVal == N.now:
return now()
normalParts = self.partition(editVal)
if normalParts is None:
return None
cast = self.rawType
return cast(*normalParts)
def tasks(self, table, kind=None):
"""Present the currently available tasks as buttons on the interface.
!!! hint "easy comments"
We also include a comment `<!-- task~!taskName:eid -->
for the ease of testing.
Parameters
----------
table: string
We must specify the table for which we want to present the
tasks: contrib, assessment, or review.
kind: string {`expert`, `final`}, optional `None`
Only if we want review attributes
Returns
-------
string(html)
"""
uid = self.uid
if not uid or table not in USER_TABLES:
return E
eid = list(self.info(table, N._id, kind=kind))[0]
taskParts = []
allowedTasks = sorted((task, taskInfo)
for (task, taskInfo) in TASKS.items()
if G(taskInfo, N.table) == table)
justNow = now()
for (task, taskInfo) in allowedTasks:
permitted = self.permission(task, kind=kind)
if not permitted:
continue
remaining = type(permitted) is timedelta and permitted
taskUntil = E
if remaining:
remainingRep = datetime.toDisplay(justNow + remaining)
taskUntil = H.span(f""" before {remainingRep}""", cls="datex")
taskMsg = G(taskInfo, N.msg)
taskCls = G(taskInfo, N.cls)
taskPart = (H.a(
[taskMsg, taskUntil],
f"""/api/task/{task}/{eid}""",
cls=f"large task {taskCls}",
) + f"""<!-- task!{task}:{eid} -->""")
taskParts.append(taskPart)
return H.join(taskParts)
def insertItem(self, table, uid, eppn, onlyIfNew, **fields):
"""Inserts a new record in a table, possibly only if it is new.
The record will be filled with the specified fields, but also with
provenance fields.
The provenance fields are the creation date, the creator,
and the start of the trail of modifiers.
Parameters
----------
table: string
The table in which the record will be inserted.
uid: ObjectId
The user that creates the record, typically the logged in user.
onlyIfNew: boolean
If `True`, it will be checked whether a record with the specified fields
already exists. If so, no record will be inserted.
eppn: string
The eppn of that same user. This is the unique identifier that comes from
the DARIAH authentication service.
**fields: dict
The field names and their contents to populate the new record with.
Returns
-------
ObjectId
The id of the newly inserted record, or the id of the first existing
record found, if `onlyIfNew` is true.
"""
if onlyIfNew:
existing = [
G(rec, N._id)
for rec in getattr(self, table, {}).values()
if all(G(rec, k) == v for (k, v) in fields.items())
]
if existing:
return existing[0]
justNow = now()
newRecord = {
N.dateCreated: justNow,
N.creator: uid,
N.modified: [MOD_FMT.format(eppn, justNow)],
**fields,
}
result = self.mongoCmd(N.insertItem, table, N.insert_one, newRecord)
if table in VALUE_TABLES:
self.recollect(table)
return result.inserted_id
def makeUserRoot(database, eppn, only=False):
"""Gives a specific user the role of root.
Parameters
----------
database: string
The name of the database in which the user lives
eppn: string
The `eppn` attribute of the user
only: boolean, optional `False`
If true, all other users with root role will be demoted to
office power.
"""
client = MongoClient()
mongo = client[database]
perms = {
perm: mongo.permissionGroup.find({"rep": perm},
{"_id": True})[0]["_id"]
for perm in {"root", "office"}
}
if only:
mongo.user.update_many(
{"group": perms["root"]},
{"$set": {
"group": perms["office"]
}},
)
mongo.user.update_one(
{"eppn": eppn},
{"$set": {
"group": perms["root"]
}},
)
mongo.collect.update_one({"table": "user"},
{"$set": {
"dateCollected": now()
}},
upsert=True)
def collect(self):
"""Collect the contents of the value tables.
Value tables have content that is needed almost all the time.
All value tables will be completely cached within Db.
!!! note
This is meant to run at start up, before the workers start.
After that, this worker will not execute it again.
See also `recollect`.
!!! warning
We must take other workers into account. They need a signal
to recollect. See `recollect`.
We store the time that this worker has collected each table
in attribute `collected`.
!!! caution
If you change the MongoDb from without, an you forget to
put an appropriate time stamp, the app will not see it untill it
is restarted.
See for example how `root.makeUserRoot` handles this.
!!! warning
This is a complicated app.
Some tables have records that specify whether other records are "actual".
After collecting a value table, the "actual" items will be recomputed.
"""
collected = self.collected
for valueTable in VALUE_TABLES:
self.cacheValueTable(valueTable)
collected[valueTable] = now()
self.collectActualItems()
if DEBUG_SYNCH:
serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""")
def permission(self, task, kind=None):
"""Checks whether a workflow task is permitted.
Note that the tasks are listed per kind of record they apply to:
contrib, assessment, review.
They are typically triggered by big workflow buttons on the interface.
When the request to execute such a task reaches the server, it will
check whether the current user is allowed to execute this task
on the records in question.
!!! hint
See above for explanation of the properties of the tasks.
!!! note
If you try to run a task on a kind of record that it is not
designed for, it will be detected and no permission will be given.
!!! note
Some tasks are designed to set a field to a value.
If that field already has that value, the task will not be permitted.
This already rules out a lot of things and relieves the burden of
prohibiting non-sensical tasks.
It may be that the task is only permitted for some limited time from now on.
Then a timedelta object with the amount of time left is returned.
Parameters
----------
table: string
In order to check permissions, we must specify the kind of record that
the task acts on: contrib, assessment, or review.
task: string
An string consisting of the name of a task.
kind: string {`expert`, `final`}, optional `None`
Only if we want review attributes
Returns
-------
boolean | timedelta
"""
db = self.db
auth = self.auth
uid = self.uid
if task not in TASKS:
return False
taskInfo = TASKS[task]
table = G(taskInfo, N.table)
if uid is None or table not in USER_TABLES:
return False
taskField = (N.selected if table == N.contrib else
N.submitted if table == N.assessment else
N.decision if table == N.review else None)
myKind = self.myKind
(
locked,
done,
frozen,
mayAdd,
stage,
stageDate,
creators,
countryId,
taskValue,
) = self.info(
table,
N.locked,
N.done,
N.frozen,
N.mayAdd,
N.stage,
N.stageDate,
N.creators,
N.country,
taskField,
kind=kind,
)
operator = G(taskInfo, N.operator)
value = G(taskInfo, N.value)
if operator == N.set:
if taskField == N.decision:
value = G(db.decisionInv, value)
(contribId, ) = self.info(N.contrib, N._id)
isOwn = uid in creators
isCoord = countryId and auth.coordinator(countryId=countryId)
isSuper = auth.superuser()
decisionDelay = G(taskInfo, N.delay)
if decisionDelay:
decisionDelay = timedelta(hours=decisionDelay)
justNow = now()
remaining = False
if decisionDelay and stageDate:
remaining = stageDate + decisionDelay - justNow
if remaining <= timedelta(hours=0):
remaining = False
forbidden = frozen or done
if forbidden and not remaining:
return False
if table == N.contrib:
if not isOwn and not isCoord and not isSuper:
return False
if task == N.startAssessment:
return not forbidden and isOwn and mayAdd
if value == taskValue:
return False
if not isCoord:
return False
answer = not frozen or remaining
if task == N.selectContrib:
return stage != N.selectYes and answer
if task == N.deselectContrib:
return stage != N.selectNo and answer
if task == N.unselectContrib:
return stage != N.selectNone and answer
return False
if table == N.assessment:
forbidden = frozen or done
if forbidden:
return False
if task == N.startReview:
return not forbidden and G(mayAdd, myKind)
if value == taskValue:
return False
if uid not in creators:
return False
answer = not locked or remaining
if not answer:
return False
if task == N.submitAssessment:
return stage == N.complete and answer
if task == N.resubmitAssessment:
return stage == N.completeWithdrawn and answer
if task == N.submitRevised:
return stage == N.completeRevised and answer
if task == N.withdrawAssessment:
return (stage in {N.submitted, N.submittedRevised} and stage
not in {N.incompleteWithdrawn, N.completeWithdrawn}
and answer)
return False
if table == N.review:
if frozen:
return False
if done and not remaining:
return False
taskKind = G(taskInfo, N.kind)
if not kind or kind != taskKind or kind != myKind:
return False
answer = remaining or not done or remaining
if not answer:
return False
(aStage, aStageDate) = self.info(N.assessment, N.stage,
N.stageDate)
(finalStage, ) = self.info(table, N.stage, kind=N.final)
(expertStage, expertStageDate) = self.info(table,
N.stage,
N.stageDate,
kind=N.expert)
xExpertStage = N.expertReviewRevoke if expertStage is None else expertStage
xFinalStage = N.finalReviewRevoke if finalStage is None else finalStage
revision = finalStage == N.reviewRevise
zFinalStage = finalStage and not revision
submitted = aStage == N.submitted
submittedRevised = aStage == N.submittedRevised
mayDecideExpert = (submitted and not finalStage
or submittedRevised and revision)
if value == taskValue:
if not revision:
return False
if task in {
N.expertReviewRevise,
N.expertReviewAccept,
N.expertReviewReject,
N.expertReviewRevoke,
} - {xExpertStage}:
return (kind == N.expert and not zFinalStage
and mayDecideExpert and answer)
if task in {
N.finalReviewRevise,
N.finalReviewAccept,
N.finalReviewReject,
N.finalReviewRevoke,
} - {xFinalStage}:
return (kind == N.final and not not expertStage
and (not aStageDate or expertStageDate > aStageDate)
and (((not finalStage and submitted) or
(revision and submittedRevised)) or remaining)
and answer)
return False
return False
def recollect(self, table=None):
"""Collect the contents of the value tables if they have changed.
For each value table it will be checked if they have been
collected (by another worker) after this worker has started and if so,
those tables and those tables only will be recollected.
!!! caution
Although the initial `collect` is done before workers start
(`gunicorn --preload`), individual workers will end up with their
own copy of the value table cache.
So when we need to recollect values for our cache, we must notify
in some way that other workers also have to recollect this table.
### Global recollection
Whenever we recollect a value table, we insert the time of recollection
in a record in the MongoDb.
Somewhere at the start of each request, these records will be checked,
and if needed, recollections will be done before the request processing.
There is a table `collect`, with records having fields `table` and
`dateCollected`. After each (re)collect of a table, the `dateCollected` of
the appropriate record will be set to the current time.
!!! note "recollect()"
A `recollect()` without arguments should be done at the start of each
request.
!!! note "recollect(table)"
A `recollect(table)` should be done whenever this worker has changed
something in that value table.
Parameters
----------
table: string, optional `None`
A recollect() without arguments collects *all* value tables that need
collecting based on the times of change as recorded in the `collect`
table.
A recollect of a single table means that this worker has made a change.
After the recollect, a timestamp will go into the `collect` table,
so that other workers can pick it up.
If table is `True`, all timestamps in the `collect` table will be set
to now, so that each worker will refresh its value cache.
"""
collected = self.collected
if table is None:
affected = set()
for valueTable in VALUE_TABLES:
record = self.mongoCmd(
N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable}
)
lastChangedGlobally = G(record, RECOLLECT_DATE)
lastChangedHere = G(collected, valueTable)
if lastChangedGlobally and (
not lastChangedHere or lastChangedHere < lastChangedGlobally
):
self.cacheValueTable(valueTable)
collected[valueTable] = now()
affected.add(valueTable)
elif table is True:
affected = set()
for valueTable in VALUE_TABLES:
self.cacheValueTable(valueTable)
collected[valueTable] = now()
affected.add(valueTable)
else:
self.cacheValueTable(table)
collected[table] = now()
affected = {table}
if affected:
justNow = now()
for aTable in affected:
self.mongoCmd(
N.collect,
N.collect,
N.update_one,
{RECOLLECT_NAME: aTable},
{M_SET: {RECOLLECT_DATE: justNow}},
upsert=True,
)
self.collectActualItems(tables=affected)
if affected:
if DEBUG_SYNCH:
serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""")
def updateField(
self, table, eid, field, data, actor, modified, nowFields=[],
):
"""Update a single field in a single record.
!!! hint
Whenever a field is updated in a record which has the field `isPristine`,
this field will be deleted from the record.
The rule is that pristine records are the ones that originate from the
legacy data and have not changed since then.
Parameters
----------
table: string
The table which holds the record to be updated.
eid: ObjectId
(Entity) id of the record to be updated.
data: mixed
The new value of for the updated field.
actor: ObjectId
The user that has triggered the update action.
modified: list of string
The current provenance trail of the record, which is a list of
strings of the form "person on date".
Here "person" is not an ID but a consolidated string representing
the name of that person.
The provenance trail will be trimmed in order to prevent excessively long
trails. On each day, only the last action by each person will be recorded.
nowFields: iterable of string, optional `[]`
The names of additional fields in which the current datetime will be stored.
For exampe, if `submitted` is modified, the current datetime will be saved in
`dateSubmitted`.
Returns
-------
dict | boolean
The updated record, if the MongoDb operation was successful, else False
"""
oid = castObjectId(eid)
if oid is None:
return False
justNow = now()
newModified = filterModified((modified or []) + [f"""{actor}{ON}{justNow}"""])
criterion = {N._id: oid}
nowItems = {nowField: justNow for nowField in nowFields}
update = {
field: data,
N.modified: newModified,
**nowItems,
}
delete = {N.isPristine: E}
instructions = {
M_SET: update,
M_UNSET: delete,
}
status = self.mongoCmd(
N.updateField, table, N.update_one, criterion, instructions
)
if not G(status.raw_result, N.ok, default=False):
return False
if table in VALUE_TABLES:
self.recollect(table)
return (
update,
set(delete.keys()),
)
def test_sidebar2(clients):
amounts = {
"All contributions": [1],
"My contributions": [({OWNER}, 1)],
f"{BELGIUM} contributions": [1],
"Contributions to be selected": [({MYCOORD}, 1)],
}
sidebar(clients, amounts)
@pytest.mark.parametrize(
("field", "value"),
(
(TITLE, TITLE1),
(YEAR, str(now().year)),
(COUNTRY, BELGIUM),
(CONTACT_PERSON_NAME, OWNER_NAME),
(CONTACT_PERSON_EMAIL, OWNER_EMAIL),
),
)
def test_fields(clientOwner, field, value):
recordInfo = startInfo["recordInfo"]
contribInfo = recordInfo[CONTRIB]
fields = G(contribInfo, "fields")
assert G(fields, field) == value
def test_makeEditorAll(clients):
valueTables = startInfo["valueTables"]
