def assertCaptions(client, expect):
"""Check whether a response text shows a certain set of captions.
Parameters
----------
client: fixture
expect: set of string
"""
url = "/"
(text, status, msgs) = accessUrl(client, url)
captionsFound = {caption: url for (caption, url) in findCaptions(text)}
for caption in captionsFound:
assert caption in expect
for caption in expect:
assert caption in captionsFound
for (caption, url) in captionsFound.items():
(expNumber, expItem) = expect[caption]
serverprint(f"CAPTION {caption}: {client.user} CLICKS {url}")
(text, status, msgs) = accessUrl(client, url)
if expNumber is None:
expItem in text
else:
(n, item) = findMainN(text)[0]
nX = f"=/={expNumber}" if n != str(expNumber) else E
iX = f"=/={expItem}" if item != expItem else E
if iX or nX:
serverprint(f"CAPTION {caption}: {n}{nX} {item}{iX}")
assert n == str(expNumber)
assert item == expItem
def illegalize(clients, url, **kwargs):
"""Append illegal/long arguments to an url and trigger a 400 response.
Parameters
----------
clients: dict
Mapping from users to client fixtures
kwargs: dict
Additional parameters to illegalize.
The url will be expanded by formatting it with the `kwargs` values.
"""
kwargsx = {k: v + "a" * 200 for (k, v) in kwargs.items()}
base = url.format(**kwargs)
basex = url.format(**kwargsx)
uxs = [
base,
basex,
f"{base}?action=xxx",
f"{base}?xxx=xxx",
f"{base}?action=" + "a" * 200,
f"{base}?" + "a" * 2000,
]
passable = {200, 301, 302, 303}
for (i, ux) in enumerate(uxs):
expectx = {
user: 400 if i > 2 or i == 1 and len(kwargsx) else passable
for user in USERS
if user in clients
}
serverprint(f"LEGAL URL ? ({ux})")
forall(clients, expectx, assertStatus, ux)
def getNames(source, val, doString=True, inner=False):
names = set()
pureNames = set()
good = True
if type(val) is str:
names = {val} if doString and Names.isName(val) else set()
elif type(val) is list:
for v in val:
if type(v) is str and Names.isName(v):
names.add(v)
elif type(v) is dict:
names |= Names.getNames(source, v, doString=False, inner=True)
elif type(val) is dict:
for (k, v) in val.items():
if inner or k != NAMES:
if type(k) is str and Names.isName(k):
names.add(k)
names |= Names.getNames(source, v, doString=False, inner=True)
else:
for val in v:
if type(val) is not str:
serverprint(
f"NAMES in {source}: "
f"WARNING: wrong type {type(val)} for {val}"
)
good = False
else:
pureNames.add(val)
if not good:
serverprint("EXIT because of FATAL ERROR")
sys.exit()
return names if inner else (pureNames, names)
def changedCallFiles(origFile):
if not os.path.exists(origFile):
serverprint(f"WARNING: caller file {origFile} not found. Skipping.")
return
changes = set()
removes = set()
with open(origFile) as of:
text = of.read()
calls = callRe.findall(text)
for (att, base, ext) in calls:
calledPath = f"{base[1:]}{ext}"
if all(not calledPath.startswith(stampDir)
for stampDir in STAMP_DIRS):
continue
if not os.path.exists(calledPath):
serverprint(
f"WARNING: called file {calledPath} not found. Skipping.")
continue
calledGlob = f"{base[1:]}-[0-9][0-9]*{ext}"
slug = int(round(os.path.getmtime(calledPath)))
(calledDir, calledFile) = os.path.split(calledPath)
calledSlugPath = f"{base[1:]}-{slug}{ext}"
for slugPath in glob(calledGlob):
if slugPath != calledSlugPath:
removes.add(slugPath)
changes.add((calledPath, calledSlugPath))
return (text, changes, removes)
def assertFieldValue(source, field, expect):
"""Verify whether a field has a certain expected value.
If we pass expect `None` we want to assert that the field is not present at all.
Parameters
----------
source: dict | (client: fixture, table: string, eid: string)
The dictionary of fields and values of a retrieved response.
If it is a tuple, the dictionary will be retrieved by looking up
the item specified by `table` and `eid`.
field: string
The name of the specific field.
expect:
The expected value for this field.
"""
if type(source) is tuple:
(client, table, eid) = source
info = getItem(client, table, eid)
fields = info["fields"]
else:
fields = source
if expect is None:
assert field not in fields
else:
assert field in fields
value = fields[field]
if value != expect:
serverprint(f"FIELDVALUE {field}={value} (=/={expect})")
assert expect == fields[field]
def collectActualItems(self, tables=None):
"""Determines which items are "actual".
Actual items are those types and criteria that are specified in a
package record that is itself actual.
A package record is actual if the current data is between its start
and end days.
!!! caution
If only value table needs to be collected that are not
involved in the concept of "actual", nothing will be done.
Parameters
----------
tables: set of string, optional `None`
"""
if tables is not None and not (tables & ACTUAL_TABLES):
return
justNow = now()
packageActual = {
G(record, N._id)
for record in self.mongoCmd(
N.collectActualItems,
N.package,
N.find,
{N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}},
)
}
for record in self.package.values():
record[N.actual] = G(record, N._id) in packageActual
typeActual = set(
chain.from_iterable(
G(record, N.typeContribution) or []
for (_id, record) in self.package.items()
if _id in packageActual
)
)
for record in self.typeContribution.values():
record[N.actual] = G(record, N._id) in typeActual
criteriaActual = {
_id
for (_id, record) in self.criteria.items()
if G(record, N.package) in packageActual
}
for record in self.criteria.values():
record[N.actual] = G(record, N._id) in criteriaActual
self.typeCriteria = {}
for (_id, record) in self.criteria.items():
if _id in criteriaActual:
for tp in G(record, N.typeContribution) or []:
self.typeCriteria.setdefault(tp, set()).add(_id)
if DEBUG_SYNCH:
serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""")
def __init__(self, regime, test=False):
"""## Initialization
Pick up the connection to MongoDb.
!!! note
Parameters
----------
regime: {"production", "development"}
See below
test: boolean
See below.
"""
self.regime = regime
"""*string* Whether the app runs in production or in development."""
self.test = test
"""*boolean* Whether to connect to the test database."""
database = G(DATABASE, N.test) if test else G(DATABASE, regime)
self.database = database
mode = f"""regime = {regime} {"test" if test else E}"""
if not self.database:
serverprint(f"""MONGO: no database configured for {mode}""")
sys.exit(1)
self.client = None
"""*object* The MongoDb client."""
self.mongo = None
"""*object* The connection to the MongoDb database.
The connnection exists before the Db singleton is initialized.
"""
self.collected = {}
"""*dict* For each value table, the time that this worker last collected it.
In the database there is a table which holds the last time for each value
table that a worker updated a value in it.
"""
self.collect()
creator = [
G(record, N._id)
for record in self.user.values()
if G(record, N.eppn) == CREATOR
]
if not creator:
serverprint(f"""DATABASE: no creator user found in {database}.user""")
sys.exit(1)
self.creatorId = creator[0]
"""*ObjectId* System user.
def test_login(clients, clientPublic):
for user in sorted(NAMED_USERS) + [E, PUBLIC, f"xxxxxx"]:
isNamed = user in NAMED_USERS
expect = 302 if isNamed else 303
serverprint(f"LOGIN {user}")
assertStatus(clientPublic, f"/login?eppn={user}", expect)
serverprint(f"LOGOUT {user}")
if user in clients:
assertStatus(clients[user], f"/logout", expect)
def computeWorkflow(regime, test):
if not regime:
serverprint("Don't know if this is development or production")
return 1
mode = f"""regime = {regime} {"test" if test else E}"""
serverprint(f"WORKFLOW RESET for {mode}")
DB = Db(regime, test)
WF = Workflow(DB)
WF.initWorkflow(drop=True)
return 0
def identify(self):
user = self.user
cl = self.cl
url = "/logout" if user == "public" else f"/login?eppn={user}"
cl.get(url)
response = cl.get("/whoami")
actualUser = response.get_data(as_text=True)
good = user == actualUser
if not good:
serverprint(f"USER={actualUser} (=/={user})")
assert good
def mongoClose(self):
"""Close connection with MongoDb.
We need this, because before we fork the process to workers,
all MongoDb connections should be closed.
"""
client = self.client
if client:
client.close()
self.client = None
self.mongo = None
serverprint("""MONGO: connection closed""")
def checkBounds(**kwargs):
"""Aggressive check on the arguments passed in an url and/or request.
First the total length of the request is counted.
If it is too much, the request will be aborted.
Each argument in request.args and `kwargs` must have a name that is allowed
and its value should have a length under an appropriate limit,
configured in `web.yaml`. There is always a fallback limit.
!!! caution "Security"
Before processing any request arg, whether from a form or from the url,
use this function to check whether the length is within limits.
If the length is exceeded, fail with a bad request,
without giving any useful feedback.
Because in this case we are dealing with a hacker.
Parameters
----------
kwargs: dict
The key-values that need to be checked.
Raises
------
HTTPException
If the length of any argument is out of bounds,
processing is aborted and a bad request response
is delivered
"""
default = G(LIMITS, N.default, default=100)
maxLen = G(LIMITS, N.request, default=default)
if request.content_length and request.content_length > maxLen:
abort(400)
n = len(request.args)
boundN = G(LIMITS, N.keys, default=default)
if len(kwargs) > boundN:
serverprint(f"""OUT-OF-BOUNDS: {n} > {boundN} KEYS IN {kwargs}""")
abort(400)
for (k, v) in chain.from_iterable((kwargs.items(), request.args.items())):
if k not in LIMITS:
abort(400)
valN = G(LIMITS, k, default=default)
if v is not None and len(v) > valN:
serverprint(f"""OUT-OF-BOUNDS: LENGTH ARG "{k}" > {valN} ({v})""")
abort(400)
def mongoOpen(self):
"""Open connection with MongoDb.
Which database we open, depends on `Db.regime` and `Db.test`.
"""
client = self.client
mongo = self.mongo
database = self.database
if not mongo:
client = MongoClient()
mongo = client[database]
self.client = client
self.mongo = mongo
serverprint(f"""MONGO: new connection to {database}""")
def assertReviewDecisions(clients, reviewId, kinds, decisions, expect):
"""Check whether the reviewers can take certain decisions.
You specify which reviewers take which decisions, and they will
all be carried out in that order.
You specifiy the expected outcomes in a dict or a boolean, telling
whether the taking of the decision is expected to succeed or not.
Parameters
----------
clients: dict
Mapping from users to client fixtures.
reviewId: dict
The review ids for the expert and final review
kinds: list of {expert, final}
At most one of each, the order is important.
decisions: list of {Reject, Revise, Accept, Revoke}
At most one of each, the order is important.
expect: bool | dict
Expected outcomes.
If it is a boolean, that is the expected outcome of all actions by all
reviewers.
Otherwise the dict is keyed by kind of reviewer.
The values are booleans or dicts.
A boolean indicates the expected outcome of all actions for that reviewer.
A dict specifies per action of that reviewer what the outcome is.
"""
for kind in kinds:
rId = G(reviewId, kind)
expectKind = (
True if expect is True else False if expect is False else G(expect, kind)
)
for decision in decisions:
decisionStr = G(G(REVIEW_DECISION, decision), kind)
url = f"/api/task/{decisionStr}/{rId}"
exp = (
True
if expectKind is True
else False
if expectKind is False
else G(expectKind, decision)
)
serverprint(f"REVIEW DECISION {decision} by {kind} expects {exp}")
assertStatus(G(clients, kind), url, exp)
def getCached(self, method, methodName, methodArgs, table, eid,
requireFresh):
"""Helper to wrap caching around a raw Db fetch method.
Only for methods that fetch single records.
Parameters
----------
method: function
The raw `control.db.Db` method.
methodName: string
The name of the raw Db method. Only used to display if cache
debugging is on.
methodNameArgs: iterable
The arguments to pass to the Db method.
table: string
The table from which the record is fetched.
eid: ObjectId
(Entity) ID of the particular record.
requireFresh: boolean, optional `False`
If True, bypass the cache and fetch the item straight from Db and put the
fetched value in the cache.
Returns
-------
mixed
Whatever the underlying fetch method returns or would return.
"""
cache = self.cache
key = eid if type(eid) is str else str(eid)
if not requireFresh:
if table in cache:
if key in cache[table]:
if DEBUG_CACHE:
serverprint(f"""CACHE HIT {methodName}({key})""")
return cache[table][key]
result = method(*methodArgs)
cache.setdefault(table, {})[key] = result
return result
def mongoCmd(self, label, table, command, *args, **kwargs):
"""Wrapper around calls to MongoDb.
All commands fired at the NongoDb go through this wrapper.
It will spit out debug information if mongo debugging is True.
Parameters
----------
label: string
A key to be mentioned in debug messages.
Very convenient to put here the name of the method that calls mongoCmd.
table: string
The table in MongoDB that is targeted by the command.
If the table does not exists, no command will be fired.
command: string
The Mongo command to execute.
The command must be listed in the mongo.yaml config file.
*args: iterable
Additional arguments will be passed straight to the Mongo command.
Returns
-------
mixed
Whatever the the MongoDb returns.
"""
self.mongoOpen()
mongo = self.mongo
method = getattr(mongo[table], command, None) if command in M_COMMANDS else None
warning = """!UNDEFINED""" if method is None else E
if DEBUG_MONGO:
argRep = args[0] if args and args[0] and command in SHOW_ARGS else E
kwargRep = COMMA.join(f"{k}={v}" for (k, v) in kwargs.items())
serverprint(
f"""MONGO<<{label}>>.{table}.{command}{warning}({argRep} {kwargRep})"""
)
if method:
return method(*args, **kwargs)
return None
def showReferences(cls):
reference = cls.reference
serverprint("""\nREFERENCE FIELD DEPENDENCIES""")
for (dep, tables) in sorted(reference.items()):
serverprint(dep)
for (table, fields) in tables.items():
serverprint(f"""\t{table:<20}: {", ".join(fields)}""")
def main():
args = sys.argv[1:]
unstamp = args and args[0] == "un"
texts = {}
mapFile = {}
changes = {}
changed = set()
removes = set()
for callFile in CALL_FILES:
(base, ext) = os.path.splitext(callFile)
origFile = f"{base}Base{ext}"
if unstamp:
label = "RESTORE"
serverprint(f"STAMP: {label} {callFile}")
copyfile(origFile, callFile)
continue
mapFile[origFile] = callFile
(text, theseChanges, theseRemoves) = changedCallFiles(origFile)
texts[origFile] = text
for (fileFrom, fileTo) in theseChanges:
changes.setdefault(origFile, set()).add((fileFrom, fileTo))
removes |= theseRemoves
for fileRem in sorted(removes):
serverprint(f"STAMP: REMOVE {fileRem}")
os.remove(fileRem)
for (origFile, changedFiles) in changes.items():
callText = texts[origFile]
for (fileFrom, fileTo) in changedFiles:
if fileFrom in changed:
continue
if not os.path.exists(fileTo):
serverprint(f"STAMP: COPY {fileFrom} => {fileTo}")
copyfile(fileFrom, fileTo)
callText = callText.replace(fileFrom, fileTo)
changed.add(fileFrom)
callFile = mapFile[origFile]
origCallText = None
if os.path.exists(callFile):
with open(callFile) as cf:
origCallText = cf.read()
if origCallText is None or origCallText != callText:
label = "WRITE" if origCallText is None else "REWRITE"
serverprint(f"STAMP: {label} {callFile}")
with open(callFile, "w") as cf:
cf.write(callText)
def collect(self):
"""Collect the contents of the value tables.
Value tables have content that is needed almost all the time.
All value tables will be completely cached within Db.
!!! note
This is meant to run at start up, before the workers start.
After that, this worker will not execute it again.
See also `recollect`.
!!! warning
We must take other workers into account. They need a signal
to recollect. See `recollect`.
We store the time that this worker has collected each table
in attribute `collected`.
!!! caution
If you change the MongoDb from without, an you forget to
put an appropriate time stamp, the app will not see it untill it
is restarted.
See for example how `root.makeUserRoot` handles this.
!!! warning
This is a complicated app.
Some tables have records that specify whether other records are "actual".
After collecting a value table, the "actual" items will be recomputed.
"""
collected = self.collected
for valueTable in VALUE_TABLES:
self.cacheValueTable(valueTable)
collected[valueTable] = now()
self.collectActualItems()
if DEBUG_SYNCH:
serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""")
def assertStatus(client, url, expect):
"""Get data and see whether that went right or wrong.
Parameters
----------
client: function
url: string(url)
The url to retrieve from the server
expect: boolean | int | set of int
If boolean: Whether it is expected to be successful
If int: status code should be exactly this
If set of int: status code should be contained in this
"""
try:
response = client.get(url)
code = response.status_code
except Exception as e:
serverprint(f"APPLICATION ERROR: {e}")
code = 4000
if type(expect) is set:
good = code in expect
if not good:
serverprint(f"STATUS {url} => {code} (not in {expect})")
assert good
elif type(expect) is int:
good = code == expect
if not good:
serverprint(f"STATUS {url} => {code} (=/= {expect})")
assert good
else:
codes = {200, 302} if expect else {400, 303}
good = code in codes
if not good:
serverprint(f"STATUS {url} => {code} (not in {codes})")
assert good
def main():
regime = sys.argv[1] if len(sys.argv) > 1 else None
test = sys.argv[2] == "test" if len(sys.argv) > 2 else False
if not regime:
serverprint("Don't know if this is development or production")
return 1
database = DATABASE["test"] if test else DATABASE.get(regime, None)
if database is None:
mode = f"""regime = {regime} {"test" if test else E}"""
serverprint(f"""ERROR: No database configured for {mode}\n"""
"""See base.yaml""")
return 1
eppn = ROOT[database]
only = len(sys.argv) > 3 and sys.argv[3] == "--only"
unique = "unique " if only else E
serverprint(f"RESETTING {eppn} as {unique}root in {database}")
makeUserRoot(database, eppn, only=only)
return 0
def getUser(self, eppn, email=None, mayCreate=False):
"""Find a user in the database.
This is called to get extra information for an authenticated user
from the database.
The resulting data will be stored in the `user` attribute of Auth.
!!! caution
Even if the user can be found, the attribute `mayLogin`
might be false, in which case it will be prevented to log in that user.
!!! tip
When assigning reviewers, office users may select people who are not yet
known to the contrib tool by specifying their email address.
When such users log in for the first time, their `eppn` and other
attributes become known, and are merged into a record in the user table.
Parameters
----------
eppn: string
The unique identifier of a user as assigned by the DARIAH identity provider.
email: string, optional `None`
New users may not have an eppn, but might already be present in the
user table by their email.
If so, the email address can be used to look up the user.
mayCreate: boolean, optional `False`
If a user cannot be found, they will be created if this flag is `True`.
This is relevant for situation where a new user has been authenticated
by the identity provider.
Returns
-------
boolean
Whether a user was authenticated and logged in.
The attributes retrieved from the database will be merged into
the `user` attribute.
If no user was logged in, the `user` attribute will be filled with
info that says that the current user is the public and nothing more.
"""
user = self.user
db = self.db
authority = self.authority
authId = self.authId
userFound = [
record
for record in db.user.values()
if (
G(record, N.authority) == authority
and (
(eppn is not None and G(record, N.eppn) == eppn)
or (
eppn is None
and email is not None
and G(record, N.eppn) is None
and G(record, N.email) == email
)
)
)
]
user.clear()
if len(userFound) > 1:
self.clearUser()
if DEBUG_AUTH:
serverprint(f"LOGIN: multiple matches in user DB: {eppn} / {email}")
return False
if len(userFound) == 1:
if not G(userFound[0], N.mayLogin, default=True):
self.clearUser()
if DEBUG_AUTH:
serverprint(f"LOGIN: existing user may not login: {eppn} / {email}")
return False
user.update({N.eppn: eppn, N.authority: authority})
if email:
user[N.email] = email
if len(userFound) == 0:
if mayCreate:
db.insertUser(user)
if DEBUG_AUTH:
serverprint(f"LOGIN: new user: {eppn} / {email}")
else:
if DEBUG_AUTH:
serverprint(f"LOGIN: may not create new user: {eppn} / {email}")
return False
else:
user.update(userFound[0])
if DEBUG_AUTH:
serverprint(f"LOGIN: existing user: {eppn} / {email}")
# new users do not have yet group information
group = user[N.group] if N.group in user else authId
if N.group not in user:
user[N.group] = group
groupRep = G(G(db.permissionGroup, group), N.rep)
result = groupRep != UNAUTH
if DEBUG_AUTH:
if result:
serverprint(f"LOGIN: user authenticated: {eppn} / {email}")
else:
serverprint(f"LOGIN: user not authenticated: {eppn} / {email}")
return result
def recollect(self, table=None):
"""Collect the contents of the value tables if they have changed.
For each value table it will be checked if they have been
collected (by another worker) after this worker has started and if so,
those tables and those tables only will be recollected.
!!! caution
Although the initial `collect` is done before workers start
(`gunicorn --preload`), individual workers will end up with their
own copy of the value table cache.
So when we need to recollect values for our cache, we must notify
in some way that other workers also have to recollect this table.
### Global recollection
Whenever we recollect a value table, we insert the time of recollection
in a record in the MongoDb.
Somewhere at the start of each request, these records will be checked,
and if needed, recollections will be done before the request processing.
There is a table `collect`, with records having fields `table` and
`dateCollected`. After each (re)collect of a table, the `dateCollected` of
the appropriate record will be set to the current time.
!!! note "recollect()"
A `recollect()` without arguments should be done at the start of each
request.
!!! note "recollect(table)"
A `recollect(table)` should be done whenever this worker has changed
something in that value table.
Parameters
----------
table: string, optional `None`
A recollect() without arguments collects *all* value tables that need
collecting based on the times of change as recorded in the `collect`
table.
A recollect of a single table means that this worker has made a change.
After the recollect, a timestamp will go into the `collect` table,
so that other workers can pick it up.
If table is `True`, all timestamps in the `collect` table will be set
to now, so that each worker will refresh its value cache.
"""
collected = self.collected
if table is None:
affected = set()
for valueTable in VALUE_TABLES:
record = self.mongoCmd(
N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable}
)
lastChangedGlobally = G(record, RECOLLECT_DATE)
lastChangedHere = G(collected, valueTable)
if lastChangedGlobally and (
not lastChangedHere or lastChangedHere < lastChangedGlobally
):
self.cacheValueTable(valueTable)
collected[valueTable] = now()
affected.add(valueTable)
elif table is True:
affected = set()
for valueTable in VALUE_TABLES:
self.cacheValueTable(valueTable)
collected[valueTable] = now()
affected.add(valueTable)
else:
self.cacheValueTable(table)
collected[table] = now()
affected = {table}
if affected:
justNow = now()
for aTable in affected:
self.mongoCmd(
N.collect,
N.collect,
N.update_one,
{RECOLLECT_NAME: aTable},
{M_SET: {RECOLLECT_DATE: justNow}},
upsert=True,
)
self.collectActualItems(tables=affected)
if affected:
if DEBUG_SYNCH:
serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""")
def checkLogin(self):
"""Checks for a currently logged in user and sets `user` accordingly.
This happens after a login action and is meant to adapt the `user` attribute
to a newly logged-in user.
Returns
-------
Whether an authenticated user has just logged in.
"""
db = self.db
user = self.user
isDevel = self.isDevel
unauthUser = self.unauthUser
contentLength = request.content_length
if contentLength is not None and contentLength > LIMIT_JSON:
abort(400)
authEnv = (
{
k[4:].lower(): utf8FromLatin1(v)
for (k, v) in request.environ.items()
if k.startswith("""AJP_""")
}
if TRANSPORT_ATTRIBUTES == N.ajp
else {k.lower(): utf8FromLatin1(v) for (k, v) in request.headers}
if TRANSPORT_ATTRIBUTES == N.http
else {k.lower(): utf8FromLatin1(v) for (k, v) in request.environ.items()}
)
if DEBUG_AUTH:
serverprint("LOGIN: auth environment/headers")
for (k, v) in authEnv.items():
serverprint(f"LOGIN: ATTRIBUTE {k} = {v}")
self.clearUser()
if isDevel:
if DEBUG_AUTH:
serverprint("LOGIN: start authentication in development mode")
eppn = G(request.args, N.eppn)
email = None
if eppn is None:
email = G(request.args, N.email) or E
if AT in email:
eppn = email.split(AT, maxsplit=1)[0]
if eppn:
if DEBUG_AUTH:
serverprint(
f"LOGIN: authentication succeeded: {eppn} / {email}"
)
return self.getUser(eppn, email=email, mayCreate=True)
user.update(unauthUser)
if DEBUG_AUTH:
serverprint("LOGIN: authentication failed: no eppn, no email")
return False
result = self.getUser(eppn, mayCreate=False)
if DEBUG_AUTH:
if result:
serverprint("LOGIN: authentication successful")
else:
serverprint("LOGIN: authentication failed")
return result
else:
if DEBUG_AUTH:
serverprint("LOGIN: start authentication with shibboleth")
authenticated = G(authEnv, SHIB_KEY)
if authenticated:
eppn = G(authEnv, N.eppn)
email = G(authEnv, N.mail)
isUser = self.getUser(eppn, email=email, mayCreate=True)
if DEBUG_AUTH:
serverprint("""LOGIN: shibboleth session found:""")
serverprint(f"""LOGIN: eppn = "{eppn}" """)
serverprint(f"""LOGIN: email = "{email}" """)
serverprint(f"""LOGIN: isUser = "******" """)
if not isUser:
# the user is refused because the database says (s)he may not login
self.clearUser()
if DEBUG_AUTH:
serverprint("LOGIN: authentication failed")
return False
# process the attributes provided by the identity server
# they may have been changed after the last login
attributes = {
toolKey: G(authEnv, envKey, default=E)
for (envKey, toolKey) in ATTRIBUTES.items()
if envKey in authEnv
}
dirty = False
for (att, val) in attributes.items():
currentVal = G(user, att)
if currentVal != val:
user[att] = val
dirty = True
if dirty:
db.updateUser(user)
if DEBUG_AUTH:
serverprint(f"LOGIN: user data updated for {eppn}/{email}")
if DEBUG_AUTH:
serverprint("LOGIN: authentication successful")
return True
user.update(unauthUser)
if DEBUG_AUTH:
serverprint("LOGIN: No shibboleth session found:")
serverprint("LOGIN: authentication failed")
return False
def test_identity(clients):
for (user, cl) in clients.items():
response = cl.get("/whoami")
actualUser = response.get_data(as_text=True)
serverprint(f"{user} says: I am {actualUser}")
assert user == actualUser
def factory(regime, test, **kwargs):
if regime not in {"production", "development"}:
serverprint(f"REGIME: illegal value: {regime}")
sys.exit()
serverprint(f"REGIME: {regime}")
return appFactory(regime, test == "test", DEBUG, **kwargs)
def main():
methodNames = Names.getMethods()
allPureNames = set()
allNames = set()
with os.scandir(CONFIG_DIR) as sd:
files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT))
for configFile in files:
section = os.path.splitext(configFile)[0]
className = cap1(section)
classObj = globals()[className]
setattr(Config, section, classObj)
with open(f"""{CONFIG_DIR}/{section}{CONFIG_EXT}""") as fh:
settings = yaml.load(fh, Loader=yaml.FullLoader)
for (subsection, subsettings) in settings.items():
if subsection != NAMES:
setattr(classObj, subsection, subsettings)
(pureNames, names) = Names.addNames(configFile, settings)
allPureNames |= pureNames
allNames |= names
N = Names
C = Config
CT = C.tables
masters = {}
for (master, details) in CT.details.items():
for detail in details:
masters.setdefault(detail, set()).add(master)
setattr(CT, "masters", masters)
with os.scandir(TABLE_DIR) as sd:
files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT))
for tableFile in files:
with open(f"""{TABLE_DIR}/{tableFile}""") as fh:
settings = yaml.load(fh, Loader=yaml.FullLoader)
(pureNames, names) = Names.addNames(configFile, settings)
allPureNames |= pureNames
allNames |= names
spuriousNames = allPureNames & allNames
if spuriousNames:
serverprint(f"NAMES: {len(spuriousNames)} spurious names")
serverprint(", ".join(sorted(spuriousNames)))
else:
if not TERSE:
serverprint("NAMES: No spurious names")
NAME_RE = re.compile(r"""\bN\.[A-Za-z0-9_]+""")
usedNames = set()
for (top, subdirs, files) in os.walk(SERVER_PATH):
for f in files:
if not f.endswith(".py"):
continue
path = f"{top}/{f}"
with open(path) as pf:
text = pf.read()
usedNames |= {name[2:] for name in set(NAME_RE.findall(text))}
unusedNames = allPureNames - usedNames
if unusedNames:
serverprint(f"NAMES: {len(unusedNames)} unused names")
serverprint(", ".join(sorted(unusedNames)))
else:
if not TERSE:
serverprint("NAMES: No unused names")
undefNames = usedNames - allPureNames - allNames - methodNames
if undefNames:
serverprint(f"NAMES: {len(undefNames)} undefined names")
serverprint(", ".join(sorted(undefNames)))
else:
if not TERSE:
serverprint("NAMES: No undefined names")
if not TERSE:
serverprint(f"NAMES: {len(allPureNames | allNames):>4} defined in yaml files")
serverprint(f"NAMES: {len(usedNames):>4} used in python code")
if undefNames:
serverprint("EXIT because of FATAL ERROR")
sys.exit()
tables = set()
MAIN_TABLE = CT.userTables[0]
USER_TABLES = set(CT.userTables)
USER_ENTRY_TABLES = set(CT.userEntryTables)
VALUE_TABLES = set(CT.valueTables)
SYSTEM_TABLES = set(CT.systemTables)
SCALAR_TYPES = CT.scalarTypes
SCALAR_TYPE_SET = set(chain.from_iterable(SCALAR_TYPES.values()))
PROV_SPECS = CT.prov
VALUE_SPECS = CT.value
CASCADE = CT.cascade
tables = tables | USER_TABLES | USER_ENTRY_TABLES | VALUE_TABLES | SYSTEM_TABLES
sortedTables = (
[MAIN_TABLE]
+ sorted(USER_TABLES - {MAIN_TABLE})
+ sorted(tables - USER_TABLES - {MAIN_TABLE})
)
reference = {}
cascade = {}
for table in tables:
specs = {}
tableFile = f"""{TABLE_DIR}/{table}{CONFIG_EXT}"""
if os.path.exists(tableFile):
with open(tableFile) as fh:
specs.update(yaml.load(fh, Loader=yaml.FullLoader))
else:
specs.update(VALUE_SPECS)
specs.update(PROV_SPECS)
for (field, fieldSpecs) in specs.items():
fieldType = G(fieldSpecs, N.type)
if fieldType and fieldType not in SCALAR_TYPE_SET:
cascaded = set(G(CASCADE, fieldType, default=[]))
if table in cascaded:
cascade.setdefault(fieldType, {}).setdefault(table, set()).add(field)
else:
reference.setdefault(fieldType, {}).setdefault(table, set()).add(field)
setattr(Tables, table, specs)
tables.add(table)
Names.addNames(table, specs)
constrainedPre = {}
for table in VALUE_TABLES:
fieldSpecs = getattr(Tables, table, {})
for (field, fieldSpec) in fieldSpecs.items():
tp = G(fieldSpec, N.type)
if tp in VALUE_TABLES and tp == field:
constrainedPre[table] = field
constrained = {}
for table in tables:
fieldSpecs = getattr(Tables, table, {})
fields = set(fieldSpecs)
for (ctable, mfield) in constrainedPre.items():
if ctable in fields and mfield in fields:
ctp = G(fieldSpecs[ctable], N.type)
if ctp == ctable:
constrained[ctable] = mfield
setattr(Tables, ALL, tables)
setattr(Tables, N.sorted, sortedTables)
setattr(Tables, N.reference, reference)
setattr(Tables, N.cascade, cascade)
setattr(Tables, N.constrained, constrained)
CF = C.workflow
TASKS = CF.tasks
taskFields = {}
for taskInfo in TASKS.values():
if G(taskInfo, N.operator) == N.set:
table = G(taskInfo, N.table)
taskFields.setdefault(table, set()).add(G(taskInfo, N.field))
dateField = G(taskInfo, N.date)
if dateField:
taskFields[table].add(dateField)
setattr(Workflow, N.taskFields, taskFields)
def showNames(cls):
serverprint("""\nNAMES""")
for (k, v) in sorted(cls.__dict__.items()):
if callable(getattr(cls, k)):
serverprint(f"""\t{k:<20} = {v}""")
