def assertCaptions(client, expect): """Check whether a response text shows a certain set of captions. Parameters ---------- client: fixture expect: set of string """ url = "/" (text, status, msgs) = accessUrl(client, url) captionsFound = {caption: url for (caption, url) in findCaptions(text)} for caption in captionsFound: assert caption in expect for caption in expect: assert caption in captionsFound for (caption, url) in captionsFound.items(): (expNumber, expItem) = expect[caption] serverprint(f"CAPTION {caption}: {client.user} CLICKS {url}") (text, status, msgs) = accessUrl(client, url) if expNumber is None: expItem in text else: (n, item) = findMainN(text)[0] nX = f"=/={expNumber}" if n != str(expNumber) else E iX = f"=/={expItem}" if item != expItem else E if iX or nX: serverprint(f"CAPTION {caption}: {n}{nX} {item}{iX}") assert n == str(expNumber) assert item == expItem
def illegalize(clients, url, **kwargs): """Append illegal/long arguments to an url and trigger a 400 response. Parameters ---------- clients: dict Mapping from users to client fixtures kwargs: dict Additional parameters to illegalize. The url will be expanded by formatting it with the `kwargs` values. """ kwargsx = {k: v + "a" * 200 for (k, v) in kwargs.items()} base = url.format(**kwargs) basex = url.format(**kwargsx) uxs = [ base, basex, f"{base}?action=xxx", f"{base}?xxx=xxx", f"{base}?action=" + "a" * 200, f"{base}?" + "a" * 2000, ] passable = {200, 301, 302, 303} for (i, ux) in enumerate(uxs): expectx = { user: 400 if i > 2 or i == 1 and len(kwargsx) else passable for user in USERS if user in clients } serverprint(f"LEGAL URL ? ({ux})") forall(clients, expectx, assertStatus, ux)
def getNames(source, val, doString=True, inner=False): names = set() pureNames = set() good = True if type(val) is str: names = {val} if doString and Names.isName(val) else set() elif type(val) is list: for v in val: if type(v) is str and Names.isName(v): names.add(v) elif type(v) is dict: names |= Names.getNames(source, v, doString=False, inner=True) elif type(val) is dict: for (k, v) in val.items(): if inner or k != NAMES: if type(k) is str and Names.isName(k): names.add(k) names |= Names.getNames(source, v, doString=False, inner=True) else: for val in v: if type(val) is not str: serverprint( f"NAMES in {source}: " f"WARNING: wrong type {type(val)} for {val}" ) good = False else: pureNames.add(val) if not good: serverprint("EXIT because of FATAL ERROR") sys.exit() return names if inner else (pureNames, names)
def changedCallFiles(origFile): if not os.path.exists(origFile): serverprint(f"WARNING: caller file {origFile} not found. Skipping.") return changes = set() removes = set() with open(origFile) as of: text = of.read() calls = callRe.findall(text) for (att, base, ext) in calls: calledPath = f"{base[1:]}{ext}" if all(not calledPath.startswith(stampDir) for stampDir in STAMP_DIRS): continue if not os.path.exists(calledPath): serverprint( f"WARNING: called file {calledPath} not found. Skipping.") continue calledGlob = f"{base[1:]}-[0-9][0-9]*{ext}" slug = int(round(os.path.getmtime(calledPath))) (calledDir, calledFile) = os.path.split(calledPath) calledSlugPath = f"{base[1:]}-{slug}{ext}" for slugPath in glob(calledGlob): if slugPath != calledSlugPath: removes.add(slugPath) changes.add((calledPath, calledSlugPath)) return (text, changes, removes)
def assertFieldValue(source, field, expect): """Verify whether a field has a certain expected value. If we pass expect `None` we want to assert that the field is not present at all. Parameters ---------- source: dict | (client: fixture, table: string, eid: string) The dictionary of fields and values of a retrieved response. If it is a tuple, the dictionary will be retrieved by looking up the item specified by `table` and `eid`. field: string The name of the specific field. expect: The expected value for this field. """ if type(source) is tuple: (client, table, eid) = source info = getItem(client, table, eid) fields = info["fields"] else: fields = source if expect is None: assert field not in fields else: assert field in fields value = fields[field] if value != expect: serverprint(f"FIELDVALUE {field}={value} (=/={expect})") assert expect == fields[field]
def collectActualItems(self, tables=None): """Determines which items are "actual". Actual items are those types and criteria that are specified in a package record that is itself actual. A package record is actual if the current data is between its start and end days. !!! caution If only value table needs to be collected that are not involved in the concept of "actual", nothing will be done. Parameters ---------- tables: set of string, optional `None` """ if tables is not None and not (tables & ACTUAL_TABLES): return justNow = now() packageActual = { G(record, N._id) for record in self.mongoCmd( N.collectActualItems, N.package, N.find, {N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}}, ) } for record in self.package.values(): record[N.actual] = G(record, N._id) in packageActual typeActual = set( chain.from_iterable( G(record, N.typeContribution) or [] for (_id, record) in self.package.items() if _id in packageActual ) ) for record in self.typeContribution.values(): record[N.actual] = G(record, N._id) in typeActual criteriaActual = { _id for (_id, record) in self.criteria.items() if G(record, N.package) in packageActual } for record in self.criteria.values(): record[N.actual] = G(record, N._id) in criteriaActual self.typeCriteria = {} for (_id, record) in self.criteria.items(): if _id in criteriaActual: for tp in G(record, N.typeContribution) or []: self.typeCriteria.setdefault(tp, set()).add(_id) if DEBUG_SYNCH: serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""")
def __init__(self, regime, test=False): """## Initialization Pick up the connection to MongoDb. !!! note Parameters ---------- regime: {"production", "development"} See below test: boolean See below. """ self.regime = regime """*string* Whether the app runs in production or in development.""" self.test = test """*boolean* Whether to connect to the test database.""" database = G(DATABASE, N.test) if test else G(DATABASE, regime) self.database = database mode = f"""regime = {regime} {"test" if test else E}""" if not self.database: serverprint(f"""MONGO: no database configured for {mode}""") sys.exit(1) self.client = None """*object* The MongoDb client.""" self.mongo = None """*object* The connection to the MongoDb database. The connnection exists before the Db singleton is initialized. """ self.collected = {} """*dict* For each value table, the time that this worker last collected it. In the database there is a table which holds the last time for each value table that a worker updated a value in it. """ self.collect() creator = [ G(record, N._id) for record in self.user.values() if G(record, N.eppn) == CREATOR ] if not creator: serverprint(f"""DATABASE: no creator user found in {database}.user""") sys.exit(1) self.creatorId = creator[0] """*ObjectId* System user.
def test_login(clients, clientPublic): for user in sorted(NAMED_USERS) + [E, PUBLIC, f"xxxxxx"]: isNamed = user in NAMED_USERS expect = 302 if isNamed else 303 serverprint(f"LOGIN {user}") assertStatus(clientPublic, f"/login?eppn={user}", expect) serverprint(f"LOGOUT {user}") if user in clients: assertStatus(clients[user], f"/logout", expect)
def computeWorkflow(regime, test): if not regime: serverprint("Don't know if this is development or production") return 1 mode = f"""regime = {regime} {"test" if test else E}""" serverprint(f"WORKFLOW RESET for {mode}") DB = Db(regime, test) WF = Workflow(DB) WF.initWorkflow(drop=True) return 0
def identify(self): user = self.user cl = self.cl url = "/logout" if user == "public" else f"/login?eppn={user}" cl.get(url) response = cl.get("/whoami") actualUser = response.get_data(as_text=True) good = user == actualUser if not good: serverprint(f"USER={actualUser} (=/={user})") assert good
def mongoClose(self): """Close connection with MongoDb. We need this, because before we fork the process to workers, all MongoDb connections should be closed. """ client = self.client if client: client.close() self.client = None self.mongo = None serverprint("""MONGO: connection closed""")
def checkBounds(**kwargs): """Aggressive check on the arguments passed in an url and/or request. First the total length of the request is counted. If it is too much, the request will be aborted. Each argument in request.args and `kwargs` must have a name that is allowed and its value should have a length under an appropriate limit, configured in `web.yaml`. There is always a fallback limit. !!! caution "Security" Before processing any request arg, whether from a form or from the url, use this function to check whether the length is within limits. If the length is exceeded, fail with a bad request, without giving any useful feedback. Because in this case we are dealing with a hacker. Parameters ---------- kwargs: dict The key-values that need to be checked. Raises ------ HTTPException If the length of any argument is out of bounds, processing is aborted and a bad request response is delivered """ default = G(LIMITS, N.default, default=100) maxLen = G(LIMITS, N.request, default=default) if request.content_length and request.content_length > maxLen: abort(400) n = len(request.args) boundN = G(LIMITS, N.keys, default=default) if len(kwargs) > boundN: serverprint(f"""OUT-OF-BOUNDS: {n} > {boundN} KEYS IN {kwargs}""") abort(400) for (k, v) in chain.from_iterable((kwargs.items(), request.args.items())): if k not in LIMITS: abort(400) valN = G(LIMITS, k, default=default) if v is not None and len(v) > valN: serverprint(f"""OUT-OF-BOUNDS: LENGTH ARG "{k}" > {valN} ({v})""") abort(400)
def mongoOpen(self): """Open connection with MongoDb. Which database we open, depends on `Db.regime` and `Db.test`. """ client = self.client mongo = self.mongo database = self.database if not mongo: client = MongoClient() mongo = client[database] self.client = client self.mongo = mongo serverprint(f"""MONGO: new connection to {database}""")
def assertReviewDecisions(clients, reviewId, kinds, decisions, expect): """Check whether the reviewers can take certain decisions. You specify which reviewers take which decisions, and they will all be carried out in that order. You specifiy the expected outcomes in a dict or a boolean, telling whether the taking of the decision is expected to succeed or not. Parameters ---------- clients: dict Mapping from users to client fixtures. reviewId: dict The review ids for the expert and final review kinds: list of {expert, final} At most one of each, the order is important. decisions: list of {Reject, Revise, Accept, Revoke} At most one of each, the order is important. expect: bool | dict Expected outcomes. If it is a boolean, that is the expected outcome of all actions by all reviewers. Otherwise the dict is keyed by kind of reviewer. The values are booleans or dicts. A boolean indicates the expected outcome of all actions for that reviewer. A dict specifies per action of that reviewer what the outcome is. """ for kind in kinds: rId = G(reviewId, kind) expectKind = ( True if expect is True else False if expect is False else G(expect, kind) ) for decision in decisions: decisionStr = G(G(REVIEW_DECISION, decision), kind) url = f"/api/task/{decisionStr}/{rId}" exp = ( True if expectKind is True else False if expectKind is False else G(expectKind, decision) ) serverprint(f"REVIEW DECISION {decision} by {kind} expects {exp}") assertStatus(G(clients, kind), url, exp)
def getCached(self, method, methodName, methodArgs, table, eid, requireFresh): """Helper to wrap caching around a raw Db fetch method. Only for methods that fetch single records. Parameters ---------- method: function The raw `control.db.Db` method. methodName: string The name of the raw Db method. Only used to display if cache debugging is on. methodNameArgs: iterable The arguments to pass to the Db method. table: string The table from which the record is fetched. eid: ObjectId (Entity) ID of the particular record. requireFresh: boolean, optional `False` If True, bypass the cache and fetch the item straight from Db and put the fetched value in the cache. Returns ------- mixed Whatever the underlying fetch method returns or would return. """ cache = self.cache key = eid if type(eid) is str else str(eid) if not requireFresh: if table in cache: if key in cache[table]: if DEBUG_CACHE: serverprint(f"""CACHE HIT {methodName}({key})""") return cache[table][key] result = method(*methodArgs) cache.setdefault(table, {})[key] = result return result
def mongoCmd(self, label, table, command, *args, **kwargs): """Wrapper around calls to MongoDb. All commands fired at the NongoDb go through this wrapper. It will spit out debug information if mongo debugging is True. Parameters ---------- label: string A key to be mentioned in debug messages. Very convenient to put here the name of the method that calls mongoCmd. table: string The table in MongoDB that is targeted by the command. If the table does not exists, no command will be fired. command: string The Mongo command to execute. The command must be listed in the mongo.yaml config file. *args: iterable Additional arguments will be passed straight to the Mongo command. Returns ------- mixed Whatever the the MongoDb returns. """ self.mongoOpen() mongo = self.mongo method = getattr(mongo[table], command, None) if command in M_COMMANDS else None warning = """!UNDEFINED""" if method is None else E if DEBUG_MONGO: argRep = args[0] if args and args[0] and command in SHOW_ARGS else E kwargRep = COMMA.join(f"{k}={v}" for (k, v) in kwargs.items()) serverprint( f"""MONGO<<{label}>>.{table}.{command}{warning}({argRep} {kwargRep})""" ) if method: return method(*args, **kwargs) return None
def showReferences(cls): reference = cls.reference serverprint("""\nREFERENCE FIELD DEPENDENCIES""") for (dep, tables) in sorted(reference.items()): serverprint(dep) for (table, fields) in tables.items(): serverprint(f"""\t{table:<20}: {", ".join(fields)}""")
def main(): args = sys.argv[1:] unstamp = args and args[0] == "un" texts = {} mapFile = {} changes = {} changed = set() removes = set() for callFile in CALL_FILES: (base, ext) = os.path.splitext(callFile) origFile = f"{base}Base{ext}" if unstamp: label = "RESTORE" serverprint(f"STAMP: {label} {callFile}") copyfile(origFile, callFile) continue mapFile[origFile] = callFile (text, theseChanges, theseRemoves) = changedCallFiles(origFile) texts[origFile] = text for (fileFrom, fileTo) in theseChanges: changes.setdefault(origFile, set()).add((fileFrom, fileTo)) removes |= theseRemoves for fileRem in sorted(removes): serverprint(f"STAMP: REMOVE {fileRem}") os.remove(fileRem) for (origFile, changedFiles) in changes.items(): callText = texts[origFile] for (fileFrom, fileTo) in changedFiles: if fileFrom in changed: continue if not os.path.exists(fileTo): serverprint(f"STAMP: COPY {fileFrom} => {fileTo}") copyfile(fileFrom, fileTo) callText = callText.replace(fileFrom, fileTo) changed.add(fileFrom) callFile = mapFile[origFile] origCallText = None if os.path.exists(callFile): with open(callFile) as cf: origCallText = cf.read() if origCallText is None or origCallText != callText: label = "WRITE" if origCallText is None else "REWRITE" serverprint(f"STAMP: {label} {callFile}") with open(callFile, "w") as cf: cf.write(callText)
def collect(self): """Collect the contents of the value tables. Value tables have content that is needed almost all the time. All value tables will be completely cached within Db. !!! note This is meant to run at start up, before the workers start. After that, this worker will not execute it again. See also `recollect`. !!! warning We must take other workers into account. They need a signal to recollect. See `recollect`. We store the time that this worker has collected each table in attribute `collected`. !!! caution If you change the MongoDb from without, an you forget to put an appropriate time stamp, the app will not see it untill it is restarted. See for example how `root.makeUserRoot` handles this. !!! warning This is a complicated app. Some tables have records that specify whether other records are "actual". After collecting a value table, the "actual" items will be recomputed. """ collected = self.collected for valueTable in VALUE_TABLES: self.cacheValueTable(valueTable) collected[valueTable] = now() self.collectActualItems() if DEBUG_SYNCH: serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""")
def assertStatus(client, url, expect): """Get data and see whether that went right or wrong. Parameters ---------- client: function url: string(url) The url to retrieve from the server expect: boolean | int | set of int If boolean: Whether it is expected to be successful If int: status code should be exactly this If set of int: status code should be contained in this """ try: response = client.get(url) code = response.status_code except Exception as e: serverprint(f"APPLICATION ERROR: {e}") code = 4000 if type(expect) is set: good = code in expect if not good: serverprint(f"STATUS {url} => {code} (not in {expect})") assert good elif type(expect) is int: good = code == expect if not good: serverprint(f"STATUS {url} => {code} (=/= {expect})") assert good else: codes = {200, 302} if expect else {400, 303} good = code in codes if not good: serverprint(f"STATUS {url} => {code} (not in {codes})") assert good
def main(): regime = sys.argv[1] if len(sys.argv) > 1 else None test = sys.argv[2] == "test" if len(sys.argv) > 2 else False if not regime: serverprint("Don't know if this is development or production") return 1 database = DATABASE["test"] if test else DATABASE.get(regime, None) if database is None: mode = f"""regime = {regime} {"test" if test else E}""" serverprint(f"""ERROR: No database configured for {mode}\n""" """See base.yaml""") return 1 eppn = ROOT[database] only = len(sys.argv) > 3 and sys.argv[3] == "--only" unique = "unique " if only else E serverprint(f"RESETTING {eppn} as {unique}root in {database}") makeUserRoot(database, eppn, only=only) return 0
def getUser(self, eppn, email=None, mayCreate=False): """Find a user in the database. This is called to get extra information for an authenticated user from the database. The resulting data will be stored in the `user` attribute of Auth. !!! caution Even if the user can be found, the attribute `mayLogin` might be false, in which case it will be prevented to log in that user. !!! tip When assigning reviewers, office users may select people who are not yet known to the contrib tool by specifying their email address. When such users log in for the first time, their `eppn` and other attributes become known, and are merged into a record in the user table. Parameters ---------- eppn: string The unique identifier of a user as assigned by the DARIAH identity provider. email: string, optional `None` New users may not have an eppn, but might already be present in the user table by their email. If so, the email address can be used to look up the user. mayCreate: boolean, optional `False` If a user cannot be found, they will be created if this flag is `True`. This is relevant for situation where a new user has been authenticated by the identity provider. Returns ------- boolean Whether a user was authenticated and logged in. The attributes retrieved from the database will be merged into the `user` attribute. If no user was logged in, the `user` attribute will be filled with info that says that the current user is the public and nothing more. """ user = self.user db = self.db authority = self.authority authId = self.authId userFound = [ record for record in db.user.values() if ( G(record, N.authority) == authority and ( (eppn is not None and G(record, N.eppn) == eppn) or ( eppn is None and email is not None and G(record, N.eppn) is None and G(record, N.email) == email ) ) ) ] user.clear() if len(userFound) > 1: self.clearUser() if DEBUG_AUTH: serverprint(f"LOGIN: multiple matches in user DB: {eppn} / {email}") return False if len(userFound) == 1: if not G(userFound[0], N.mayLogin, default=True): self.clearUser() if DEBUG_AUTH: serverprint(f"LOGIN: existing user may not login: {eppn} / {email}") return False user.update({N.eppn: eppn, N.authority: authority}) if email: user[N.email] = email if len(userFound) == 0: if mayCreate: db.insertUser(user) if DEBUG_AUTH: serverprint(f"LOGIN: new user: {eppn} / {email}") else: if DEBUG_AUTH: serverprint(f"LOGIN: may not create new user: {eppn} / {email}") return False else: user.update(userFound[0]) if DEBUG_AUTH: serverprint(f"LOGIN: existing user: {eppn} / {email}") # new users do not have yet group information group = user[N.group] if N.group in user else authId if N.group not in user: user[N.group] = group groupRep = G(G(db.permissionGroup, group), N.rep) result = groupRep != UNAUTH if DEBUG_AUTH: if result: serverprint(f"LOGIN: user authenticated: {eppn} / {email}") else: serverprint(f"LOGIN: user not authenticated: {eppn} / {email}") return result
def recollect(self, table=None): """Collect the contents of the value tables if they have changed. For each value table it will be checked if they have been collected (by another worker) after this worker has started and if so, those tables and those tables only will be recollected. !!! caution Although the initial `collect` is done before workers start (`gunicorn --preload`), individual workers will end up with their own copy of the value table cache. So when we need to recollect values for our cache, we must notify in some way that other workers also have to recollect this table. ### Global recollection Whenever we recollect a value table, we insert the time of recollection in a record in the MongoDb. Somewhere at the start of each request, these records will be checked, and if needed, recollections will be done before the request processing. There is a table `collect`, with records having fields `table` and `dateCollected`. After each (re)collect of a table, the `dateCollected` of the appropriate record will be set to the current time. !!! note "recollect()" A `recollect()` without arguments should be done at the start of each request. !!! note "recollect(table)" A `recollect(table)` should be done whenever this worker has changed something in that value table. Parameters ---------- table: string, optional `None` A recollect() without arguments collects *all* value tables that need collecting based on the times of change as recorded in the `collect` table. A recollect of a single table means that this worker has made a change. After the recollect, a timestamp will go into the `collect` table, so that other workers can pick it up. If table is `True`, all timestamps in the `collect` table will be set to now, so that each worker will refresh its value cache. """ collected = self.collected if table is None: affected = set() for valueTable in VALUE_TABLES: record = self.mongoCmd( N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable} ) lastChangedGlobally = G(record, RECOLLECT_DATE) lastChangedHere = G(collected, valueTable) if lastChangedGlobally and ( not lastChangedHere or lastChangedHere < lastChangedGlobally ): self.cacheValueTable(valueTable) collected[valueTable] = now() affected.add(valueTable) elif table is True: affected = set() for valueTable in VALUE_TABLES: self.cacheValueTable(valueTable) collected[valueTable] = now() affected.add(valueTable) else: self.cacheValueTable(table) collected[table] = now() affected = {table} if affected: justNow = now() for aTable in affected: self.mongoCmd( N.collect, N.collect, N.update_one, {RECOLLECT_NAME: aTable}, {M_SET: {RECOLLECT_DATE: justNow}}, upsert=True, ) self.collectActualItems(tables=affected) if affected: if DEBUG_SYNCH: serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""")
def checkLogin(self): """Checks for a currently logged in user and sets `user` accordingly. This happens after a login action and is meant to adapt the `user` attribute to a newly logged-in user. Returns ------- Whether an authenticated user has just logged in. """ db = self.db user = self.user isDevel = self.isDevel unauthUser = self.unauthUser contentLength = request.content_length if contentLength is not None and contentLength > LIMIT_JSON: abort(400) authEnv = ( { k[4:].lower(): utf8FromLatin1(v) for (k, v) in request.environ.items() if k.startswith("""AJP_""") } if TRANSPORT_ATTRIBUTES == N.ajp else {k.lower(): utf8FromLatin1(v) for (k, v) in request.headers} if TRANSPORT_ATTRIBUTES == N.http else {k.lower(): utf8FromLatin1(v) for (k, v) in request.environ.items()} ) if DEBUG_AUTH: serverprint("LOGIN: auth environment/headers") for (k, v) in authEnv.items(): serverprint(f"LOGIN: ATTRIBUTE {k} = {v}") self.clearUser() if isDevel: if DEBUG_AUTH: serverprint("LOGIN: start authentication in development mode") eppn = G(request.args, N.eppn) email = None if eppn is None: email = G(request.args, N.email) or E if AT in email: eppn = email.split(AT, maxsplit=1)[0] if eppn: if DEBUG_AUTH: serverprint( f"LOGIN: authentication succeeded: {eppn} / {email}" ) return self.getUser(eppn, email=email, mayCreate=True) user.update(unauthUser) if DEBUG_AUTH: serverprint("LOGIN: authentication failed: no eppn, no email") return False result = self.getUser(eppn, mayCreate=False) if DEBUG_AUTH: if result: serverprint("LOGIN: authentication successful") else: serverprint("LOGIN: authentication failed") return result else: if DEBUG_AUTH: serverprint("LOGIN: start authentication with shibboleth") authenticated = G(authEnv, SHIB_KEY) if authenticated: eppn = G(authEnv, N.eppn) email = G(authEnv, N.mail) isUser = self.getUser(eppn, email=email, mayCreate=True) if DEBUG_AUTH: serverprint("""LOGIN: shibboleth session found:""") serverprint(f"""LOGIN: eppn = "{eppn}" """) serverprint(f"""LOGIN: email = "{email}" """) serverprint(f"""LOGIN: isUser = "******" """) if not isUser: # the user is refused because the database says (s)he may not login self.clearUser() if DEBUG_AUTH: serverprint("LOGIN: authentication failed") return False # process the attributes provided by the identity server # they may have been changed after the last login attributes = { toolKey: G(authEnv, envKey, default=E) for (envKey, toolKey) in ATTRIBUTES.items() if envKey in authEnv } dirty = False for (att, val) in attributes.items(): currentVal = G(user, att) if currentVal != val: user[att] = val dirty = True if dirty: db.updateUser(user) if DEBUG_AUTH: serverprint(f"LOGIN: user data updated for {eppn}/{email}") if DEBUG_AUTH: serverprint("LOGIN: authentication successful") return True user.update(unauthUser) if DEBUG_AUTH: serverprint("LOGIN: No shibboleth session found:") serverprint("LOGIN: authentication failed") return False
def test_identity(clients): for (user, cl) in clients.items(): response = cl.get("/whoami") actualUser = response.get_data(as_text=True) serverprint(f"{user} says: I am {actualUser}") assert user == actualUser
def factory(regime, test, **kwargs): if regime not in {"production", "development"}: serverprint(f"REGIME: illegal value: {regime}") sys.exit() serverprint(f"REGIME: {regime}") return appFactory(regime, test == "test", DEBUG, **kwargs)
def main(): methodNames = Names.getMethods() allPureNames = set() allNames = set() with os.scandir(CONFIG_DIR) as sd: files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) for configFile in files: section = os.path.splitext(configFile)[0] className = cap1(section) classObj = globals()[className] setattr(Config, section, classObj) with open(f"""{CONFIG_DIR}/{section}{CONFIG_EXT}""") as fh: settings = yaml.load(fh, Loader=yaml.FullLoader) for (subsection, subsettings) in settings.items(): if subsection != NAMES: setattr(classObj, subsection, subsettings) (pureNames, names) = Names.addNames(configFile, settings) allPureNames |= pureNames allNames |= names N = Names C = Config CT = C.tables masters = {} for (master, details) in CT.details.items(): for detail in details: masters.setdefault(detail, set()).add(master) setattr(CT, "masters", masters) with os.scandir(TABLE_DIR) as sd: files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) for tableFile in files: with open(f"""{TABLE_DIR}/{tableFile}""") as fh: settings = yaml.load(fh, Loader=yaml.FullLoader) (pureNames, names) = Names.addNames(configFile, settings) allPureNames |= pureNames allNames |= names spuriousNames = allPureNames & allNames if spuriousNames: serverprint(f"NAMES: {len(spuriousNames)} spurious names") serverprint(", ".join(sorted(spuriousNames))) else: if not TERSE: serverprint("NAMES: No spurious names") NAME_RE = re.compile(r"""\bN\.[A-Za-z0-9_]+""") usedNames = set() for (top, subdirs, files) in os.walk(SERVER_PATH): for f in files: if not f.endswith(".py"): continue path = f"{top}/{f}" with open(path) as pf: text = pf.read() usedNames |= {name[2:] for name in set(NAME_RE.findall(text))} unusedNames = allPureNames - usedNames if unusedNames: serverprint(f"NAMES: {len(unusedNames)} unused names") serverprint(", ".join(sorted(unusedNames))) else: if not TERSE: serverprint("NAMES: No unused names") undefNames = usedNames - allPureNames - allNames - methodNames if undefNames: serverprint(f"NAMES: {len(undefNames)} undefined names") serverprint(", ".join(sorted(undefNames))) else: if not TERSE: serverprint("NAMES: No undefined names") if not TERSE: serverprint(f"NAMES: {len(allPureNames | allNames):>4} defined in yaml files") serverprint(f"NAMES: {len(usedNames):>4} used in python code") if undefNames: serverprint("EXIT because of FATAL ERROR") sys.exit() tables = set() MAIN_TABLE = CT.userTables[0] USER_TABLES = set(CT.userTables) USER_ENTRY_TABLES = set(CT.userEntryTables) VALUE_TABLES = set(CT.valueTables) SYSTEM_TABLES = set(CT.systemTables) SCALAR_TYPES = CT.scalarTypes SCALAR_TYPE_SET = set(chain.from_iterable(SCALAR_TYPES.values())) PROV_SPECS = CT.prov VALUE_SPECS = CT.value CASCADE = CT.cascade tables = tables | USER_TABLES | USER_ENTRY_TABLES | VALUE_TABLES | SYSTEM_TABLES sortedTables = ( [MAIN_TABLE] + sorted(USER_TABLES - {MAIN_TABLE}) + sorted(tables - USER_TABLES - {MAIN_TABLE}) ) reference = {} cascade = {} for table in tables: specs = {} tableFile = f"""{TABLE_DIR}/{table}{CONFIG_EXT}""" if os.path.exists(tableFile): with open(tableFile) as fh: specs.update(yaml.load(fh, Loader=yaml.FullLoader)) else: specs.update(VALUE_SPECS) specs.update(PROV_SPECS) for (field, fieldSpecs) in specs.items(): fieldType = G(fieldSpecs, N.type) if fieldType and fieldType not in SCALAR_TYPE_SET: cascaded = set(G(CASCADE, fieldType, default=[])) if table in cascaded: cascade.setdefault(fieldType, {}).setdefault(table, set()).add(field) else: reference.setdefault(fieldType, {}).setdefault(table, set()).add(field) setattr(Tables, table, specs) tables.add(table) Names.addNames(table, specs) constrainedPre = {} for table in VALUE_TABLES: fieldSpecs = getattr(Tables, table, {}) for (field, fieldSpec) in fieldSpecs.items(): tp = G(fieldSpec, N.type) if tp in VALUE_TABLES and tp == field: constrainedPre[table] = field constrained = {} for table in tables: fieldSpecs = getattr(Tables, table, {}) fields = set(fieldSpecs) for (ctable, mfield) in constrainedPre.items(): if ctable in fields and mfield in fields: ctp = G(fieldSpecs[ctable], N.type) if ctp == ctable: constrained[ctable] = mfield setattr(Tables, ALL, tables) setattr(Tables, N.sorted, sortedTables) setattr(Tables, N.reference, reference) setattr(Tables, N.cascade, cascade) setattr(Tables, N.constrained, constrained) CF = C.workflow TASKS = CF.tasks taskFields = {} for taskInfo in TASKS.values(): if G(taskInfo, N.operator) == N.set: table = G(taskInfo, N.table) taskFields.setdefault(table, set()).add(G(taskInfo, N.field)) dateField = G(taskInfo, N.date) if dateField: taskFields[table].add(dateField) setattr(Workflow, N.taskFields, taskFields)
def showNames(cls): serverprint("""\nNAMES""") for (k, v) in sorted(cls.__dict__.items()): if callable(getattr(cls, k)): serverprint(f"""\t{k:<20} = {v}""")