def update_user_visas(self, user, db_session=current_session):
"""
Updates user's RAS refresh token and uses the new access token to retrieve new visas from
RAS's /userinfo endpoint and update the db with the new visa.
- delete user's visas from db if we're not able to get a new access_token
- delete user's visas from db if we're not able to get a new visa
"""
user.ga4gh_visas_v1 = []
db_session.commit()
try:
token_endpoint = self.get_value_from_discovery_doc(
"token_endpoint", "")
userinfo_endpoint = self.get_value_from_discovery_doc(
"userinfo_endpoint", "")
token = self.get_access_token(user, token_endpoint, db_session)
userinfo = self.get_userinfo(token, userinfo_endpoint)
encoded_visas = userinfo.get("ga4gh_passport_v1", [])
except Exception as e:
err_msg = "Could not retrieve visa"
self.logger.exception("{}: {}".format(err_msg, e))
raise
for encoded_visa in encoded_visas:
try:
# TODO: These visas must be validated!!!
decoded_visa = jwt.decode(encoded_visa, verify=False)
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_db_session = db_session.object_session(visa)
current_db_session.add(visa)
except Exception as e:
err_msg = (
f"Could not process visa '{encoded_visa}' - skipping this visa"
)
self.logger.exception("{}: {}".format(err_msg, e),
exc_info=True)
db_session.commit()
def post_login(self, user, token_result):
# TODO: I'm not convinced this code should be in post_login.
# Just putting it in here for now, but might refactor later.
# This saves us a call to RAS /userinfo, but will not make sense
# when there is more than one visa issuer.
# Clear all of user's visas, to avoid having duplicate visas
# where only iss/exp/jti differ
# TODO: This is not IdP-specific and will need a rethink when
# we have multiple IdPs
user.ga4gh_visas_v1 = []
current_session.commit()
encoded_visas = flask.g.userinfo.get("ga4gh_passport_v1", [])
for encoded_visa in encoded_visas:
# TODO: These visas must be validated!!!
# i.e. (Remove `verify=False` in jwt.decode call)
# But: need a routine for getting public keys per visa.
# And we probably want to cache them.
# Also needs any ga4gh-specific validation.
# For now just read them without validation:
decoded_visa = jwt.decode(encoded_visa, verify=False)
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_session.add(visa)
current_session.commit()
# Store refresh token in db
refresh_token = flask.g.tokens.get("refresh_token")
id_token = flask.g.tokens.get("id_token")
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
expires = int(decoded_id.get("iat")) + config["RAS_REFRESH_EXPIRATION"]
flask.current_app.ras_client.store_refresh_token(
user=user, refresh_token=refresh_token, expires=expires)
def add_visa_manually(db_session,
user,
rsa_private_key,
kid,
expires=None,
sub=None):
expires = expires or int(time.time()) + 1000
make_invalid = False
if getattr(user, "username", user) == "expired_visa_user":
expires -= 100000
if getattr(user, "username", user) == "invalid_visa_user":
make_invalid = True
if getattr(user, "username", user) == "TESTUSERD":
make_invalid = True
encoded_visa, decoded_visa, expires = get_test_encoded_decoded_visa_and_exp(
db_session,
user,
rsa_private_key,
kid,
expires=expires,
make_invalid=make_invalid,
sub=sub,
)
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=expires,
ga4gh_visa=encoded_visa,
)
db_session.add(visa)
db_session.commit()
return encoded_visa, visa
def add_visa_manually(db_session, user, rsa_private_key, kid):
headers = {"kid": kid}
decoded_visa = {
"iss": "https://stsstg.nih.gov",
"sub": "abcde12345aspdij",
"iat": int(time.time()),
"exp": int(time.time()) + 1000,
"scope": "openid ga4gh_passport_v1 email profile",
"jti": "jtiajoidasndokmasdl",
"txn": "sapidjspa.asipidja",
"name": "",
"ga4gh_visa_v1": {
"type": "https://ras/visa/v1",
"asserted": int(time.time()),
"value": "https://nig/passport/dbgap",
"source": "https://ncbi/gap",
},
}
encoded_visa = jwt.encode(decoded_visa,
key=rsa_private_key,
headers=headers,
algorithm="RS256").decode("utf-8")
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
db_session.add(visa)
db_session.commit()
def post_login(self, user=None, token_result=None):
# TODO: I'm not convinced this code should be in post_login.
# Just putting it in here for now, but might refactor later.
# This saves us a call to RAS /userinfo, but will not make sense
# when there is more than one visa issuer.
# Clear all of user's visas, to avoid having duplicate visas
# where only iss/exp/jti differ
# TODO: This is not IdP-specific and will need a rethink when
# we have multiple IdPs
user.ga4gh_visas_v1 = []
current_session.commit()
encoded_visas = flask.g.userinfo.get("ga4gh_passport_v1", [])
for encoded_visa in encoded_visas:
# TODO: These visas must be validated!!!
# i.e. (Remove `verify=False` in jwt.decode call)
# But: need a routine for getting public keys per visa.
# And we probably want to cache them.
# Also needs any ga4gh-specific validation.
# For now just read them without validation:
decoded_visa = jwt.decode(encoded_visa, verify=False)
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_session.add(visa)
current_session.commit()
# Store refresh token in db
assert "refresh_token" in flask.g.tokens, "No refresh_token in user tokens"
refresh_token = flask.g.tokens["refresh_token"]
assert "id_token" in flask.g.tokens, "No id_token in user tokens"
id_token = flask.g.tokens["id_token"]
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
issued_time = int(decoded_id.get("iat"))
expires = config["RAS_REFRESH_EXPIRATION"]
# User definied RAS refresh token expiration time
parsed_url = urlparse(flask.session.get("redirect"))
query_params = parse_qs(parsed_url.query)
if query_params.get("upstream_expires_in"):
custom_refresh_expiration = query_params.get(
"upstream_expires_in")[0]
expires = get_valid_expiration(
custom_refresh_expiration,
expires,
expires,
)
flask.current_app.ras_client.store_refresh_token(
user=user,
refresh_token=refresh_token,
expires=expires + issued_time)
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
# Check if user has any project_access from a previous session or from usersync AND if fence is configured to use visas as authZ source
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
if not user.project_access and sync_from_visas:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()
DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
sync = init_syncer(
dbGaP,
None,
DB,
)
sync.sync_single_user_visas(user, current_session)
super(RASCallback, self).post_login()
def add_visa_manually(db_session, user, rsa_private_key, kid):
headers = {"kid": kid}
decoded_visa = {
"iss": "https://stsstg.nih.gov",
"sub": "abcde12345aspdij",
"iat": int(time.time()),
"exp": int(time.time()) + 1000,
"scope": "openid ga4gh_passport_v1 email profile",
"jti": "jtiajoidasndokmasdl",
"txn": "sapidjspa.asipidja",
"name": "",
"ga4gh_visa_v1": {
"type": "https://ras.nih.gov/visas/v1",
"asserted": int(time.time()),
"value": "https://nig/passport/dbgap",
"source": "https://ncbi/gap",
},
"ras_dbgap_permissions": [
{
"consent_name": "Health/Medical/Biomedical",
"phs_id": "phs000991",
"version": "v1",
"participant_set": "p1",
"consent_group": "c1",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
{
"consent_name": "General Research Use (IRB, PUB)",
"phs_id": "phs000961",
"version": "v1",
"participant_set": "p1",
"consent_group": "c1",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
{
"consent_name": "Disease-Specific (Cardiovascular Disease)",
"phs_id": "phs000279",
"version": "v2",
"participant_set": "p1",
"consent_group": "c1",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
{
"consent_name": "Health/Medical/Biomedical (IRB)",
"phs_id": "phs000286",
"version": "v6",
"participant_set": "p2",
"consent_group": "c3",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
{
"consent_name": "Disease-Specific (Focused Disease Only, IRB, NPU)",
"phs_id": "phs000289",
"version": "v6",
"participant_set": "p2",
"consent_group": "c2",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
{
"consent_name": "Disease-Specific (Autism Spectrum Disorder)",
"phs_id": "phs000298",
"version": "v4",
"participant_set": "p3",
"consent_group": "c1",
"role": "designated user",
"expiration": "2020-11-14 00:00:00",
},
],
}
encoded_visa = jwt.encode(
decoded_visa, key=rsa_private_key, headers=headers, algorithm="RS256"
).decode("utf-8")
expires = int(decoded_visa["exp"])
if user.username == "expired_visa_user":
expires -= 100000
if user.username == "invalid_visa_user":
encoded_visa = encoded_visa[: len(encoded_visa) // 2]
if user.username == "TESTUSERD":
encoded_visa = encoded_visa[: len(encoded_visa) // 2]
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=expires,
ga4gh_visa=encoded_visa,
)
db_session.add(visa)
db_session.commit()
def sync_gen3_users_authz_from_ga4gh_passports(
passports,
pkey_cache=None,
db_session=None,
):
"""
Validate passports and embedded visas, using each valid visa's identity
established by <iss, sub> combination to possibly create and definitely
determine a Fence user who is added to the list returned by this
function. In the process of determining Fence users from visas, visa
authorization information is also persisted in Fence and synced to
Arborist.
Args:
passports (list): a list of raw encoded passport strings, each
including header, payload, and signature
Return:
list: a list of users, each corresponding to a valid visa identity
embedded within the passports passed in
"""
db_session = db_session or current_session
# {"username": user, "username2": user2}
users_from_all_passports = {}
for passport in passports:
try:
cached_usernames = get_gen3_usernames_for_passport_from_cache(
passport=passport, db_session=db_session)
if cached_usernames:
# there's a chance a given username exists in the cache but no longer in
# the database. if not all are in db, ignore the cache and actually parse
# and validate the passport
all_users_exist_in_db = True
usernames_to_update = {}
for username in cached_usernames:
user = query_for_user(session=db_session,
username=username)
if not user:
all_users_exist_in_db = False
continue
usernames_to_update[user.username] = user
if all_users_exist_in_db:
users_from_all_passports.update(usernames_to_update)
# existence in the cache and a user in db means that this passport
# was validated previously (expiration was also checked)
continue
# below function also validates passport (or raises exception)
raw_visas = get_unvalidated_visas_from_valid_passport(
passport, pkey_cache=pkey_cache)
except Exception as exc:
logger.warning(
f"Invalid passport provided, ignoring. Error: {exc}")
continue
# an empty raw_visas list means that either the current passport is
# invalid or that it has no visas. in both cases, the current passport
# is ignored and we move on to the next passport
if not raw_visas:
continue
identity_to_visas = collections.defaultdict(list)
min_visa_expiration = int(
time.time()) + datetime.timedelta(hours=1).seconds
for raw_visa in raw_visas:
try:
validated_decoded_visa = validate_visa(raw_visa,
pkey_cache=pkey_cache)
identity_to_visas[(
validated_decoded_visa.get("iss"),
validated_decoded_visa.get("sub"),
)].append((raw_visa, validated_decoded_visa))
min_visa_expiration = min(min_visa_expiration,
validated_decoded_visa.get("exp"))
except Exception as exc:
logger.warning(
f"Invalid visa provided, ignoring. Error: {exc}")
continue
expired_authz_removal_job_freq_in_seconds = config[
"EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS"]
min_visa_expiration -= expired_authz_removal_job_freq_in_seconds
if min_visa_expiration <= int(time.time()):
logger.warning(
"The passport's earliest valid visa expiration time is set to "
f"occur within {expired_authz_removal_job_freq_in_seconds} "
"seconds from now, which is too soon an expiration to handle.")
continue
users_from_current_passport = []
for (issuer, subject_id), visas in identity_to_visas.items():
gen3_user = get_or_create_gen3_user_from_iss_sub(
issuer, subject_id, db_session=db_session)
ga4gh_visas = [
GA4GHVisaV1(
user=gen3_user,
source=validated_decoded_visa["ga4gh_visa_v1"]["source"],
type=validated_decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(
validated_decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(validated_decoded_visa["exp"]),
ga4gh_visa=raw_visa,
) for raw_visa, validated_decoded_visa in visas
]
# NOTE: does not validate, assumes validation occurs above.
# This adds the visas to the database session but doesn't commit until
# the end of this function
_sync_validated_visa_authorization(
gen3_user=gen3_user,
ga4gh_visas=ga4gh_visas,
expiration=min_visa_expiration,
db_session=db_session,
)
users_from_current_passport.append(gen3_user)
for user in users_from_current_passport:
users_from_all_passports[user.username] = user
put_gen3_usernames_for_passport_into_cache(
passport=passport,
user_ids_from_passports=list(users_from_all_passports.keys()),
expires_at=min_visa_expiration,
db_session=db_session,
)
db_session.commit()
logger.info(
f"Got Gen3 usernames from passport(s): {list(users_from_all_passports.keys())}"
)
return users_from_all_passports
def post_login(self, user=None, token_result=None):
# TODO: I'm not convinced this code should be in post_login.
# Just putting it in here for now, but might refactor later.
# This saves us a call to RAS /userinfo, but will not make sense
# when there is more than one visa issuer.
# Clear all of user's visas, to avoid having duplicate visas
# where only iss/exp/jti differ
# TODO: This is not IdP-specific and will need a rethink when
# we have multiple IdPs
user.ga4gh_visas_v1 = []
current_session.commit()
encoded_visas = []
try:
encoded_visas = flask.current_app.ras_client.get_encoded_visas_v11_userinfo(
flask.g.userinfo
)
except Exception as e:
err_msg = "Could not retrieve visas"
logger.error("{}: {}".format(e, err_msg))
raise
for encoded_visa in encoded_visas:
try:
# Do not move out of loop unless we can assume every visa has same issuer and kid
public_key = get_public_key_for_token(
encoded_visa, attempt_refresh=True
)
except Exception as e:
# (But don't log the visa contents!)
logger.error(
"Could not get public key to validate visa: {}. Discarding visa.".format(
e
)
)
continue
try:
# Validate the visa per GA4GH AAI "Embedded access token" format rules.
# pyjwt also validates signature and expiration.
decoded_visa = validate_jwt(
encoded_visa,
public_key,
# Embedded token must not contain aud claim
aud=None,
# Embedded token must contain scope claim, which must include openid
scope={"openid"},
issuers=config.get("GA4GH_VISA_ISSUER_ALLOWLIST", []),
# Embedded token must contain iss, sub, iat, exp claims
# options={"require": ["iss", "sub", "iat", "exp"]},
# ^ FIXME 2021-05-13: Above needs pyjwt>=v2.0.0, which requires cryptography>=3.
# Once we can unpin and upgrade cryptography and pyjwt, switch to above "options" arg.
# For now, pyjwt 1.7.1 is able to require iat and exp;
# authutils' validate_jwt (i.e. the function being called) checks issuers already (see above);
# and we will check separately for sub below.
options={
"require_iat": True,
"require_exp": True,
},
)
# Also require 'sub' claim (see note above about pyjwt and the options arg).
if "sub" not in decoded_visa:
raise JWTError("Visa is missing the 'sub' claim.")
except Exception as e:
logger.error("Visa failed validation: {}. Discarding visa.".format(e))
continue
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_session.add(visa)
current_session.commit()
# Store refresh token in db
assert "refresh_token" in flask.g.tokens, "No refresh_token in user tokens"
refresh_token = flask.g.tokens["refresh_token"]
assert "id_token" in flask.g.tokens, "No id_token in user tokens"
id_token = flask.g.tokens["id_token"]
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
issued_time = int(decoded_id.get("iat"))
expires = config["RAS_REFRESH_EXPIRATION"]
# User definied RAS refresh token expiration time
parsed_url = urlparse(flask.session.get("redirect"))
query_params = parse_qs(parsed_url.query)
if query_params.get("upstream_expires_in"):
custom_refresh_expiration = query_params.get("upstream_expires_in")[0]
expires = get_valid_expiration(
custom_refresh_expiration,
expires,
expires,
)
flask.current_app.ras_client.store_refresh_token(
user=user, refresh_token=refresh_token, expires=expires + issued_time
)
global_parse_visas_on_login = config["GLOBAL_PARSE_VISAS_ON_LOGIN"]
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
parse_visas = global_parse_visas_on_login or (
global_parse_visas_on_login == None
and (
strtobool(query_params.get("parse_visas")[0])
if query_params.get("parse_visas")
else False
)
)
# if sync_from_visas and (global_parse_visas_on_login or global_parse_visas_on_login == None):
# Check if user has any project_access from a previous session or from usersync AND if fence is configured to use visas as authZ source
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
# If GLOBAL_PARSE_VISAS_ON_LOGIN is true then we want to run it regardless of whether or not the client sent parse_visas on request
if sync_from_visas and parse_visas and not user.project_access:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()
DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
arborist = ArboristClient(
arborist_base_url=config["ARBORIST"],
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
sync = init_syncer(
dbGaP,
None,
DB,
arborist=arborist,
)
sync.sync_single_user_visas(user, current_session)
super(RASCallback, self).post_login()
def update_user_visas(self, user, pkey_cache, db_session=current_session):
"""
Updates user's RAS refresh token and uses the new access token to retrieve new visas from
RAS's /userinfo endpoint and update the db with the new visa.
- delete user's visas from db if we're not able to get a new access_token
- delete user's visas from db if we're not able to get new visas
- only visas which pass validation are added to the database
"""
# Note: in the cronjob this is called per-user per-visa.
# So it should be noted that when there are more clients than just RAS,
# this code as it stands can remove visas that the user has from other clients.
user.ga4gh_visas_v1 = []
db_session.commit()
try:
token_endpoint = self.get_value_from_discovery_doc(
"token_endpoint", "")
token = self.get_access_token(user, token_endpoint, db_session)
userinfo = self.get_userinfo(token)
encoded_visas = self.get_encoded_visas_v11_userinfo(
userinfo, pkey_cache)
except Exception as e:
err_msg = "Could not retrieve visas"
self.logger.exception("{}: {}".format(err_msg, e))
raise
for encoded_visa in encoded_visas:
try:
visa_issuer = get_iss(encoded_visa)
visa_kid = get_kid(encoded_visa)
except Exception as e:
self.logger.error(
"Could not get issuer or kid from visa: {}. Discarding visa."
.format(e))
continue # Not raise: If visa malformed, does not make sense to retry
# See if pkey is in cronjob cache; if not, update cache.
public_key = pkey_cache.get(visa_issuer, {}).get(visa_kid)
if not public_key:
try:
public_key = self.refresh_cronjob_pkey_cache(
visa_issuer, visa_kid, pkey_cache)
except Exception as e:
self.logger.error(
"Could not refresh public key cache: {}".format(e))
continue
if not public_key:
self.logger.error(
"Could not get public key to validate visa: Successfully fetched "
"issuer's keys but did not find the visa's key id among them. Discarding visa."
)
continue # Not raise: If issuer not publishing pkey, does not make sense to retry
try:
# Validate the visa per GA4GH AAI "Embedded access token" format rules.
# pyjwt also validates signature and expiration.
decoded_visa = validate_jwt(
encoded_visa,
public_key,
# Embedded token must not contain aud claim
aud=None,
# Embedded token must contain scope claim, which must include openid
scope={"openid"},
issuers=config.get("GA4GH_VISA_ISSUER_ALLOWLIST", []),
# Embedded token must contain iss, sub, iat, exp claims
# options={"require": ["iss", "sub", "iat", "exp"]},
# ^ FIXME 2021-05-13: Above needs pyjwt>=v2.0.0, which requires cryptography>=3.
# Once we can unpin and upgrade cryptography and pyjwt, switch to above "options" arg.
# For now, pyjwt 1.7.1 is able to require iat and exp;
# authutils' validate_jwt (i.e. the function being called) checks issuers already (see above);
# and we will check separately for sub below.
options={
"require_iat": True,
"require_exp": True,
},
)
# Also require 'sub' claim (see note above about pyjwt and the options arg).
if "sub" not in decoded_visa:
raise JWTError("Visa is missing the 'sub' claim.")
except Exception as e:
self.logger.error(
"Visa failed validation: {}. Discarding visa.".format(e))
continue
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_db_session = db_session.object_session(visa)
current_db_session.add(visa)
db_session.commit()