def generate_multipart_upload_presigned_url():
"""
Generate multipart upload presigned url
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "partNumber"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"presigned_url": BlankIndex.generate_aws_presigned_url_for_part(
params["key"],
params["uploadId"],
params["partNumber"],
expires_in=expires_in,
)
}
return flask.jsonify(response), 200
def complete_multipart_upload():
"""
Complete multipart upload
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "parts"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
try:
BlankIndex.complete_multipart_upload(
params["key"], params["uploadId"], params["parts"], expires_in=expires_in
),
except InternalError as e:
return flask.jsonify({"message": e.message}), e.code
return flask.jsonify({"message": "OK"}), 200
def init_multipart_upload():
"""
Initialize a multipart upload request
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"guid": blank_index.guid,
"uploadId": BlankIndex.init_multipart_upload(
blank_index.guid + "/" + params["file_name"], expires_in=expires_in
),
}
return flask.jsonify(response), 201
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
authorized = False
authz_err_msg = "Auth error when attempting to get a presigned URL for upload. User must have '{}' access on '{}'."
authz = params.get("authz")
uploader = None
if authz:
# if requesting an authz field, using new authorization method which doesn't
# rely on uploader field, so clear it out
uploader = ""
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["create", "write-storage"],
resources=authz,
)
if not authorized:
logger.error(authz_err_msg.format("create' and 'write-storage", authz))
else:
# no 'authz' was provided, so fall back on 'file_upload' logic
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="fence",
methods=["file_upload"],
resources=["/data_file"],
)
if not authorized:
logger.error(authz_err_msg.format("file_upload", "/data_file"))
if not authorized:
raise Forbidden(
"You do not have access to upload data. You either need "
"general file uploader permissions or create and write-storage permissions "
"on the authz resources you specified (if you specified any)."
)
blank_index = BlankIndex(
file_name=params["file_name"], authz=params.get("authz"), uploader=uploader
)
default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration(
params.get("expires_in"),
max_limit=default_expires_in,
default=default_expires_in,
)
response = {
"guid": blank_index.guid,
"url": blank_index.make_signed_url(params["file_name"], expires_in=expires_in),
}
return flask.jsonify(response), 201
def post_login(self, user=None, token_result=None):
# TODO: I'm not convinced this code should be in post_login.
# Just putting it in here for now, but might refactor later.
# This saves us a call to RAS /userinfo, but will not make sense
# when there is more than one visa issuer.
# Clear all of user's visas, to avoid having duplicate visas
# where only iss/exp/jti differ
# TODO: This is not IdP-specific and will need a rethink when
# we have multiple IdPs
user.ga4gh_visas_v1 = []
current_session.commit()
encoded_visas = flask.g.userinfo.get("ga4gh_passport_v1", [])
for encoded_visa in encoded_visas:
# TODO: These visas must be validated!!!
# i.e. (Remove `verify=False` in jwt.decode call)
# But: need a routine for getting public keys per visa.
# And we probably want to cache them.
# Also needs any ga4gh-specific validation.
# For now just read them without validation:
decoded_visa = jwt.decode(encoded_visa, verify=False)
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_session.add(visa)
current_session.commit()
# Store refresh token in db
assert "refresh_token" in flask.g.tokens, "No refresh_token in user tokens"
refresh_token = flask.g.tokens["refresh_token"]
assert "id_token" in flask.g.tokens, "No id_token in user tokens"
id_token = flask.g.tokens["id_token"]
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
issued_time = int(decoded_id.get("iat"))
expires = config["RAS_REFRESH_EXPIRATION"]
# User definied RAS refresh token expiration time
parsed_url = urlparse(flask.session.get("redirect"))
query_params = parse_qs(parsed_url.query)
if query_params.get("upstream_expires_in"):
custom_refresh_expiration = query_params.get(
"upstream_expires_in")[0]
expires = get_valid_expiration(
custom_refresh_expiration,
expires,
expires,
)
flask.current_app.ras_client.store_refresh_token(
user=user,
refresh_token=refresh_token,
expires=expires + issued_time)
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
# Check if user has any project_access from a previous session or from usersync AND if fence is configured to use visas as authZ source
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
if not user.project_access and sync_from_visas:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()
DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
sync = init_syncer(
dbGaP,
None,
DB,
)
sync.sync_single_user_visas(user, current_session)
super(RASCallback, self).post_login()
def post_login(self, user=None, token_result=None):
# TODO: I'm not convinced this code should be in post_login.
# Just putting it in here for now, but might refactor later.
# This saves us a call to RAS /userinfo, but will not make sense
# when there is more than one visa issuer.
# Clear all of user's visas, to avoid having duplicate visas
# where only iss/exp/jti differ
# TODO: This is not IdP-specific and will need a rethink when
# we have multiple IdPs
user.ga4gh_visas_v1 = []
current_session.commit()
encoded_visas = []
try:
encoded_visas = flask.current_app.ras_client.get_encoded_visas_v11_userinfo(
flask.g.userinfo
)
except Exception as e:
err_msg = "Could not retrieve visas"
logger.error("{}: {}".format(e, err_msg))
raise
for encoded_visa in encoded_visas:
try:
# Do not move out of loop unless we can assume every visa has same issuer and kid
public_key = get_public_key_for_token(
encoded_visa, attempt_refresh=True
)
except Exception as e:
# (But don't log the visa contents!)
logger.error(
"Could not get public key to validate visa: {}. Discarding visa.".format(
e
)
)
continue
try:
# Validate the visa per GA4GH AAI "Embedded access token" format rules.
# pyjwt also validates signature and expiration.
decoded_visa = validate_jwt(
encoded_visa,
public_key,
# Embedded token must not contain aud claim
aud=None,
# Embedded token must contain scope claim, which must include openid
scope={"openid"},
issuers=config.get("GA4GH_VISA_ISSUER_ALLOWLIST", []),
# Embedded token must contain iss, sub, iat, exp claims
# options={"require": ["iss", "sub", "iat", "exp"]},
# ^ FIXME 2021-05-13: Above needs pyjwt>=v2.0.0, which requires cryptography>=3.
# Once we can unpin and upgrade cryptography and pyjwt, switch to above "options" arg.
# For now, pyjwt 1.7.1 is able to require iat and exp;
# authutils' validate_jwt (i.e. the function being called) checks issuers already (see above);
# and we will check separately for sub below.
options={
"require_iat": True,
"require_exp": True,
},
)
# Also require 'sub' claim (see note above about pyjwt and the options arg).
if "sub" not in decoded_visa:
raise JWTError("Visa is missing the 'sub' claim.")
except Exception as e:
logger.error("Visa failed validation: {}. Discarding visa.".format(e))
continue
visa = GA4GHVisaV1(
user=user,
source=decoded_visa["ga4gh_visa_v1"]["source"],
type=decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(decoded_visa["exp"]),
ga4gh_visa=encoded_visa,
)
current_session.add(visa)
current_session.commit()
# Store refresh token in db
assert "refresh_token" in flask.g.tokens, "No refresh_token in user tokens"
refresh_token = flask.g.tokens["refresh_token"]
assert "id_token" in flask.g.tokens, "No id_token in user tokens"
id_token = flask.g.tokens["id_token"]
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
issued_time = int(decoded_id.get("iat"))
expires = config["RAS_REFRESH_EXPIRATION"]
# User definied RAS refresh token expiration time
parsed_url = urlparse(flask.session.get("redirect"))
query_params = parse_qs(parsed_url.query)
if query_params.get("upstream_expires_in"):
custom_refresh_expiration = query_params.get("upstream_expires_in")[0]
expires = get_valid_expiration(
custom_refresh_expiration,
expires,
expires,
)
flask.current_app.ras_client.store_refresh_token(
user=user, refresh_token=refresh_token, expires=expires + issued_time
)
global_parse_visas_on_login = config["GLOBAL_PARSE_VISAS_ON_LOGIN"]
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
parse_visas = global_parse_visas_on_login or (
global_parse_visas_on_login == None
and (
strtobool(query_params.get("parse_visas")[0])
if query_params.get("parse_visas")
else False
)
)
# if sync_from_visas and (global_parse_visas_on_login or global_parse_visas_on_login == None):
# Check if user has any project_access from a previous session or from usersync AND if fence is configured to use visas as authZ source
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
# If GLOBAL_PARSE_VISAS_ON_LOGIN is true then we want to run it regardless of whether or not the client sent parse_visas on request
if sync_from_visas and parse_visas and not user.project_access:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()
DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
arborist = ArboristClient(
arborist_base_url=config["ARBORIST"],
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
sync = init_syncer(
dbGaP,
None,
DB,
arborist=arborist,
)
sync.sync_single_user_visas(user, current_session)
super(RASCallback, self).post_login()
def post_login(self, user=None, token_result=None, id_from_idp=None):
parsed_url = urlparse(flask.session.get("redirect"))
query_params = parse_qs(parsed_url.query)
userinfo = flask.g.userinfo
global_parse_visas_on_login = config["GLOBAL_PARSE_VISAS_ON_LOGIN"]
parse_visas = global_parse_visas_on_login or (
global_parse_visas_on_login == None and
(strtobool(query_params.get("parse_visas")[0])
if query_params.get("parse_visas") else False))
# do an on-the-fly usersync for this user to give them instant access after logging in through RAS
# if GLOBAL_PARSE_VISAS_ON_LOGIN is true then we want to run it regardless of whether or not the client sent parse_visas on request
if parse_visas:
# get passport then call sync on it
try:
passport = (flask.current_app.ras_client.
get_encoded_passport_v11_userinfo(userinfo))
except Exception as e:
err_msg = "Could not retrieve passport or visas"
logger.error("{}: {}".format(e, err_msg))
raise
# now sync authz updates
users_from_passports = fence.resources.ga4gh.passports.sync_gen3_users_authz_from_ga4gh_passports(
[passport],
pkey_cache=PKEY_CACHE,
db_session=current_session,
)
user_ids_from_passports = list(users_from_passports.keys())
# TODO?
# put_gen3_usernames_for_passport_into_cache(
# passport, usernames_from_current_passport
# )
# Store refresh token in db
assert "refresh_token" in flask.g.tokens, "No refresh_token in user tokens"
refresh_token = flask.g.tokens["refresh_token"]
assert "id_token" in flask.g.tokens, "No id_token in user tokens"
id_token = flask.g.tokens["id_token"]
decoded_id = jwt.decode(id_token, verify=False)
# Add 15 days to iat to calculate refresh token expiration time
# TODO do they really not provide exp?
issued_time = int(decoded_id.get("iat"))
expires = config["RAS_REFRESH_EXPIRATION"]
# User definied RAS refresh token expiration time
if query_params.get("upstream_expires_in"):
custom_refresh_expiration = query_params.get(
"upstream_expires_in")[0]
expires = get_valid_expiration(
custom_refresh_expiration,
expires,
expires,
)
flask.current_app.ras_client.store_refresh_token(
user=user,
refresh_token=refresh_token,
expires=expires + issued_time)
super(RASCallback, self).post_login(id_from_idp=id_from_idp)
