def get_signed_url_for_file(action, file_id, file_name=None):
requested_protocol = flask.request.args.get("protocol", None)
r_pays_project = flask.request.args.get("userProject", None)
# default to signing the url even if it's a public object
# this will work so long as we're provided a user token
force_signed_url = True
no_force_sign_param = flask.request.args.get("no_force_sign")
if no_force_sign_param and no_force_sign_param.lower() == "true":
force_signed_url = False
indexed_file = IndexedFile(file_id)
default_expires_in = config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration_from_request(
max_limit=default_expires_in,
default=default_expires_in,
)
signed_url = indexed_file.get_signed_url(
requested_protocol,
action,
expires_in,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
file_name=file_name,
)
return {"url": signed_url}
def create_authorization_code(client, grant_user, request):
"""
Create an ``AuthorizationCode`` model for the current OAuth request
from the given client and user.
Certain parameters in the ``AuthorizationCode`` are filled out using
the arguments passed from the OAuth request (the redirect URI, scope,
and nonce).
"""
# requested lifetime (in seconds) for the refresh token
refresh_token_expires_in = get_valid_expiration_from_request(
expiry_param="refresh_token_expires_in",
max_limit=config["REFRESH_TOKEN_EXPIRES_IN"],
default=config["REFRESH_TOKEN_EXPIRES_IN"],
)
code = AuthorizationCode(
code=generate_token(50),
client_id=client.client_id,
redirect_uri=request.redirect_uri,
scope=request.scope,
user_id=grant_user.id,
nonce=request.data.get("nonce"),
refresh_token_expires_in=refresh_token_expires_in,
)
with flask.current_app.db.session as session:
session.add(code)
session.commit()
return code.code
def handle_user_service_account_creds(self, key, service_account):
"""
Add the service account creds for the user into our db. Actual
Oauth Client SAs are handled separately. This function assigns
the same expiration to the user's generated key but the mechanism
for expiration uses our db instead of checking google directly.
See fence-create scripting functions for expiration logic.
The reason for this difference is due to the fact that fence itself
uses the user's primary service account for url signing (in addition
to the user themselves). Since the expirations are different, a
different mechanism than the Client SAs was required.
"""
# requested time (in seconds) during which the access key will be valid
# x days * 24 hr/day * 60 min/hr * 60 s/min = y seconds
expires_in = cirrus_config.SERVICE_KEY_EXPIRATION_IN_DAYS * 24 * 60 * 60
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
expires_in = min(expires_in, requested_expires_in)
expiration_time = int(time.time()) + int(expires_in)
key_id = key.get("private_key_id")
add_custom_service_account_key_expiration(key_id,
service_account.id,
expires=expiration_time)
def extend_service_account_access(service_account_email, db=None):
"""
Extend the Google service accounts access to data by extending the
expiration time for each of the Google Bucket Access Groups it's in.
WARNING: This does NOT do any AuthZ, do before this.
Args:
service_account_email (str): service account email
db(str): db connection string
"""
session = get_db_session(db)
service_account = (
session.query(UserServiceAccount).filter_by(email=service_account_email).first()
)
if service_account:
bucket_access_groups = get_google_access_groups_for_service_account(
service_account
)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration_time = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
requested_expiration = int(time.time()) + requested_expires_in
expiration_time = min(expiration_time, requested_expiration)
logger.debug(
"Service Account ({}) access extended to {}.".format(
service_account.email, expiration_time
)
)
for access_group in bucket_access_groups:
bucket_access = (
session.query(ServiceAccountToGoogleBucketAccessGroup)
.filter_by(
service_account_id=service_account.id,
access_group_id=access_group.id,
)
.first()
)
if not bucket_access:
bucket_access = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
access_group_id=access_group.id,
expires=expiration_time,
)
session.add(bucket_access)
bucket_access.expires = expiration_time
session.commit()
def add_user_service_account_to_db(session, to_add_project_ids, service_account):
"""
Add user service account to service account
access privilege and service account bucket access group
Args:
sess(current_session): db session
to_add_project_ids(List(int)): List of project id
service_account(UserServiceAccount): user service account
requested_expires_in(int): requested time (in seconds) during which
the SA has bucket access
Returns:
None
Contrains:
The service account is not in DB yet
"""
for project_id in to_add_project_ids:
session.add(
ServiceAccountAccessPrivilege(
project_id=project_id, service_account_id=service_account.id
)
)
access_groups = _get_google_access_groups(session, project_id)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration_time = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
requested_expires_in = (
get_valid_expiration_from_request()
) # requested time (in seconds)
if requested_expires_in:
# convert it to timestamp
requested_expiration = int(time.time()) + requested_expires_in
expiration_time = min(expiration_time, requested_expiration)
for access_group in access_groups:
sa_to_group = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=expiration_time,
access_group_id=access_group.id,
)
session.add(sa_to_group)
session.commit()
def add_user_service_account_to_db(session, to_add_project_ids,
service_account):
"""
Add user service account to service account
access privilege and service account bucket access group
Args:
sess(current_session): db session
to_add_project_ids(List(int)): List of project id
service_account(UserServiceAccount): user service account
requested_expires_in(int): requested time (in seconds) during which
the SA has bucket access
Returns:
None
Contrains:
The service account is not in DB yet
"""
for project_id in to_add_project_ids:
session.add(
ServiceAccountAccessPrivilege(
project_id=project_id, service_account_id=service_account.id))
access_groups = _get_google_access_groups(session, project_id)
# time until the SA will lose bucket access
# by default: use configured time or 7 days
default_expires_in = config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800)
# use expires_in from request query params if it was provided and
# it was not greater than the default
expires_in = get_valid_expiration_from_request(
max_limit=default_expires_in,
default=default_expires_in,
)
# convert expires_in to timestamp
expiration_time = int(time.time() + expires_in)
for access_group in access_groups:
sa_to_group = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=expiration_time,
access_group_id=access_group.id,
)
session.add(sa_to_group)
session.commit()
def _extend_account_expiration():
user_id = current_token["sub"]
google_email = get_users_linked_google_email(user_id)
proxy_group = get_or_create_proxy_group_id()
# requested time (in seconds) during which the link will be valid
requested_expires_in = get_valid_expiration_from_request()
access_expiration = _force_update_user_google_account(
user_id,
google_email,
proxy_group,
_allow_new=False,
requested_expires_in=requested_expires_in,
)
return {"exp": access_expiration}, 200
def get_signed_url_for_file(action, file_id):
requested_protocol = flask.request.args.get("protocol", None)
# default to signing the url even if it's a public object
# this will work so long as we're provided a user token
force_signed_url = True
if flask.request.args.get("no_force_sign"):
force_signed_url = False
indexed_file = IndexedFile(file_id)
expires_in = config.get("MAX_PRESIGNED_URL_TTL", 3600)
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
expires_in = min(requested_expires_in, expires_in)
signed_url = indexed_file.get_signed_url(requested_protocol,
action,
expires_in,
force_signed_url=force_signed_url)
return {"url": signed_url}
def get_signed_url_for_file(action, file_id, file_name=None):
requested_protocol = flask.request.args.get("protocol", None)
r_pays_project = flask.request.args.get("userProject", None)
# default to signing the url even if it's a public object
# this will work so long as we're provided a user token
force_signed_url = True
no_force_sign_param = flask.request.args.get("no_force_sign")
if no_force_sign_param and no_force_sign_param.lower() == "true":
force_signed_url = False
indexed_file = IndexedFile(file_id)
default_expires_in = config.get("MAX_PRESIGNED_URL_TTL", 3600)
expires_in = get_valid_expiration_from_request(
max_limit=default_expires_in,
default=default_expires_in,
)
signed_url = indexed_file.get_signed_url(
requested_protocol,
action,
expires_in,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
file_name=file_name,
)
# increment counter for gen3-metrics
counter = flask.current_app.prometheus_counters.get("pre_signed_url_req")
if counter:
counter.labels(requested_protocol).inc()
if action == "download": # for now only record download requests
create_presigned_url_audit_log(protocol=requested_protocol,
indexed_file=indexed_file,
action=action)
return {"url": signed_url}
def _get_service_account_error_status(sa):
"""
Get a dictionary describing any errors that will occur if attempting
to give service account specified permissions fails.
Args:
sa (
fence.resources.google.service_account.GoogleServiceAccountRegistration
): the service account object with its email, project_access, a google project,
and optionally a user who is attempting to modify/add
Returns:
dict: error information if unsuccessful, { "success": True } otherwise
Example:
{
"success": False,
"errors": {
"service_account_email": {
"status": 200,
"error": None,
"error_description": None
},
"google_project_id": {
"status": 200,
"error": None,
"error_description": None
},
"project_access": {
"projectA": {
"status": 200,
"error": None,
"error_description": None
},
"projectB": {
"status": 403,
"error": "unauthorized",
"error_description": "Not all users have access requested"
}
},
"expires_in": {
"status": 400,
"error": "user_error",
"error_description": "expires_in must be a positive integer"
}
}
}
"""
response = {
"success": False,
"errors": {
"service_account_email": None,
"google_project_id": None,
"project_access": None,
"expires_in": {
"status": 200,
"error": None,
"error_description": None
},
},
}
try:
get_valid_expiration_from_request()
except UserError as e:
response["errors"]["expires_in"] = {
"status": e.code,
"error": "user_error",
"error_description": e.message,
}
project_validity = GoogleProjectValidity(
google_project_id=sa.google_project_id,
new_service_account=sa.email,
new_service_account_access=sa.project_access,
user_id=sa.user_id,
)
project_validity.check_validity(early_return=False)
response["errors"][
"google_project_id"] = _get_google_project_id_error_status(
project_validity)
response["errors"][
"service_account_email"] = _get_service_account_email_error_status(
project_validity)
response["errors"]["project_access"] = _get_project_access_error_status(
project_validity)
# if we cannot find the monitoring service account, the other checks statuses should
# not be 200 and should be populated with relevant information
if (response["errors"]["google_project_id"]["error"] ==
ValidationErrors.MONITOR_NOT_FOUND):
if response["errors"]["service_account_email"].get("status") == 200:
response["errors"]["service_account_email"]["status"] = 400
response["errors"]["service_account_email"][
"error"] = ValidationErrors.MONITOR_NOT_FOUND
response["errors"]["service_account_email"]["error_description"] = (
"Fence's monitoring service account was not found on the project so we "
"were unable to complete the necessary validation checks.")
if response["errors"]["project_access"].get("status") == 200:
response["errors"]["project_access"]["status"] = 400
response["errors"]["project_access"][
"error"] = ValidationErrors.MONITOR_NOT_FOUND
response["errors"]["project_access"]["error_description"] = (
"Fence's monitoring service account was not found on the project so we "
"were unable to complete the necessary validation checks.")
# all statuses must be 200 to be successful
if (response["errors"]["service_account_email"].get("status") == 200
and response["errors"]["google_project_id"].get("status") == 200
and response["errors"]["project_access"].get("status") == 200
and response["errors"]["expires_in"].get("status") == 200):
response["success"] = True
return response
def _link_google_account():
provided_redirect = flask.request.args.get("redirect")
# will raise UserError if invalid
validate_redirect(provided_redirect)
if not provided_redirect:
raise UserError({"error": "No redirect provided."})
user_id = current_token["sub"]
google_email = get_users_linked_google_email(user_id)
proxy_group = get_or_create_proxy_group_id()
# Set session flag to signify that we're linking and not logging in
# Save info needed for linking in session since we need to AuthN first
flask.session["google_link"] = True
flask.session["user_id"] = user_id
flask.session["google_proxy_group_id"] = proxy_group
flask.session["linked_google_email"] = google_email
if not google_email:
# save off provided redirect in session and initiate Google AuthN
flask.session["redirect"] = provided_redirect
# requested time (in seconds) during which the link will be valid
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
flask.session["google_link_expires_in"] = requested_expires_in
# if we're mocking Google login, skip to callback
if config.get("MOCK_GOOGLE_AUTH", False):
flask.redirect_url = (config["BASE_URL"].strip("/") +
"/link/google/callback?code=abc")
response = flask.redirect(flask.redirect_url)
# pass-through the authorization header. The user's username
# MUST be a Google email for MOCK_GOOGLE_AUTH to actually link that
# email correctly
response.headers["Authorization"] = flask.request.headers.get(
"Authorization")
return response
flask.redirect_url = flask.current_app.google_client.get_auth_url()
# Tell Google to let user select an account
flask.redirect_url = append_query_params(flask.redirect_url,
prompt="select_account")
else:
# double check that the token isn't stale by hitting db
linked_email_in_db = get_linked_google_account_email(user_id)
if linked_email_in_db:
# skip Google AuthN, already linked, error
redirect_with_errors = append_query_params(
provided_redirect,
error="g_acnt_link_error",
error_description=
"User already has a linked Google account.",
)
flask.redirect_url = redirect_with_errors
_clear_google_link_info_from_session()
else:
# TODO can we handle this error?
redirect_with_errors = append_query_params(
provided_redirect,
error="g_acnt_link_error",
error_description="Stale access token, please refresh.",
)
flask.redirect_url = redirect_with_errors
_clear_google_link_info_from_session()
return flask.redirect(flask.redirect_url)
