def create_keypair(user, current_session, encryption_key, expire=86400):
existing_keypairs = 0
for keypair in user.hmac_keypairs:
if not keypair.check_and_archive(current_session):
existing_keypairs += 1
if existing_keypairs >= 2:
raise UserError("You can only have at most 2 keypairs")
key = Fernet(encryption_key)
# default to 1 day, max to 30 days
try:
expire = int(expire)
except ValueError:
raise UserError("Expiration has to be an integer representing"
" expiration time in seconds")
if expire > 2592000:
raise UserError("Max expiration time is 30 days(2592000 seconds)")
expire = min(expire, 2592000)
result = dict(access_key=random_str(20), secret_key=random_str(40))
keypair = HMACKeyPair(
access_key=result["access_key"],
secret_key=key.encrypt(result["secret_key"]),
expire=expire,
user_id=user.id,
)
current_session.add(keypair)
current_session.commit()
return result
def generate_multipart_upload_presigned_url():
"""
Generate multipart upload presigned url
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "partNumber"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
expires_in = 3600
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"presigned_url": BlankIndex.generate_aws_presigned_url_for_part(
params["key"],
params["uploadId"],
params["partNumber"],
expires_in=expires_in,
)
}
return flask.jsonify(response), 200
def create_user(current_session, username, role, email):
"""
Create a user for all the projects or groups in the list.
If the user already exists, to avoid unadvertedly changing it, we suggest update
Returns a dictionary.
"""
if not username:
raise UserError(("Error: Please provide a username"))
try:
usr = us.get_user(current_session, username)
raise UserError(("Error: user already exist. If this is not a"
" mistake, please, retry using update"))
except NotFound:
user_list = [
user["name"].upper()
for user in get_all_users(current_session)["users"]
]
if username.upper() in user_list:
raise UserError(
("Error: user with a name with the same combination/order "
"of characters already exists. Please remove this other user"
" or modify the new one. Contact us in case of doubt"))
is_admin = role == "admin"
email_add = email
usr = User(username=username,
active=True,
is_admin=is_admin,
email=email_add)
current_session.add(usr)
return us.get_user_info(current_session, username)
def __create_policy__(self,
policy_name,
policy_document,
path=None,
description=None):
"""
Create policy with name and policies specified in policy_document.
:param policy_name: Name of policy in AWS.
:param policy_document: Document specified the policy rule.
:param path:
:param description:
:return:
"""
try:
aws_kwargs = dict(Path=path, Description=description)
aws_kwargs = {
k: v
for k, v in list(aws_kwargs.items()) if v is not None
}
policy = self.iam.create_policy(PolicyName=policy_name,
PolicyDocument=policy_document,
**aws_kwargs)
self.iam.attach_group_policy(GroupName=policy_name,
PolicyArn=policy["Policy"]["Arn"])
except Exception as ex:
self.logger.exception(ex)
raise UserError("Fail to create policy: {}".format(ex))
return policy
def create_provider(
current_session,
provider_name,
backend=None,
service=None,
endpoint=None,
description=None,
):
"""
Create a new provider on the table
"""
check = (current_session.query(CloudProvider).filter(
CloudProvider.name == provider_name).first())
if check:
msg = (
"provider name {} already in use; please choose a different name"
" and try again").format(provider_name)
raise UserError(msg)
provider = CloudProvider(
name=provider_name,
backend=backend,
service=service,
endpoint=endpoint,
description=description,
)
current_session.add(provider)
msg = {"result": "success"}
return msg
def is_valid_expiration(expires_in):
"""
Throw an error if expires_in is not a positive integer.
"""
try:
expires_in = int(flask.request.args["expires_in"])
assert expires_in > 0
except (ValueError, AssertionError):
raise UserError("expires_in must be a positive integer")
def create_group(current_session, groupname, description):
group = udm.get_group(current_session, groupname)
if group:
raise UserError("Group already exists")
group = udm.get_empty_group()
group.name = groupname
group.description = description
current_session.add(group)
return {"result": "success"}
def create_user_group(self, group_name, path=None):
try:
group = self.iam.create_group(GroupName=group_name)["Group"]
self.__create_policy__(
group_name,
self.__get_policy_document_by_group_name__(group_name))
except Exception as ex:
self.logger.exception(ex)
raise UserError("Fail to create group {}: {}".format(
group_name, ex))
return group
def get_group_info(current_session, groupname):
group = get_group(current_session, groupname)
if not group:
raise UserError("Error: group doesn't exist")
projects = get_group_projects(current_session, groupname)
return {
"name": group.name,
"description": group.description,
"projects": projects
}
def get_user_group(self, group_names):
try:
groups = self.iam.list_groups()["Groups"]
res = {}
for group in groups:
if group["GroupName"] in group_names:
res[group["GroupName"]] = group
except Exception as ex:
self.logger.exception(ex)
raise UserError("Fail to get list of groups {}: {}".format(
group_names, ex))
return res
def init_multipart_upload():
"""
Initialize a multipart upload request
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
blank_index = BlankIndex(file_name=params["file_name"])
expires_in = 3600
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid": blank_index.guid,
"uploadId": BlankIndex.init_multipart_upload(
blank_index.guid + "/" + params["file_name"], expires_in=expires_in
),
}
return flask.jsonify(response), 201
def add_user_to_group(self, groups, username):
"""
Add user to the list of group which have association membership.
:param groups:
:param username:
:return:
"""
try:
for group in list(groups.values()):
self.iam.add_user_to_group(GroupName=group["GroupName"],
UserName=username)
except Exception as ex:
self.logger.exception(ex)
raise UserError("Fail to add user to group: {}".format(ex))
def complete_multipart_upload():
"""
Complete multipart upload
"""
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
missing = {"key", "uploadId", "parts"}.difference(set(params))
if missing:
raise UserError("missing required arguments: {}".format(list(missing)))
expires_in = 3600
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
try:
BlankIndex.complete_multipart_upload(
params["key"], params["uploadId"], params["parts"], expires_in=expires_in
),
except InternalError as e:
return flask.jsonify({"message": e.message}), e.code
return flask.jsonify({"message": "OK"}), 200
def remove_projects_from_group(current_session, groupname, projects=None):
if not projects:
projects = []
grp = gp.get_group(current_session, groupname)
usrs = get_group_users(current_session, groupname)
users_names = [x["name"] for x in usrs["users"]]
if not grp:
raise UserError("Error: group does not exist")
responses = []
for proj in projects:
for usr in users_names:
update_user_projects_within_group(current_session, usr, groupname,
proj)
response = disconnect_project_from_group(current_session, grp, proj)
responses.append(response)
return {"result": responses}
def update_user(current_session, username, role, email, new_name):
usr = us.get_user(current_session, username)
user_list = [
user["name"].upper()
for user in get_all_users(current_session)["users"]
]
if (new_name and new_name.upper() in user_list
and not username.upper() == new_name.upper()):
raise UserError(
("Error: user with a name with the same combination/order "
"of characters already exists. Please remove this other user"
" or modify the new one. Contact us in case of doubt"))
usr.email = email or usr.email
if role:
usr.is_admin = role == "admin"
usr.username = new_name or usr.username
return us.get_user_info(current_session, usr.username)
def add_projects_to_group(current_session, groupname, projects=None):
if not projects:
projects = []
grp = gp.get_group(current_session, groupname)
usrs = gp.get_group_users(current_session, groupname)
if not grp:
raise UserError("Error: group does not exist")
responses = []
for proj in projects:
try:
response = connect_project_to_group(current_session, grp, proj)
responses.append(response)
update_group_users_projects(current_session, grp, proj, usrs)
except Exception as e:
current_session.rollback()
raise e
return {"result": responses}
def get_all_groups(self, list_group_name):
"""
Get all group listed in the list_group_name.
If group does not exist, add as new group and include in the return list
:param list_group_name:
:return:
"""
try:
groups = self.get_user_group(list_group_name)
if len(groups) < len(list_group_name):
for group_name in list_group_name:
if group_name not in groups:
groups[group_name] = self.create_user_group(group_name)
except Exception as ex:
self.logger.exception(ex)
raise UserError("Fail to create list of groups: {}".format(ex))
return groups
def update_user_resource(username, resource):
with flask.current_app.db.session as session:
user = find_user(username, session)
if not user.application:
raise UserError("User haven't started the application")
resources = set(user.application.resources_granted or [])
resources.add(resource)
user.application.resources_granted = list(resources)
if "EMAIL_SERVER" in config:
content = "You have been granted {} resources in Bionimbus Cloud.".format(
", ".join(resources))
send_mail(
config["SEND_FROM"],
[user.email],
"Account update from Bionimbus Cloud",
text=content,
server=config["EMAIL_SERVER"],
)
return get_user_info(user, session)
def connect_user_to_group(current_session, usr, groupname=None):
grp = gp.get_group(current_session, groupname)
if not grp:
raise UserError(("Group {0} doesn't exist".format(groupname)))
else:
responses = []
responses.append(gp.connect_user_to_group(current_session, usr, grp))
projects = gp.get_group_projects(current_session, groupname)
projects_data = [
pj.get_project(current_session, project).auth_id
for project in projects
]
projects_list = [{
"auth_id": auth_id,
"privilege": ["read"]
} for auth_id in projects_data]
for project in projects_list:
connect_user_to_project(current_session, usr, project)
return responses
def delete_provider(current_session, provider_name):
"""
Delete a cloud provider if it has not
ongoing relationships
"""
provider = (current_session.query(CloudProvider).filter(
CloudProvider.name == provider_name).first())
if not provider:
msg = "provider name {}, not found"
raise NotFound(msg.format(provider_name))
projects = (current_session.query(StorageAccess).filter(
StorageAccess.provider_id == provider.id).first())
if projects:
msg = ("Provider name {} in use in projects."
" Please remove these references and retry")
raise UserError(msg.format(provider_name))
current_session.delete(provider)
return {"response": "success"}
def create_bucket_on_project(current_session, project_name, bucket_name,
provider_name):
"""
Create a bucket and assign it to a project
"""
project = (current_session.query(Project).filter(
Project.name == project_name).first())
if not project:
msg = "".join(["Project ", project_name, " not found"])
raise NotFound(msg)
provider = (current_session.query(CloudProvider).filter(
CloudProvider.name == provider_name).first())
if not provider:
msg = "".join(["Provider ", provider_name, " not found"])
raise NotFound(msg)
bucket = (current_session.query(Bucket).filter(
Bucket.name == bucket_name, Bucket.provider_id == provider.id).first())
if not bucket:
bucket = Bucket(name=bucket_name, provider_id=provider.id)
current_session.add(bucket)
current_session.flush()
proj_to_bucket = ProjectToBucket(project_id=project.id,
bucket_id=bucket.id,
privilege=["owner"])
current_session.add(proj_to_bucket)
# Find the users that need to be updated
users_in_project = current_session.query(AccessPrivilege).filter(
AccessPrivilege.project_id == project.id)
users_to_update = []
for row in users_in_project:
usr = current_session.query(User).filter(
User.id == row.user_id).first()
users_to_update.append((usr, row.privilege))
return {
"result": "success",
"provider": provider,
"bucket": bucket,
"users_to_update": users_to_update,
}
else:
raise UserError("Error, name already in use for that storage system")
def presigned_url(self, bucket, key, expires, config, method="get_object"):
"""
Args:
bucket (str): bucket name
key (str): key in bucket
expires (int): presigned URL expiration time, in seconds
config (dict): additional parameters if necessary (e.g. updating access key)
method (str): "get_object" or "put_object" (ClientMethod argument to boto)
"""
if method not in ["get_object", "put_object"]:
raise UserError("method {} not allowed".format(method))
if "aws_access_key_id" in config:
self.s3_client = client("s3", **config)
expires = int(expires) or self.URL_EXPIRATION_DEFAULT
expires = min(expires, self.URL_EXPIRATION_MAX)
params = {"Bucket": bucket, "Key": key}
if method == "put_object":
params["ServerSideEncryption"] = "AES256"
return self.s3_client.generate_presigned_url(ClientMethod=method,
Params=params,
ExpiresIn=expires)
def upload_certificate(certificate):
extension = flask.request.args.get("extension")
allowed_extension = ["pdf", "png", "jpg", "jpeg", "txt"]
if not extension or extension not in allowed_extension:
raise UserError(
"Invalid extension in parameter, acceptable extensions are {}".
format(", ".join(allowed_extension)))
if not flask.g.user.application:
flask.g.user.application = Application()
current_session.merge(flask.g.user)
cert = (current_session.query(Certificate).filter(
Certificate.name == certificate).filter(
Certificate.application_id == flask.g.user.application.id).first())
if not cert:
cert = Certificate(name=certificate)
cert.application_id = flask.g.user.application.id
cert.extension = extension
cert.data = flask.request.data
current_session.merge(cert)
certificates = flask.g.user.application.certificates_uploaded
if set(REQUIRED_CERTIFICATES.keys()).issubset(
set(c.name for c in certificates)):
title = "User application for {}".format(flask.g.user.username)
if getattr(flask.g, "client"):
title += " from {}".format(flask.g.client)
if "EMAIL_SERVER" in config:
content = "Application for user: {}\n" "email: {}".format(
flask.g.user.username, flask.g.user.email)
send_mail(
config["SEND_FROM"],
config["SEND_TO"],
title,
text=content,
server=config["EMAIL_SERVER"],
certificates=certificates,
)
return "", 201
def _link_google_account():
provided_redirect = flask.request.args.get("redirect")
if not provided_redirect:
raise UserError({"error": "No redirect provided."})
user_id = current_token["sub"]
google_email = get_users_linked_google_email(user_id)
proxy_group = get_or_create_proxy_group_id()
# Set session flag to signify that we're linking and not logging in
# Save info needed for linking in session since we need to AuthN first
flask.session["google_link"] = True
flask.session["user_id"] = user_id
flask.session["google_proxy_group_id"] = proxy_group
flask.session["linked_google_email"] = google_email
if not google_email:
# save off provided redirect in session and initiate Google AuthN
flask.session["redirect"] = provided_redirect
# requested time (in seconds) during which the link will be valid
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
flask.session["google_link_expires_in"] = requested_expires_in
# if we're mocking Google login, skip to callback
if config.get("MOCK_GOOGLE_AUTH", False):
flask.redirect_url = (
config["BASE_URL"].strip("/") + "/link/google/callback?code=abc"
)
response = flask.redirect(flask.redirect_url)
# pass-through the authorization header. The user's username
# MUST be a Google email for MOCK_GOOGLE_AUTH to actually link that
# email correctly
response.headers["Authorization"] = flask.request.headers.get(
"Authorization"
)
return response
flask.redirect_url = flask.current_app.google_client.get_auth_url()
# Tell Google to let user select an account
flask.redirect_url = append_query_params(
flask.redirect_url, prompt="select_account"
)
else:
# double check that the token isn't stale by hitting db
linked_email_in_db = get_linked_google_account_email(user_id)
if linked_email_in_db:
# skip Google AuthN, already linked, error
redirect_with_errors = append_query_params(
provided_redirect,
error="g_acnt_link_error",
error_description="User already has a linked Google account.",
)
flask.redirect_url = redirect_with_errors
_clear_google_link_info_from_session()
else:
# TODO can we handle this error?
redirect_with_errors = append_query_params(
provided_redirect,
error="g_acnt_link_error",
error_description="Stale access token, please refresh.",
)
flask.redirect_url = redirect_with_errors
_clear_google_link_info_from_session()
return flask.redirect(flask.redirect_url)
def get(self):
"""
Link a user's Google account after AuthN.
This is Google's callback that occurs after oauth2 flow and does
the actual linkage/creation in our db.
This will redirect with `error` and `error_description` query params
if any issues arise.
Raises:
UserError: No redirect provided
"""
provided_redirect = flask.session.get("redirect")
code = flask.request.args.get("code")
if not config.get("MOCK_GOOGLE_AUTH", False):
google_response = flask.current_app.google_client.get_user_id(code)
email = google_response.get("email")
else:
# if we're mocking google auth, mock response to include the email
# from the provided access token
try:
token = validate_request({"user"})
email = get_user_from_claims(token).username
except Exception as exc:
logger.info(
"Unable to parse Google email from token, using default mock value. "
"Error: {}".format(exc)
)
email = "*****@*****.**"
error = ""
error_description = ""
# get info from session and then clear it
user_id = flask.session.get("user_id")
proxy_group = flask.session.get("google_proxy_group_id")
expires_in = flask.session.get("google_link_expires_in")
_clear_google_link_info_from_session()
if not email:
error = "g_acnt_auth_failure"
error_description = google_response
else:
error, error_description = get_errors_update_user_google_account_dry_run(
user_id, email, proxy_group, _already_authed=True
)
if not error:
exp = _force_update_user_google_account(
user_id,
email,
proxy_group,
_allow_new=True,
requested_expires_in=expires_in,
)
# TODO: perhaps this is problematic??
# keep linked email in session so when session refreshes access
# token, we don't have to hit db to see if user has linked acnt
# NOTE: This only saves us from a db hit if they maintain their
# session
flask.session["linked_google_email"] = email
# if we have a redirect, follow it and add any errors
if provided_redirect:
if error:
redirect_with_params = append_query_params(
provided_redirect, error=error, error_description=error_description
)
else:
redirect_with_params = append_query_params(
provided_redirect, linked_email=email, exp=exp
)
return flask.redirect(redirect_with_params)
else:
# we don't have a redirect, so the endpoint was probably hit
# without the actual auth flow. Raise with error info
if error:
raise UserError({error: error_description})
else:
raise UserError({"error": "No redirect provided."})
def connect_project_to_group(current_session, grp, project=None):
prj = pj.get_project(current_session, project)
if not prj:
raise UserError(("Project {0} doesn't exist".format(project)))
return gp.connect_project_to_group(current_session, grp, prj)
def upload_data_file():
"""
Return a presigned URL for use with uploading a data file.
See the documentation on the entire flow here for more info:
https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload
"""
# make new record in indexd, with just the `uploader` field (and a GUID)
params = flask.request.get_json()
if not params:
raise UserError("wrong Content-Type; expected application/json")
if "file_name" not in params:
raise UserError("missing required argument `file_name`")
authorized = False
authz_err_msg = "Auth error when attempting to get a presigned URL for upload. User must have '{}' access on '{}'."
authz = params.get("authz")
uploader = None
if authz:
# if requesting an authz field, using new authorization method which doesn't
# rely on uploader field, so clear it out
uploader = ""
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="amanuensis",
methods=["create", "write-storage"],
resources=authz,
)
if not authorized:
logger.error(authz_err_msg.format("create' and 'write-storage", authz))
else:
# no 'authz' was provided, so fall back on 'file_upload' logic
authorized = flask.current_app.arborist.auth_request(
jwt=get_jwt(),
service="amanuensis",
methods=["file_upload"],
resources=["/data_file"],
)
if not authorized:
logger.error(authz_err_msg.format("file_upload", "/data_file"))
if not authorized:
raise Forbidden(
"You do not have access to upload data. You either need "
"general file uploader permissions or create and write-storage permissions "
"on the authz resources you specified (if you specified any)."
)
blank_index = BlankIndex(
file_name=params["file_name"], authz=params.get("authz"), uploader=uploader
)
expires_in = 3600
if "expires_in" in params:
is_valid_expiration(params["expires_in"])
expires_in = min(params["expires_in"], expires_in)
response = {
"guid": blank_index.guid,
"url": blank_index.make_signed_url(params["file_name"], expires_in=expires_in),
}
return flask.jsonify(response), 201
def delete_group(current_session, groupname):
group = udm.get_group(current_session, groupname)
if not group:
raise UserError("Group doesn't exist")
else:
current_session.delete(group)