def privacy_policy():
# Check if we want to redirect out for the privacy policy.
privacy_policy_url = config.get("PRIVACY_POLICY_URL") or os.environ.get(
"PRIVACY_POLICY_URL")
if privacy_policy_url:
return flask.redirect(privacy_policy_url)
global cache
if not cache.has("privacy-policy-md"):
file_contents = pkgutil.get_data(
"fence", "static/privacy_policy.md").decode("utf-8")
if not file_contents:
raise NotFound("this endpoint is not configured")
cache.add("privacy-policy-md", file_contents)
if "text/markdown" in str(flask.request.accept_mimetypes).lower():
return flask.Response(cache.get("privacy-policy-md"),
mimetype="text/markdown")
else:
if not cache.has("privacy-policy-html"):
html = Markdown().convert(cache.get("privacy-policy-md"))
if not html:
raise NotFound("this endpoint is not configured")
cache.add("privacy-policy-html", html)
return flask.Response(cache.get("privacy-policy-html"),
mimetype="text/html")
def _get_signed_url(self, protocol, action, expires_in):
signed_url = None
if protocol:
for file_location in self.indexed_file_locations:
# allow file location to be https, even if they specific http
if (file_location.protocol
== protocol) or (protocol == "http" and
file_location.protocol == "https"):
signed_url = file_location.get_signed_url(
action, expires_in, public_data=self.public)
if not signed_url:
raise NotFound(
"File {} does not have a location with specified "
"protocol {}.".format(self.file_id, protocol))
# no protocol specified, return first location as signed url
elif len(self.indexed_file_locations) > 0:
signed_url = self.indexed_file_locations[0].get_signed_url(
action, expires_in, public_data=self.public)
else:
# at this point, they haven't specified a protocol and we don't
# have any actual locations, error out
raise NotFound("Can't find any file locations.")
return signed_url
def _get_signed_url(self, protocol, action, expires_in, force_signed_url,
r_pays_project):
if not protocol:
# no protocol specified, return first location as signed url
try:
return self.indexed_file_locations[0].get_signed_url(
action,
expires_in,
public_data=self.public,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
)
except IndexError:
raise NotFound("Can't find any file locations.")
for file_location in self.indexed_file_locations:
# allow file location to be https, even if they specific http
if (file_location.protocol
== protocol) or (protocol == "http"
and file_location.protocol == "https"):
return file_location.get_signed_url(
action,
expires_in,
public_data=self.public,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
)
raise NotFound("File {} does not have a location with specified "
"protocol {}.".format(self.file_id, protocol))
def return_link(action, urls):
protocol = flask.request.args.get('protocol', None)
expires = min(int(flask.request.args.get('expires_in', 3600)), 3600)
if (protocol is not None) and (protocol not in SUPPORTED_PROTOCOLS):
raise NotSupported("The specified protocol is not supported")
if len(urls) == 0:
raise NotFound("Can't find any location for the data")
for url in urls:
location = urlparse(url)
if check_protocol(protocol, location.scheme):
return resolve_url(url, location, expires, action)
raise NotFound(
"Can't find a location for the data with given request arguments.")
def _get_or_create_storage_user(self, username, provider, session):
"""
Return a user.
Depending on the provider, may call to get or create or just
search fence's db.
Args:
username (str): User's name
provider (str): backend provider
session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db
session to query for Users
Returns:
fence.models.User: User with username
"""
if provider == GOOGLE_PROVIDER:
user = query_for_user(session=session, username=username.lower())
if not user:
raise NotFound(
"User not found with username {}. For Google Storage "
"Backend user's must already exist in the db and have a "
"Google Proxy Group.".format(username))
return user
return self.clients[provider].get_or_create_user(username)
def get_project_info(current_session, project_name):
"""
Get project info from userdatamodel
from its name
"""
proj = get_project(current_session, project_name)
if not proj:
msg = "".join(["Error: project ", project_name, " not found"])
raise NotFound(msg)
info = {
"id": proj.id,
"name": proj.name,
"auth_id": proj.auth_id,
"description": proj.description,
"associated buckets": [],
}
buckets = current_session.query(ProjectToBucket).filter(
ProjectToBucket.project_id == proj.id
)
for row in buckets:
bucket = (
current_session.query(Bucket).filter(Bucket.id == row.bucket_id).first()
)
info["associated buckets"].append(bucket.name)
return info
def list_buckets_on_project(current_session, project_name):
"""
List all the buckets assigned to a project
"""
project = (
current_session.query(Project).filter(Project.name == project_name).first()
)
if not project:
msg = "".join(["Project name ", project_name, " not found"])
raise NotFound(msg)
buckets = current_session.query(ProjectToBucket).filter(
ProjectToBucket.project_id == project.id
)
response = {"buckets": []}
for bucket in buckets:
buck = (
current_session.query(Bucket).filter(Bucket.id == bucket.bucket_id).first()
)
provider = (
current_session.query(CloudProvider)
.filter(CloudProvider.id == buck.provider_id)
.first()
)
new_buck = {"name": buck.name, "provider": provider.name}
response["buckets"].append(new_buck)
return response
def get_users_from_google_members(members, db=None):
"""
Get User objects for all members on a Google project by checking db.
Args:
members(List[cirrus.google_cloud.iam.GooglePolicyMember]): Members on
the google project who are of type User
Return:
List[fence.models.User]: Users from our db for members on Google project
Raises:
NotFound: Member on google project doesn't exist in our db
"""
result = []
for member in members:
user = get_user_from_google_member(member, db=db)
if user:
result.append(user)
else:
raise NotFound(
"Google member {} does not exist as a linked Google Account.".format(
member
)
)
return result
def get_index_document(file_id):
indexd_server = (current_app.config.get('INDEXD')
or current_app.config['BASE_URL'] + '/index')
url = indexd_server + '/index/'
try:
res = requests.get(url + file_id)
except Exception as e:
current_app.logger.error("failed to reach indexd at {0}: {1}".format(
url + file_id, e))
raise UnavailableError(
"Fail to reach id service to find data location")
if res.status_code == 200:
try:
json_response = res.json()
if 'urls' not in json_response or 'metadata' not in json_response:
current_app.logger.error(
'URLs and metadata are not included in response from indexd: {}'
.format(url + file_id))
raise InternalError('URLs and metadata not found')
return res.json()
except Exception as e:
flask.current_app.logger.error(
'indexd response missing JSON field {}'.format(url + file_id))
raise InternalError('internal error from indexd: {}'.format(e))
elif res.status_code == 404:
flask.current_app.logger.error(
'indexd did not find find {}; {}'.format(url + file_id, res.text))
raise NotFound("Can't find a location for the data")
else:
raise UnavailableError(res.text)
def index_document(self):
indexd_server = config.get("INDEXD") or config["BASE_URL"] + "/index"
url = indexd_server + "/index/"
try:
res = requests.get(url + self.file_id)
except Exception as e:
logger.error(
"failed to reach indexd at {0}: {1}".format(url + self.file_id, e)
)
raise UnavailableError("Fail to reach id service to find data location")
if res.status_code == 200:
try:
json_response = res.json()
if "urls" not in json_response:
logger.error(
"URLs are not included in response from "
"indexd: {}".format(url + self.file_id)
)
raise InternalError("URLs and metadata not found")
return res.json()
except Exception as e:
logger.error(
"indexd response missing JSON field {}".format(url + self.file_id)
)
raise InternalError("internal error from indexd: {}".format(e))
elif res.status_code == 404:
logger.error(
"Not Found. indexd could not find {}: {}".format(
url + self.file_id, res.text
)
)
raise NotFound("No indexed document found with id {}".format(self.file_id))
else:
raise UnavailableError(res.text)
def delete_keypair(user, current_session, access_key):
result = (current_session.query(HMACKeyPair).filter(
HMACKeyPair.access_key == access_key).filter(
HMACKeyPair.user_id == user.id).first())
if not result:
raise NotFound("Access key doesn't exist")
result.archive_keypair(current_session)
def delete(self, id_):
"""
Delete a service account
Args:
id_ (str): Google service account email to delete
"""
user_id = current_token["sub"]
service_account_email = get_service_account_email(id_)
registered_service_account = get_registered_service_account_from_email(
service_account_email)
if not registered_service_account:
raise NotFound(
"Could not find a registered service account from given email {}"
.format(service_account_email))
google_project_id = registered_service_account.google_project_id
# check if user has permission to delete the service account
with GoogleCloudManager(google_project_id) as gcm:
authorized = is_user_member_of_google_project(user_id, gcm)
if not authorized:
return (
'User "{}" does not have permission to delete the provided '
'service account "{}".'.format(user_id, id_),
403,
)
return self._delete(id_)
def create_bucket_on_project(current_session, project_name, bucket_name, provider_name):
"""
Create a bucket and assign it to a project
"""
project = (
current_session.query(Project).filter(Project.name == project_name).first()
)
if not project:
msg = "".join(["Project ", project_name, " not found"])
raise NotFound(msg)
provider = (
current_session.query(CloudProvider)
.filter(CloudProvider.name == provider_name)
.first()
)
if not provider:
msg = "".join(["Provider ", provider_name, " not found"])
raise NotFound(msg)
bucket = (
current_session.query(Bucket)
.filter(Bucket.name == bucket_name, Bucket.provider_id == provider.id)
.first()
)
if not bucket:
bucket = Bucket(name=bucket_name, provider_id=provider.id)
current_session.add(bucket)
current_session.flush()
proj_to_bucket = ProjectToBucket(
project_id=project.id, bucket_id=bucket.id, privilege=["owner"]
)
current_session.add(proj_to_bucket)
# Find the users that need to be updated
users_in_project = current_session.query(AccessPrivilege).filter(
AccessPrivilege.project_id == project.id
)
users_to_update = []
for row in users_in_project:
usr = current_session.query(User).filter(User.id == row.user_id).first()
users_to_update.append((usr, row.privilege))
return {
"result": "success",
"provider": provider,
"bucket": bucket,
"users_to_update": users_to_update,
}
else:
raise UserError("Error, name already in use for that storage system")
def remove_user_from_project(current_session, user, project):
access = udm.get_user_project_access_privilege(current_session, user,
project)
if access:
current_session.delete(access)
else:
raise NotFound("Project {0} not connected to user {1}".format(
project.name, user.username))
def find_user(username, session):
user = (
session.query(User)
.filter(func.lower(User.username) == username.lower())
.first()
)
if not user:
raise NotFound("user {} not found".format(username))
return user
def _get_service_account_for_patch(id_):
user_id = current_token["sub"]
service_account_email = get_service_account_email(id_)
registered_service_account = get_registered_service_account_from_email(
service_account_email)
if not registered_service_account:
raise NotFound(
"Could not find a registered service account from given email {}".
format(service_account_email))
payload = flask.request.get_json(silent=True) or {}
# check if the user requested to update more than project_access
project_access = payload.pop("project_access", None)
# if they're trying to patch more fields, error out, we only support the above
if payload:
raise Forbidden("Cannot update provided fields: {}".format(payload))
# if the field is not provided at all, use service accounts current access
# NOTE: the user can provide project_access=[] to remove all datasets so checking
# `if not project_access` here will NOT work
#
# In other words, to extend access you don't provide the field. To remove all
# access you provide it as an empty list
if project_access is None:
project_access = [
access_privilege.project.auth_id for access_privilege in
registered_service_account.access_privileges
]
if len(project_access) > config["SERVICE_ACCOUNT_LIMIT"]:
response = {
"success": False,
"errors": {
"service_account_limit": {
"status":
400,
"error":
"project_limit",
"error_description":
"Exceeded Allowable Number of Projects. Maximum {} Projects allowed per account."
.format(config["SERVICE_ACCOUNT_LIMIT"]),
}
},
}
return response, 400
google_project_id = registered_service_account.google_project_id
return GoogleServiceAccountRegistration(service_account_email,
project_access,
google_project_id,
user_id=user_id)
def _update_access_to_bucket(
self, bucket, provider, storage_user, storage_username, access, session
):
# Need different logic for google (since buckets can have multiple
# access groups)
if not provider == GOOGLE_PROVIDER:
self.clients[provider].add_bucket_acl(
bucket.name, storage_username, access=access
)
return
if not bucket.google_bucket_access_groups:
raise NotFound(
"Google bucket {} does not have any access groups.".format(bucket.name)
)
access = StorageManager._get_bucket_access_privileges(access)
for bucket_access_group in bucket.google_bucket_access_groups:
bucket_privileges = bucket_access_group.privileges or []
if set(bucket_privileges).issubset(access):
bucket_name = bucket_access_group.email
# NOTE: bucket_name for Google is the Google Access Group's
# email address.
# TODO Update storageclient API for more clarity
self.clients[provider].add_bucket_acl(bucket_name, storage_username)
self.logger.info(
"User {}'s Google proxy group ({}) added to Google Bucket Access Group {}.".format(
storage_user.email, storage_username, bucket_name
)
)
StorageManager._add_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
)
else:
# In the case of google, since we have multiple groups
# with access to the bucket, we need to also remove access
# here in case a users permissions change from read & write
# to just read.
StorageManager._remove_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
)
bucket_name = bucket_access_group.email
self.clients[provider].delete_bucket_acl(bucket_name, storage_username)
self.logger.info(
"User {}'s Google proxy group ({}) removed or never existed in Google Bucket Access Group {}.".format(
storage_user.email, storage_username, bucket_name
)
)
def _get_signed_url(
self, protocol, action, expires_in, force_signed_url, r_pays_project, file_name
):
if action == "upload":
# NOTE: self.index_document ensures the GUID exists in indexd and raises
# an error if not (which is expected to be caught upstream in the
# app)
blank_record = BlankIndex(uploader="", guid=self.index_document.get("did"))
return blank_record.make_signed_url(
file_name=file_name, expires_in=expires_in
)
if not protocol:
# no protocol specified, return first location as signed url
try:
return self.indexed_file_locations[0].get_signed_url(
action,
expires_in,
public_data=self.public,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
)
except IndexError:
raise NotFound("Can't find any file locations.")
for file_location in self.indexed_file_locations:
# allow file location to be https, even if they specific http
if (file_location.protocol == protocol) or (
protocol == "http" and file_location.protocol == "https"
):
return file_location.get_signed_url(
action,
expires_in,
public_data=self.public,
force_signed_url=force_signed_url,
r_pays_project=r_pays_project,
)
raise NotFound(
"File {} does not have a location with specified "
"protocol {}.".format(self.file_id, protocol)
)
def remove_project_from_group(current_session, group, project):
to_be_removed = udm.get_project_group_access_privilege(
current_session, project, group)
if to_be_removed:
current_session.delete(to_be_removed)
msg = "Project: {0} SUCCESFULLY removed from Group: {1}".format(
project.name, group.name)
return {"result": msg}
else:
raise NotFound("Project {0} and Group {1} are not linked".format(
project.name, group.name))
def revoke_user_policies(user_id):
"""
Revoke all the policies which this user has access to.
"""
with flask.current_app.db.session as session:
user = session.query(User).filter_by(id=user_id).first()
if not user:
raise NotFound("no user exists with ID: {}".format(user_id))
# Set user's policies to empty list.
user.policies = []
session.commit()
return "", 204
def revoke_user_policy(user_id, policy_id):
"""
Revoke a specific policy (the policy ID in the path) granted to a user
(specified by user ID in the path).
"""
with flask.current_app.db.session as session:
user = session.query(User).filter_by(User.id == user_id).first()
if not user:
raise NotFound("no user exists with ID: {}".format(user_id))
user.policies.remove(policy_id)
session.flush()
return "", 204
def remove_user_from_group(current_session, user, group):
to_be_removed = udm.get_user_group_access_privilege(
current_session, user, group)
if to_be_removed:
current_session.delete(to_be_removed)
return {
"result":
("User: {0} SUCCESFULLY "
"removed from Group: {1}".format(user.username, group.name))
}
else:
raise NotFound("User {0} and Group {1} are not linked".format(
user.username, group.name))
def delete_bucket_on_project(current_session, project_name, bucket_name):
"""
Remove a bucket and its relationship to a project
"""
bucket = current_session.query(Bucket).filter_by(name=bucket_name).first()
if not bucket:
msg = "".join(["Bucket name ", bucket_name, " not found"])
raise NotFound(msg)
provider = (
current_session.query(CloudProvider)
.filter(CloudProvider.id == bucket.provider_id)
.first()
)
project = (
current_session.query(Project).filter(Project.name == project_name).first()
)
if not project:
msg = "".join(["Project name ", project_name, " not found"])
raise NotFound(msg)
proj_to_bucket = (
current_session.query(ProjectToBucket)
.filter(
ProjectToBucket.bucket_id == bucket.id,
ProjectToBucket.project_id == project.id,
)
.first()
)
if proj_to_bucket:
current_session.delete(proj_to_bucket)
current_session.delete(bucket)
return {"result": "success", "provider": provider}
else:
current_session.delete(bucket)
msg = (
"WARNING: Project-to-bucket "
"relationship not found, deleting bucket anyway"
)
return {"result": msg, "provider": provider}
def download_certificate(certificate):
if not flask.g.user.application:
flask.g.user.application = Application()
current_session.merge(flask.g.user)
cert = (current_session.query(Certificate).filter(
Certificate.name == certificate).filter(
Certificate.application_id == flask.g.user.application.id).first())
if cert:
resp = flask.make_response(cert.data)
resp.headers['Content-Type'] = 'application/octet-stream'
resp.headers['Content-Disposition'] =\
'attachment; filename={}.{}'.format(cert.name, cert.extension)
return resp
else:
raise NotFound('No certificate with name {} found'.format(certificate))
def send_email(from_email, to_emails, subject, text, smtp_domain):
"""
Send email to group of emails using mail gun api.
https://app.mailgun.com/
Args:
from_email(str): from email
to_emails(list): list of emails to receive the messages
text(str): the text message
smtp_domain(dict): smtp domain server
{
"smtp_hostname": "smtp.mailgun.org",
"default_login": "******",
"api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net",
"smtp_password": "******",
"api_key": "api key"
}
Returns:
Http response
Exceptions:
KeyError
"""
if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get(
smtp_domain).get("smtp_password"):
raise NotFound(
"SMTP Domain '{}' does not exist in configuration for GUN_MAIL or "
"smtp_password was not provided. "
"Cannot send email.".format(smtp_domain))
api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "")
email_url = config["GUN_MAIL"][smtp_domain].get("api_url",
"") + "/messages"
return requests.post(
email_url,
auth=("api", api_key),
data={
"from": from_email,
"to": to_emails,
"subject": subject,
"text": text
},
)
def replace_user_policies(user_id):
"""
Overwrite the user's existing policies and replace them with the ones
provided in the request.
"""
policy_ids = _validate_policy_ids(flask.request.get_json().get("policies"))
with flask.current_app.db.session as session:
policies = lookup_policies(policy_ids)
user = session.query(User).filter_by(id=user_id).first()
if not user:
raise NotFound("no user exists with ID: {}".format(user_id))
user.policies = policies
session.commit()
return "", 204
def _get_user_policy_ids(user_id):
"""
List the IDs for policies which are granted to this user, according to the
fence database.
Args:
user_id (str): the id for a user
Return:
List[str]: list of policies granted to the user
"""
with flask.current_app.db.session as session:
user = session.query(User).filter(User.id == user_id).first()
if not user:
raise NotFound("no user exists with ID: {}".format(user_id))
return [policy.id for policy in user.policies]
def get_provider(current_session, provider_name):
"""
Get the provider info from the userdatamodel
"""
provider = (current_session.query(CloudProvider).filter(
CloudProvider.name == provider_name).first())
if not provider:
msg = "".join(["error, cloud provider ", provider_name, " not found"])
raise NotFound(msg)
info = {
"name": provider.name,
"backend": provider.backend,
"endpoint": provider.endpoint,
"description": provider.description,
"service": provider.service,
}
return info
def create_project(current_session, name, auth_id, storage_accesses):
"""
Creates a project with an associated auth_id and storage access
"""
new_project = Project(name=name, auth_id=auth_id)
current_session.add(new_project)
current_session.flush()
for storage in storage_accesses:
provider = current_session.query(CloudProvider).filter(
CloudProvider.name == storage).first()
if provider:
new_storage_access = StorageAccess(provider_id=provider.id,
project_id=new_project.id)
current_session.add(new_storage_access)
else:
raise NotFound()
return new_project
def get_service_account_policy(account, google_cloud_manager):
"""
Get the policy for the service account identified by `account`,
using the provided cloud_manager
Args:
account(str): service account identifier
google_cloud_manager: cloud_manager instance
Returns:
(Response): returns response from Google API
"""
sa_policy = google_cloud_manager.get_service_account_policy(account)
if sa_policy.status_code != 200:
raise NotFound(
"Unable to get Service Account policy (status: {})".format(
sa_policy.status_code))
else:
return sa_policy