def privacy_policy(): # Check if we want to redirect out for the privacy policy. privacy_policy_url = config.get("PRIVACY_POLICY_URL") or os.environ.get( "PRIVACY_POLICY_URL") if privacy_policy_url: return flask.redirect(privacy_policy_url) global cache if not cache.has("privacy-policy-md"): file_contents = pkgutil.get_data( "fence", "static/privacy_policy.md").decode("utf-8") if not file_contents: raise NotFound("this endpoint is not configured") cache.add("privacy-policy-md", file_contents) if "text/markdown" in str(flask.request.accept_mimetypes).lower(): return flask.Response(cache.get("privacy-policy-md"), mimetype="text/markdown") else: if not cache.has("privacy-policy-html"): html = Markdown().convert(cache.get("privacy-policy-md")) if not html: raise NotFound("this endpoint is not configured") cache.add("privacy-policy-html", html) return flask.Response(cache.get("privacy-policy-html"), mimetype="text/html")
def _get_signed_url(self, protocol, action, expires_in): signed_url = None if protocol: for file_location in self.indexed_file_locations: # allow file location to be https, even if they specific http if (file_location.protocol == protocol) or (protocol == "http" and file_location.protocol == "https"): signed_url = file_location.get_signed_url( action, expires_in, public_data=self.public) if not signed_url: raise NotFound( "File {} does not have a location with specified " "protocol {}.".format(self.file_id, protocol)) # no protocol specified, return first location as signed url elif len(self.indexed_file_locations) > 0: signed_url = self.indexed_file_locations[0].get_signed_url( action, expires_in, public_data=self.public) else: # at this point, they haven't specified a protocol and we don't # have any actual locations, error out raise NotFound("Can't find any file locations.") return signed_url
def _get_signed_url(self, protocol, action, expires_in, force_signed_url, r_pays_project): if not protocol: # no protocol specified, return first location as signed url try: return self.indexed_file_locations[0].get_signed_url( action, expires_in, public_data=self.public, force_signed_url=force_signed_url, r_pays_project=r_pays_project, ) except IndexError: raise NotFound("Can't find any file locations.") for file_location in self.indexed_file_locations: # allow file location to be https, even if they specific http if (file_location.protocol == protocol) or (protocol == "http" and file_location.protocol == "https"): return file_location.get_signed_url( action, expires_in, public_data=self.public, force_signed_url=force_signed_url, r_pays_project=r_pays_project, ) raise NotFound("File {} does not have a location with specified " "protocol {}.".format(self.file_id, protocol))
def return_link(action, urls): protocol = flask.request.args.get('protocol', None) expires = min(int(flask.request.args.get('expires_in', 3600)), 3600) if (protocol is not None) and (protocol not in SUPPORTED_PROTOCOLS): raise NotSupported("The specified protocol is not supported") if len(urls) == 0: raise NotFound("Can't find any location for the data") for url in urls: location = urlparse(url) if check_protocol(protocol, location.scheme): return resolve_url(url, location, expires, action) raise NotFound( "Can't find a location for the data with given request arguments.")
def _get_or_create_storage_user(self, username, provider, session): """ Return a user. Depending on the provider, may call to get or create or just search fence's db. Args: username (str): User's name provider (str): backend provider session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db session to query for Users Returns: fence.models.User: User with username """ if provider == GOOGLE_PROVIDER: user = query_for_user(session=session, username=username.lower()) if not user: raise NotFound( "User not found with username {}. For Google Storage " "Backend user's must already exist in the db and have a " "Google Proxy Group.".format(username)) return user return self.clients[provider].get_or_create_user(username)
def get_project_info(current_session, project_name): """ Get project info from userdatamodel from its name """ proj = get_project(current_session, project_name) if not proj: msg = "".join(["Error: project ", project_name, " not found"]) raise NotFound(msg) info = { "id": proj.id, "name": proj.name, "auth_id": proj.auth_id, "description": proj.description, "associated buckets": [], } buckets = current_session.query(ProjectToBucket).filter( ProjectToBucket.project_id == proj.id ) for row in buckets: bucket = ( current_session.query(Bucket).filter(Bucket.id == row.bucket_id).first() ) info["associated buckets"].append(bucket.name) return info
def list_buckets_on_project(current_session, project_name): """ List all the buckets assigned to a project """ project = ( current_session.query(Project).filter(Project.name == project_name).first() ) if not project: msg = "".join(["Project name ", project_name, " not found"]) raise NotFound(msg) buckets = current_session.query(ProjectToBucket).filter( ProjectToBucket.project_id == project.id ) response = {"buckets": []} for bucket in buckets: buck = ( current_session.query(Bucket).filter(Bucket.id == bucket.bucket_id).first() ) provider = ( current_session.query(CloudProvider) .filter(CloudProvider.id == buck.provider_id) .first() ) new_buck = {"name": buck.name, "provider": provider.name} response["buckets"].append(new_buck) return response
def get_users_from_google_members(members, db=None): """ Get User objects for all members on a Google project by checking db. Args: members(List[cirrus.google_cloud.iam.GooglePolicyMember]): Members on the google project who are of type User Return: List[fence.models.User]: Users from our db for members on Google project Raises: NotFound: Member on google project doesn't exist in our db """ result = [] for member in members: user = get_user_from_google_member(member, db=db) if user: result.append(user) else: raise NotFound( "Google member {} does not exist as a linked Google Account.".format( member ) ) return result
def get_index_document(file_id): indexd_server = (current_app.config.get('INDEXD') or current_app.config['BASE_URL'] + '/index') url = indexd_server + '/index/' try: res = requests.get(url + file_id) except Exception as e: current_app.logger.error("failed to reach indexd at {0}: {1}".format( url + file_id, e)) raise UnavailableError( "Fail to reach id service to find data location") if res.status_code == 200: try: json_response = res.json() if 'urls' not in json_response or 'metadata' not in json_response: current_app.logger.error( 'URLs and metadata are not included in response from indexd: {}' .format(url + file_id)) raise InternalError('URLs and metadata not found') return res.json() except Exception as e: flask.current_app.logger.error( 'indexd response missing JSON field {}'.format(url + file_id)) raise InternalError('internal error from indexd: {}'.format(e)) elif res.status_code == 404: flask.current_app.logger.error( 'indexd did not find find {}; {}'.format(url + file_id, res.text)) raise NotFound("Can't find a location for the data") else: raise UnavailableError(res.text)
def index_document(self): indexd_server = config.get("INDEXD") or config["BASE_URL"] + "/index" url = indexd_server + "/index/" try: res = requests.get(url + self.file_id) except Exception as e: logger.error( "failed to reach indexd at {0}: {1}".format(url + self.file_id, e) ) raise UnavailableError("Fail to reach id service to find data location") if res.status_code == 200: try: json_response = res.json() if "urls" not in json_response: logger.error( "URLs are not included in response from " "indexd: {}".format(url + self.file_id) ) raise InternalError("URLs and metadata not found") return res.json() except Exception as e: logger.error( "indexd response missing JSON field {}".format(url + self.file_id) ) raise InternalError("internal error from indexd: {}".format(e)) elif res.status_code == 404: logger.error( "Not Found. indexd could not find {}: {}".format( url + self.file_id, res.text ) ) raise NotFound("No indexed document found with id {}".format(self.file_id)) else: raise UnavailableError(res.text)
def delete_keypair(user, current_session, access_key): result = (current_session.query(HMACKeyPair).filter( HMACKeyPair.access_key == access_key).filter( HMACKeyPair.user_id == user.id).first()) if not result: raise NotFound("Access key doesn't exist") result.archive_keypair(current_session)
def delete(self, id_): """ Delete a service account Args: id_ (str): Google service account email to delete """ user_id = current_token["sub"] service_account_email = get_service_account_email(id_) registered_service_account = get_registered_service_account_from_email( service_account_email) if not registered_service_account: raise NotFound( "Could not find a registered service account from given email {}" .format(service_account_email)) google_project_id = registered_service_account.google_project_id # check if user has permission to delete the service account with GoogleCloudManager(google_project_id) as gcm: authorized = is_user_member_of_google_project(user_id, gcm) if not authorized: return ( 'User "{}" does not have permission to delete the provided ' 'service account "{}".'.format(user_id, id_), 403, ) return self._delete(id_)
def create_bucket_on_project(current_session, project_name, bucket_name, provider_name): """ Create a bucket and assign it to a project """ project = ( current_session.query(Project).filter(Project.name == project_name).first() ) if not project: msg = "".join(["Project ", project_name, " not found"]) raise NotFound(msg) provider = ( current_session.query(CloudProvider) .filter(CloudProvider.name == provider_name) .first() ) if not provider: msg = "".join(["Provider ", provider_name, " not found"]) raise NotFound(msg) bucket = ( current_session.query(Bucket) .filter(Bucket.name == bucket_name, Bucket.provider_id == provider.id) .first() ) if not bucket: bucket = Bucket(name=bucket_name, provider_id=provider.id) current_session.add(bucket) current_session.flush() proj_to_bucket = ProjectToBucket( project_id=project.id, bucket_id=bucket.id, privilege=["owner"] ) current_session.add(proj_to_bucket) # Find the users that need to be updated users_in_project = current_session.query(AccessPrivilege).filter( AccessPrivilege.project_id == project.id ) users_to_update = [] for row in users_in_project: usr = current_session.query(User).filter(User.id == row.user_id).first() users_to_update.append((usr, row.privilege)) return { "result": "success", "provider": provider, "bucket": bucket, "users_to_update": users_to_update, } else: raise UserError("Error, name already in use for that storage system")
def remove_user_from_project(current_session, user, project): access = udm.get_user_project_access_privilege(current_session, user, project) if access: current_session.delete(access) else: raise NotFound("Project {0} not connected to user {1}".format( project.name, user.username))
def find_user(username, session): user = ( session.query(User) .filter(func.lower(User.username) == username.lower()) .first() ) if not user: raise NotFound("user {} not found".format(username)) return user
def _get_service_account_for_patch(id_): user_id = current_token["sub"] service_account_email = get_service_account_email(id_) registered_service_account = get_registered_service_account_from_email( service_account_email) if not registered_service_account: raise NotFound( "Could not find a registered service account from given email {}". format(service_account_email)) payload = flask.request.get_json(silent=True) or {} # check if the user requested to update more than project_access project_access = payload.pop("project_access", None) # if they're trying to patch more fields, error out, we only support the above if payload: raise Forbidden("Cannot update provided fields: {}".format(payload)) # if the field is not provided at all, use service accounts current access # NOTE: the user can provide project_access=[] to remove all datasets so checking # `if not project_access` here will NOT work # # In other words, to extend access you don't provide the field. To remove all # access you provide it as an empty list if project_access is None: project_access = [ access_privilege.project.auth_id for access_privilege in registered_service_account.access_privileges ] if len(project_access) > config["SERVICE_ACCOUNT_LIMIT"]: response = { "success": False, "errors": { "service_account_limit": { "status": 400, "error": "project_limit", "error_description": "Exceeded Allowable Number of Projects. Maximum {} Projects allowed per account." .format(config["SERVICE_ACCOUNT_LIMIT"]), } }, } return response, 400 google_project_id = registered_service_account.google_project_id return GoogleServiceAccountRegistration(service_account_email, project_access, google_project_id, user_id=user_id)
def _update_access_to_bucket( self, bucket, provider, storage_user, storage_username, access, session ): # Need different logic for google (since buckets can have multiple # access groups) if not provider == GOOGLE_PROVIDER: self.clients[provider].add_bucket_acl( bucket.name, storage_username, access=access ) return if not bucket.google_bucket_access_groups: raise NotFound( "Google bucket {} does not have any access groups.".format(bucket.name) ) access = StorageManager._get_bucket_access_privileges(access) for bucket_access_group in bucket.google_bucket_access_groups: bucket_privileges = bucket_access_group.privileges or [] if set(bucket_privileges).issubset(access): bucket_name = bucket_access_group.email # NOTE: bucket_name for Google is the Google Access Group's # email address. # TODO Update storageclient API for more clarity self.clients[provider].add_bucket_acl(bucket_name, storage_username) self.logger.info( "User {}'s Google proxy group ({}) added to Google Bucket Access Group {}.".format( storage_user.email, storage_username, bucket_name ) ) StorageManager._add_google_db_entry_for_bucket_access( storage_user, bucket_access_group, session ) else: # In the case of google, since we have multiple groups # with access to the bucket, we need to also remove access # here in case a users permissions change from read & write # to just read. StorageManager._remove_google_db_entry_for_bucket_access( storage_user, bucket_access_group, session ) bucket_name = bucket_access_group.email self.clients[provider].delete_bucket_acl(bucket_name, storage_username) self.logger.info( "User {}'s Google proxy group ({}) removed or never existed in Google Bucket Access Group {}.".format( storage_user.email, storage_username, bucket_name ) )
def _get_signed_url( self, protocol, action, expires_in, force_signed_url, r_pays_project, file_name ): if action == "upload": # NOTE: self.index_document ensures the GUID exists in indexd and raises # an error if not (which is expected to be caught upstream in the # app) blank_record = BlankIndex(uploader="", guid=self.index_document.get("did")) return blank_record.make_signed_url( file_name=file_name, expires_in=expires_in ) if not protocol: # no protocol specified, return first location as signed url try: return self.indexed_file_locations[0].get_signed_url( action, expires_in, public_data=self.public, force_signed_url=force_signed_url, r_pays_project=r_pays_project, ) except IndexError: raise NotFound("Can't find any file locations.") for file_location in self.indexed_file_locations: # allow file location to be https, even if they specific http if (file_location.protocol == protocol) or ( protocol == "http" and file_location.protocol == "https" ): return file_location.get_signed_url( action, expires_in, public_data=self.public, force_signed_url=force_signed_url, r_pays_project=r_pays_project, ) raise NotFound( "File {} does not have a location with specified " "protocol {}.".format(self.file_id, protocol) )
def remove_project_from_group(current_session, group, project): to_be_removed = udm.get_project_group_access_privilege( current_session, project, group) if to_be_removed: current_session.delete(to_be_removed) msg = "Project: {0} SUCCESFULLY removed from Group: {1}".format( project.name, group.name) return {"result": msg} else: raise NotFound("Project {0} and Group {1} are not linked".format( project.name, group.name))
def revoke_user_policies(user_id): """ Revoke all the policies which this user has access to. """ with flask.current_app.db.session as session: user = session.query(User).filter_by(id=user_id).first() if not user: raise NotFound("no user exists with ID: {}".format(user_id)) # Set user's policies to empty list. user.policies = [] session.commit() return "", 204
def revoke_user_policy(user_id, policy_id): """ Revoke a specific policy (the policy ID in the path) granted to a user (specified by user ID in the path). """ with flask.current_app.db.session as session: user = session.query(User).filter_by(User.id == user_id).first() if not user: raise NotFound("no user exists with ID: {}".format(user_id)) user.policies.remove(policy_id) session.flush() return "", 204
def remove_user_from_group(current_session, user, group): to_be_removed = udm.get_user_group_access_privilege( current_session, user, group) if to_be_removed: current_session.delete(to_be_removed) return { "result": ("User: {0} SUCCESFULLY " "removed from Group: {1}".format(user.username, group.name)) } else: raise NotFound("User {0} and Group {1} are not linked".format( user.username, group.name))
def delete_bucket_on_project(current_session, project_name, bucket_name): """ Remove a bucket and its relationship to a project """ bucket = current_session.query(Bucket).filter_by(name=bucket_name).first() if not bucket: msg = "".join(["Bucket name ", bucket_name, " not found"]) raise NotFound(msg) provider = ( current_session.query(CloudProvider) .filter(CloudProvider.id == bucket.provider_id) .first() ) project = ( current_session.query(Project).filter(Project.name == project_name).first() ) if not project: msg = "".join(["Project name ", project_name, " not found"]) raise NotFound(msg) proj_to_bucket = ( current_session.query(ProjectToBucket) .filter( ProjectToBucket.bucket_id == bucket.id, ProjectToBucket.project_id == project.id, ) .first() ) if proj_to_bucket: current_session.delete(proj_to_bucket) current_session.delete(bucket) return {"result": "success", "provider": provider} else: current_session.delete(bucket) msg = ( "WARNING: Project-to-bucket " "relationship not found, deleting bucket anyway" ) return {"result": msg, "provider": provider}
def download_certificate(certificate): if not flask.g.user.application: flask.g.user.application = Application() current_session.merge(flask.g.user) cert = (current_session.query(Certificate).filter( Certificate.name == certificate).filter( Certificate.application_id == flask.g.user.application.id).first()) if cert: resp = flask.make_response(cert.data) resp.headers['Content-Type'] = 'application/octet-stream' resp.headers['Content-Disposition'] =\ 'attachment; filename={}.{}'.format(cert.name, cert.extension) return resp else: raise NotFound('No certificate with name {} found'.format(certificate))
def send_email(from_email, to_emails, subject, text, smtp_domain): """ Send email to group of emails using mail gun api. https://app.mailgun.com/ Args: from_email(str): from email to_emails(list): list of emails to receive the messages text(str): the text message smtp_domain(dict): smtp domain server { "smtp_hostname": "smtp.mailgun.org", "default_login": "******", "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net", "smtp_password": "******", "api_key": "api key" } Returns: Http response Exceptions: KeyError """ if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get( smtp_domain).get("smtp_password"): raise NotFound( "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or " "smtp_password was not provided. " "Cannot send email.".format(smtp_domain)) api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "") email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages" return requests.post( email_url, auth=("api", api_key), data={ "from": from_email, "to": to_emails, "subject": subject, "text": text }, )
def replace_user_policies(user_id): """ Overwrite the user's existing policies and replace them with the ones provided in the request. """ policy_ids = _validate_policy_ids(flask.request.get_json().get("policies")) with flask.current_app.db.session as session: policies = lookup_policies(policy_ids) user = session.query(User).filter_by(id=user_id).first() if not user: raise NotFound("no user exists with ID: {}".format(user_id)) user.policies = policies session.commit() return "", 204
def _get_user_policy_ids(user_id): """ List the IDs for policies which are granted to this user, according to the fence database. Args: user_id (str): the id for a user Return: List[str]: list of policies granted to the user """ with flask.current_app.db.session as session: user = session.query(User).filter(User.id == user_id).first() if not user: raise NotFound("no user exists with ID: {}".format(user_id)) return [policy.id for policy in user.policies]
def get_provider(current_session, provider_name): """ Get the provider info from the userdatamodel """ provider = (current_session.query(CloudProvider).filter( CloudProvider.name == provider_name).first()) if not provider: msg = "".join(["error, cloud provider ", provider_name, " not found"]) raise NotFound(msg) info = { "name": provider.name, "backend": provider.backend, "endpoint": provider.endpoint, "description": provider.description, "service": provider.service, } return info
def create_project(current_session, name, auth_id, storage_accesses): """ Creates a project with an associated auth_id and storage access """ new_project = Project(name=name, auth_id=auth_id) current_session.add(new_project) current_session.flush() for storage in storage_accesses: provider = current_session.query(CloudProvider).filter( CloudProvider.name == storage).first() if provider: new_storage_access = StorageAccess(provider_id=provider.id, project_id=new_project.id) current_session.add(new_storage_access) else: raise NotFound() return new_project
def get_service_account_policy(account, google_cloud_manager): """ Get the policy for the service account identified by `account`, using the provided cloud_manager Args: account(str): service account identifier google_cloud_manager: cloud_manager instance Returns: (Response): returns response from Google API """ sa_policy = google_cloud_manager.get_service_account_policy(account) if sa_policy.status_code != 200: raise NotFound( "Unable to get Service Account policy (status: {})".format( sa_policy.status_code)) else: return sa_policy