def test_sync_revoke(syncer, db_session, storage_client): phsids = { "userA": { "phs000178": {"read", "read-storage"}, "phs000179": {"read", "read-storage", "write-storage"}, }, "userB": {"phs000179": {"read", "read-storage", "write-storage"}}, } userinfo = { "userA": {"email": "a@b", "tags": {}}, "userB": {"email": "a@b", "tags": {}}, } phsids2 = {"userA": {"phs000179": {"read", "read-storage", "write-storage"}}} syncer.sync_to_db_and_storage_backend(phsids, userinfo, db_session) syncer.sync_to_db_and_storage_backend(phsids2, userinfo, db_session) user_B = models.query_for_user(session=db_session, username="******") n_access_privilege = ( db_session.query(models.AccessPrivilege).filter_by(user_id=user_B.id).count() ) if n_access_privilege: raise AssertionError()
def _get_or_create_storage_user(self, username, provider, session): """ Return a user. Depending on the provider, may call to get or create or just search fence's db. Args: username (str): User's name provider (str): backend provider session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db session to query for Users Returns: fence.models.User: User with username """ if provider == GOOGLE_PROVIDER: user = query_for_user(session=session, username=username.lower()) if not user: raise NotFound( "User not found with username {}. For Google Storage " "Backend user's must already exist in the db and have a " "Google Proxy Group.".format(username)) return user return self.clients[provider].get_or_create_user(username)
def test_map_iss_sub_pair_to_user_with_no_prior_DRS_access(db_session): """ Test RASOauth2Client.map_iss_sub_pair_to_user when the username passed in (e.g. eRA username) does not already exist in the Fence database and that user's <iss, sub> combination has not already been mapped through a prior DRS access request. """ # reset users table db_session.query(User).delete() db_session.commit() iss = "https://domain.tld" sub = "123_abc" username = "******" email = "*****@*****.**" oidc = config.get("OPENID_CONNECT", {}) ras_client = RASClient( oidc["ras"], HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger, ) assert not query_for_user(db_session, username) iss_sub_pair_to_user_records = db_session.query(IssSubPairToUser).all() assert len(iss_sub_pair_to_user_records) == 0 username_to_log_in = ras_client.map_iss_sub_pair_to_user( iss, sub, username, email, db_session=db_session) assert username_to_log_in == username iss_sub_pair_to_user = db_session.query(IssSubPairToUser).get((iss, sub)) assert iss_sub_pair_to_user.user.username == username assert iss_sub_pair_to_user.user.email == email iss_sub_pair_to_user_records = db_session.query(IssSubPairToUser).all() assert len(iss_sub_pair_to_user_records) == 1
def create_user(users, db_session, is_admin=False): s = db_session for username in list(users.keys()): user = query_for_user(session=s, username=username) if not user: user = User(username=username, is_admin=is_admin) s.add(user) for project_data in users[username]["projects"]: privilege = project_data["privilege"] auth_id = project_data["auth_id"] p_name = project_data.get("name", auth_id) project = s.query(Project).filter( Project.auth_id == auth_id).first() if not project: project = Project(name=p_name, auth_id=auth_id) s.add(project) ap = (s.query(AccessPrivilege).join( AccessPrivilege.project).join(AccessPrivilege.user).filter( Project.name == p_name, User.username == user.username).first()) if not ap: ap = AccessPrivilege(project=project, user=user, privilege=privilege) s.add(ap) else: ap.privilege = privilege return user.id, user.username
def _grant_from_db(self, sess, to_add, user_info, user_project, auth_provider_list): """ Grant user access to projects in the auth database Args: sess: sqlalchemy session to_add: a set of (username, project.auth_id) to be granted user_project: a dictionary of {username: {project: {'read','write'}} Return: None """ for (username, project_auth_id) in to_add: u = query_for_user(session=sess, username=username) auth_provider = auth_provider_list[0] if "dbgap_role" not in user_info[username]["tags"]: auth_provider = auth_provider_list[1] user_access = AccessPrivilege( user=u, project=self._projects[project_auth_id], privilege=list(user_project[username][project_auth_id]), auth_provider=auth_provider, ) self.logger.info("grant user {} to {} with access {}".format( username, user_access.project, user_access.privilege)) sess.add(user_access)
def create_users_with_group(DB, s, data): providers = {} data_groups = data.get("groups", {}) users = data.get("users", {}) for username, data in users.items(): is_existing_user = True user = query_for_user(session=s, username=username) admin = data.get("admin", False) if not user: is_existing_user = False provider_name = data.get("provider", "google") provider = providers.get(provider_name) if not provider: provider = (s.query(IdentityProvider).filter( IdentityProvider.name == provider_name).first()) providers[provider_name] = provider if not provider: raise Exception( "provider {} not found".format(provider_name)) user = User(username=username, idp_id=provider.id, is_admin=admin) user.is_admin = admin group_names = data.get("groups", []) for group_name in group_names: assign_group_to_user(s, user, group_name, data_groups[group_name]) projects = data.get("projects", []) for project in projects: grant_project_to_group_or_user(s, project, user=user) if not is_existing_user: s.add(user) for client in data.get("clients", []): create_client_action(DB, username=username, **client)
def create_refresh_token(self): """ Create a new refresh token and add its entry to the database. Return: JWTResult: the refresh token result """ driver = SQLAlchemyDriver(self.db) with driver.session as current_session: user = query_for_user(session=current_session, username=self.username) if not user: raise EnvironmentError( "no user found with given username: "******"jti"], userid=user.id, expires=jwt_result.claims["exp"], ) ) return jwt_result
def create_client( username, urls, DB, name="", description="", auto_approve=False, is_admin=False, grant_types=None, confidential=True, arborist=None, policies=None, allowed_scopes=None, ): client_id = random_str(40) if arborist is not None: arborist.create_client(client_id, policies) grant_types = grant_types driver = SQLAlchemyDriver(DB) client_secret = None hashed_secret = None if confidential: client_secret = random_str(55) hashed_secret = bcrypt.hashpw(client_secret.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") auth_method = "client_secret_basic" if confidential else "none" allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"] if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])): raise ValueError("Each allowed scope must be one of: {}".format( config["CLIENT_ALLOWED_SCOPES"])) if "openid" not in allowed_scopes: allowed_scopes.append("openid") logger.warning( 'Adding required "openid" scope to list of allowed scopes.') with driver.session as s: user = query_for_user(session=s, username=username) if not user: user = User(username=username, is_admin=is_admin) s.add(user) if s.query(Client).filter(Client.name == name).first(): if arborist is not None: arborist.delete_client(client_id) raise Exception("client {} already exists".format(name)) client = Client( client_id=client_id, client_secret=hashed_secret, user=user, redirect_uris=urls, _allowed_scopes=" ".join(allowed_scopes), description=description, name=name, auto_approve=auto_approve, grant_types=grant_types, is_confidential=confidential, token_endpoint_auth_method=auth_method, ) s.add(client) s.commit() return client_id, client_secret
def test_sync_from_files(syncer, db_session, storage_client): sess = db_session phsids = { "userA": { "phs000178": {"read", "read-storage"}, "phs000179": {"read", "read-storage", "write-storage"}, }, "userB": { "phs000179": {"read", "read-storage", "write-storage"} }, } userinfo = { "userA": { "email": "a@b", "tags": {} }, "userB": { "email": "a@b", "tags": {} }, } syncer.sync_to_db_and_storage_backend(phsids, userinfo, sess) u = models.query_for_user(session=db_session, username="******") assert equal_project_access( u.project_access, {"phs000179": ["read", "read-storage", "write-storage"]})
def create_awg_user(users, db_session): s = db_session for username in list(users.keys()): user = query_for_user(session=s, username=username) if not user: user = User(username=username) s.add(user) projects = {} for project_data in users[username]["projects"]: auth_id = project_data["auth_id"] p_name = project_data.get("name", auth_id) project = s.query(Project).filter(Project.auth_id == auth_id).first() if not project: project = Project(name=p_name, auth_id=auth_id) s.add(project) projects[p_name] = project groups = users[username].get("groups", []) for group in groups: group_name = group["name"] group_desc = group["description"] grp = s.query(Group).filter(Group.name == group_name).first() if not grp: grp = Group() grp.name = group_name grp.description = group_desc s.add(grp) s.flush() UserToGroup(group=grp, user=user) for projectname in group["projects"]: gap = ( s.query(AccessPrivilege) .join(AccessPrivilege.project) .join(AccessPrivilege.group) .filter(Project.name == projectname, Group.name == group_name) .first() ) if not gap: project = projects[projectname] gap = AccessPrivilege(project_id=project.id, group_id=grp.id) s.add(gap) s.flush() ap = ( s.query(AccessPrivilege) .join(AccessPrivilege.project) .join(AccessPrivilege.user) .filter(Project.name == projectname, User.username == user.username) .first() ) privilege = {"read"} if not ap: project = projects[projectname] ap = AccessPrivilege( project=project, user=user, privilege=privilege ) s.add(ap) s.flush() return user.id, user.username
def test_sync_in_login( syncer, db_session, storage_client, rsa_private_key, kid, monkeypatch, ): user = models.query_for_user( session=db_session, username="******") # contains no information syncer.sync_single_user_visas(user, db_session) user = models.query_for_user( session=db_session, username="******") # contains only visa information user1 = models.query_for_user(session=db_session, username="******") assert len(user1.project_access) == 0 # other users are not affected assert len(user.project_access) == 6
def get_current_user(flask_session=None): flask_session = flask_session or flask.session username = flask_session.get("username") if config.get("MOCK_AUTH", False) is True: username = "******" if not username: raise Unauthorized("User not logged in") return query_for_user(session=current_session, username=username)
def login_user(username, provider, fence_idp=None, shib_idp=None, email=None): """ Login a user with the given username and provider. Set values in Flask session to indicate the user being logged in. In addition, commit the user and associated idp information to the db. Args: username (str): specific username of user to be logged in provider (str): specfic idp of user to be logged in fence_idp (str, optional): Downstreawm fence IdP shib_idp (str, optional): Downstreawm shibboleth IdP email (str, optional): email of user (may or may not match username depending on the IdP) """ def set_flask_session_values(user): """ Helper fuction to set user values in the session. Args: user (User): User object """ flask.session["username"] = user.username flask.session["user_id"] = str(user.id) flask.session["provider"] = user.identity_provider.name if fence_idp: flask.session["fence_idp"] = fence_idp if shib_idp: flask.session["shib_idp"] = shib_idp flask.g.user = user flask.g.scopes = ["_all"] flask.g.token = None user = query_for_user(session=current_session, username=username) if user: _update_users_email(user, email) # This expression is relevant to those users who already have user and # idp info persisted to the database. We return early to avoid # unnecessarily re-saving that user and idp info. if user.identity_provider and user.identity_provider.name == provider: set_flask_session_values(user) return else: if email: user = User(username=username, email=email) else: user = User(username=username) idp = (current_session.query(IdentityProvider).filter( IdentityProvider.name == provider).first()) if not idp: idp = IdentityProvider(name=provider) user.identity_provider = idp current_session.add(user) current_session.commit() set_flask_session_values(user)
def get_or_create_gen3_user_from_iss_sub(issuer, subject_id, db_session=None): """ Get a user from the Fence database corresponding to the visa identity indicated by the <issuer, subject_id> combination. If a Fence user has not yet been created for the given <issuer, subject_id> combination, create and return such a user. Args: issuer (str): the issuer of a given visa subject_id (str): the subject of a given visa Return: userdatamodel.user.User: the Fence user corresponding to issuer and subject_id """ db_session = db_session or current_session logger.debug( f"get_or_create_gen3_user_from_iss_sub: issuer: {issuer} & subject_id: {subject_id}" ) iss_sub_pair_to_user = db_session.query(IssSubPairToUser).get( (issuer, subject_id)) if not iss_sub_pair_to_user: username = subject_id + issuer[len("https://"):] gen3_user = query_for_user(session=db_session, username=username) idp_name = IssSubPairToUser.ISSUER_TO_IDP.get(issuer) logger.debug(f"issuer_to_idp: {IssSubPairToUser.ISSUER_TO_IDP}") if not gen3_user: gen3_user = create_user(db_session, logger, username, idp_name=idp_name) if not idp_name: logger.info( f"The user (id:{gen3_user.id}) was created without a linked identity " f"provider since it could not be determined based on " f"the issuer {issuer}") # ensure user has an associated identity provider if not gen3_user.identity_provider: idp = (db_session.query(IdentityProvider).filter( IdentityProvider.name == idp_name).first()) if not idp: idp = IdentityProvider(name=idp_name) gen3_user.identity_provider = idp logger.info(f'Mapping subject id ("{subject_id}") and issuer ' f'("{issuer}") combination to Fence user ' f'"{gen3_user.username}" with IdP = "{idp_name}"') iss_sub_pair_to_user = IssSubPairToUser(iss=issuer, sub=subject_id) iss_sub_pair_to_user.user = gen3_user db_session.add(iss_sub_pair_to_user) db_session.commit() return iss_sub_pair_to_user.user
def _upsert_userinfo(self, sess, user_info): """ update user info to database. Args: sess: sqlalchemy session user_info: a dict of {username: {display_name, phone_number, tags, admin} Return: None """ for username in user_info: u = query_for_user(session=sess, username=username) if u is None: self.logger.info("create user {}".format(username)) u = User(username=username) sess.add(u) if self.arborist_client: self.arborist_client.create_user({"name": username}) u.email = user_info[username].get("email", "") u.display_name = user_info[username].get("display_name", "") u.phone_number = user_info[username].get("phone_number", "") u.is_admin = user_info[username].get("admin", False) # do not update if there is no tag if user_info[username]["tags"] == {}: continue # remove user db tags if they are not shown in new tags for tag in u.tags: if tag.key not in user_info[username]["tags"]: u.tags.remove(tag) # sync for k, v in user_info[username]["tags"].items(): found = False for tag in u.tags: if tag.key == k: found = True tag.value = v # create new tag if not found if not found: tag = Tag(key=k, value=v) u.tags.append(tag)
def login_user(request, username, provider): user = query_for_user(session=current_session, username=username) if not user: user = User(username=username) idp = (current_session.query(IdentityProvider).filter( IdentityProvider.name == provider).first()) if not idp: idp = IdentityProvider(name=provider) user.identity_provider = idp current_session.add(user) current_session.commit() flask.session["username"] = username flask.session["provider"] = provider flask.session["user_id"] = str(user.id) flask.g.user = user flask.g.scopes = ["_all"] flask.g.token = None
def create_client( username, urls, DB, name="", description="", auto_approve=False, is_admin=False, grant_types=None, confidential=True, ): grant_types = grant_types driver = SQLAlchemyDriver(DB) client_id = random_str(40) client_secret = None hashed_secret = None if confidential: client_secret = random_str(55) hashed_secret = bcrypt.hashpw(client_secret, bcrypt.gensalt()) auth_method = "client_secret_basic" if confidential else "none" with driver.session as s: user = query_for_user(session=s, username=username) if not user: user = User(username=username, is_admin=is_admin) s.add(user) if s.query(Client).filter(Client.name == name).first(): raise Exception("client {} already exists".format(name)) client = Client( client_id=client_id, client_secret=hashed_secret, user=user, redirect_uris=urls, _allowed_scopes=" ".join(config["CLIENT_ALLOWED_SCOPES"]), description=description, name=name, auto_approve=auto_approve, grant_types=grant_types, is_confidential=confidential, token_endpoint_auth_method=auth_method, ) s.add(client) s.commit() return client_id, client_secret
def _get_storage_user(self, username, provider, session): """ Return a user. Depending on the provider, may call to get or just search fence's db. Args: username (str): User's name provider (str): backend provider session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db session to query for Users Returns: fence.models.User: User with username """ if provider == GOOGLE_PROVIDER: return query_for_user(session=session, username=username) return self.clients[provider].get_user(username)
def create_access_token(self): """ Create a new access token. Return: JWTResult: result containing the encoded token and claims """ driver = SQLAlchemyDriver(self.db) with driver.session as current_session: user = query_for_user(session=current_session, username=self.username) if not user: raise EnvironmentError( "no user found with given username: " + self.username ) return generate_signed_access_token( self.kid, self.private_key, user, self.expires_in, self.scopes, iss=self.base_url, )
def _get_proxy_group_id(user_id=None, username=None): """ Get users proxy group id from the current token, if possible. Otherwise, check the database for it. Returnns: int: id of proxy group associated with user """ proxy_group_id = get_users_proxy_group_from_token() if not proxy_group_id: user_id = user_id or current_token["sub"] try: user = query_for_user_by_id(current_session, user_id) if not user: user = query_for_user(current_session, username) except Exception: user = None if user: proxy_group_id = user.google_proxy_group_id return proxy_group_id
def test_sync(syncer, db_session, storage_client, parse_consent_code_config, monkeypatch): # patch the sync to use the parameterized config value monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config) monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) syncer.sync() users = db_session.query(models.User).all() assert len(users) == 11 if parse_consent_code_config: user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000178.c1": ["read", "read-storage"], "phs000178.c2": ["read", "read-storage"], "phs000178.c999": ["read", "read-storage"], "phs000179.c1": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000178.c1": ["read", "read-storage"], "phs000178.c2": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000179.c1": ["read", "read-storage"], "phs000178.c1": ["read", "read-storage"], }, ) else: user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000178": ["read", "read-storage"], "TCGA-PCAWG": ["read", "read-storage"], "phs000179": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000178": ["read", "read-storage"], "TCGA-PCAWG": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") assert equal_project_access( user.project_access, { "phs000178": ["read", "read-storage"], "TCGA-PCAWG": ["read", "read-storage"], "phs000179": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") assert user.display_name == "USER D" assert user.phone_number == "123-456-789" user = models.query_for_user(session=db_session, username="******") user_access = db_session.query( models.AccessPrivilege).filter_by(user=user).all() assert set(user_access[0].privilege) == { "create", "read", "update", "delete", "upload", } assert len(user_access) == 1 # TODO: check user policy access (add in user sync changes) user = models.query_for_user(session=db_session, username="******") assert not user.is_admin user_access = db_session.query( models.AccessPrivilege).filter_by(user=user).all() assert not user_access
def test_dbgap_consent_codes( syncer, db_session, storage_client, enable_common_exchange_area, parse_consent_code_config, monkeypatch, ): # patch the sync to use the parameterized value for whether or not to parse exchange # area data # we moved to support multiple dbgap sftp servers, the config file has a list of dbgap # for local file dir, we only use the parameters from first dbgap config # hence only those are mocked here monkeypatch.setitem( syncer.dbGaP[0], "enable_common_exchange_area_access", enable_common_exchange_area, ) monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config) monkeypatch.setattr(syncer, "project_mapping", {}) syncer.sync() user = models.query_for_user(session=db_session, username="******") if parse_consent_code_config: if enable_common_exchange_area: # b/c user has c999, ensure they have access to all consents, study-specific # exchange area (via .c999) and the common exchange area configured assert equal_project_access( user.project_access, { "phs000179.c1": ["read", "read-storage"], "phs000178.c1": ["read", "read-storage"], "phs000178.c2": ["read", "read-storage"], "phs000178.c999": ["read", "read-storage"], # should additionally include the study-specific exchange area access and # access to the common exchange area "test_common_exchange_area": ["read", "read-storage"], }, ) else: # b/c user has c999 but common exchange area is disabled, ensure they have # access to all consents, study-specific exchange area (via .c999) assert equal_project_access( user.project_access, { "phs000179.c1": ["read", "read-storage"], # c999 gives access to all consents "phs000178.c1": ["read", "read-storage"], "phs000178.c2": ["read", "read-storage"], "phs000178.c999": ["read", "read-storage"], }, ) else: # with consent code parsing off, ensure users have access to just phsids assert equal_project_access( user.project_access, { "phs000178": ["read", "read-storage"], "phs000179": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") if parse_consent_code_config: assert equal_project_access( user.project_access, { "phs000178.c1": ["read", "read-storage"], "phs000178.c2": ["read", "read-storage"], }, ) else: assert equal_project_access(user.project_access, {"phs000178": ["read", "read-storage"]}) user = models.query_for_user(session=db_session, username="******") if parse_consent_code_config: assert equal_project_access( user.project_access, { "phs000178.c1": ["read", "read-storage"], "phs000179.c1": ["read", "read-storage"], }, ) else: assert equal_project_access( user.project_access, { "phs000178": ["read", "read-storage"], "phs000179": ["read", "read-storage"], }, ) user = models.query_for_user(session=db_session, username="******") if parse_consent_code_config: assert equal_project_access(user.project_access, {"phs000179.c1": ["read", "read-storage"]}) else: assert equal_project_access(user.project_access, {"phs000179": ["read", "read-storage"]}) resource_to_parent_paths = {} for call in syncer.arborist_client.update_resource.call_args_list: args, kwargs = call parent_path = args[0] resource = args[1].get("name") resource_to_parent_paths.setdefault(resource, []).append(parent_path) if parse_consent_code_config: if enable_common_exchange_area: # b/c user has c999, ensure they have access to all consents, study-specific # exchange area (via .c999) and the common exchange area configured assert "phs000178.c999" in resource_to_parent_paths assert resource_to_parent_paths["phs000178.c999"] == [ "/orgA/programs/" ] assert "test_common_exchange_area" in resource_to_parent_paths assert resource_to_parent_paths["test_common_exchange_area"] == [ "/dbgap/programs/" ] assert "phs000178.c1" in resource_to_parent_paths assert resource_to_parent_paths["phs000178.c1"] == ["/orgA/programs/"] # NOTE: this study+consent is configured to have multiple names in the dbgap config assert "phs000178.c2" in resource_to_parent_paths assert resource_to_parent_paths["phs000178.c2"] == [ "/orgA/programs/", "/orgB/programs/", "/programs/", ] assert "phs000178.c999" in resource_to_parent_paths assert resource_to_parent_paths["phs000178.c999"] == [ "/orgA/programs/" ] assert "phs000179.c1" in resource_to_parent_paths assert resource_to_parent_paths["phs000179.c1"] == ["/orgA/programs/"] else: assert "phs000178" in resource_to_parent_paths # NOTE: this study is configured to have multiple names in the dbgap config assert resource_to_parent_paths["phs000178"] == [ "/orgA/programs/", "/orgB/programs/", "/programs/", ] assert "phs000179" in resource_to_parent_paths assert resource_to_parent_paths["phs000179"] == ["/orgA/programs/"]
def delete_user(current_session, username): """ Remove a user from both the userdatamodel and the associated storage for that project/bucket. Returns a dictionary. The Fence db may not always be in perfect sync with Google. We err on the side of safety (we prioritise making sure the user is really cleared out of Google to prevent unauthorized data access issues; we prefer cirrus/Google over the Fence db as the source of truth.) So, if the Fence-Google sync situation changes, do edit this code accordingly. """ logger.debug("Beginning delete user.") with GoogleCloudManager() as gcm: # Delete user's service accounts, SA keys, user proxy group from Google. # Noop if Google not in use. user = query_for_user(session=current_session, username=username) if not user: raise NotFound("user name {} not found".format(username)) logger.debug("Found user in Fence db: {}".format(user)) # First: Find this user's proxy group. google_proxy_group_from_fence_db = ( current_session.query(GoogleProxyGroup).filter( GoogleProxyGroup.id == user.google_proxy_group_id).first() # one_or_none() would be better, but is only in sqlalchemy 1.0.9 ) if google_proxy_group_from_fence_db: gpg_email = google_proxy_group_from_fence_db.email logger.debug( "Found Google proxy group in Fence db: {}".format(gpg_email)) else: # Construct the proxy group name that would have been used # and check if it exists in cirrus, in case Fence db just # didn't know about it. logger.debug( "Could not find Google proxy group for this user in Fence db. Checking cirrus..." ) pgname = get_proxy_group_name_for_user( user.id, user.username, prefix=config["GOOGLE_GROUP_PREFIX"]) google_proxy_group_from_google = gcm.get_group(pgname) gpg_email = (google_proxy_group_from_google.get("email") if google_proxy_group_from_google else None) if not gpg_email: logger.info( "Could not find Google proxy group for user in Fence db or in cirrus. " "Assuming Google not in use as IdP. Proceeding with Fence deletes." ) else: logger.debug( "Found Google proxy group email of user to delete: {}." "Proceeding with Google deletions.".format(gpg_email)) # Note: Fence db deletes here are interleaved with Google deletes. # This is so that if (for example) Google succeeds in deleting one SA # and then fails on the next, and the deletion process aborts, there # will not remain a record in Fence of the first, now-nonexistent SA. delete_google_service_accounts_and_keys(current_session, gcm, gpg_email) delete_google_proxy_group(current_session, gcm, gpg_email, google_proxy_group_from_fence_db, user) logger.debug("Deleting all user data from Fence database...") current_session.delete(user) current_session.commit() logger.info("Deleted all user data from Fence database. Returning.") return {"result": "success"}
def test_user_sync_with_visas( syncer, db_session, storage_client, parse_consent_code_config, fallback_to_dbgap_sftp, monkeypatch, ): # patch the sync to use the parameterized config value monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config) monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) monkeypatch.setattr(syncer, "fallback_to_dbgap_sftp", fallback_to_dbgap_sftp) monkeypatch.setattr(syncer, "sync_from_visas", True) syncer.sync_visas() users = db_session.query(models.User).all() user = models.query_for_user( session=db_session, username="******") # contains only visa information backup_user = models.query_for_user( session=db_session, username="******" ) # Contains invalid visa and also in telemetry file expired_user = models.query_for_user( session=db_session, username="******", ) invalid_user = models.query_for_user(session=db_session, username="******") assert len(invalid_user.project_access) == 0 assert len(expired_user.project_access) == 0 assert len(invalid_user.ga4gh_visas_v1) == 0 assert len(expired_user.ga4gh_visas_v1) == 0 if fallback_to_dbgap_sftp: assert len(users) == 13 if parse_consent_code_config: assert equal_project_access( user.project_access, { "phs000991.c1": ["read", "read-storage"], "phs000961.c1": ["read", "read-storage"], "phs000279.c1": ["read", "read-storage"], "phs000286.c3": ["read", "read-storage"], "phs000289.c2": ["read", "read-storage"], "phs000298.c1": ["read", "read-storage"], }, ) assert equal_project_access( backup_user.project_access, { "phs000179.c1": ["read", "read-storage"], }, ) else: assert equal_project_access( user.project_access, { "phs000991": ["read", "read-storage"], "phs000961": ["read", "read-storage"], "phs000279": ["read", "read-storage"], "phs000286": ["read", "read-storage"], "phs000289": ["read", "read-storage"], "phs000298": ["read", "read-storage"], }, ) assert equal_project_access( backup_user.project_access, { "phs000179": ["read", "read-storage"], }, ) else: assert len(users) == 11 assert len(backup_user.project_access) == 0 if parse_consent_code_config: assert equal_project_access( user.project_access, { "phs000991.c1": ["read", "read-storage"], "phs000961.c1": ["read", "read-storage"], "phs000279.c1": ["read", "read-storage"], "phs000286.c3": ["read", "read-storage"], "phs000289.c2": ["read", "read-storage"], "phs000298.c1": ["read", "read-storage"], }, ) else: assert equal_project_access( user.project_access, { "phs000991": ["read", "read-storage"], "phs000961": ["read", "read-storage"], "phs000279": ["read", "read-storage"], "phs000286": ["read", "read-storage"], "phs000289": ["read", "read-storage"], "phs000298": ["read", "read-storage"], }, )
def find_user(username, session): user = query_for_user(session=session, username=username) if not user: raise NotFound("user {} not found".format(username)) return user
def force_update_google_link(DB, username, google_email, expires_in=None): """ WARNING: This function circumvents Google Auth flow, and should only be used for internal testing! WARNING: This function assumes that a user already has a proxy group! Adds user's google account to proxy group and/or updates expiration for that google account's access. WARNING: This assumes that provided arguments represent valid information. This BLINDLY adds without verification. Do verification before this. Specifically, this ASSUMES that the proxy group provided belongs to the given user and that the user has ALREADY authenticated to prove the provided google_email is also their's. Args: DB username (str): Username to link with google_email (str): Google email to link to Raises: NotFound: Linked Google account not found Unauthorized: Couldn't determine user Returns: Expiration time of the newly updated google account's access """ cirrus_config.update(**config["CIRRUS_CFG"]) db = SQLAlchemyDriver(DB) with db.session as session: user_account = query_for_user(session=session, username=username) if user_account: user_id = user_account.id proxy_group_id = user_account.google_proxy_group_id else: raise Unauthorized( "Could not determine authed user " "from session. Unable to link Google account." ) user_google_account = ( session.query(UserGoogleAccount) .filter(UserGoogleAccount.email == google_email) .first() ) if not user_google_account: user_google_account = add_new_user_google_account( user_id, google_email, session ) # timestamp at which the SA will lose bucket access # by default: use configured time or 7 days expiration = int(time.time()) + config.get( "GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800 ) if expires_in: is_valid_expiration(expires_in) # convert it to timestamp requested_expiration = int(time.time()) + expires_in expiration = min(expiration, requested_expiration) force_update_user_google_account_expiration( user_google_account, proxy_group_id, google_email, expiration, session ) session.commit() return expiration
def sync_gen3_users_authz_from_ga4gh_passports( passports, pkey_cache=None, db_session=None, ): """ Validate passports and embedded visas, using each valid visa's identity established by <iss, sub> combination to possibly create and definitely determine a Fence user who is added to the list returned by this function. In the process of determining Fence users from visas, visa authorization information is also persisted in Fence and synced to Arborist. Args: passports (list): a list of raw encoded passport strings, each including header, payload, and signature Return: list: a list of users, each corresponding to a valid visa identity embedded within the passports passed in """ db_session = db_session or current_session # {"username": user, "username2": user2} users_from_all_passports = {} for passport in passports: try: cached_usernames = get_gen3_usernames_for_passport_from_cache( passport=passport, db_session=db_session) if cached_usernames: # there's a chance a given username exists in the cache but no longer in # the database. if not all are in db, ignore the cache and actually parse # and validate the passport all_users_exist_in_db = True usernames_to_update = {} for username in cached_usernames: user = query_for_user(session=db_session, username=username) if not user: all_users_exist_in_db = False continue usernames_to_update[user.username] = user if all_users_exist_in_db: users_from_all_passports.update(usernames_to_update) # existence in the cache and a user in db means that this passport # was validated previously (expiration was also checked) continue # below function also validates passport (or raises exception) raw_visas = get_unvalidated_visas_from_valid_passport( passport, pkey_cache=pkey_cache) except Exception as exc: logger.warning( f"Invalid passport provided, ignoring. Error: {exc}") continue # an empty raw_visas list means that either the current passport is # invalid or that it has no visas. in both cases, the current passport # is ignored and we move on to the next passport if not raw_visas: continue identity_to_visas = collections.defaultdict(list) min_visa_expiration = int( time.time()) + datetime.timedelta(hours=1).seconds for raw_visa in raw_visas: try: validated_decoded_visa = validate_visa(raw_visa, pkey_cache=pkey_cache) identity_to_visas[( validated_decoded_visa.get("iss"), validated_decoded_visa.get("sub"), )].append((raw_visa, validated_decoded_visa)) min_visa_expiration = min(min_visa_expiration, validated_decoded_visa.get("exp")) except Exception as exc: logger.warning( f"Invalid visa provided, ignoring. Error: {exc}") continue expired_authz_removal_job_freq_in_seconds = config[ "EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS"] min_visa_expiration -= expired_authz_removal_job_freq_in_seconds if min_visa_expiration <= int(time.time()): logger.warning( "The passport's earliest valid visa expiration time is set to " f"occur within {expired_authz_removal_job_freq_in_seconds} " "seconds from now, which is too soon an expiration to handle.") continue users_from_current_passport = [] for (issuer, subject_id), visas in identity_to_visas.items(): gen3_user = get_or_create_gen3_user_from_iss_sub( issuer, subject_id, db_session=db_session) ga4gh_visas = [ GA4GHVisaV1( user=gen3_user, source=validated_decoded_visa["ga4gh_visa_v1"]["source"], type=validated_decoded_visa["ga4gh_visa_v1"]["type"], asserted=int( validated_decoded_visa["ga4gh_visa_v1"]["asserted"]), expires=int(validated_decoded_visa["exp"]), ga4gh_visa=raw_visa, ) for raw_visa, validated_decoded_visa in visas ] # NOTE: does not validate, assumes validation occurs above. # This adds the visas to the database session but doesn't commit until # the end of this function _sync_validated_visa_authorization( gen3_user=gen3_user, ga4gh_visas=ga4gh_visas, expiration=min_visa_expiration, db_session=db_session, ) users_from_current_passport.append(gen3_user) for user in users_from_current_passport: users_from_all_passports[user.username] = user put_gen3_usernames_for_passport_into_cache( passport=passport, user_ids_from_passports=list(users_from_all_passports.keys()), expires_at=min_visa_expiration, db_session=db_session, ) db_session.commit() logger.info( f"Got Gen3 usernames from passport(s): {list(users_from_all_passports.keys())}" ) return users_from_all_passports
def sync_to_db_and_storage_backend(self, user_project, user_info, user_policies, sess): """ sync user access control to database and storage backend Args: user_project (dict): a dictionary of { username: { 'project1': {'read-storage','write-storage'}, 'project2': {'read-storage'} } } user_info (dict): a dictionary of {username: user_info{}} user_policies (List[str]): list of policies sess: a sqlalchemy session Return: None """ self._init_projects(user_project, sess) auth_provider_list = [ self._get_or_create(sess, AuthorizationProvider, name="dbGaP"), self._get_or_create(sess, AuthorizationProvider, name="fence"), ] cur_db_user_project_list = { (ua.user.username.lower(), ua.project.auth_id) for ua in sess.query(AccessPrivilege).all() } # we need to compare db -> whitelist case-insensitively for username # db stores case-sensitively, but we need to query case-insensitively user_project_lowercase = {} syncing_user_project_list = set() for username, projects in user_project.iteritems(): user_project_lowercase[username.lower()] = projects for project, _ in projects.iteritems(): syncing_user_project_list.add((username.lower(), project)) user_info_lowercase = { username.lower(): info for username, info in user_info.iteritems() } to_delete = set.difference(cur_db_user_project_list, syncing_user_project_list) to_add = set.difference(syncing_user_project_list, cur_db_user_project_list) to_update = set.intersection(cur_db_user_project_list, syncing_user_project_list) # when updating users we want to maintain case sesitivity in the username so # pass the original, non-lowered user_info dict self._upsert_userinfo(sess, user_info) self._revoke_from_storage(to_delete, sess) self._revoke_from_db(sess, to_delete) self._grant_from_storage(to_add, user_project_lowercase, sess) self._grant_from_db( sess, to_add, user_info_lowercase, user_project_lowercase, auth_provider_list, ) # re-grant self._grant_from_storage(to_update, user_project_lowercase, sess) self._update_from_db(sess, to_update, user_project_lowercase) self._validate_and_update_user_admin(sess, user_info_lowercase) # Add policies to user models in the database. These will show up in users' # JWTs; services can send the JWTs to arborist. if user_policies: self.logger.info("populating RBAC information from YAML file") for username, policies in user_policies.iteritems(): user = query_for_user(session=sess, username=username) for policy_id in policies: policy = self._get_or_create_policy(sess, policy_id) if policy not in user.policies: user.policies.append(policy) self.logger.info( "granted policy `{}` to user `{}` ({})".format( policy_id, username, user.id)) sess.commit()
def _update_arborist(self, session, user_yaml): """ Create roles and resources in arborist from the information in ``user_projects``. The projects are sent to arborist as resources with paths like ``/projects/{project}``. Roles are created with just the original names for the privileges like ``"read-storage"`` etc. Args: session (sqlalchemy.Session) user_yaml (UserYAML) Return: bool: success """ if not self.arborist_client: self.logger.warn("no arborist client set; skipping arborist sync") return False if not self.arborist_client.healthy(): # TODO (rudyardrichter, 2019-01-07): add backoff/retry here self.logger.error( "arborist service is unavailable; skipping arborist sync") return False # Set up the resource tree in arborist resources = user_yaml.rbac.get("resources", []) for resource in resources: try: self.arborist_client.create_resource("/", resource, overwrite=True) except ArboristError as e: self.logger.error(e) # keep going; maybe just some conflicts from things existing already created_roles = set() roles = user_yaml.rbac.get("roles", []) for role in roles: try: response = self.arborist_client.create_role(role) if response: created_roles.add(role["id"]) except ArboristError as e: self.logger.error(e) # keep going; maybe just some conflicts from things existing already created_policies = set() policies = user_yaml.rbac.get("policies", []) for policy in policies: try: response = self.arborist_client.create_policy(policy) if response: created_policies.add(policy["id"]) except ArboristError as e: self.logger.error(e) # keep going; maybe just some conflicts from things existing already user_projects = user_yaml.user_rbac for username, user_resources in user_projects.iteritems(): self.logger.info("processing user `{}`".format(username)) user = query_for_user(session=session, username=username) for path, permissions in user_resources.iteritems(): for permission in permissions: # "permission" in the dbgap sense, not the arborist sense if permission not in created_roles: try: self.arborist_client.create_role( arborist_role_for_permission(permission)) except ArboristError as e: self.logger.info( "not creating role for permission `{}`; {}". format(permission, str(e))) created_roles.add(permission) # If everything was created fine, grant a policy to # this user which contains exactly just this resource, # with this permission as a role. # format project '/x/y/z' -> 'x.y.z' # so the policy id will be something like 'x.y.z-create' policy_id = _format_policy_id(path, permission) if policy_id not in created_policies: try: self.arborist_client.create_policy({ "id": policy_id, "description": "policy created by fence sync", "role_ids": [permission], "resource_paths": [path], }) except ArboristError as e: self.logger.info( "not creating policy in arborist; {}".format( str(e))) created_policies.add(policy_id) policy = self._get_or_create_policy(session, policy_id) user.policies.append(policy) self.logger.info("granted policy `{}` to user `{}`".format( policy_id, user.username)) return True
def get_or_create_proxy_group_id(expires=None, user_id=None, username=None): """ If no username returned from token or database, create a new proxy group for the given user. Also, add the access privileges. Returns: int: id of (possibly newly created) proxy group associated with user """ proxy_group_id = _get_proxy_group_id(user_id=user_id, username=username) if not proxy_group_id: try: user_by_id = query_for_user_by_id(current_session, user_id) user_by_username = query_for_user( session=current_session, username=username ) except Exception: user_by_id = None user_by_username = None if user_by_id: user_id = user_id username = user_by_id.username elif user_by_username: user_id = user_by_username.id username = username elif current_token: user_id = current_token["sub"] username = current_token.get("context", {}).get("user", {}).get("name", "") else: raise Exception( f"could not find user given input user_id={user_id} or " f"username={username}, nor was there a current_token" ) proxy_group_id = _create_proxy_group(user_id, username).id privileges = current_session.query(AccessPrivilege).filter( AccessPrivilege.user_id == user_id ) for p in privileges: storage_accesses = p.project.storage_access for sa in storage_accesses: if sa.provider.name == STORAGE_ACCESS_PROVIDER_NAME: flask.current_app.storage_manager.logger.info( "grant {} access {} to {} in {}".format( username, p.privilege, p.project_id, p.auth_provider ) ) flask.current_app.storage_manager.grant_access( provider=(sa.provider.name), username=username, project=p.project, access=p.privilege, session=current_session, expires=expires, ) return proxy_group_id