def test_sync_revoke(syncer, db_session, storage_client):
phsids = {
"userA": {
"phs000178": {"read", "read-storage"},
"phs000179": {"read", "read-storage", "write-storage"},
},
"userB": {"phs000179": {"read", "read-storage", "write-storage"}},
}
userinfo = {
"userA": {"email": "a@b", "tags": {}},
"userB": {"email": "a@b", "tags": {}},
}
phsids2 = {"userA": {"phs000179": {"read", "read-storage", "write-storage"}}}
syncer.sync_to_db_and_storage_backend(phsids, userinfo, db_session)
syncer.sync_to_db_and_storage_backend(phsids2, userinfo, db_session)
user_B = models.query_for_user(session=db_session, username="******")
n_access_privilege = (
db_session.query(models.AccessPrivilege).filter_by(user_id=user_B.id).count()
)
if n_access_privilege:
raise AssertionError()
def _get_or_create_storage_user(self, username, provider, session):
"""
Return a user.
Depending on the provider, may call to get or create or just
search fence's db.
Args:
username (str): User's name
provider (str): backend provider
session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db
session to query for Users
Returns:
fence.models.User: User with username
"""
if provider == GOOGLE_PROVIDER:
user = query_for_user(session=session, username=username.lower())
if not user:
raise NotFound(
"User not found with username {}. For Google Storage "
"Backend user's must already exist in the db and have a "
"Google Proxy Group.".format(username))
return user
return self.clients[provider].get_or_create_user(username)
def test_map_iss_sub_pair_to_user_with_no_prior_DRS_access(db_session):
"""
Test RASOauth2Client.map_iss_sub_pair_to_user when the username passed in
(e.g. eRA username) does not already exist in the Fence database and that
user's <iss, sub> combination has not already been mapped through a prior
DRS access request.
"""
# reset users table
db_session.query(User).delete()
db_session.commit()
iss = "https://domain.tld"
sub = "123_abc"
username = "******"
email = "*****@*****.**"
oidc = config.get("OPENID_CONNECT", {})
ras_client = RASClient(
oidc["ras"],
HTTP_PROXY=config.get("HTTP_PROXY"),
logger=logger,
)
assert not query_for_user(db_session, username)
iss_sub_pair_to_user_records = db_session.query(IssSubPairToUser).all()
assert len(iss_sub_pair_to_user_records) == 0
username_to_log_in = ras_client.map_iss_sub_pair_to_user(
iss, sub, username, email, db_session=db_session)
assert username_to_log_in == username
iss_sub_pair_to_user = db_session.query(IssSubPairToUser).get((iss, sub))
assert iss_sub_pair_to_user.user.username == username
assert iss_sub_pair_to_user.user.email == email
iss_sub_pair_to_user_records = db_session.query(IssSubPairToUser).all()
assert len(iss_sub_pair_to_user_records) == 1
def create_user(users, db_session, is_admin=False):
s = db_session
for username in list(users.keys()):
user = query_for_user(session=s, username=username)
if not user:
user = User(username=username, is_admin=is_admin)
s.add(user)
for project_data in users[username]["projects"]:
privilege = project_data["privilege"]
auth_id = project_data["auth_id"]
p_name = project_data.get("name", auth_id)
project = s.query(Project).filter(
Project.auth_id == auth_id).first()
if not project:
project = Project(name=p_name, auth_id=auth_id)
s.add(project)
ap = (s.query(AccessPrivilege).join(
AccessPrivilege.project).join(AccessPrivilege.user).filter(
Project.name == p_name,
User.username == user.username).first())
if not ap:
ap = AccessPrivilege(project=project,
user=user,
privilege=privilege)
s.add(ap)
else:
ap.privilege = privilege
return user.id, user.username
def _grant_from_db(self, sess, to_add, user_info, user_project,
auth_provider_list):
"""
Grant user access to projects in the auth database
Args:
sess: sqlalchemy session
to_add: a set of (username, project.auth_id) to be granted
user_project:
a dictionary of {username: {project: {'read','write'}}
Return:
None
"""
for (username, project_auth_id) in to_add:
u = query_for_user(session=sess, username=username)
auth_provider = auth_provider_list[0]
if "dbgap_role" not in user_info[username]["tags"]:
auth_provider = auth_provider_list[1]
user_access = AccessPrivilege(
user=u,
project=self._projects[project_auth_id],
privilege=list(user_project[username][project_auth_id]),
auth_provider=auth_provider,
)
self.logger.info("grant user {} to {} with access {}".format(
username, user_access.project, user_access.privilege))
sess.add(user_access)
def create_users_with_group(DB, s, data):
providers = {}
data_groups = data.get("groups", {})
users = data.get("users", {})
for username, data in users.items():
is_existing_user = True
user = query_for_user(session=s, username=username)
admin = data.get("admin", False)
if not user:
is_existing_user = False
provider_name = data.get("provider", "google")
provider = providers.get(provider_name)
if not provider:
provider = (s.query(IdentityProvider).filter(
IdentityProvider.name == provider_name).first())
providers[provider_name] = provider
if not provider:
raise Exception(
"provider {} not found".format(provider_name))
user = User(username=username, idp_id=provider.id, is_admin=admin)
user.is_admin = admin
group_names = data.get("groups", [])
for group_name in group_names:
assign_group_to_user(s, user, group_name, data_groups[group_name])
projects = data.get("projects", [])
for project in projects:
grant_project_to_group_or_user(s, project, user=user)
if not is_existing_user:
s.add(user)
for client in data.get("clients", []):
create_client_action(DB, username=username, **client)
def create_refresh_token(self):
"""
Create a new refresh token and add its entry to the database.
Return:
JWTResult: the refresh token result
"""
driver = SQLAlchemyDriver(self.db)
with driver.session as current_session:
user = query_for_user(session=current_session, username=self.username)
if not user:
raise EnvironmentError(
"no user found with given username: "******"jti"],
userid=user.id,
expires=jwt_result.claims["exp"],
)
)
return jwt_result
def create_client(
username,
urls,
DB,
name="",
description="",
auto_approve=False,
is_admin=False,
grant_types=None,
confidential=True,
arborist=None,
policies=None,
allowed_scopes=None,
):
client_id = random_str(40)
if arborist is not None:
arborist.create_client(client_id, policies)
grant_types = grant_types
driver = SQLAlchemyDriver(DB)
client_secret = None
hashed_secret = None
if confidential:
client_secret = random_str(55)
hashed_secret = bcrypt.hashpw(client_secret.encode("utf-8"),
bcrypt.gensalt()).decode("utf-8")
auth_method = "client_secret_basic" if confidential else "none"
allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"]
if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])):
raise ValueError("Each allowed scope must be one of: {}".format(
config["CLIENT_ALLOWED_SCOPES"]))
if "openid" not in allowed_scopes:
allowed_scopes.append("openid")
logger.warning(
'Adding required "openid" scope to list of allowed scopes.')
with driver.session as s:
user = query_for_user(session=s, username=username)
if not user:
user = User(username=username, is_admin=is_admin)
s.add(user)
if s.query(Client).filter(Client.name == name).first():
if arborist is not None:
arborist.delete_client(client_id)
raise Exception("client {} already exists".format(name))
client = Client(
client_id=client_id,
client_secret=hashed_secret,
user=user,
redirect_uris=urls,
_allowed_scopes=" ".join(allowed_scopes),
description=description,
name=name,
auto_approve=auto_approve,
grant_types=grant_types,
is_confidential=confidential,
token_endpoint_auth_method=auth_method,
)
s.add(client)
s.commit()
return client_id, client_secret
def test_sync_from_files(syncer, db_session, storage_client):
sess = db_session
phsids = {
"userA": {
"phs000178": {"read", "read-storage"},
"phs000179": {"read", "read-storage", "write-storage"},
},
"userB": {
"phs000179": {"read", "read-storage", "write-storage"}
},
}
userinfo = {
"userA": {
"email": "a@b",
"tags": {}
},
"userB": {
"email": "a@b",
"tags": {}
},
}
syncer.sync_to_db_and_storage_backend(phsids, userinfo, sess)
u = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
u.project_access,
{"phs000179": ["read", "read-storage", "write-storage"]})
def create_awg_user(users, db_session):
s = db_session
for username in list(users.keys()):
user = query_for_user(session=s, username=username)
if not user:
user = User(username=username)
s.add(user)
projects = {}
for project_data in users[username]["projects"]:
auth_id = project_data["auth_id"]
p_name = project_data.get("name", auth_id)
project = s.query(Project).filter(Project.auth_id == auth_id).first()
if not project:
project = Project(name=p_name, auth_id=auth_id)
s.add(project)
projects[p_name] = project
groups = users[username].get("groups", [])
for group in groups:
group_name = group["name"]
group_desc = group["description"]
grp = s.query(Group).filter(Group.name == group_name).first()
if not grp:
grp = Group()
grp.name = group_name
grp.description = group_desc
s.add(grp)
s.flush()
UserToGroup(group=grp, user=user)
for projectname in group["projects"]:
gap = (
s.query(AccessPrivilege)
.join(AccessPrivilege.project)
.join(AccessPrivilege.group)
.filter(Project.name == projectname, Group.name == group_name)
.first()
)
if not gap:
project = projects[projectname]
gap = AccessPrivilege(project_id=project.id, group_id=grp.id)
s.add(gap)
s.flush()
ap = (
s.query(AccessPrivilege)
.join(AccessPrivilege.project)
.join(AccessPrivilege.user)
.filter(Project.name == projectname, User.username == user.username)
.first()
)
privilege = {"read"}
if not ap:
project = projects[projectname]
ap = AccessPrivilege(
project=project, user=user, privilege=privilege
)
s.add(ap)
s.flush()
return user.id, user.username
def test_sync_in_login(
syncer,
db_session,
storage_client,
rsa_private_key,
kid,
monkeypatch,
):
user = models.query_for_user(
session=db_session, username="******") # contains no information
syncer.sync_single_user_visas(user, db_session)
user = models.query_for_user(
session=db_session,
username="******") # contains only visa information
user1 = models.query_for_user(session=db_session, username="******")
assert len(user1.project_access) == 0 # other users are not affected
assert len(user.project_access) == 6
def get_current_user(flask_session=None):
flask_session = flask_session or flask.session
username = flask_session.get("username")
if config.get("MOCK_AUTH", False) is True:
username = "******"
if not username:
raise Unauthorized("User not logged in")
return query_for_user(session=current_session, username=username)
def login_user(username, provider, fence_idp=None, shib_idp=None, email=None):
"""
Login a user with the given username and provider. Set values in Flask
session to indicate the user being logged in. In addition, commit the user
and associated idp information to the db.
Args:
username (str): specific username of user to be logged in
provider (str): specfic idp of user to be logged in
fence_idp (str, optional): Downstreawm fence IdP
shib_idp (str, optional): Downstreawm shibboleth IdP
email (str, optional): email of user (may or may not match username depending
on the IdP)
"""
def set_flask_session_values(user):
"""
Helper fuction to set user values in the session.
Args:
user (User): User object
"""
flask.session["username"] = user.username
flask.session["user_id"] = str(user.id)
flask.session["provider"] = user.identity_provider.name
if fence_idp:
flask.session["fence_idp"] = fence_idp
if shib_idp:
flask.session["shib_idp"] = shib_idp
flask.g.user = user
flask.g.scopes = ["_all"]
flask.g.token = None
user = query_for_user(session=current_session, username=username)
if user:
_update_users_email(user, email)
# This expression is relevant to those users who already have user and
# idp info persisted to the database. We return early to avoid
# unnecessarily re-saving that user and idp info.
if user.identity_provider and user.identity_provider.name == provider:
set_flask_session_values(user)
return
else:
if email:
user = User(username=username, email=email)
else:
user = User(username=username)
idp = (current_session.query(IdentityProvider).filter(
IdentityProvider.name == provider).first())
if not idp:
idp = IdentityProvider(name=provider)
user.identity_provider = idp
current_session.add(user)
current_session.commit()
set_flask_session_values(user)
def get_or_create_gen3_user_from_iss_sub(issuer, subject_id, db_session=None):
"""
Get a user from the Fence database corresponding to the visa identity
indicated by the <issuer, subject_id> combination. If a Fence user has
not yet been created for the given <issuer, subject_id> combination,
create and return such a user.
Args:
issuer (str): the issuer of a given visa
subject_id (str): the subject of a given visa
Return:
userdatamodel.user.User: the Fence user corresponding to issuer and subject_id
"""
db_session = db_session or current_session
logger.debug(
f"get_or_create_gen3_user_from_iss_sub: issuer: {issuer} & subject_id: {subject_id}"
)
iss_sub_pair_to_user = db_session.query(IssSubPairToUser).get(
(issuer, subject_id))
if not iss_sub_pair_to_user:
username = subject_id + issuer[len("https://"):]
gen3_user = query_for_user(session=db_session, username=username)
idp_name = IssSubPairToUser.ISSUER_TO_IDP.get(issuer)
logger.debug(f"issuer_to_idp: {IssSubPairToUser.ISSUER_TO_IDP}")
if not gen3_user:
gen3_user = create_user(db_session,
logger,
username,
idp_name=idp_name)
if not idp_name:
logger.info(
f"The user (id:{gen3_user.id}) was created without a linked identity "
f"provider since it could not be determined based on "
f"the issuer {issuer}")
# ensure user has an associated identity provider
if not gen3_user.identity_provider:
idp = (db_session.query(IdentityProvider).filter(
IdentityProvider.name == idp_name).first())
if not idp:
idp = IdentityProvider(name=idp_name)
gen3_user.identity_provider = idp
logger.info(f'Mapping subject id ("{subject_id}") and issuer '
f'("{issuer}") combination to Fence user '
f'"{gen3_user.username}" with IdP = "{idp_name}"')
iss_sub_pair_to_user = IssSubPairToUser(iss=issuer, sub=subject_id)
iss_sub_pair_to_user.user = gen3_user
db_session.add(iss_sub_pair_to_user)
db_session.commit()
return iss_sub_pair_to_user.user
def _upsert_userinfo(self, sess, user_info):
"""
update user info to database.
Args:
sess: sqlalchemy session
user_info:
a dict of {username: {display_name, phone_number, tags, admin}
Return:
None
"""
for username in user_info:
u = query_for_user(session=sess, username=username)
if u is None:
self.logger.info("create user {}".format(username))
u = User(username=username)
sess.add(u)
if self.arborist_client:
self.arborist_client.create_user({"name": username})
u.email = user_info[username].get("email", "")
u.display_name = user_info[username].get("display_name", "")
u.phone_number = user_info[username].get("phone_number", "")
u.is_admin = user_info[username].get("admin", False)
# do not update if there is no tag
if user_info[username]["tags"] == {}:
continue
# remove user db tags if they are not shown in new tags
for tag in u.tags:
if tag.key not in user_info[username]["tags"]:
u.tags.remove(tag)
# sync
for k, v in user_info[username]["tags"].items():
found = False
for tag in u.tags:
if tag.key == k:
found = True
tag.value = v
# create new tag if not found
if not found:
tag = Tag(key=k, value=v)
u.tags.append(tag)
def login_user(request, username, provider):
user = query_for_user(session=current_session, username=username)
if not user:
user = User(username=username)
idp = (current_session.query(IdentityProvider).filter(
IdentityProvider.name == provider).first())
if not idp:
idp = IdentityProvider(name=provider)
user.identity_provider = idp
current_session.add(user)
current_session.commit()
flask.session["username"] = username
flask.session["provider"] = provider
flask.session["user_id"] = str(user.id)
flask.g.user = user
flask.g.scopes = ["_all"]
flask.g.token = None
def create_client(
username,
urls,
DB,
name="",
description="",
auto_approve=False,
is_admin=False,
grant_types=None,
confidential=True,
):
grant_types = grant_types
driver = SQLAlchemyDriver(DB)
client_id = random_str(40)
client_secret = None
hashed_secret = None
if confidential:
client_secret = random_str(55)
hashed_secret = bcrypt.hashpw(client_secret, bcrypt.gensalt())
auth_method = "client_secret_basic" if confidential else "none"
with driver.session as s:
user = query_for_user(session=s, username=username)
if not user:
user = User(username=username, is_admin=is_admin)
s.add(user)
if s.query(Client).filter(Client.name == name).first():
raise Exception("client {} already exists".format(name))
client = Client(
client_id=client_id,
client_secret=hashed_secret,
user=user,
redirect_uris=urls,
_allowed_scopes=" ".join(config["CLIENT_ALLOWED_SCOPES"]),
description=description,
name=name,
auto_approve=auto_approve,
grant_types=grant_types,
is_confidential=confidential,
token_endpoint_auth_method=auth_method,
)
s.add(client)
s.commit()
return client_id, client_secret
def _get_storage_user(self, username, provider, session):
"""
Return a user.
Depending on the provider, may call to get or just search fence's db.
Args:
username (str): User's name
provider (str): backend provider
session (userdatamodel.driver.SQLAlchemyDriver.session): fence's db
session to query for Users
Returns:
fence.models.User: User with username
"""
if provider == GOOGLE_PROVIDER:
return query_for_user(session=session, username=username)
return self.clients[provider].get_user(username)
def create_access_token(self):
"""
Create a new access token.
Return:
JWTResult: result containing the encoded token and claims
"""
driver = SQLAlchemyDriver(self.db)
with driver.session as current_session:
user = query_for_user(session=current_session, username=self.username)
if not user:
raise EnvironmentError(
"no user found with given username: " + self.username
)
return generate_signed_access_token(
self.kid,
self.private_key,
user,
self.expires_in,
self.scopes,
iss=self.base_url,
)
def _get_proxy_group_id(user_id=None, username=None):
"""
Get users proxy group id from the current token, if possible.
Otherwise, check the database for it.
Returnns:
int: id of proxy group associated with user
"""
proxy_group_id = get_users_proxy_group_from_token()
if not proxy_group_id:
user_id = user_id or current_token["sub"]
try:
user = query_for_user_by_id(current_session, user_id)
if not user:
user = query_for_user(current_session, username)
except Exception:
user = None
if user:
proxy_group_id = user.google_proxy_group_id
return proxy_group_id
def test_sync(syncer, db_session, storage_client, parse_consent_code_config,
monkeypatch):
# patch the sync to use the parameterized config value
monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code",
parse_consent_code_config)
monkeypatch.setattr(syncer, "parse_consent_code",
parse_consent_code_config)
syncer.sync()
users = db_session.query(models.User).all()
assert len(users) == 11
if parse_consent_code_config:
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000178.c1": ["read", "read-storage"],
"phs000178.c2": ["read", "read-storage"],
"phs000178.c999": ["read", "read-storage"],
"phs000179.c1": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000178.c1": ["read", "read-storage"],
"phs000178.c2": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000179.c1": ["read", "read-storage"],
"phs000178.c1": ["read", "read-storage"],
},
)
else:
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000178": ["read", "read-storage"],
"TCGA-PCAWG": ["read", "read-storage"],
"phs000179": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000178": ["read", "read-storage"],
"TCGA-PCAWG": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
assert equal_project_access(
user.project_access,
{
"phs000178": ["read", "read-storage"],
"TCGA-PCAWG": ["read", "read-storage"],
"phs000179": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
assert user.display_name == "USER D"
assert user.phone_number == "123-456-789"
user = models.query_for_user(session=db_session,
username="******")
user_access = db_session.query(
models.AccessPrivilege).filter_by(user=user).all()
assert set(user_access[0].privilege) == {
"create",
"read",
"update",
"delete",
"upload",
}
assert len(user_access) == 1
# TODO: check user policy access (add in user sync changes)
user = models.query_for_user(session=db_session,
username="******")
assert not user.is_admin
user_access = db_session.query(
models.AccessPrivilege).filter_by(user=user).all()
assert not user_access
def test_dbgap_consent_codes(
syncer,
db_session,
storage_client,
enable_common_exchange_area,
parse_consent_code_config,
monkeypatch,
):
# patch the sync to use the parameterized value for whether or not to parse exchange
# area data
# we moved to support multiple dbgap sftp servers, the config file has a list of dbgap
# for local file dir, we only use the parameters from first dbgap config
# hence only those are mocked here
monkeypatch.setitem(
syncer.dbGaP[0],
"enable_common_exchange_area_access",
enable_common_exchange_area,
)
monkeypatch.setattr(syncer, "parse_consent_code",
parse_consent_code_config)
monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code",
parse_consent_code_config)
monkeypatch.setattr(syncer, "project_mapping", {})
syncer.sync()
user = models.query_for_user(session=db_session, username="******")
if parse_consent_code_config:
if enable_common_exchange_area:
# b/c user has c999, ensure they have access to all consents, study-specific
# exchange area (via .c999) and the common exchange area configured
assert equal_project_access(
user.project_access,
{
"phs000179.c1": ["read", "read-storage"],
"phs000178.c1": ["read", "read-storage"],
"phs000178.c2": ["read", "read-storage"],
"phs000178.c999": ["read", "read-storage"],
# should additionally include the study-specific exchange area access and
# access to the common exchange area
"test_common_exchange_area": ["read", "read-storage"],
},
)
else:
# b/c user has c999 but common exchange area is disabled, ensure they have
# access to all consents, study-specific exchange area (via .c999)
assert equal_project_access(
user.project_access,
{
"phs000179.c1": ["read", "read-storage"],
# c999 gives access to all consents
"phs000178.c1": ["read", "read-storage"],
"phs000178.c2": ["read", "read-storage"],
"phs000178.c999": ["read", "read-storage"],
},
)
else:
# with consent code parsing off, ensure users have access to just phsids
assert equal_project_access(
user.project_access,
{
"phs000178": ["read", "read-storage"],
"phs000179": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
if parse_consent_code_config:
assert equal_project_access(
user.project_access,
{
"phs000178.c1": ["read", "read-storage"],
"phs000178.c2": ["read", "read-storage"],
},
)
else:
assert equal_project_access(user.project_access,
{"phs000178": ["read", "read-storage"]})
user = models.query_for_user(session=db_session, username="******")
if parse_consent_code_config:
assert equal_project_access(
user.project_access,
{
"phs000178.c1": ["read", "read-storage"],
"phs000179.c1": ["read", "read-storage"],
},
)
else:
assert equal_project_access(
user.project_access,
{
"phs000178": ["read", "read-storage"],
"phs000179": ["read", "read-storage"],
},
)
user = models.query_for_user(session=db_session, username="******")
if parse_consent_code_config:
assert equal_project_access(user.project_access,
{"phs000179.c1": ["read", "read-storage"]})
else:
assert equal_project_access(user.project_access,
{"phs000179": ["read", "read-storage"]})
resource_to_parent_paths = {}
for call in syncer.arborist_client.update_resource.call_args_list:
args, kwargs = call
parent_path = args[0]
resource = args[1].get("name")
resource_to_parent_paths.setdefault(resource, []).append(parent_path)
if parse_consent_code_config:
if enable_common_exchange_area:
# b/c user has c999, ensure they have access to all consents, study-specific
# exchange area (via .c999) and the common exchange area configured
assert "phs000178.c999" in resource_to_parent_paths
assert resource_to_parent_paths["phs000178.c999"] == [
"/orgA/programs/"
]
assert "test_common_exchange_area" in resource_to_parent_paths
assert resource_to_parent_paths["test_common_exchange_area"] == [
"/dbgap/programs/"
]
assert "phs000178.c1" in resource_to_parent_paths
assert resource_to_parent_paths["phs000178.c1"] == ["/orgA/programs/"]
# NOTE: this study+consent is configured to have multiple names in the dbgap config
assert "phs000178.c2" in resource_to_parent_paths
assert resource_to_parent_paths["phs000178.c2"] == [
"/orgA/programs/",
"/orgB/programs/",
"/programs/",
]
assert "phs000178.c999" in resource_to_parent_paths
assert resource_to_parent_paths["phs000178.c999"] == [
"/orgA/programs/"
]
assert "phs000179.c1" in resource_to_parent_paths
assert resource_to_parent_paths["phs000179.c1"] == ["/orgA/programs/"]
else:
assert "phs000178" in resource_to_parent_paths
# NOTE: this study is configured to have multiple names in the dbgap config
assert resource_to_parent_paths["phs000178"] == [
"/orgA/programs/",
"/orgB/programs/",
"/programs/",
]
assert "phs000179" in resource_to_parent_paths
assert resource_to_parent_paths["phs000179"] == ["/orgA/programs/"]
def delete_user(current_session, username):
"""
Remove a user from both the userdatamodel
and the associated storage for that project/bucket.
Returns a dictionary.
The Fence db may not always be in perfect sync with Google. We err on the
side of safety (we prioritise making sure the user is really cleared out of
Google to prevent unauthorized data access issues; we prefer cirrus/Google
over the Fence db as the source of truth.) So, if the Fence-Google sync
situation changes, do edit this code accordingly.
"""
logger.debug("Beginning delete user.")
with GoogleCloudManager() as gcm:
# Delete user's service accounts, SA keys, user proxy group from Google.
# Noop if Google not in use.
user = query_for_user(session=current_session, username=username)
if not user:
raise NotFound("user name {} not found".format(username))
logger.debug("Found user in Fence db: {}".format(user))
# First: Find this user's proxy group.
google_proxy_group_from_fence_db = (
current_session.query(GoogleProxyGroup).filter(
GoogleProxyGroup.id == user.google_proxy_group_id).first()
# one_or_none() would be better, but is only in sqlalchemy 1.0.9
)
if google_proxy_group_from_fence_db:
gpg_email = google_proxy_group_from_fence_db.email
logger.debug(
"Found Google proxy group in Fence db: {}".format(gpg_email))
else:
# Construct the proxy group name that would have been used
# and check if it exists in cirrus, in case Fence db just
# didn't know about it.
logger.debug(
"Could not find Google proxy group for this user in Fence db. Checking cirrus..."
)
pgname = get_proxy_group_name_for_user(
user.id, user.username, prefix=config["GOOGLE_GROUP_PREFIX"])
google_proxy_group_from_google = gcm.get_group(pgname)
gpg_email = (google_proxy_group_from_google.get("email")
if google_proxy_group_from_google else None)
if not gpg_email:
logger.info(
"Could not find Google proxy group for user in Fence db or in cirrus. "
"Assuming Google not in use as IdP. Proceeding with Fence deletes."
)
else:
logger.debug(
"Found Google proxy group email of user to delete: {}."
"Proceeding with Google deletions.".format(gpg_email))
# Note: Fence db deletes here are interleaved with Google deletes.
# This is so that if (for example) Google succeeds in deleting one SA
# and then fails on the next, and the deletion process aborts, there
# will not remain a record in Fence of the first, now-nonexistent SA.
delete_google_service_accounts_and_keys(current_session, gcm,
gpg_email)
delete_google_proxy_group(current_session, gcm, gpg_email,
google_proxy_group_from_fence_db, user)
logger.debug("Deleting all user data from Fence database...")
current_session.delete(user)
current_session.commit()
logger.info("Deleted all user data from Fence database. Returning.")
return {"result": "success"}
def test_user_sync_with_visas(
syncer,
db_session,
storage_client,
parse_consent_code_config,
fallback_to_dbgap_sftp,
monkeypatch,
):
# patch the sync to use the parameterized config value
monkeypatch.setitem(syncer.dbGaP[0], "parse_consent_code",
parse_consent_code_config)
monkeypatch.setattr(syncer, "parse_consent_code",
parse_consent_code_config)
monkeypatch.setattr(syncer, "fallback_to_dbgap_sftp",
fallback_to_dbgap_sftp)
monkeypatch.setattr(syncer, "sync_from_visas", True)
syncer.sync_visas()
users = db_session.query(models.User).all()
user = models.query_for_user(
session=db_session,
username="******") # contains only visa information
backup_user = models.query_for_user(
session=db_session, username="******"
) # Contains invalid visa and also in telemetry file
expired_user = models.query_for_user(
session=db_session,
username="******",
)
invalid_user = models.query_for_user(session=db_session,
username="******")
assert len(invalid_user.project_access) == 0
assert len(expired_user.project_access) == 0
assert len(invalid_user.ga4gh_visas_v1) == 0
assert len(expired_user.ga4gh_visas_v1) == 0
if fallback_to_dbgap_sftp:
assert len(users) == 13
if parse_consent_code_config:
assert equal_project_access(
user.project_access,
{
"phs000991.c1": ["read", "read-storage"],
"phs000961.c1": ["read", "read-storage"],
"phs000279.c1": ["read", "read-storage"],
"phs000286.c3": ["read", "read-storage"],
"phs000289.c2": ["read", "read-storage"],
"phs000298.c1": ["read", "read-storage"],
},
)
assert equal_project_access(
backup_user.project_access,
{
"phs000179.c1": ["read", "read-storage"],
},
)
else:
assert equal_project_access(
user.project_access,
{
"phs000991": ["read", "read-storage"],
"phs000961": ["read", "read-storage"],
"phs000279": ["read", "read-storage"],
"phs000286": ["read", "read-storage"],
"phs000289": ["read", "read-storage"],
"phs000298": ["read", "read-storage"],
},
)
assert equal_project_access(
backup_user.project_access,
{
"phs000179": ["read", "read-storage"],
},
)
else:
assert len(users) == 11
assert len(backup_user.project_access) == 0
if parse_consent_code_config:
assert equal_project_access(
user.project_access,
{
"phs000991.c1": ["read", "read-storage"],
"phs000961.c1": ["read", "read-storage"],
"phs000279.c1": ["read", "read-storage"],
"phs000286.c3": ["read", "read-storage"],
"phs000289.c2": ["read", "read-storage"],
"phs000298.c1": ["read", "read-storage"],
},
)
else:
assert equal_project_access(
user.project_access,
{
"phs000991": ["read", "read-storage"],
"phs000961": ["read", "read-storage"],
"phs000279": ["read", "read-storage"],
"phs000286": ["read", "read-storage"],
"phs000289": ["read", "read-storage"],
"phs000298": ["read", "read-storage"],
},
)
def find_user(username, session):
user = query_for_user(session=session, username=username)
if not user:
raise NotFound("user {} not found".format(username))
return user
def force_update_google_link(DB, username, google_email, expires_in=None):
"""
WARNING: This function circumvents Google Auth flow, and should only be
used for internal testing!
WARNING: This function assumes that a user already has a proxy group!
Adds user's google account to proxy group and/or updates expiration for
that google account's access.
WARNING: This assumes that provided arguments represent valid information.
This BLINDLY adds without verification. Do verification
before this.
Specifically, this ASSUMES that the proxy group provided belongs to the
given user and that the user has ALREADY authenticated to prove the
provided google_email is also their's.
Args:
DB
username (str): Username to link with
google_email (str): Google email to link to
Raises:
NotFound: Linked Google account not found
Unauthorized: Couldn't determine user
Returns:
Expiration time of the newly updated google account's access
"""
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(DB)
with db.session as session:
user_account = query_for_user(session=session, username=username)
if user_account:
user_id = user_account.id
proxy_group_id = user_account.google_proxy_group_id
else:
raise Unauthorized(
"Could not determine authed user "
"from session. Unable to link Google account."
)
user_google_account = (
session.query(UserGoogleAccount)
.filter(UserGoogleAccount.email == google_email)
.first()
)
if not user_google_account:
user_google_account = add_new_user_google_account(
user_id, google_email, session
)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
if expires_in:
is_valid_expiration(expires_in)
# convert it to timestamp
requested_expiration = int(time.time()) + expires_in
expiration = min(expiration, requested_expiration)
force_update_user_google_account_expiration(
user_google_account, proxy_group_id, google_email, expiration, session
)
session.commit()
return expiration
def sync_gen3_users_authz_from_ga4gh_passports(
passports,
pkey_cache=None,
db_session=None,
):
"""
Validate passports and embedded visas, using each valid visa's identity
established by <iss, sub> combination to possibly create and definitely
determine a Fence user who is added to the list returned by this
function. In the process of determining Fence users from visas, visa
authorization information is also persisted in Fence and synced to
Arborist.
Args:
passports (list): a list of raw encoded passport strings, each
including header, payload, and signature
Return:
list: a list of users, each corresponding to a valid visa identity
embedded within the passports passed in
"""
db_session = db_session or current_session
# {"username": user, "username2": user2}
users_from_all_passports = {}
for passport in passports:
try:
cached_usernames = get_gen3_usernames_for_passport_from_cache(
passport=passport, db_session=db_session)
if cached_usernames:
# there's a chance a given username exists in the cache but no longer in
# the database. if not all are in db, ignore the cache and actually parse
# and validate the passport
all_users_exist_in_db = True
usernames_to_update = {}
for username in cached_usernames:
user = query_for_user(session=db_session,
username=username)
if not user:
all_users_exist_in_db = False
continue
usernames_to_update[user.username] = user
if all_users_exist_in_db:
users_from_all_passports.update(usernames_to_update)
# existence in the cache and a user in db means that this passport
# was validated previously (expiration was also checked)
continue
# below function also validates passport (or raises exception)
raw_visas = get_unvalidated_visas_from_valid_passport(
passport, pkey_cache=pkey_cache)
except Exception as exc:
logger.warning(
f"Invalid passport provided, ignoring. Error: {exc}")
continue
# an empty raw_visas list means that either the current passport is
# invalid or that it has no visas. in both cases, the current passport
# is ignored and we move on to the next passport
if not raw_visas:
continue
identity_to_visas = collections.defaultdict(list)
min_visa_expiration = int(
time.time()) + datetime.timedelta(hours=1).seconds
for raw_visa in raw_visas:
try:
validated_decoded_visa = validate_visa(raw_visa,
pkey_cache=pkey_cache)
identity_to_visas[(
validated_decoded_visa.get("iss"),
validated_decoded_visa.get("sub"),
)].append((raw_visa, validated_decoded_visa))
min_visa_expiration = min(min_visa_expiration,
validated_decoded_visa.get("exp"))
except Exception as exc:
logger.warning(
f"Invalid visa provided, ignoring. Error: {exc}")
continue
expired_authz_removal_job_freq_in_seconds = config[
"EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS"]
min_visa_expiration -= expired_authz_removal_job_freq_in_seconds
if min_visa_expiration <= int(time.time()):
logger.warning(
"The passport's earliest valid visa expiration time is set to "
f"occur within {expired_authz_removal_job_freq_in_seconds} "
"seconds from now, which is too soon an expiration to handle.")
continue
users_from_current_passport = []
for (issuer, subject_id), visas in identity_to_visas.items():
gen3_user = get_or_create_gen3_user_from_iss_sub(
issuer, subject_id, db_session=db_session)
ga4gh_visas = [
GA4GHVisaV1(
user=gen3_user,
source=validated_decoded_visa["ga4gh_visa_v1"]["source"],
type=validated_decoded_visa["ga4gh_visa_v1"]["type"],
asserted=int(
validated_decoded_visa["ga4gh_visa_v1"]["asserted"]),
expires=int(validated_decoded_visa["exp"]),
ga4gh_visa=raw_visa,
) for raw_visa, validated_decoded_visa in visas
]
# NOTE: does not validate, assumes validation occurs above.
# This adds the visas to the database session but doesn't commit until
# the end of this function
_sync_validated_visa_authorization(
gen3_user=gen3_user,
ga4gh_visas=ga4gh_visas,
expiration=min_visa_expiration,
db_session=db_session,
)
users_from_current_passport.append(gen3_user)
for user in users_from_current_passport:
users_from_all_passports[user.username] = user
put_gen3_usernames_for_passport_into_cache(
passport=passport,
user_ids_from_passports=list(users_from_all_passports.keys()),
expires_at=min_visa_expiration,
db_session=db_session,
)
db_session.commit()
logger.info(
f"Got Gen3 usernames from passport(s): {list(users_from_all_passports.keys())}"
)
return users_from_all_passports
def sync_to_db_and_storage_backend(self, user_project, user_info,
user_policies, sess):
"""
sync user access control to database and storage backend
Args:
user_project (dict): a dictionary of
{
username: {
'project1': {'read-storage','write-storage'},
'project2': {'read-storage'}
}
}
user_info (dict): a dictionary of {username: user_info{}}
user_policies (List[str]): list of policies
sess: a sqlalchemy session
Return:
None
"""
self._init_projects(user_project, sess)
auth_provider_list = [
self._get_or_create(sess, AuthorizationProvider, name="dbGaP"),
self._get_or_create(sess, AuthorizationProvider, name="fence"),
]
cur_db_user_project_list = {
(ua.user.username.lower(), ua.project.auth_id)
for ua in sess.query(AccessPrivilege).all()
}
# we need to compare db -> whitelist case-insensitively for username
# db stores case-sensitively, but we need to query case-insensitively
user_project_lowercase = {}
syncing_user_project_list = set()
for username, projects in user_project.iteritems():
user_project_lowercase[username.lower()] = projects
for project, _ in projects.iteritems():
syncing_user_project_list.add((username.lower(), project))
user_info_lowercase = {
username.lower(): info
for username, info in user_info.iteritems()
}
to_delete = set.difference(cur_db_user_project_list,
syncing_user_project_list)
to_add = set.difference(syncing_user_project_list,
cur_db_user_project_list)
to_update = set.intersection(cur_db_user_project_list,
syncing_user_project_list)
# when updating users we want to maintain case sesitivity in the username so
# pass the original, non-lowered user_info dict
self._upsert_userinfo(sess, user_info)
self._revoke_from_storage(to_delete, sess)
self._revoke_from_db(sess, to_delete)
self._grant_from_storage(to_add, user_project_lowercase, sess)
self._grant_from_db(
sess,
to_add,
user_info_lowercase,
user_project_lowercase,
auth_provider_list,
)
# re-grant
self._grant_from_storage(to_update, user_project_lowercase, sess)
self._update_from_db(sess, to_update, user_project_lowercase)
self._validate_and_update_user_admin(sess, user_info_lowercase)
# Add policies to user models in the database. These will show up in users'
# JWTs; services can send the JWTs to arborist.
if user_policies:
self.logger.info("populating RBAC information from YAML file")
for username, policies in user_policies.iteritems():
user = query_for_user(session=sess, username=username)
for policy_id in policies:
policy = self._get_or_create_policy(sess, policy_id)
if policy not in user.policies:
user.policies.append(policy)
self.logger.info(
"granted policy `{}` to user `{}` ({})".format(
policy_id, username, user.id))
sess.commit()
def _update_arborist(self, session, user_yaml):
"""
Create roles and resources in arborist from the information in
``user_projects``.
The projects are sent to arborist as resources with paths like
``/projects/{project}``. Roles are created with just the original names
for the privileges like ``"read-storage"`` etc.
Args:
session (sqlalchemy.Session)
user_yaml (UserYAML)
Return:
bool: success
"""
if not self.arborist_client:
self.logger.warn("no arborist client set; skipping arborist sync")
return False
if not self.arborist_client.healthy():
# TODO (rudyardrichter, 2019-01-07): add backoff/retry here
self.logger.error(
"arborist service is unavailable; skipping arborist sync")
return False
# Set up the resource tree in arborist
resources = user_yaml.rbac.get("resources", [])
for resource in resources:
try:
self.arborist_client.create_resource("/",
resource,
overwrite=True)
except ArboristError as e:
self.logger.error(e)
# keep going; maybe just some conflicts from things existing already
created_roles = set()
roles = user_yaml.rbac.get("roles", [])
for role in roles:
try:
response = self.arborist_client.create_role(role)
if response:
created_roles.add(role["id"])
except ArboristError as e:
self.logger.error(e)
# keep going; maybe just some conflicts from things existing already
created_policies = set()
policies = user_yaml.rbac.get("policies", [])
for policy in policies:
try:
response = self.arborist_client.create_policy(policy)
if response:
created_policies.add(policy["id"])
except ArboristError as e:
self.logger.error(e)
# keep going; maybe just some conflicts from things existing already
user_projects = user_yaml.user_rbac
for username, user_resources in user_projects.iteritems():
self.logger.info("processing user `{}`".format(username))
user = query_for_user(session=session, username=username)
for path, permissions in user_resources.iteritems():
for permission in permissions:
# "permission" in the dbgap sense, not the arborist sense
if permission not in created_roles:
try:
self.arborist_client.create_role(
arborist_role_for_permission(permission))
except ArboristError as e:
self.logger.info(
"not creating role for permission `{}`; {}".
format(permission, str(e)))
created_roles.add(permission)
# If everything was created fine, grant a policy to
# this user which contains exactly just this resource,
# with this permission as a role.
# format project '/x/y/z' -> 'x.y.z'
# so the policy id will be something like 'x.y.z-create'
policy_id = _format_policy_id(path, permission)
if policy_id not in created_policies:
try:
self.arborist_client.create_policy({
"id":
policy_id,
"description":
"policy created by fence sync",
"role_ids": [permission],
"resource_paths": [path],
})
except ArboristError as e:
self.logger.info(
"not creating policy in arborist; {}".format(
str(e)))
created_policies.add(policy_id)
policy = self._get_or_create_policy(session, policy_id)
user.policies.append(policy)
self.logger.info("granted policy `{}` to user `{}`".format(
policy_id, user.username))
return True
def get_or_create_proxy_group_id(expires=None, user_id=None, username=None):
"""
If no username returned from token or database, create a new proxy group
for the given user. Also, add the access privileges.
Returns:
int: id of (possibly newly created) proxy group associated with user
"""
proxy_group_id = _get_proxy_group_id(user_id=user_id, username=username)
if not proxy_group_id:
try:
user_by_id = query_for_user_by_id(current_session, user_id)
user_by_username = query_for_user(
session=current_session, username=username
)
except Exception:
user_by_id = None
user_by_username = None
if user_by_id:
user_id = user_id
username = user_by_id.username
elif user_by_username:
user_id = user_by_username.id
username = username
elif current_token:
user_id = current_token["sub"]
username = current_token.get("context", {}).get("user", {}).get("name", "")
else:
raise Exception(
f"could not find user given input user_id={user_id} or "
f"username={username}, nor was there a current_token"
)
proxy_group_id = _create_proxy_group(user_id, username).id
privileges = current_session.query(AccessPrivilege).filter(
AccessPrivilege.user_id == user_id
)
for p in privileges:
storage_accesses = p.project.storage_access
for sa in storage_accesses:
if sa.provider.name == STORAGE_ACCESS_PROVIDER_NAME:
flask.current_app.storage_manager.logger.info(
"grant {} access {} to {} in {}".format(
username, p.privilege, p.project_id, p.auth_provider
)
)
flask.current_app.storage_manager.grant_access(
provider=(sa.provider.name),
username=username,
project=p.project,
access=p.privilege,
session=current_session,
expires=expires,
)
return proxy_group_id
