def link_external_bucket(db, name):
"""
Link with bucket owned by an external party. This will create the bucket
in fence database and create a google group to access the bucket in both
Google and fence database.
The external party will need to add the google group read access to bucket
afterwards.
"""
cirrus_config.update(**config["CIRRUS_CFG"])
google_project_id = cirrus_config.GOOGLE_PROJECT_ID
db = SQLAlchemyDriver(db)
with db.session as current_session:
google_cloud_provider = _get_or_create_google_provider(current_session)
bucket_db_entry = Bucket(name=name, provider_id=google_cloud_provider.id)
current_session.add(bucket_db_entry)
current_session.commit()
privileges = ["read"]
access_group = _create_google_bucket_access_group(
current_session, name, bucket_db_entry.id, google_project_id, privileges
)
logger.info("bucket access group email: {}".format(access_group.email))
return access_group.email
def delete_expired_service_accounts(DB):
"""
Delete all expired service accounts.
"""
cirrus_config.update(**config["CIRRUS_CFG"])
driver = SQLAlchemyDriver(DB)
with driver.session as session:
current_time = int(time.time())
records_to_delete = (
session.query(ServiceAccountToGoogleBucketAccessGroup)
.filter(ServiceAccountToGoogleBucketAccessGroup.expires < current_time)
.all()
)
if len(records_to_delete):
with GoogleCloudManager() as manager:
for record in records_to_delete:
try:
manager.remove_member_from_group(
record.service_account.email, record.access_group.email
)
session.delete(record)
logger.info(
"Removed expired service account: {}".format(
record.service_account.email
)
)
except Exception as e:
logger.error(
"ERROR: Could not delete service account {}. Details: {}".format(
record.service_account.email, e
)
)
session.commit()
def verify_user_registration(DB):
"""
Validate user registration
"""
cirrus_config.update(**config["CIRRUS_CFG"])
validation_check(DB)
def create_google_logging_bucket(name, storage_class=None, google_project_id=None):
cirrus_config.update(**config["CIRRUS_CFG"])
# determine project where buckets are located if not provided, default
# to configured project if checking creds doesn't work
storage_creds_project_id = (
google_project_id
or _get_storage_project_id()
or cirrus_config.GOOGLE_PROJECT_ID
)
manager = GoogleCloudManager(
storage_creds_project_id, creds=cirrus_config.configs["GOOGLE_STORAGE_CREDS"]
)
with manager as g_mgr:
g_mgr.create_or_update_bucket(
name,
storage_class=storage_class,
public=False,
requester_pays=False,
for_logging=True,
)
logger.info(
"Successfully created Google Bucket {} "
"to store Access Logs.".format(name)
)
def delete_client_action(DB, client_name):
try:
cirrus_config.update(**config["CIRRUS_CFG"])
except AttributeError:
# no cirrus config, continue anyway. we don't have client service accounts
# to delete
pass
try:
driver = SQLAlchemyDriver(DB)
with driver.session as current_session:
if (
not current_session.query(Client)
.filter(Client.name == client_name)
.first()
):
raise Exception("client {} does not exist".format(client_name))
clients = (
current_session.query(Client).filter(Client.name == client_name).all()
)
for client in clients:
_remove_client_service_accounts(current_session, client)
current_session.delete(client)
current_session.commit()
logger.info("Client {} deleted".format(client_name))
except Exception as e:
logger.error(str(e))
def delete_client_action(DB, client_name):
import fence.settings
try:
cirrus_config.update(**fence.settings.CIRRUS_CFG)
except AttributeError:
# no cirrus config, continue anyway. we don't have client service accounts
# to delete
pass
try:
driver = SQLAlchemyDriver(DB)
with driver.session as current_session:
if (not current_session.query(Client).filter(
Client.name == client_name).first()):
raise Exception("client {} does not exist".format(client_name))
clients = current_session.query(Client).filter(
Client.name == client_name)
for client in clients:
_remove_client_service_accounts(current_session, client)
clients.delete()
current_session.commit()
print("Client {} deleted".format(client_name))
except Exception as e:
print(e.message)
def force_update_google_link(DB, username, google_email):
"""
WARNING: This function circumvents Google Auth flow, and should only be
used for internal testing!
WARNING: This function assumes that a user already has a proxy group!
Adds user's google account to proxy group and/or updates expiration for
that google account's access.
WARNING: This assumes that provided arguments represent valid information.
This BLINDLY adds without verification. Do verification
before this.
Specifically, this ASSUMES that the proxy group provided belongs to the
given user and that the user has ALREADY authenticated to prove the
provided google_email is also their's.
Args:
DB
username (str): Username to link with
google_email (str): Google email to link to
Raises:
NotFound: Linked Google account not found
Unauthorized: Couldn't determine user
Returns:
Expiration time of the newly updated google account's access
"""
import fence.settings
cirrus_config.update(**fence.settings.CIRRUS_CFG)
db = SQLAlchemyDriver(DB)
with db.session as session:
user_account = session.query(User).filter(
User.username == username).first()
if user_account:
user_id = user_account.id
proxy_group_id = user_account.google_proxy_group_id
else:
raise Unauthorized("Could not determine authed user "
"from session. Unable to link Google account.")
user_google_account = (session.query(UserGoogleAccount).filter(
UserGoogleAccount.email == google_email).first())
if not user_google_account:
user_google_account = add_new_user_google_account(
user_id, google_email, session)
now = int(time.time())
expiration = now + GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN
force_update_user_google_account_expiration(user_google_account,
proxy_group_id,
google_email, expiration,
session)
session.commit()
return expiration
def verify_user_registration(DB, config):
"""
Validate user registration
"""
import fence.settings
cirrus_config.update(**fence.settings.CIRRUS_CFG)
validation_check(DB, config)
def remove_expired_google_service_account_keys(db):
import fence.settings
cirrus_config.update(**fence.settings.CIRRUS_CFG)
db = SQLAlchemyDriver(db)
with db.session as current_session:
client_service_accounts = current_session.query(
GoogleServiceAccount,
Client).filter(GoogleServiceAccount.client_id == Client.client_id)
current_time = int(time.time())
print("Current time: {}\n".format(current_time))
expired_sa_keys_for_users = current_session.query(
GoogleServiceAccountKey).filter(
GoogleServiceAccountKey.expires <= current_time)
# handle service accounts with default max expiration
for service_account, client in client_service_accounts:
with GoogleCloudManager() as g_mgr:
g_mgr.handle_expired_service_account_keys(
service_account.google_unique_id)
# handle service accounts with custom expiration
for expired_user_key in expired_sa_keys_for_users:
sa = (current_session.query(GoogleServiceAccount).filter(
GoogleServiceAccount.id ==
expired_user_key.service_account_id).first())
response = g_mgr.delete_service_account_key(
account=sa.google_unique_id,
key_name=expired_user_key.key_id)
response_error_code = response.get("error", {}).get("code")
if not response_error_code:
current_session.delete(expired_user_key)
print(
"INFO: Removed expired service account key {} "
"for service account {} (owned by user with id {}).\n".
format(expired_user_key.key_id, sa.email, sa.user_id))
elif response_error_code == 404:
print(
"INFO: Service account key {} for service account {} "
"(owned by user with id {}) does not exist in Google. "
"Removing from database...\n".format(
expired_user_key.key_id, sa.email, sa.user_id))
current_session.delete(expired_user_key)
else:
print("ERROR: Google returned an error when attempting to "
"remove service account key {} "
"for service account {} (owned by user with id {}). "
"Error:\n{}\n".format(expired_user_key.key_id,
sa.email, sa.user_id,
response))
def remove_expired_google_accounts_from_proxy_groups(db):
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(db)
with db.session as current_session:
current_time = int(time.time())
logger.info("Current time: {}".format(current_time))
expired_accounts = current_session.query(UserGoogleAccountToProxyGroup).filter(
UserGoogleAccountToProxyGroup.expires <= current_time
)
with GoogleCloudManager() as g_mgr:
for expired_account_access in expired_accounts:
g_account = (
current_session.query(UserGoogleAccount)
.filter(
UserGoogleAccount.id
== expired_account_access.user_google_account_id
)
.first()
)
try:
response = g_mgr.remove_member_from_group(
member_email=g_account.email,
group_id=expired_account_access.proxy_group_id,
)
response_error_code = response.get("error", {}).get("code")
if not response_error_code:
current_session.delete(expired_account_access)
logger.info(
"INFO: Removed {} from proxy group with id {}.\n".format(
g_account.email, expired_account_access.proxy_group_id
)
)
else:
logger.error(
"ERROR: Google returned an error when attempting to "
"remove member {} from proxy group {}. Error:\n{}\n".format(
g_account.email,
expired_account_access.proxy_group_id,
response,
)
)
except Exception as exc:
logger.error(
"ERROR: Google returned an error when attempting to "
"remove member {} from proxy group {}. Error:\n{}\n".format(
g_account.email, expired_account_access.proxy_group_id, exc
)
)
def verify_bucket_access_group(DB):
"""
Go through all the google group members, remove them from Google group and Google
user service account if they are not in Fence
Args:
DB(str): db connection string
Returns:
None
"""
cirrus_config.update(**config["CIRRUS_CFG"])
driver = SQLAlchemyDriver(DB)
with driver.session as session:
access_groups = session.query(GoogleBucketAccessGroup).all()
with GoogleCloudManager() as manager:
for access_group in access_groups:
try:
members = manager.get_group_members(access_group.email)
logger.debug(f"google group members response: {members}")
except GoogleAuthError as e:
logger.error("ERROR: Authentication error!!!. Detail {}".format(e))
return
except Exception as e:
logger.error(
"ERROR: Could not list group members of {}. Detail {}".format(
access_group.email, e
)
)
return
for member in members:
if member.get("type") == "GROUP":
_verify_google_group_member(session, access_group, member)
elif member.get("type") == "USER":
_verify_google_service_account_member(
session, access_group, member
)
def init_syncer(
dbGaP,
STORAGE_CREDENTIALS,
DB,
projects=None,
is_sync_from_dbgap_server=False,
sync_from_local_csv_dir=None,
sync_from_local_yaml_file=None,
arborist=None,
folder=None,
):
"""
sync ACL files from dbGap to auth db and storage backends
imports from config is done here because dbGap is
an optional requirement for fence so it might not be specified
in config
Args:
projects: path to project_mapping yaml file which contains mapping
from dbgap phsids to projects in fence database
Returns:
None
Examples:
the expected yaml structure sould look like:
.. code-block:: yaml
phs000178:
- name: TCGA
auth_id: phs000178
- name: TCGA-PCAWG
auth_id: TCGA-PCAWG
phs000235:
- name: CGCI
auth_id: phs000235
"""
try:
cirrus_config.update(**config["CIRRUS_CFG"])
except AttributeError:
# no cirrus config, continue anyway. Google APIs will probably fail.
# this is okay if users don't need access to Google buckets
pass
if projects is not None and not os.path.exists(projects):
logger.error("====={} is not found!!!=======".format(projects))
return
if sync_from_local_csv_dir and not os.path.exists(sync_from_local_csv_dir):
logger.error("====={} is not found!!!=======".format(sync_from_local_csv_dir))
return
if sync_from_local_yaml_file and not os.path.exists(sync_from_local_yaml_file):
logger.error("====={} is not found!!!=======".format(sync_from_local_yaml_file))
return
project_mapping = None
if projects:
try:
with open(projects, "r") as f:
project_mapping = safe_load(f)
except IOError:
pass
return UserSyncer(
dbGaP,
DB,
project_mapping=project_mapping,
storage_credentials=STORAGE_CREDENTIALS,
is_sync_from_dbgap_server=is_sync_from_dbgap_server,
sync_from_local_csv_dir=sync_from_local_csv_dir,
sync_from_local_yaml_file=sync_from_local_yaml_file,
arborist=arborist,
folder=folder,
)
def force_update_google_link(DB, username, google_email, expires_in=None):
"""
WARNING: This function circumvents Google Auth flow, and should only be
used for internal testing!
WARNING: This function assumes that a user already has a proxy group!
Adds user's google account to proxy group and/or updates expiration for
that google account's access.
WARNING: This assumes that provided arguments represent valid information.
This BLINDLY adds without verification. Do verification
before this.
Specifically, this ASSUMES that the proxy group provided belongs to the
given user and that the user has ALREADY authenticated to prove the
provided google_email is also their's.
Args:
DB
username (str): Username to link with
google_email (str): Google email to link to
Raises:
NotFound: Linked Google account not found
Unauthorized: Couldn't determine user
Returns:
Expiration time of the newly updated google account's access
"""
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(DB)
with db.session as session:
user_account = query_for_user(session=session, username=username)
if user_account:
user_id = user_account.id
proxy_group_id = user_account.google_proxy_group_id
else:
raise Unauthorized(
"Could not determine authed user "
"from session. Unable to link Google account."
)
user_google_account = (
session.query(UserGoogleAccount)
.filter(UserGoogleAccount.email == google_email)
.first()
)
if not user_google_account:
user_google_account = add_new_user_google_account(
user_id, google_email, session
)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
if expires_in:
is_valid_expiration(expires_in)
# convert it to timestamp
requested_expiration = int(time.time()) + expires_in
expiration = min(expiration, requested_expiration)
force_update_user_google_account_expiration(
user_google_account, proxy_group_id, google_email, expiration, session
)
session.commit()
return expiration
def create_or_update_google_bucket(
db,
name,
storage_class=None,
public=None,
requester_pays=False,
google_project_id=None,
project_auth_id=None,
access_logs_bucket=None,
allowed_privileges=None,
):
"""
Create a Google bucket and populate database with necessary information.
If the bucket is not public, this will also create a Google Bucket Access
Group(s) to control access to the new bucket. In order to give access
to a new user, simply add them to the Google Bucket Access Group.
NOTE: At the moment, a different Google Bucket Access Group is created
for each different privilege in allowed_privileges (which defaults
to ['read', 'write']). So there will be separate Google Groups for
each access level.
Args:
db (TYPE): database
name (str): name for the bucket, must be globally unique throughout Google
storage_class (str): enum, one of the cirrus's GOOGLE_STORAGE_CLASSES
public (bool or None, optional): whether or not the bucket should be public.
None means leave IAM on the bucket unchanged.
requester_pays (bool, optional): Whether or not to enable requester_pays
on the bucket
google_project_id (str, optional): Google project this bucket should be
associated with
project_auth_id (str, optional): a Project.auth_id to associate this
bucket with. The project must exist in the db already.
access_logs_bucket (str, optional): Enables logging. Must provide a
Google bucket name which will store the access logs
allowed_privileges (List(str), optional): privileges to allow on
the bucket. Defaults to ['read', 'write']. Also allows:
['admin'] for all permission on the bucket including delete,
['read'] for viewing access,
['write'] for creation rights but not viewing access
"""
cirrus_config.update(**config["CIRRUS_CFG"])
google_project_id = google_project_id or cirrus_config.GOOGLE_PROJECT_ID
# determine project where buckets are located
# default to same project, try to get storage creds project from key file
storage_creds_project_id = _get_storage_project_id() or google_project_id
# default to read access
allowed_privileges = allowed_privileges or ["read", "write"]
driver = SQLAlchemyDriver(db)
with driver.session as current_session:
# use storage creds to create bucket
# (default creds don't have permission)
bucket_db_entry = _create_or_update_google_bucket_and_db(
db_session=current_session,
name=name,
storage_class=storage_class,
requester_pays=requester_pays,
storage_creds_project_id=storage_creds_project_id,
public=public,
project_auth_id=project_auth_id,
access_logs_bucket=access_logs_bucket,
)
if public is not None and not public:
for privilege in allowed_privileges:
_setup_google_bucket_access_group(
db_session=current_session,
google_bucket_name=name,
bucket_db_id=bucket_db_entry.id,
google_project_id=google_project_id,
storage_creds_project_id=storage_creds_project_id,
privileges=[privilege],
)
