def link_external_bucket(db, name): """ Link with bucket owned by an external party. This will create the bucket in fence database and create a google group to access the bucket in both Google and fence database. The external party will need to add the google group read access to bucket afterwards. """ cirrus_config.update(**config["CIRRUS_CFG"]) google_project_id = cirrus_config.GOOGLE_PROJECT_ID db = SQLAlchemyDriver(db) with db.session as current_session: google_cloud_provider = _get_or_create_google_provider(current_session) bucket_db_entry = Bucket(name=name, provider_id=google_cloud_provider.id) current_session.add(bucket_db_entry) current_session.commit() privileges = ["read"] access_group = _create_google_bucket_access_group( current_session, name, bucket_db_entry.id, google_project_id, privileges ) logger.info("bucket access group email: {}".format(access_group.email)) return access_group.email
def delete_expired_service_accounts(DB): """ Delete all expired service accounts. """ cirrus_config.update(**config["CIRRUS_CFG"]) driver = SQLAlchemyDriver(DB) with driver.session as session: current_time = int(time.time()) records_to_delete = ( session.query(ServiceAccountToGoogleBucketAccessGroup) .filter(ServiceAccountToGoogleBucketAccessGroup.expires < current_time) .all() ) if len(records_to_delete): with GoogleCloudManager() as manager: for record in records_to_delete: try: manager.remove_member_from_group( record.service_account.email, record.access_group.email ) session.delete(record) logger.info( "Removed expired service account: {}".format( record.service_account.email ) ) except Exception as e: logger.error( "ERROR: Could not delete service account {}. Details: {}".format( record.service_account.email, e ) ) session.commit()
def verify_user_registration(DB): """ Validate user registration """ cirrus_config.update(**config["CIRRUS_CFG"]) validation_check(DB)
def create_google_logging_bucket(name, storage_class=None, google_project_id=None): cirrus_config.update(**config["CIRRUS_CFG"]) # determine project where buckets are located if not provided, default # to configured project if checking creds doesn't work storage_creds_project_id = ( google_project_id or _get_storage_project_id() or cirrus_config.GOOGLE_PROJECT_ID ) manager = GoogleCloudManager( storage_creds_project_id, creds=cirrus_config.configs["GOOGLE_STORAGE_CREDS"] ) with manager as g_mgr: g_mgr.create_or_update_bucket( name, storage_class=storage_class, public=False, requester_pays=False, for_logging=True, ) logger.info( "Successfully created Google Bucket {} " "to store Access Logs.".format(name) )
def delete_client_action(DB, client_name): try: cirrus_config.update(**config["CIRRUS_CFG"]) except AttributeError: # no cirrus config, continue anyway. we don't have client service accounts # to delete pass try: driver = SQLAlchemyDriver(DB) with driver.session as current_session: if ( not current_session.query(Client) .filter(Client.name == client_name) .first() ): raise Exception("client {} does not exist".format(client_name)) clients = ( current_session.query(Client).filter(Client.name == client_name).all() ) for client in clients: _remove_client_service_accounts(current_session, client) current_session.delete(client) current_session.commit() logger.info("Client {} deleted".format(client_name)) except Exception as e: logger.error(str(e))
def delete_client_action(DB, client_name): import fence.settings try: cirrus_config.update(**fence.settings.CIRRUS_CFG) except AttributeError: # no cirrus config, continue anyway. we don't have client service accounts # to delete pass try: driver = SQLAlchemyDriver(DB) with driver.session as current_session: if (not current_session.query(Client).filter( Client.name == client_name).first()): raise Exception("client {} does not exist".format(client_name)) clients = current_session.query(Client).filter( Client.name == client_name) for client in clients: _remove_client_service_accounts(current_session, client) clients.delete() current_session.commit() print("Client {} deleted".format(client_name)) except Exception as e: print(e.message)
def force_update_google_link(DB, username, google_email): """ WARNING: This function circumvents Google Auth flow, and should only be used for internal testing! WARNING: This function assumes that a user already has a proxy group! Adds user's google account to proxy group and/or updates expiration for that google account's access. WARNING: This assumes that provided arguments represent valid information. This BLINDLY adds without verification. Do verification before this. Specifically, this ASSUMES that the proxy group provided belongs to the given user and that the user has ALREADY authenticated to prove the provided google_email is also their's. Args: DB username (str): Username to link with google_email (str): Google email to link to Raises: NotFound: Linked Google account not found Unauthorized: Couldn't determine user Returns: Expiration time of the newly updated google account's access """ import fence.settings cirrus_config.update(**fence.settings.CIRRUS_CFG) db = SQLAlchemyDriver(DB) with db.session as session: user_account = session.query(User).filter( User.username == username).first() if user_account: user_id = user_account.id proxy_group_id = user_account.google_proxy_group_id else: raise Unauthorized("Could not determine authed user " "from session. Unable to link Google account.") user_google_account = (session.query(UserGoogleAccount).filter( UserGoogleAccount.email == google_email).first()) if not user_google_account: user_google_account = add_new_user_google_account( user_id, google_email, session) now = int(time.time()) expiration = now + GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN force_update_user_google_account_expiration(user_google_account, proxy_group_id, google_email, expiration, session) session.commit() return expiration
def verify_user_registration(DB, config): """ Validate user registration """ import fence.settings cirrus_config.update(**fence.settings.CIRRUS_CFG) validation_check(DB, config)
def remove_expired_google_service_account_keys(db): import fence.settings cirrus_config.update(**fence.settings.CIRRUS_CFG) db = SQLAlchemyDriver(db) with db.session as current_session: client_service_accounts = current_session.query( GoogleServiceAccount, Client).filter(GoogleServiceAccount.client_id == Client.client_id) current_time = int(time.time()) print("Current time: {}\n".format(current_time)) expired_sa_keys_for_users = current_session.query( GoogleServiceAccountKey).filter( GoogleServiceAccountKey.expires <= current_time) # handle service accounts with default max expiration for service_account, client in client_service_accounts: with GoogleCloudManager() as g_mgr: g_mgr.handle_expired_service_account_keys( service_account.google_unique_id) # handle service accounts with custom expiration for expired_user_key in expired_sa_keys_for_users: sa = (current_session.query(GoogleServiceAccount).filter( GoogleServiceAccount.id == expired_user_key.service_account_id).first()) response = g_mgr.delete_service_account_key( account=sa.google_unique_id, key_name=expired_user_key.key_id) response_error_code = response.get("error", {}).get("code") if not response_error_code: current_session.delete(expired_user_key) print( "INFO: Removed expired service account key {} " "for service account {} (owned by user with id {}).\n". format(expired_user_key.key_id, sa.email, sa.user_id)) elif response_error_code == 404: print( "INFO: Service account key {} for service account {} " "(owned by user with id {}) does not exist in Google. " "Removing from database...\n".format( expired_user_key.key_id, sa.email, sa.user_id)) current_session.delete(expired_user_key) else: print("ERROR: Google returned an error when attempting to " "remove service account key {} " "for service account {} (owned by user with id {}). " "Error:\n{}\n".format(expired_user_key.key_id, sa.email, sa.user_id, response))
def remove_expired_google_accounts_from_proxy_groups(db): cirrus_config.update(**config["CIRRUS_CFG"]) db = SQLAlchemyDriver(db) with db.session as current_session: current_time = int(time.time()) logger.info("Current time: {}".format(current_time)) expired_accounts = current_session.query(UserGoogleAccountToProxyGroup).filter( UserGoogleAccountToProxyGroup.expires <= current_time ) with GoogleCloudManager() as g_mgr: for expired_account_access in expired_accounts: g_account = ( current_session.query(UserGoogleAccount) .filter( UserGoogleAccount.id == expired_account_access.user_google_account_id ) .first() ) try: response = g_mgr.remove_member_from_group( member_email=g_account.email, group_id=expired_account_access.proxy_group_id, ) response_error_code = response.get("error", {}).get("code") if not response_error_code: current_session.delete(expired_account_access) logger.info( "INFO: Removed {} from proxy group with id {}.\n".format( g_account.email, expired_account_access.proxy_group_id ) ) else: logger.error( "ERROR: Google returned an error when attempting to " "remove member {} from proxy group {}. Error:\n{}\n".format( g_account.email, expired_account_access.proxy_group_id, response, ) ) except Exception as exc: logger.error( "ERROR: Google returned an error when attempting to " "remove member {} from proxy group {}. Error:\n{}\n".format( g_account.email, expired_account_access.proxy_group_id, exc ) )
def verify_bucket_access_group(DB): """ Go through all the google group members, remove them from Google group and Google user service account if they are not in Fence Args: DB(str): db connection string Returns: None """ cirrus_config.update(**config["CIRRUS_CFG"]) driver = SQLAlchemyDriver(DB) with driver.session as session: access_groups = session.query(GoogleBucketAccessGroup).all() with GoogleCloudManager() as manager: for access_group in access_groups: try: members = manager.get_group_members(access_group.email) logger.debug(f"google group members response: {members}") except GoogleAuthError as e: logger.error("ERROR: Authentication error!!!. Detail {}".format(e)) return except Exception as e: logger.error( "ERROR: Could not list group members of {}. Detail {}".format( access_group.email, e ) ) return for member in members: if member.get("type") == "GROUP": _verify_google_group_member(session, access_group, member) elif member.get("type") == "USER": _verify_google_service_account_member( session, access_group, member )
def init_syncer( dbGaP, STORAGE_CREDENTIALS, DB, projects=None, is_sync_from_dbgap_server=False, sync_from_local_csv_dir=None, sync_from_local_yaml_file=None, arborist=None, folder=None, ): """ sync ACL files from dbGap to auth db and storage backends imports from config is done here because dbGap is an optional requirement for fence so it might not be specified in config Args: projects: path to project_mapping yaml file which contains mapping from dbgap phsids to projects in fence database Returns: None Examples: the expected yaml structure sould look like: .. code-block:: yaml phs000178: - name: TCGA auth_id: phs000178 - name: TCGA-PCAWG auth_id: TCGA-PCAWG phs000235: - name: CGCI auth_id: phs000235 """ try: cirrus_config.update(**config["CIRRUS_CFG"]) except AttributeError: # no cirrus config, continue anyway. Google APIs will probably fail. # this is okay if users don't need access to Google buckets pass if projects is not None and not os.path.exists(projects): logger.error("====={} is not found!!!=======".format(projects)) return if sync_from_local_csv_dir and not os.path.exists(sync_from_local_csv_dir): logger.error("====={} is not found!!!=======".format(sync_from_local_csv_dir)) return if sync_from_local_yaml_file and not os.path.exists(sync_from_local_yaml_file): logger.error("====={} is not found!!!=======".format(sync_from_local_yaml_file)) return project_mapping = None if projects: try: with open(projects, "r") as f: project_mapping = safe_load(f) except IOError: pass return UserSyncer( dbGaP, DB, project_mapping=project_mapping, storage_credentials=STORAGE_CREDENTIALS, is_sync_from_dbgap_server=is_sync_from_dbgap_server, sync_from_local_csv_dir=sync_from_local_csv_dir, sync_from_local_yaml_file=sync_from_local_yaml_file, arborist=arborist, folder=folder, )
def force_update_google_link(DB, username, google_email, expires_in=None): """ WARNING: This function circumvents Google Auth flow, and should only be used for internal testing! WARNING: This function assumes that a user already has a proxy group! Adds user's google account to proxy group and/or updates expiration for that google account's access. WARNING: This assumes that provided arguments represent valid information. This BLINDLY adds without verification. Do verification before this. Specifically, this ASSUMES that the proxy group provided belongs to the given user and that the user has ALREADY authenticated to prove the provided google_email is also their's. Args: DB username (str): Username to link with google_email (str): Google email to link to Raises: NotFound: Linked Google account not found Unauthorized: Couldn't determine user Returns: Expiration time of the newly updated google account's access """ cirrus_config.update(**config["CIRRUS_CFG"]) db = SQLAlchemyDriver(DB) with db.session as session: user_account = query_for_user(session=session, username=username) if user_account: user_id = user_account.id proxy_group_id = user_account.google_proxy_group_id else: raise Unauthorized( "Could not determine authed user " "from session. Unable to link Google account." ) user_google_account = ( session.query(UserGoogleAccount) .filter(UserGoogleAccount.email == google_email) .first() ) if not user_google_account: user_google_account = add_new_user_google_account( user_id, google_email, session ) # timestamp at which the SA will lose bucket access # by default: use configured time or 7 days expiration = int(time.time()) + config.get( "GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800 ) if expires_in: is_valid_expiration(expires_in) # convert it to timestamp requested_expiration = int(time.time()) + expires_in expiration = min(expiration, requested_expiration) force_update_user_google_account_expiration( user_google_account, proxy_group_id, google_email, expiration, session ) session.commit() return expiration
def create_or_update_google_bucket( db, name, storage_class=None, public=None, requester_pays=False, google_project_id=None, project_auth_id=None, access_logs_bucket=None, allowed_privileges=None, ): """ Create a Google bucket and populate database with necessary information. If the bucket is not public, this will also create a Google Bucket Access Group(s) to control access to the new bucket. In order to give access to a new user, simply add them to the Google Bucket Access Group. NOTE: At the moment, a different Google Bucket Access Group is created for each different privilege in allowed_privileges (which defaults to ['read', 'write']). So there will be separate Google Groups for each access level. Args: db (TYPE): database name (str): name for the bucket, must be globally unique throughout Google storage_class (str): enum, one of the cirrus's GOOGLE_STORAGE_CLASSES public (bool or None, optional): whether or not the bucket should be public. None means leave IAM on the bucket unchanged. requester_pays (bool, optional): Whether or not to enable requester_pays on the bucket google_project_id (str, optional): Google project this bucket should be associated with project_auth_id (str, optional): a Project.auth_id to associate this bucket with. The project must exist in the db already. access_logs_bucket (str, optional): Enables logging. Must provide a Google bucket name which will store the access logs allowed_privileges (List(str), optional): privileges to allow on the bucket. Defaults to ['read', 'write']. Also allows: ['admin'] for all permission on the bucket including delete, ['read'] for viewing access, ['write'] for creation rights but not viewing access """ cirrus_config.update(**config["CIRRUS_CFG"]) google_project_id = google_project_id or cirrus_config.GOOGLE_PROJECT_ID # determine project where buckets are located # default to same project, try to get storage creds project from key file storage_creds_project_id = _get_storage_project_id() or google_project_id # default to read access allowed_privileges = allowed_privileges or ["read", "write"] driver = SQLAlchemyDriver(db) with driver.session as current_session: # use storage creds to create bucket # (default creds don't have permission) bucket_db_entry = _create_or_update_google_bucket_and_db( db_session=current_session, name=name, storage_class=storage_class, requester_pays=requester_pays, storage_creds_project_id=storage_creds_project_id, public=public, project_auth_id=project_auth_id, access_logs_bucket=access_logs_bucket, ) if public is not None and not public: for privilege in allowed_privileges: _setup_google_bucket_access_group( db_session=current_session, google_bucket_name=name, bucket_db_id=bucket_db_entry.id, google_project_id=google_project_id, storage_creds_project_id=storage_creds_project_id, privileges=[privilege], )