def test_delete_expired_google_access(cloud_manager, app, db_session): """ Test deleting all expired service accounts """ import fence fence.settings = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.return_value = ( {}) current_time = int(time.time()) # 1 expired, 2 not expired access_1_expires = current_time - 3600 access_2_expires = current_time + 3600 setup_results = _setup_google_access(db_session, access_1_expires=access_1_expires, access_2_expires=access_2_expires) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database to make sure all the service accounts exist pre_deletion_google_access_size = len(google_access) pre_deletion_google_proxy_groups_size = len(google_proxy_groups) pre_deletion_google_bucket_access_grps_size = len( google_bucket_access_grps) # call function to delete expired service account delete_expired_google_access(config["DB"]) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database again. Expect 1 access is deleted - proxy group and gbag should be intact assert len(google_access) == pre_deletion_google_access_size - 1 remaining_ids = [ str(gpg_to_gbag.proxy_group_id) for gpg_to_gbag in google_access ] # b/c expired assert str( setup_results["google_proxy_group_ids"]["1"]) not in remaining_ids # b/c not expired assert str(setup_results["google_proxy_group_ids"]["2"]) in remaining_ids assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size assert len(google_bucket_access_grps ) == pre_deletion_google_bucket_access_grps_size
def test_delete_expired_google_access_with_one_fail_first( cloud_manager, app, db_session): """ Test the case that there is a failure of removing from google group in GCP. In this case, we still want the expired record to exist in the db so we can try to remove it again. """ from googleapiclient.errors import HttpError import fence fence.settings = MagicMock() cirrus.config.update = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), {}, ] current_time = int(time.time()) # 1 expired, 2 not expired access_1_expires = current_time - 3600 access_2_expires = current_time + 3600 _setup_google_access(db_session, access_1_expires=access_1_expires, access_2_expires=access_2_expires) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database to make sure all the service accounts exist pre_deletion_google_access_size = len(google_access) pre_deletion_google_proxy_groups_size = len(google_proxy_groups) pre_deletion_google_bucket_access_grps_size = len( google_bucket_access_grps) # call function to delete expired service account delete_expired_google_access(config["DB"]) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database again. Expect nothing is deleted assert len(google_access) == pre_deletion_google_access_size assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size assert len(google_bucket_access_grps ) == pre_deletion_google_bucket_access_grps_size
def test_delete_not_specified_expiration_google_access(app, db_session): """ Test the case that there is no expiration time specified in the db for google access In this case, we expect backwards compatible behavior, e.g. they are NOT removed """ import fence fence.settings = MagicMock() current_time = int(time.time()) access_1_expires = None access_2_expires = None _setup_google_access(db_session, access_1_expires=access_1_expires, access_2_expires=access_2_expires) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database to make sure all the service accounts exist pre_deletion_google_access_size = len(google_access) pre_deletion_google_proxy_groups_size = len(google_proxy_groups) pre_deletion_google_bucket_access_grps_size = len( google_bucket_access_grps) # call function to delete expired service account delete_expired_google_access(config["DB"]) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database again. Expect nothing is deleted assert len(google_access) == pre_deletion_google_access_size assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size assert len(google_bucket_access_grps ) == pre_deletion_google_bucket_access_grps_size
def test_delete_not_expired_google_access(app, db_session): """ Test the case that there is no expired google access """ import fence fence.settings = MagicMock() current_time = int(time.time()) # 1 not expired, 2 not expired access_1_expires = current_time + 3600 access_2_expires = current_time + 3600 _setup_google_access(db_session, access_1_expires=access_1_expires, access_2_expires=access_2_expires) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database to make sure all the service accounts exist pre_deletion_google_access_size = len(google_access) pre_deletion_google_proxy_groups_size = len(google_proxy_groups) pre_deletion_google_bucket_access_grps_size = len( google_bucket_access_grps) # call function to delete expired service account delete_expired_google_access(config["DB"]) google_access = db_session.query( GoogleProxyGroupToGoogleBucketAccessGroup).all() google_proxy_groups = db_session.query(GoogleProxyGroup).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() # check database again. Expect nothing is deleted assert len(google_access) == pre_deletion_google_access_size assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size assert len(google_bucket_access_grps ) == pre_deletion_google_bucket_access_grps_size
def main(): args = parse_arguments() # get database information sys.path.append(args.path) # replicate cfg loading done in flask app to maintain backwards compatibility # TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in # settings/local_settings is deprecated import flask settings_cfg = flask.Config(".") settings_cfg.from_object("fence.settings") config.update(dict(settings_cfg)) # END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE config.load(search_folders=CONFIG_SEARCH_FOLDERS) DB = os.environ.get("FENCE_DB") or config.get("DB") # attempt to get from settings, this is backwards-compatibility for integration # tests if DB is None: try: from fence.settings import DB except ImportError: pass BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL") ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname( os.path.dirname(os.path.realpath(__file__))) dbGaP = os.environ.get("dbGaP") or config.get("dbGaP") if not isinstance(dbGaP, list): dbGaP = [dbGaP] STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get( "STORAGE_CREDENTIALS") arborist = None if args.arborist: arborist = ArboristClient( arborist_base_url=args.arborist, logger=get_logger("user_syncer.arborist_client"), authz_provider="user-sync", ) if args.action == "create": yaml_input = args.__dict__["yaml-file-path"] create_sample_data(DB, yaml_input) elif args.action == "client-create": confidential = not args.public create_client_action( DB, username=args.username, client=args.client, urls=args.urls, auto_approve=args.auto_approve, grant_types=args.grant_types, confidential=confidential, arborist=arborist, policies=args.policies, allowed_scopes=args.allowed_scopes, ) elif args.action == "client-modify": modify_client_action( DB, client=args.client, delete_urls=args.delete_urls, urls=args.urls, name=args.name, description=args.description, set_auto_approve=args.set_auto_approve, unset_auto_approve=args.unset_auto_approve, arborist=arborist, policies=args.policies, allowed_scopes=args.allowed_scopes, append=args.append, ) elif args.action == "client-delete": delete_client_action(DB, args.client) elif args.action == "client-list": list_client_action(DB) elif args.action == "user-delete": delete_users(DB, args.users) elif args.action == "expired-service-account-delete": delete_expired_service_accounts(DB) elif args.action == "bucket-access-group-verify": verify_bucket_access_group(DB) elif args.action == "delete-expired-google-access": delete_expired_google_access(DB) elif args.action == "cleanup-expired-ga4gh-information": cleanup_expired_ga4gh_information(DB) elif args.action == "sync": sync_users( dbGaP, STORAGE_CREDENTIALS, DB, projects=args.project_mapping, is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap), sync_from_local_csv_dir=args.csv_dir, sync_from_local_yaml_file=args.yaml, folder=args.folder, arborist=arborist, ) elif args.action == "dbgap-download-access-files": download_dbgap_files( dbGaP, STORAGE_CREDENTIALS, DB, folder=args.folder, ) elif args.action == "google-manage-keys": remove_expired_google_service_account_keys(DB) elif args.action == "google-init": google_init(DB) elif args.action == "google-manage-user-registrations": verify_user_registration(DB) elif args.action == "google-manage-account-access": remove_expired_google_accounts_from_proxy_groups(DB) elif args.action == "google-bucket-create": # true if true provided, false if anything else provided, leave as # None if not provided at all (policy will remain unchanged) if args.public and args.public.lower().strip() == "true": args.public = True elif args.public is not None: args.public = False create_or_update_google_bucket( DB, args.unique_name, storage_class=args.storage_class, public=args.public, requester_pays=args.requester_pays, google_project_id=args.google_project_id, project_auth_id=args.project_auth_id, access_logs_bucket=args.access_logs_bucket, allowed_privileges=args.allowed_privileges, ) elif args.action == "google-logging-bucket-create": create_google_logging_bucket( args.unique_name, storage_class=args.storage_class, google_project_id=args.google_project_id, ) elif args.action == "link-external-bucket": link_external_bucket(DB, name=args.bucket_name) elif args.action == "link-bucket-to-project": link_bucket_to_project( DB, bucket_id=args.bucket_id, bucket_provider=args.bucket_provider, project_auth_id=args.project_auth_id, ) elif args.action == "google-list-authz-groups": google_list_authz_groups(DB) elif args.action == "token-create": keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys")) keypairs = keys.load_keypairs(keys_path) # Default to the most recent one, but try to find the keypair with # matching ``kid`` to the argument provided. keypair = keypairs[-1] kid = getattr(args, "kid") if kid: for try_keypair in keypairs: if try_keypair.kid == kid: keypair = try_keypair break jwt_creator = JWTCreator( DB, BASE_URL, kid=keypair.kid, private_key=keypair.private_key, username=args.username, scopes=args.scopes, expires_in=args.exp, ) token_type = str(args.type).strip().lower() if token_type == "access_token" or token_type == "access": print(jwt_creator.create_access_token().token) elif token_type == "refresh_token" or token_type == "refresh": print(jwt_creator.create_refresh_token().token) else: print('invalid token type "{}"; expected "access" or "refresh"'. format(token_type)) sys.exit(1) elif args.action == "force-link-google": exp = force_update_google_link( DB, username=args.username, google_email=args.google_email, expires_in=args.expires_in, ) print(exp) elif args.action == "notify-problem-users": notify_problem_users(DB, args.emails, args.auth_ids, args.check_linking, args.google_project_id) elif args.action == "migrate": migrate_database(DB) elif args.action == "update-visas": access_token_polling_job( DB, chunk_size=args.chunk_size, concurrency=args.concurrency, thread_pool_size=args.thread_pool_size, buffer_size=args.buffer_size, )