def test_delete_not_expired_service_account(app, db_session): """ Test the case that there is no expired service account """ import fence fence.settings = MagicMock() _setup_service_account_to_google_bucket_access_group(db_session) service_account = db_session.query(UserServiceAccount).first() google_bucket_access_grp1 = db_session.query( GoogleBucketAccessGroup).first() current_time = int(time.time()) # Add non-expired service account service_account = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_account.id, expires=current_time + 3600, access_group_id=google_bucket_access_grp1.id, ) db_session.add(service_account) db_session.commit() records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 1 # call function to delete expired service account delete_expired_service_accounts(app.config["DB"]) # check db again to make sure the record still exists records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 1
def test_delete_expired_service_accounts_with_one_fail_second( cloud_manager, app, db_session ): """ Test the case that there is a failure of removing service account from google group """ from googleapiclient.errors import HttpError import fence fence.settings = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ {}, HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), ] _setup_service_account_to_google_bucket_access_group(db_session) service_accounts = db_session.query(UserServiceAccount).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() current_time = int(time.time()) # Add two expired service account. They are both supposed to be deleted but # only one is deleted due to a raise exception service_account1 = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[0].id, expires=current_time - 3600, access_group_id=google_bucket_access_grps[0].id, ) service_account2 = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[1].id, expires=current_time - 3600, access_group_id=google_bucket_access_grps[1].id, ) db_session.add(service_account1) db_session.add(service_account2) db_session.commit() # check database to make sure there are two records in DB records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 2 # call function to delete expired service account delete_expired_service_accounts(config["DB"]) # check database to make sure only the first one is deleted, the second one # still exists because of the raised exception records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 1 assert records[0].id == service_account2.id
def test_delete_expired_service_accounts_with_one_fail_first( cloud_manager, app, db_session ): """ Test the case that there is a failure of removing service account from google group """ from googleapiclient.errors import HttpError import fence fence.settings = MagicMock() cirrus.config.update = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), {}, ] _setup_service_account_to_google_bucket_access_group(db_session) service_accounts = db_session.query(UserServiceAccount).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() current_time = int(time.time()) # Add expired service account. This acccount is supposed to be deleted db_session.add( ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[0].id, expires=current_time - 3600, access_group_id=google_bucket_access_grps[0].id, ) ) # Add non-expired service account. db_session.add( ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[1].id, expires=current_time + 3600, access_group_id=google_bucket_access_grps[1].id, ) ) db_session.commit() # check database to make sure all the service accounts exist records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 2 # call function to delete expired service account delete_expired_service_accounts(config["DB"]) # check database again. Expect no service account is deleted records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 2
def test_delete_expired_service_accounts(cloud_manager, app, db_session): """ Test deleting all expired service accounts """ import fence fence.settings = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.return_value = ( {} ) _setup_service_account_to_google_bucket_access_group(db_session) service_accounts = db_session.query(UserServiceAccount).all() google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all() current_time = int(time.time()) # Add 2 expired and 1 not expired accounts service_account1 = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[0].id, expires=current_time - 3600, access_group_id=google_bucket_access_grps[0].id, ) service_account2 = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[0].id, expires=current_time - 3600, access_group_id=google_bucket_access_grps[1].id, ) service_account3 = ServiceAccountToGoogleBucketAccessGroup( service_account_id=service_accounts[1].id, expires=current_time + 3600, access_group_id=google_bucket_access_grps[1].id, ) db_session.add(service_account1) db_session.add(service_account2) db_session.add(service_account3) db_session.commit() records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 3 # call function to delete expired service account delete_expired_service_accounts(config["DB"]) # check database. Expect 2 deleted records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all() assert len(records) == 1 assert records[0].id == service_account3.id
def main(): args = parse_arguments() # get database information sys.path.append(args.path) # replicate cfg loading done in flask app to maintain backwards compatibility # TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in # settings/local_settings is deprecated import flask settings_cfg = flask.Config(".") settings_cfg.from_object("fence.settings") config.update(dict(settings_cfg)) # END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE config.load(search_folders=CONFIG_SEARCH_FOLDERS) DB = os.environ.get("FENCE_DB") or config.get("DB") # attempt to get from settings, this is backwards-compatibility for integration # tests if DB is None: try: from fence.settings import DB except ImportError: pass BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL") ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname( os.path.dirname(os.path.realpath(__file__))) dbGaP = os.environ.get("dbGaP") or config.get("dbGaP") if not isinstance(dbGaP, list): dbGaP = [dbGaP] STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get( "STORAGE_CREDENTIALS") usersync = config.get("USERSYNC", {}) sync_from_visas = usersync.get("sync_from_visas", False) fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False) arborist = None if args.arborist: arborist = ArboristClient( arborist_base_url=args.arborist, logger=get_logger("user_syncer.arborist_client"), authz_provider="user-sync", ) if args.action == "create": yaml_input = args.__dict__["yaml-file-path"] create_sample_data(DB, yaml_input) elif args.action == "client-create": confidential = not args.public create_client_action( DB, username=args.username, client=args.client, urls=args.urls, auto_approve=args.auto_approve, grant_types=args.grant_types, confidential=confidential, arborist=arborist, policies=args.policies, allowed_scopes=args.allowed_scopes, ) elif args.action == "client-modify": modify_client_action( DB, client=args.client, delete_urls=args.delete_urls, urls=args.urls, name=args.name, description=args.description, set_auto_approve=args.set_auto_approve, unset_auto_approve=args.unset_auto_approve, arborist=arborist, policies=args.policies, allowed_scopes=args.allowed_scopes, append=args.append, ) elif args.action == "client-delete": delete_client_action(DB, args.client) elif args.action == "client-list": list_client_action(DB) elif args.action == "user-delete": delete_users(DB, args.users) elif args.action == "expired-service-account-delete": delete_expired_service_accounts(DB) elif args.action == "bucket-access-group-verify": verify_bucket_access_group(DB) elif args.action == "sync": sync_users( dbGaP, STORAGE_CREDENTIALS, DB, projects=args.project_mapping, is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap), sync_from_local_csv_dir=args.csv_dir, sync_from_local_yaml_file=args.yaml, folder=args.folder, arborist=arborist, sync_from_visas=sync_from_visas, fallback_to_dbgap_sftp=fallback_to_dbgap_sftp, ) elif args.action == "dbgap-download-access-files": download_dbgap_files( dbGaP, STORAGE_CREDENTIALS, DB, folder=args.folder, ) elif args.action == "google-manage-keys": remove_expired_google_service_account_keys(DB) elif args.action == "google-init": google_init(DB) elif args.action == "google-manage-user-registrations": verify_user_registration(DB) elif args.action == "google-manage-account-access": remove_expired_google_accounts_from_proxy_groups(DB) elif args.action == "google-bucket-create": # true if true provided, false if anything else provided, leave as # None if not provided at all (policy will remain unchanged) if args.public and args.public.lower().strip() == "true": args.public = True elif args.public is not None: args.public = False create_or_update_google_bucket( DB, args.unique_name, storage_class=args.storage_class, public=args.public, requester_pays=args.requester_pays, google_project_id=args.google_project_id, project_auth_id=args.project_auth_id, access_logs_bucket=args.access_logs_bucket, allowed_privileges=args.allowed_privileges, ) elif args.action == "google-logging-bucket-create": create_google_logging_bucket( args.unique_name, storage_class=args.storage_class, google_project_id=args.google_project_id, ) elif args.action == "link-external-bucket": link_external_bucket(DB, name=args.bucket_name) elif args.action == "link-bucket-to-project": link_bucket_to_project( DB, bucket_id=args.bucket_id, bucket_provider=args.bucket_provider, project_auth_id=args.project_auth_id, ) elif args.action == "google-list-authz-groups": google_list_authz_groups(DB) elif args.action == "token-create": keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys")) keypairs = keys.load_keypairs(keys_path) # Default to the most recent one, but try to find the keypair with # matching ``kid`` to the argument provided. keypair = keypairs[-1] kid = getattr(args, "kid") if kid: for try_keypair in keypairs: if try_keypair.kid == kid: keypair = try_keypair break jwt_creator = JWTCreator( DB, BASE_URL, kid=keypair.kid, private_key=keypair.private_key, username=args.username, scopes=args.scopes, expires_in=args.exp, ) token_type = str(args.type).strip().lower() if token_type == "access_token" or token_type == "access": print(jwt_creator.create_access_token().token) elif token_type == "refresh_token" or token_type == "refresh": print(jwt_creator.create_refresh_token().token) else: print('invalid token type "{}"; expected "access" or "refresh"'. format(token_type)) sys.exit(1) elif args.action == "force-link-google": exp = force_update_google_link( DB, username=args.username, google_email=args.google_email, expires_in=args.expires_in, ) print(exp) elif args.action == "notify-problem-users": notify_problem_users(DB, args.emails, args.auth_ids, args.check_linking, args.google_project_id) elif args.action == "migrate": migrate_database(DB) elif args.action == "update-visas": update_user_visas( DB, chunk_size=args.chunk_size, concurrency=args.concurrency, thread_pool_size=args.thread_pool_size, buffer_size=args.buffer_size, )