def test_delete_expired_google_access(cloud_manager, app, db_session):
"""
Test deleting all expired service accounts
"""
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.return_value = (
{})
current_time = int(time.time())
# 1 expired, 2 not expired
access_1_expires = current_time - 3600
access_2_expires = current_time + 3600
setup_results = _setup_google_access(db_session,
access_1_expires=access_1_expires,
access_2_expires=access_2_expires)
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database to make sure all the service accounts exist
pre_deletion_google_access_size = len(google_access)
pre_deletion_google_proxy_groups_size = len(google_proxy_groups)
pre_deletion_google_bucket_access_grps_size = len(
google_bucket_access_grps)
# call function to delete expired service account
delete_expired_google_access(config["DB"])
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database again. Expect 1 access is deleted - proxy group and gbag should be intact
assert len(google_access) == pre_deletion_google_access_size - 1
remaining_ids = [
str(gpg_to_gbag.proxy_group_id) for gpg_to_gbag in google_access
]
# b/c expired
assert str(
setup_results["google_proxy_group_ids"]["1"]) not in remaining_ids
# b/c not expired
assert str(setup_results["google_proxy_group_ids"]["2"]) in remaining_ids
assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size
assert len(google_bucket_access_grps
) == pre_deletion_google_bucket_access_grps_size
def test_delete_expired_google_access_with_one_fail_first(
cloud_manager, app, db_session):
"""
Test the case that there is a failure of removing from google group in GCP.
In this case, we still want the expired record to exist in the db so we can try to
remove it again.
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cirrus.config.update = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
{},
]
current_time = int(time.time())
# 1 expired, 2 not expired
access_1_expires = current_time - 3600
access_2_expires = current_time + 3600
_setup_google_access(db_session,
access_1_expires=access_1_expires,
access_2_expires=access_2_expires)
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database to make sure all the service accounts exist
pre_deletion_google_access_size = len(google_access)
pre_deletion_google_proxy_groups_size = len(google_proxy_groups)
pre_deletion_google_bucket_access_grps_size = len(
google_bucket_access_grps)
# call function to delete expired service account
delete_expired_google_access(config["DB"])
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database again. Expect nothing is deleted
assert len(google_access) == pre_deletion_google_access_size
assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size
assert len(google_bucket_access_grps
) == pre_deletion_google_bucket_access_grps_size
def test_delete_not_specified_expiration_google_access(app, db_session):
"""
Test the case that there is no expiration time specified in the db for google access
In this case, we expect backwards compatible behavior, e.g. they are NOT removed
"""
import fence
fence.settings = MagicMock()
current_time = int(time.time())
access_1_expires = None
access_2_expires = None
_setup_google_access(db_session,
access_1_expires=access_1_expires,
access_2_expires=access_2_expires)
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database to make sure all the service accounts exist
pre_deletion_google_access_size = len(google_access)
pre_deletion_google_proxy_groups_size = len(google_proxy_groups)
pre_deletion_google_bucket_access_grps_size = len(
google_bucket_access_grps)
# call function to delete expired service account
delete_expired_google_access(config["DB"])
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database again. Expect nothing is deleted
assert len(google_access) == pre_deletion_google_access_size
assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size
assert len(google_bucket_access_grps
) == pre_deletion_google_bucket_access_grps_size
def test_delete_not_expired_google_access(app, db_session):
"""
Test the case that there is no expired google access
"""
import fence
fence.settings = MagicMock()
current_time = int(time.time())
# 1 not expired, 2 not expired
access_1_expires = current_time + 3600
access_2_expires = current_time + 3600
_setup_google_access(db_session,
access_1_expires=access_1_expires,
access_2_expires=access_2_expires)
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database to make sure all the service accounts exist
pre_deletion_google_access_size = len(google_access)
pre_deletion_google_proxy_groups_size = len(google_proxy_groups)
pre_deletion_google_bucket_access_grps_size = len(
google_bucket_access_grps)
# call function to delete expired service account
delete_expired_google_access(config["DB"])
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database again. Expect nothing is deleted
assert len(google_access) == pre_deletion_google_access_size
assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size
assert len(google_bucket_access_grps
) == pre_deletion_google_bucket_access_grps_size
def main():
args = parse_arguments()
# get database information
sys.path.append(args.path)
# replicate cfg loading done in flask app to maintain backwards compatibility
# TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in
# settings/local_settings is deprecated
import flask
settings_cfg = flask.Config(".")
settings_cfg.from_object("fence.settings")
config.update(dict(settings_cfg))
# END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE
config.load(search_folders=CONFIG_SEARCH_FOLDERS)
DB = os.environ.get("FENCE_DB") or config.get("DB")
# attempt to get from settings, this is backwards-compatibility for integration
# tests
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL")
ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
"STORAGE_CREDENTIALS")
arborist = None
if args.arborist:
arborist = ArboristClient(
arborist_base_url=args.arborist,
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
if args.action == "create":
yaml_input = args.__dict__["yaml-file-path"]
create_sample_data(DB, yaml_input)
elif args.action == "client-create":
confidential = not args.public
create_client_action(
DB,
username=args.username,
client=args.client,
urls=args.urls,
auto_approve=args.auto_approve,
grant_types=args.grant_types,
confidential=confidential,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
)
elif args.action == "client-modify":
modify_client_action(
DB,
client=args.client,
delete_urls=args.delete_urls,
urls=args.urls,
name=args.name,
description=args.description,
set_auto_approve=args.set_auto_approve,
unset_auto_approve=args.unset_auto_approve,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
append=args.append,
)
elif args.action == "client-delete":
delete_client_action(DB, args.client)
elif args.action == "client-list":
list_client_action(DB)
elif args.action == "user-delete":
delete_users(DB, args.users)
elif args.action == "expired-service-account-delete":
delete_expired_service_accounts(DB)
elif args.action == "bucket-access-group-verify":
verify_bucket_access_group(DB)
elif args.action == "delete-expired-google-access":
delete_expired_google_access(DB)
elif args.action == "cleanup-expired-ga4gh-information":
cleanup_expired_ga4gh_information(DB)
elif args.action == "sync":
sync_users(
dbGaP,
STORAGE_CREDENTIALS,
DB,
projects=args.project_mapping,
is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap),
sync_from_local_csv_dir=args.csv_dir,
sync_from_local_yaml_file=args.yaml,
folder=args.folder,
arborist=arborist,
)
elif args.action == "dbgap-download-access-files":
download_dbgap_files(
dbGaP,
STORAGE_CREDENTIALS,
DB,
folder=args.folder,
)
elif args.action == "google-manage-keys":
remove_expired_google_service_account_keys(DB)
elif args.action == "google-init":
google_init(DB)
elif args.action == "google-manage-user-registrations":
verify_user_registration(DB)
elif args.action == "google-manage-account-access":
remove_expired_google_accounts_from_proxy_groups(DB)
elif args.action == "google-bucket-create":
# true if true provided, false if anything else provided, leave as
# None if not provided at all (policy will remain unchanged)
if args.public and args.public.lower().strip() == "true":
args.public = True
elif args.public is not None:
args.public = False
create_or_update_google_bucket(
DB,
args.unique_name,
storage_class=args.storage_class,
public=args.public,
requester_pays=args.requester_pays,
google_project_id=args.google_project_id,
project_auth_id=args.project_auth_id,
access_logs_bucket=args.access_logs_bucket,
allowed_privileges=args.allowed_privileges,
)
elif args.action == "google-logging-bucket-create":
create_google_logging_bucket(
args.unique_name,
storage_class=args.storage_class,
google_project_id=args.google_project_id,
)
elif args.action == "link-external-bucket":
link_external_bucket(DB, name=args.bucket_name)
elif args.action == "link-bucket-to-project":
link_bucket_to_project(
DB,
bucket_id=args.bucket_id,
bucket_provider=args.bucket_provider,
project_auth_id=args.project_auth_id,
)
elif args.action == "google-list-authz-groups":
google_list_authz_groups(DB)
elif args.action == "token-create":
keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys"))
keypairs = keys.load_keypairs(keys_path)
# Default to the most recent one, but try to find the keypair with
# matching ``kid`` to the argument provided.
keypair = keypairs[-1]
kid = getattr(args, "kid")
if kid:
for try_keypair in keypairs:
if try_keypair.kid == kid:
keypair = try_keypair
break
jwt_creator = JWTCreator(
DB,
BASE_URL,
kid=keypair.kid,
private_key=keypair.private_key,
username=args.username,
scopes=args.scopes,
expires_in=args.exp,
)
token_type = str(args.type).strip().lower()
if token_type == "access_token" or token_type == "access":
print(jwt_creator.create_access_token().token)
elif token_type == "refresh_token" or token_type == "refresh":
print(jwt_creator.create_refresh_token().token)
else:
print('invalid token type "{}"; expected "access" or "refresh"'.
format(token_type))
sys.exit(1)
elif args.action == "force-link-google":
exp = force_update_google_link(
DB,
username=args.username,
google_email=args.google_email,
expires_in=args.expires_in,
)
print(exp)
elif args.action == "notify-problem-users":
notify_problem_users(DB, args.emails, args.auth_ids,
args.check_linking, args.google_project_id)
elif args.action == "migrate":
migrate_database(DB)
elif args.action == "update-visas":
access_token_polling_job(
DB,
chunk_size=args.chunk_size,
concurrency=args.concurrency,
thread_pool_size=args.thread_pool_size,
buffer_size=args.buffer_size,
)