def test_delete_not_expired_service_account(app, db_session):
"""
Test the case that there is no expired service account
"""
import fence
fence.settings = MagicMock()
_setup_service_account_to_google_bucket_access_group(db_session)
service_account = db_session.query(UserServiceAccount).first()
google_bucket_access_grp1 = db_session.query(
GoogleBucketAccessGroup).first()
current_time = int(time.time())
# Add non-expired service account
service_account = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grp1.id,
)
db_session.add(service_account)
db_session.commit()
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
# call function to delete expired service account
delete_expired_service_accounts(app.config["DB"])
# check db again to make sure the record still exists
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
def test_delete_expired_service_accounts_with_one_fail_second(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
{},
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add two expired service account. They are both supposed to be deleted but
# only one is deleted due to a raise exception
service_account1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
service_account2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[1].id,
)
db_session.add(service_account1)
db_session.add(service_account2)
db_session.commit()
# check database to make sure there are two records in DB
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database to make sure only the first one is deleted, the second one
# still exists because of the raised exception
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
assert records[0].id == service_account2.id
def test_delete_expired_service_accounts_with_one_fail_first(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cirrus.config.update = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
{},
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add expired service account. This acccount is supposed to be deleted
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
)
# Add non-expired service account.
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grps[1].id,
)
)
db_session.commit()
# check database to make sure all the service accounts exist
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database again. Expect no service account is deleted
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
def test_delete_expired_service_accounts(cloud_manager, app, db_session):
"""
Test deleting all expired service accounts
"""
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.return_value = (
{}
)
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add 2 expired and 1 not expired accounts
service_account1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
service_account2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[1].id,
)
service_account3 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grps[1].id,
)
db_session.add(service_account1)
db_session.add(service_account2)
db_session.add(service_account3)
db_session.commit()
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 3
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database. Expect 2 deleted
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
assert records[0].id == service_account3.id
def main():
args = parse_arguments()
# get database information
sys.path.append(args.path)
# replicate cfg loading done in flask app to maintain backwards compatibility
# TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in
# settings/local_settings is deprecated
import flask
settings_cfg = flask.Config(".")
settings_cfg.from_object("fence.settings")
config.update(dict(settings_cfg))
# END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE
config.load(search_folders=CONFIG_SEARCH_FOLDERS)
DB = os.environ.get("FENCE_DB") or config.get("DB")
# attempt to get from settings, this is backwards-compatibility for integration
# tests
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL")
ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
"STORAGE_CREDENTIALS")
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False)
arborist = None
if args.arborist:
arborist = ArboristClient(
arborist_base_url=args.arborist,
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
if args.action == "create":
yaml_input = args.__dict__["yaml-file-path"]
create_sample_data(DB, yaml_input)
elif args.action == "client-create":
confidential = not args.public
create_client_action(
DB,
username=args.username,
client=args.client,
urls=args.urls,
auto_approve=args.auto_approve,
grant_types=args.grant_types,
confidential=confidential,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
)
elif args.action == "client-modify":
modify_client_action(
DB,
client=args.client,
delete_urls=args.delete_urls,
urls=args.urls,
name=args.name,
description=args.description,
set_auto_approve=args.set_auto_approve,
unset_auto_approve=args.unset_auto_approve,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
append=args.append,
)
elif args.action == "client-delete":
delete_client_action(DB, args.client)
elif args.action == "client-list":
list_client_action(DB)
elif args.action == "user-delete":
delete_users(DB, args.users)
elif args.action == "expired-service-account-delete":
delete_expired_service_accounts(DB)
elif args.action == "bucket-access-group-verify":
verify_bucket_access_group(DB)
elif args.action == "sync":
sync_users(
dbGaP,
STORAGE_CREDENTIALS,
DB,
projects=args.project_mapping,
is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap),
sync_from_local_csv_dir=args.csv_dir,
sync_from_local_yaml_file=args.yaml,
folder=args.folder,
arborist=arborist,
sync_from_visas=sync_from_visas,
fallback_to_dbgap_sftp=fallback_to_dbgap_sftp,
)
elif args.action == "dbgap-download-access-files":
download_dbgap_files(
dbGaP,
STORAGE_CREDENTIALS,
DB,
folder=args.folder,
)
elif args.action == "google-manage-keys":
remove_expired_google_service_account_keys(DB)
elif args.action == "google-init":
google_init(DB)
elif args.action == "google-manage-user-registrations":
verify_user_registration(DB)
elif args.action == "google-manage-account-access":
remove_expired_google_accounts_from_proxy_groups(DB)
elif args.action == "google-bucket-create":
# true if true provided, false if anything else provided, leave as
# None if not provided at all (policy will remain unchanged)
if args.public and args.public.lower().strip() == "true":
args.public = True
elif args.public is not None:
args.public = False
create_or_update_google_bucket(
DB,
args.unique_name,
storage_class=args.storage_class,
public=args.public,
requester_pays=args.requester_pays,
google_project_id=args.google_project_id,
project_auth_id=args.project_auth_id,
access_logs_bucket=args.access_logs_bucket,
allowed_privileges=args.allowed_privileges,
)
elif args.action == "google-logging-bucket-create":
create_google_logging_bucket(
args.unique_name,
storage_class=args.storage_class,
google_project_id=args.google_project_id,
)
elif args.action == "link-external-bucket":
link_external_bucket(DB, name=args.bucket_name)
elif args.action == "link-bucket-to-project":
link_bucket_to_project(
DB,
bucket_id=args.bucket_id,
bucket_provider=args.bucket_provider,
project_auth_id=args.project_auth_id,
)
elif args.action == "google-list-authz-groups":
google_list_authz_groups(DB)
elif args.action == "token-create":
keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys"))
keypairs = keys.load_keypairs(keys_path)
# Default to the most recent one, but try to find the keypair with
# matching ``kid`` to the argument provided.
keypair = keypairs[-1]
kid = getattr(args, "kid")
if kid:
for try_keypair in keypairs:
if try_keypair.kid == kid:
keypair = try_keypair
break
jwt_creator = JWTCreator(
DB,
BASE_URL,
kid=keypair.kid,
private_key=keypair.private_key,
username=args.username,
scopes=args.scopes,
expires_in=args.exp,
)
token_type = str(args.type).strip().lower()
if token_type == "access_token" or token_type == "access":
print(jwt_creator.create_access_token().token)
elif token_type == "refresh_token" or token_type == "refresh":
print(jwt_creator.create_refresh_token().token)
else:
print('invalid token type "{}"; expected "access" or "refresh"'.
format(token_type))
sys.exit(1)
elif args.action == "force-link-google":
exp = force_update_google_link(
DB,
username=args.username,
google_email=args.google_email,
expires_in=args.expires_in,
)
print(exp)
elif args.action == "notify-problem-users":
notify_problem_users(DB, args.emails, args.auth_ids,
args.check_linking, args.google_project_id)
elif args.action == "migrate":
migrate_database(DB)
elif args.action == "update-visas":
update_user_visas(
DB,
chunk_size=args.chunk_size,
concurrency=args.concurrency,
thread_pool_size=args.thread_pool_size,
buffer_size=args.buffer_size,
)
