def app_config(
app,
settings="fence.settings",
root_dir=None,
config_path=None,
file_name=None,
):
"""
Set up the config for the Flask app.
"""
if root_dir is None:
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
logger.info("Loading settings...")
# not using app.config.from_object because we don't want all the extra flask cfg
# vars inside our singleton when we pass these through in the next step
settings_cfg = flask.Config(app.config.root_path)
settings_cfg.from_object(settings)
# dump the settings into the config singleton before loading a configuration file
config.update(dict(settings_cfg))
# load the configuration file, this overwrites anything from settings/local_settings
config.load(
config_path=config_path,
search_folders=CONFIG_SEARCH_FOLDERS,
file_name=file_name,
)
# load all config back into flask app config for now, we should PREFER getting config
# directly from the fence config singleton in the code though.
app.config.update(**config._configs)
_setup_arborist_client(app)
_setup_audit_service_client(app)
_setup_data_endpoint_and_boto(app)
_load_keys(app, root_dir)
_set_authlib_cfgs(app)
app.prometheus_counters = {}
if config["ENABLE_PROMETHEUS_METRICS"]:
logger.info("Enabling Prometheus metrics...")
_setup_prometheus(app)
else:
logger.info("Prometheus metrics are NOT enabled.")
app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
logger=logger)
app.debug = config["DEBUG"]
# Following will update logger level, propagate, and handlers
get_logger(__name__,
log_level="debug" if config["DEBUG"] is True else "info")
_setup_oidc_clients(app)
with app.app_context():
_check_aws_creds_and_region(app)
_check_azure_storage(app)
def app_config(app,
settings="fence.settings",
root_dir=None,
config_path=None,
file_name=None):
"""
Set up the config for the Flask app.
"""
if root_dir is None:
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
logger.info("Loading settings...")
# not using app.config.from_object because we don't want all the extra flask cfg
# vars inside our singleton when we pass these through in the next step
settings_cfg = flask.Config(app.config.root_path)
settings_cfg.from_object(settings)
# dump the settings into the config singleton before loading a configuration file
config.update(dict(settings_cfg))
# load the configuration file, this overwrites anything from settings/local_settings
config.load(config_path, file_name)
# load all config back into flask app config for now, we should PREFER getting config
# directly from the fence config singleton in the code though.
app.config.update(**config._configs)
_setup_arborist_client(app)
_setup_data_endpoint_and_boto(app)
_load_keys(app, root_dir)
_set_authlib_cfgs(app)
app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
logger=logger)
app.debug = config["DEBUG"]
# Following will update logger level, propagate, and handlers
get_logger(__name__,
log_level="debug" if config["DEBUG"] == True else "info")
_setup_oidc_clients(app)
def main():
args = parse_arguments()
# get database information
sys.path.append(args.path)
# replicate cfg loading done in flask app to maintain backwards compatibility
# TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in
# settings/local_settings is deprecated
import flask
settings_cfg = flask.Config(".")
settings_cfg.from_object("fence.settings")
config.update(dict(settings_cfg))
# END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE
config.load(search_folders=CONFIG_SEARCH_FOLDERS)
DB = os.environ.get("FENCE_DB") or config.get("DB")
# attempt to get from settings, this is backwards-compatibility for integration
# tests
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL")
ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
"STORAGE_CREDENTIALS")
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False)
arborist = None
if args.arborist:
arborist = ArboristClient(
arborist_base_url=args.arborist,
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
if args.action == "create":
yaml_input = args.__dict__["yaml-file-path"]
create_sample_data(DB, yaml_input)
elif args.action == "client-create":
confidential = not args.public
create_client_action(
DB,
username=args.username,
client=args.client,
urls=args.urls,
auto_approve=args.auto_approve,
grant_types=args.grant_types,
confidential=confidential,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
)
elif args.action == "client-modify":
modify_client_action(
DB,
client=args.client,
delete_urls=args.delete_urls,
urls=args.urls,
name=args.name,
description=args.description,
set_auto_approve=args.set_auto_approve,
unset_auto_approve=args.unset_auto_approve,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
append=args.append,
)
elif args.action == "client-delete":
delete_client_action(DB, args.client)
elif args.action == "client-list":
list_client_action(DB)
elif args.action == "user-delete":
delete_users(DB, args.users)
elif args.action == "expired-service-account-delete":
delete_expired_service_accounts(DB)
elif args.action == "bucket-access-group-verify":
verify_bucket_access_group(DB)
elif args.action == "sync":
sync_users(
dbGaP,
STORAGE_CREDENTIALS,
DB,
projects=args.project_mapping,
is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap),
sync_from_local_csv_dir=args.csv_dir,
sync_from_local_yaml_file=args.yaml,
folder=args.folder,
arborist=arborist,
sync_from_visas=sync_from_visas,
fallback_to_dbgap_sftp=fallback_to_dbgap_sftp,
)
elif args.action == "dbgap-download-access-files":
download_dbgap_files(
dbGaP,
STORAGE_CREDENTIALS,
DB,
folder=args.folder,
)
elif args.action == "google-manage-keys":
remove_expired_google_service_account_keys(DB)
elif args.action == "google-init":
google_init(DB)
elif args.action == "google-manage-user-registrations":
verify_user_registration(DB)
elif args.action == "google-manage-account-access":
remove_expired_google_accounts_from_proxy_groups(DB)
elif args.action == "google-bucket-create":
# true if true provided, false if anything else provided, leave as
# None if not provided at all (policy will remain unchanged)
if args.public and args.public.lower().strip() == "true":
args.public = True
elif args.public is not None:
args.public = False
create_or_update_google_bucket(
DB,
args.unique_name,
storage_class=args.storage_class,
public=args.public,
requester_pays=args.requester_pays,
google_project_id=args.google_project_id,
project_auth_id=args.project_auth_id,
access_logs_bucket=args.access_logs_bucket,
allowed_privileges=args.allowed_privileges,
)
elif args.action == "google-logging-bucket-create":
create_google_logging_bucket(
args.unique_name,
storage_class=args.storage_class,
google_project_id=args.google_project_id,
)
elif args.action == "link-external-bucket":
link_external_bucket(DB, name=args.bucket_name)
elif args.action == "link-bucket-to-project":
link_bucket_to_project(
DB,
bucket_id=args.bucket_id,
bucket_provider=args.bucket_provider,
project_auth_id=args.project_auth_id,
)
elif args.action == "google-list-authz-groups":
google_list_authz_groups(DB)
elif args.action == "token-create":
keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys"))
keypairs = keys.load_keypairs(keys_path)
# Default to the most recent one, but try to find the keypair with
# matching ``kid`` to the argument provided.
keypair = keypairs[-1]
kid = getattr(args, "kid")
if kid:
for try_keypair in keypairs:
if try_keypair.kid == kid:
keypair = try_keypair
break
jwt_creator = JWTCreator(
DB,
BASE_URL,
kid=keypair.kid,
private_key=keypair.private_key,
username=args.username,
scopes=args.scopes,
expires_in=args.exp,
)
token_type = str(args.type).strip().lower()
if token_type == "access_token" or token_type == "access":
print(jwt_creator.create_access_token().token)
elif token_type == "refresh_token" or token_type == "refresh":
print(jwt_creator.create_refresh_token().token)
else:
print('invalid token type "{}"; expected "access" or "refresh"'.
format(token_type))
sys.exit(1)
elif args.action == "force-link-google":
exp = force_update_google_link(
DB,
username=args.username,
google_email=args.google_email,
expires_in=args.expires_in,
)
print(exp)
elif args.action == "notify-problem-users":
notify_problem_users(DB, args.emails, args.auth_ids,
args.check_linking, args.google_project_id)
elif args.action == "migrate":
migrate_database(DB)
elif args.action == "update-visas":
update_user_visas(
DB,
chunk_size=args.chunk_size,
concurrency=args.concurrency,
thread_pool_size=args.thread_pool_size,
buffer_size=args.buffer_size,
)
from userdatamodel.driver import SQLAlchemyDriver
from werkzeug.middleware.dispatcher import DispatcherMiddleware
from azure.storage.blob import BlobServiceClient
from azure.core.exceptions import ResourceNotFoundError
from urllib.parse import urlparse
# Can't read config yet. Just set to debug for now, else no handlers.
# Later, in app_config(), will actually set level based on config
logger = get_logger(__name__, log_level="debug")
# Load the configuration *before* importing modules that rely on it
from fence.config import config
from fence.settings import CONFIG_SEARCH_FOLDERS
config.load(
config_path=os.environ.get("FENCE_CONFIG_PATH"),
search_folders=CONFIG_SEARCH_FOLDERS,
)
from fence.auth import logout, build_redirect_url
from fence.blueprints.data.indexd import S3IndexedFileLocation
from fence.blueprints.login.utils import allowed_login_redirects, domain
from fence.errors import UserError
from fence.jwt import keys
from fence.models import migrate, IdentityProvider
from fence.oidc.client import query_client
from fence.oidc.server import server
from fence.resources.audit.client import AuditServiceClient
from fence.resources.aws.boto_manager import BotoManager
from fence.resources.openid.idp_oauth2 import Oauth2ClientBase
from fence.resources.openid.cilogon_oauth2 import CilogonOauth2Client
from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client