Example #1
0
def app_config(
    app,
    settings="fence.settings",
    root_dir=None,
    config_path=None,
    file_name=None,
):
    """
    Set up the config for the Flask app.
    """
    if root_dir is None:
        root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    logger.info("Loading settings...")
    # not using app.config.from_object because we don't want all the extra flask cfg
    # vars inside our singleton when we pass these through in the next step
    settings_cfg = flask.Config(app.config.root_path)
    settings_cfg.from_object(settings)

    # dump the settings into the config singleton before loading a configuration file
    config.update(dict(settings_cfg))

    # load the configuration file, this overwrites anything from settings/local_settings
    config.load(
        config_path=config_path,
        search_folders=CONFIG_SEARCH_FOLDERS,
        file_name=file_name,
    )

    # load all config back into flask app config for now, we should PREFER getting config
    # directly from the fence config singleton in the code though.
    app.config.update(**config._configs)

    _setup_arborist_client(app)
    _setup_audit_service_client(app)
    _setup_data_endpoint_and_boto(app)
    _load_keys(app, root_dir)
    _set_authlib_cfgs(app)

    app.prometheus_counters = {}
    if config["ENABLE_PROMETHEUS_METRICS"]:
        logger.info("Enabling Prometheus metrics...")
        _setup_prometheus(app)
    else:
        logger.info("Prometheus metrics are NOT enabled.")

    app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
                                         logger=logger)

    app.debug = config["DEBUG"]
    # Following will update logger level, propagate, and handlers
    get_logger(__name__,
               log_level="debug" if config["DEBUG"] is True else "info")

    _setup_oidc_clients(app)

    with app.app_context():
        _check_aws_creds_and_region(app)
        _check_azure_storage(app)
Example #2
0
def app_config(app,
               settings="fence.settings",
               root_dir=None,
               config_path=None,
               file_name=None):
    """
    Set up the config for the Flask app.
    """
    if root_dir is None:
        root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    logger.info("Loading settings...")
    # not using app.config.from_object because we don't want all the extra flask cfg
    # vars inside our singleton when we pass these through in the next step
    settings_cfg = flask.Config(app.config.root_path)
    settings_cfg.from_object(settings)

    # dump the settings into the config singleton before loading a configuration file
    config.update(dict(settings_cfg))

    # load the configuration file, this overwrites anything from settings/local_settings
    config.load(config_path, file_name)

    # load all config back into flask app config for now, we should PREFER getting config
    # directly from the fence config singleton in the code though.
    app.config.update(**config._configs)

    _setup_arborist_client(app)
    _setup_data_endpoint_and_boto(app)
    _load_keys(app, root_dir)
    _set_authlib_cfgs(app)

    app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
                                         logger=logger)

    app.debug = config["DEBUG"]
    # Following will update logger level, propagate, and handlers
    get_logger(__name__,
               log_level="debug" if config["DEBUG"] == True else "info")

    _setup_oidc_clients(app)
Example #3
0
def main():
    args = parse_arguments()

    # get database information
    sys.path.append(args.path)

    # replicate cfg loading done in flask app to maintain backwards compatibility
    # TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in
    # settings/local_settings is deprecated
    import flask

    settings_cfg = flask.Config(".")
    settings_cfg.from_object("fence.settings")
    config.update(dict(settings_cfg))

    # END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE

    config.load(search_folders=CONFIG_SEARCH_FOLDERS)

    DB = os.environ.get("FENCE_DB") or config.get("DB")

    # attempt to get from settings, this is backwards-compatibility for integration
    # tests
    if DB is None:
        try:
            from fence.settings import DB
        except ImportError:
            pass

    BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL")
    ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname(
        os.path.dirname(os.path.realpath(__file__)))
    dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
    if not isinstance(dbGaP, list):
        dbGaP = [dbGaP]
    STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
        "STORAGE_CREDENTIALS")
    usersync = config.get("USERSYNC", {})
    sync_from_visas = usersync.get("sync_from_visas", False)
    fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False)

    arborist = None
    if args.arborist:
        arborist = ArboristClient(
            arborist_base_url=args.arborist,
            logger=get_logger("user_syncer.arborist_client"),
            authz_provider="user-sync",
        )

    if args.action == "create":
        yaml_input = args.__dict__["yaml-file-path"]
        create_sample_data(DB, yaml_input)
    elif args.action == "client-create":
        confidential = not args.public
        create_client_action(
            DB,
            username=args.username,
            client=args.client,
            urls=args.urls,
            auto_approve=args.auto_approve,
            grant_types=args.grant_types,
            confidential=confidential,
            arborist=arborist,
            policies=args.policies,
            allowed_scopes=args.allowed_scopes,
        )
    elif args.action == "client-modify":
        modify_client_action(
            DB,
            client=args.client,
            delete_urls=args.delete_urls,
            urls=args.urls,
            name=args.name,
            description=args.description,
            set_auto_approve=args.set_auto_approve,
            unset_auto_approve=args.unset_auto_approve,
            arborist=arborist,
            policies=args.policies,
            allowed_scopes=args.allowed_scopes,
            append=args.append,
        )
    elif args.action == "client-delete":
        delete_client_action(DB, args.client)
    elif args.action == "client-list":
        list_client_action(DB)
    elif args.action == "user-delete":
        delete_users(DB, args.users)
    elif args.action == "expired-service-account-delete":
        delete_expired_service_accounts(DB)
    elif args.action == "bucket-access-group-verify":
        verify_bucket_access_group(DB)
    elif args.action == "sync":
        sync_users(
            dbGaP,
            STORAGE_CREDENTIALS,
            DB,
            projects=args.project_mapping,
            is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap),
            sync_from_local_csv_dir=args.csv_dir,
            sync_from_local_yaml_file=args.yaml,
            folder=args.folder,
            arborist=arborist,
            sync_from_visas=sync_from_visas,
            fallback_to_dbgap_sftp=fallback_to_dbgap_sftp,
        )
    elif args.action == "dbgap-download-access-files":
        download_dbgap_files(
            dbGaP,
            STORAGE_CREDENTIALS,
            DB,
            folder=args.folder,
        )
    elif args.action == "google-manage-keys":
        remove_expired_google_service_account_keys(DB)
    elif args.action == "google-init":
        google_init(DB)
    elif args.action == "google-manage-user-registrations":
        verify_user_registration(DB)
    elif args.action == "google-manage-account-access":
        remove_expired_google_accounts_from_proxy_groups(DB)
    elif args.action == "google-bucket-create":
        # true if true provided, false if anything else provided, leave as
        # None if not provided at all (policy will remain unchanged)
        if args.public and args.public.lower().strip() == "true":
            args.public = True
        elif args.public is not None:
            args.public = False

        create_or_update_google_bucket(
            DB,
            args.unique_name,
            storage_class=args.storage_class,
            public=args.public,
            requester_pays=args.requester_pays,
            google_project_id=args.google_project_id,
            project_auth_id=args.project_auth_id,
            access_logs_bucket=args.access_logs_bucket,
            allowed_privileges=args.allowed_privileges,
        )
    elif args.action == "google-logging-bucket-create":
        create_google_logging_bucket(
            args.unique_name,
            storage_class=args.storage_class,
            google_project_id=args.google_project_id,
        )
    elif args.action == "link-external-bucket":
        link_external_bucket(DB, name=args.bucket_name)
    elif args.action == "link-bucket-to-project":
        link_bucket_to_project(
            DB,
            bucket_id=args.bucket_id,
            bucket_provider=args.bucket_provider,
            project_auth_id=args.project_auth_id,
        )
    elif args.action == "google-list-authz-groups":
        google_list_authz_groups(DB)
    elif args.action == "token-create":
        keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys"))
        keypairs = keys.load_keypairs(keys_path)
        # Default to the most recent one, but try to find the keypair with
        # matching ``kid`` to the argument provided.
        keypair = keypairs[-1]
        kid = getattr(args, "kid")
        if kid:
            for try_keypair in keypairs:
                if try_keypair.kid == kid:
                    keypair = try_keypair
                    break
        jwt_creator = JWTCreator(
            DB,
            BASE_URL,
            kid=keypair.kid,
            private_key=keypair.private_key,
            username=args.username,
            scopes=args.scopes,
            expires_in=args.exp,
        )
        token_type = str(args.type).strip().lower()
        if token_type == "access_token" or token_type == "access":
            print(jwt_creator.create_access_token().token)
        elif token_type == "refresh_token" or token_type == "refresh":
            print(jwt_creator.create_refresh_token().token)
        else:
            print('invalid token type "{}"; expected "access" or "refresh"'.
                  format(token_type))
            sys.exit(1)
    elif args.action == "force-link-google":
        exp = force_update_google_link(
            DB,
            username=args.username,
            google_email=args.google_email,
            expires_in=args.expires_in,
        )
        print(exp)
    elif args.action == "notify-problem-users":
        notify_problem_users(DB, args.emails, args.auth_ids,
                             args.check_linking, args.google_project_id)
    elif args.action == "migrate":
        migrate_database(DB)
    elif args.action == "update-visas":
        update_user_visas(
            DB,
            chunk_size=args.chunk_size,
            concurrency=args.concurrency,
            thread_pool_size=args.thread_pool_size,
            buffer_size=args.buffer_size,
        )
Example #4
0
from userdatamodel.driver import SQLAlchemyDriver
from werkzeug.middleware.dispatcher import DispatcherMiddleware
from azure.storage.blob import BlobServiceClient
from azure.core.exceptions import ResourceNotFoundError
from urllib.parse import urlparse

# Can't read config yet. Just set to debug for now, else no handlers.
# Later, in app_config(), will actually set level based on config
logger = get_logger(__name__, log_level="debug")

# Load the configuration *before* importing modules that rely on it
from fence.config import config
from fence.settings import CONFIG_SEARCH_FOLDERS

config.load(
    config_path=os.environ.get("FENCE_CONFIG_PATH"),
    search_folders=CONFIG_SEARCH_FOLDERS,
)

from fence.auth import logout, build_redirect_url
from fence.blueprints.data.indexd import S3IndexedFileLocation
from fence.blueprints.login.utils import allowed_login_redirects, domain
from fence.errors import UserError
from fence.jwt import keys
from fence.models import migrate, IdentityProvider
from fence.oidc.client import query_client
from fence.oidc.server import server
from fence.resources.audit.client import AuditServiceClient
from fence.resources.aws.boto_manager import BotoManager
from fence.resources.openid.idp_oauth2 import Oauth2ClientBase
from fence.resources.openid.cilogon_oauth2 import CilogonOauth2Client
from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client