def magpie_update_services_conflict(conflict_services, services_dict, request_cookies): # type: (List[Str], ConfigDict, Dict[Str, Str]) -> Dict[Str, int] """ Resolve conflicting services by name during registration by updating them only if pointing to different URL. """ magpie_url = get_magpie_url() statuses = dict() for svc_name in conflict_services: statuses[svc_name] = 409 svc_url_new = services_dict[svc_name]["url"] svc_url_db = "{magpie}/services/{svc}".format(magpie=magpie_url, svc=svc_name) svc_resp = requests.get(svc_url_db, cookies=request_cookies) svc_info = get_json(svc_resp).get(svc_name) svc_url_old = svc_info["service_url"] if svc_url_old != svc_url_new: svc_info["service_url"] = svc_url_new res_svc_put = requests.put(svc_url_db, data=svc_info, cookies=request_cookies) statuses[svc_name] = res_svc_put.status_code print_log( "[{url_old}] => [{url_new}] Service URL update ({svc}): {resp}" .format(svc=svc_name, url_old=svc_url_old, url_new=svc_url_new, resp=res_svc_put.status_code), logger=LOGGER) return statuses
def phoenix_remove_services(): # type: () -> bool """ Removes the Phoenix services using temporary cookies retrieved from login with defined `PHOENIX` constants. :returns: success status of the procedure. """ phoenix_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_phoenix") error = 0 try: if not phoenix_login(phoenix_cookies): print_log("Login unsuccessful from post-login check, aborting...", logger=LOGGER) return False phoenix_url = get_phoenix_url() remove_services_url = phoenix_url + "/clear_services" error, http_code = request_curl(remove_services_url, cookies=phoenix_cookies, msg="Phoenix remove services") except Exception as e: print_log("Exception during phoenix remove services: [{!r}]".format(e), logger=LOGGER) finally: if os.path.isfile(phoenix_cookies): os.remove(phoenix_cookies) return error == 0
def register_user_with_group(user_name, group_name, email, password, db_session): # type: (Str, Str, Str, Str, Session) -> None """ Registers the user if missing and associate him to a group specified by name, also created if missing. :param user_name: name of the user to create (if missing) and to make part of the group (if specified) :param group_name: name of the group to create (if missing and specified) and to make the user join (if not already) :param email: email of the user to be created (if missing) :param password: password of the user to be created (if missing) :param db_session: database connexion to apply changes .. warning:: Should be employed only for **special** users/groups in this module as other expected API behaviour and operations will not be applied (ex: create additional permissions or user-group references). """ if not GroupService.by_group_name(group_name, db_session=db_session): # noinspection PyArgumentList new_group = models.Group(group_name=group_name) db_session.add(new_group) registered_group = GroupService.by_group_name(group_name=group_name, db_session=db_session) registered_user = UserService.by_user_name(user_name, db_session=db_session) if not registered_user: # noinspection PyArgumentList new_user = models.User(user_name=user_name, email=email) UserService.set_password(new_user, password) UserService.regenerate_security_code(new_user) db_session.add(new_user) if group_name is not None: registered_user = UserService.by_user_name(user_name, db_session=db_session) else: print_log("User '{}' already exist".format(user_name), level=logging.DEBUG) # noinspection PyBroadException try: # ensure the reference between user/group exists (user joined the group) user_group_refs = BaseService.all(models.UserGroup, db_session=db_session) user_group_refs_tup = [(ref.group_id, ref.user_id) for ref in user_group_refs] if (registered_group.id, registered_user.id) not in user_group_refs_tup: # noinspection PyArgumentList group_entry = models.UserGroup(group_id=registered_group.id, user_id=registered_user.id) db_session.add(group_entry) except Exception: # in case reference already exists, avoid duplicate error db_session.rollback()
def phoenix_update_services(services_dict): if not phoenix_remove_services(): print_log( "Could not remove services, aborting register sync services to Phoenix", logger=LOGGER) return False if not phoenix_register_services(services_dict): print_log( "Failed services registration from Magpie to Phoenix\n" + "[warning: services could have been removed but could not be re-added]", logger=LOGGER) return False return True
def init_users_group(db_session, settings=None): # type: (Session, Optional[AnySettingsContainer]) -> None """ Registers into database the group matching :py:data:`magpie.constants.MAGPIE_USERS_GROUP` if not defined. """ usr_grp_name = get_constant("MAGPIE_USERS_GROUP", settings_container=settings) if not GroupService.by_group_name(usr_grp_name, db_session=db_session): user_group = models.Group(group_name=usr_grp_name) # noqa db_session.add(user_group) else: print_log("MAGPIE_USERS_GROUP already initialized", level=logging.DEBUG)
def init_users_group(db_session, settings=None): # type: (Session, Optional[AnySettingsContainer]) -> None """ Registers in db the group matching ``MAGPIE_USERS_GROUP`` if not defined. """ usr_grp_name = get_constant("MAGPIE_USERS_GROUP", settings_container=settings) if not GroupService.by_group_name(usr_grp_name, db_session=db_session): # noinspection PyArgumentList user_group = models.Group(group_name=usr_grp_name) db_session.add(user_group) else: print_log("MAGPIE_USERS_GROUP already initialized", level=logging.DEBUG)
def get_version(request): """ Version information of the API. """ version_db = None try: version_db = get_database_revision(request.db) except Exception as exc: print_log("Failed to retrieve database revision: [{!r}]".format(exc), LOGGER, logging.WARNING) version = {"version": __meta__.__version__, "db_version": version_db} return ax.valid_http(http_success=HTTPOk, content=version, content_type=CONTENT_TYPE_JSON, detail=s.Version_GET_OkResponseSchema.description)
def is_database_ready(db_session=None, container=None): # type: (Optional[Session], Optional[AnySettingsContainer]) -> bool """ Obtains the database status against expected table names to ensure it is ready for use. """ if isinstance(db_session, Session): engine = db_session.bind else: engine = get_engine(container=container) inspector = Inspector.from_engine(engine) table_names = inspector.get_table_names() for _, obj in inspect.getmembers(models): if inspect.isclass(obj) and hasattr(obj, "__tablename__"): if obj.__tablename__ not in table_names: print_log( "Database table (or its associated parent) is missing for '{}' object" .format(obj), logger=LOGGER, level=logging.ERROR) return False return True
def request_curl(url, cookie_jar=None, cookies=None, form_params=None, msg="Response"): # type: (Str, Optional[Str], Optional[Str], Optional[Str], Optional[Str]) -> Tuple[int, int] """ Executes a request using cURL. :returns: tuple of the returned system command code and the response http code """ # arg -k allows to ignore insecure SSL errors, ie: access 'https' page not configured for it # curl_cmd = 'curl -k -L -s -o /dev/null -w "{msg_out} : %{{http_code}}\\n" {params} {url}' # curl_cmd = curl_cmd.format(msg_out=msg, params=params, url=url) msg_sep = msg + ": " params = [ "curl", "-k", "-L", "-s", "-o", "/dev/null", "-w", msg_sep + "%{http_code}" ] if cookie_jar is not None and cookies is not None: raise ValueError( "CookiesType and Cookie_Jar cannot be both set simultaneously") if cookie_jar is not None: params.extend(["--cookie-jar", cookie_jar]) # save cookies if cookies is not None: params.extend(["--cookie", cookies]) # use cookies if form_params is not None: params.extend(["--data", form_params]) params.extend([url]) curl_out = subprocess.Popen(params, stdout=subprocess.PIPE) curl_msg = curl_out.communicate()[0] # type: Str curl_err = curl_out.returncode # type: int http_code = int(curl_msg.split(msg_sep)[1]) print_log("[{url}] {response}".format(response=curl_msg, url=url), logger=LOGGER) return curl_err, http_code
def phoenix_register_services(services_dict, allowed_service_types=None): phoenix_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_phoenix") success = False statuses = dict() try: allowed_service_types = SERVICES_PHOENIX_ALLOWED if allowed_service_types is None else allowed_service_types allowed_service_types = [svc.upper() for svc in allowed_service_types] if not phoenix_login(phoenix_cookies): print_log("Login unsuccessful from post-login check, aborting...", logger=LOGGER, level=logging.WARN) return False # Filter specific services to push filtered_services_dict = {} for svc in services_dict: if str(services_dict[svc].get( "type")).upper() in allowed_service_types: filtered_services_dict[svc] = services_dict[svc] filtered_services_dict[svc]["type"] = filtered_services_dict[ svc]["type"].upper() # Register services success, statuses = register_services(SERVICES_PHOENIX, filtered_services_dict, phoenix_cookies, "Phoenix register service") except Exception as e: print_log( "Exception during phoenix register services: [{!r}]".format(e), logger=LOGGER, level=logging.ERROR) finally: if os.path.isfile(phoenix_cookies): os.remove(phoenix_cookies) return success, statuses
def run_database_migration_when_ready(settings, db_session=None): # type: (SettingsType, Optional[Session]) -> None """ Runs db migration if requested by config and need from revisions. """ db_ready = False if asbool( get_constant("MAGPIE_DB_MIGRATION", settings, "magpie.db_migration", default_value=True, raise_missing=False, raise_not_set=False, print_missing=True)): attempts = int( get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS", settings, "magpie.db_migration_attempts", default_value=5, raise_missing=False, raise_not_set=False, print_missing=True)) print_log("Running database migration (as required)...") attempts = max( attempts, 2 ) # enforce at least 2 attempts, 1 for db creation and one for actual migration for i in range(1, attempts + 1): try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=sa_exc.SAWarning) run_database_migration(db_session) except ImportError as e: print_log( "Database migration produced [{!r}] (ignored).".format(e), level=logging.WARNING) pass except Exception as e: if i <= attempts: print_log( "Database migration failed [{!r}]. Retrying... ({}/{})" .format(e, i, attempts)) time.sleep(2) continue else: raise_log("Database migration failed [{!r}]".format(e), exception=RuntimeError) db_ready = is_database_ready(db_session) if not db_ready: print_log("Database not ready. Retrying... ({}/{})".format( i, attempts)) time.sleep(2) continue break else: db_ready = is_database_ready(db_session) if not db_ready: time.sleep(2) raise_log("Database not ready", exception=RuntimeError)
def main(global_config=None, **settings): # noqa: F811 """ This function returns a Pyramid WSGI application. """ import magpie.constants # pylint: disable=C0415 # avoid circular import # override magpie ini if provided with --paste to gunicorn, otherwise use environment variable config_env = get_constant("MAGPIE_INI_FILE_PATH", settings, raise_missing=True) config_ini = (global_config or {}).get("__file__", config_env) if config_ini != config_env: magpie.constants.MAGPIE_INI_FILE_PATH = config_ini settings["magpie.ini_file_path"] = config_ini print_log("Setting up loggers...", LOGGER) log_lvl = get_constant("MAGPIE_LOG_LEVEL", settings, "magpie.log_level", default_value="INFO", raise_missing=False, raise_not_set=False, print_missing=True) # apply proper value in case it was in ini AND env since up until then, only env was check # we want to prioritize the ini definition magpie.constants.MAGPIE_LOG_LEVEL = log_lvl LOGGER.setLevel(log_lvl) sa_settings = set_sqlalchemy_log_level(log_lvl) print_log("Looking for db migration requirement...", LOGGER) run_database_migration_when_ready( settings) # cannot pass db session as it might not even exist yet! # NOTE: # migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set # to 'echo=False' because engines are re-created as needed... (ie: missing db) # apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`""" set_sqlalchemy_log_level(log_lvl) # fetch db session here, otherwise, any following db engine connection will re-initialize # with a new engine class and logging settings don't get re-evaluated/applied db_session = get_db_session_from_config_ini(config_ini, settings_override=sa_settings) print_log("Validate settings that require explicit definitions...", LOGGER) for req_config in [ "MAGPIE_SECRET", "MAGPIE_ADMIN_USER", "MAGPIE_ADMIN_PASSWORD" ]: get_constant(req_config, settings_container=settings, raise_missing=True, raise_not_set=True) print_log("Register default users...", LOGGER) register_defaults(db_session=db_session, settings=settings) print_log("Register service providers...", logger=LOGGER) combined_config = get_constant("MAGPIE_CONFIG_PATH", settings, default_value=None, raise_missing=False, raise_not_set=False, print_missing=True) push_phoenix = asbool( get_constant("PHOENIX_PUSH", settings, settings_name="phoenix.push", default_value=False, raise_missing=False, raise_not_set=False, print_missing=True)) prov_cfg = combined_config or get_constant("MAGPIE_PROVIDERS_CONFIG_PATH", settings, default_value="", raise_missing=False, raise_not_set=False, print_missing=True) magpie_register_services_from_config(prov_cfg, push_to_phoenix=push_phoenix, force_update=True, disable_getcapabilities=False, db_session=db_session) print_log("Register configuration permissions...", LOGGER) perm_cfg = combined_config or get_constant( "MAGPIE_PERMISSIONS_CONFIG_PATH", settings, default_value="", raise_missing=False, raise_not_set=False, print_missing=True) magpie_register_permissions_from_config(perm_cfg, db_session=db_session) print_log("Running configurations setup...", LOGGER) patch_magpie_url(settings) # avoid cornice conflicting with magpie exception views settings["handle_exceptions"] = False config = get_auth_config(settings) set_cache_regions_from_settings(settings) # don't use scan otherwise modules like 'magpie.adapter' are # automatically found and cause import errors on missing packages print_log("Including Magpie modules...", LOGGER) config.include("magpie") # config.scan("magpie") print_log("Starting Magpie app...", LOGGER) wsgi_app = config.make_wsgi_app() return wsgi_app
def main(global_config=None, **settings): """ This function returns a Pyramid WSGI application. """ config_ini = get_constant("MAGPIE_INI_FILE_PATH", raise_missing=True) print_log("Setting up loggers...", LOGGER) log_lvl = get_constant("MAGPIE_LOG_LEVEL", settings, "magpie.log_level", default_value="INFO", raise_missing=False, raise_not_set=False, print_missing=True) LOGGER.setLevel(log_lvl) sa_settings = db.set_sqlalchemy_log_level(log_lvl) print_log("Looking for db migration requirement...", LOGGER) db.run_database_migration_when_ready(settings) # cannot pass db session as it might not even exist yet! # HACK: # migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set # to 'echo=False' because engines are re-created as needed... (ie: missing db) # apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`""" db.set_sqlalchemy_log_level(log_lvl) # fetch db session here, otherwise, any following db engine connection will re-initialize # with a new engine class and logging settings don't get re-evaluated/applied db_session = db.get_db_session_from_config_ini(config_ini, settings_override=sa_settings) print_log("Register default users...", LOGGER) register_default_users(db_session=db_session, settings=settings) print_log("Register configuration providers...", logger=LOGGER) push_phoenix = asbool(get_constant("PHOENIX_PUSH", settings, settings_name="magpie.phoenix_push", raise_missing=False, raise_not_set=False, print_missing=True)) prov_cfg = get_constant("MAGPIE_PROVIDERS_CONFIG_PATH", default_value="", raise_missing=False, raise_not_set=False, print_missing=True) if os.path.isfile(prov_cfg): magpie_register_services_from_config(prov_cfg, push_to_phoenix=push_phoenix, force_update=True, disable_getcapabilities=False, db_session=db_session) else: print_log("No configuration file found for providers registration, skipping...", LOGGER, logging.WARN) print_log("Register configuration permissions...", LOGGER) perm_cfg = get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH", default_value="", raise_missing=False, raise_not_set=False, print_missing=True) if os.path.isfile(perm_cfg): magpie_register_permissions_from_config(get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH"), db_session=db_session) else: print_log("No configuration file found for permissions registration, skipping...", LOGGER, logging.WARN) print_log("Running configurations setup...", LOGGER) patch_magpie_url(settings) # avoid cornice conflicting with magpie exception views settings["handle_exceptions"] = False config = get_auth_config(settings) set_cache_regions_from_settings(settings) # Don't use scan otherwise modules like 'magpie.adapter' are # automatically found and cause import errors on missing packages config.include("magpie") # config.scan("magpie") print_log("Starting Magpie app...", LOGGER) wsgi_app = config.make_wsgi_app() return wsgi_app
def get_constant(constant_name, # type: Str settings_container=None, # type: Optional[AnySettingsContainer] settings_name=None, # type: Optional[Str] default_value=None, # type: Optional[SettingValue] raise_missing=True, # type: bool print_missing=False, # type: bool raise_not_set=True # type: bool ): # type: (...) -> SettingValue """ Search in order for matched value of :paramref:`constant_name`: 1. search in :py:data:`MAGPIE_CONSTANTS` 2. search in settings if specified 3. search alternative setting names (see below) 4. search in :mod:`magpie.constants` definitions 5. search in environment variables Parameter :paramref:`constant_name` is expected to have the format ``MAGPIE_[VARIABLE_NAME]`` although any value can be passed to retrieve generic settings from all above mentioned search locations. If :paramref:`settings_name` is provided as alternative name, it is used as is to search for results if :paramref:`constant_name` was not found. Otherwise, ``magpie.[variable_name]`` is used for additional search when the format ``MAGPIE_[VARIABLE_NAME]`` was used for :paramref:`constant_name` (i.e.: ``MAGPIE_ADMIN_USER`` will also search for ``magpie.admin_user`` and so on for corresponding constants). :param constant_name: key to search for a value :param settings_container: WSGI application settings container (if not provided, uses found one in current thread) :param settings_name: alternative name for `settings` if specified :param default_value: default value to be returned if not found anywhere, and exception raises are disabled. :param raise_missing: raise exception if key is not found anywhere :param print_missing: print message if key is not found anywhere, return ``None`` :param raise_not_set: raise an exception if the found key is ``None``, search until last case if others are ``None`` :returns: found value or `default_value` :raises ValueError: if resulting value is invalid based on options (by default raise missing/``None`` value) :raises LookupError: if no appropriate value could be found from all search locations (according to options) """ from magpie.utils import get_settings, print_log, raise_log # pylint: disable=C0415 # avoid circular import error if constant_name in MAGPIE_CONSTANTS: return globals()[constant_name] missing = True magpie_value = None if settings_container: settings = get_settings(settings_container) else: # note: this will work only after include of magpie will have triggered configurator setup print_log("Using settings from local thread.", level=logging.DEBUG) settings = get_settings(get_current_registry()) if settings and constant_name in settings: # pylint: disable=E1135 missing = False magpie_value = settings.get(constant_name) if magpie_value is not None: print_log("Constant found in settings with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if not settings_name: settings_name = get_constant_setting_name(constant_name) print_log("Constant alternate search: {}".format(settings_name), level=logging.DEBUG) if settings and settings_name and settings_name in settings: # pylint: disable=E1135 missing = False magpie_value = settings.get(settings_name) if magpie_value is not None: print_log("Constant found in settings with: {}".format(settings_name), level=logging.DEBUG) return magpie_value magpie_globals = globals() if constant_name in magpie_globals: missing = False magpie_value = magpie_globals.get(constant_name) if magpie_value is not None: print_log("Constant found in definitions with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if constant_name in os.environ: missing = False magpie_value = os.environ.get(constant_name) if magpie_value is not None: print_log("Constant found in environment with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if not missing and raise_not_set: raise_log("Constant was found but was not set: {}".format(constant_name), level=logging.ERROR, exception=ValueError) if missing and raise_missing: raise_log("Constant could not be found: {}".format(constant_name), level=logging.ERROR, exception=LookupError) if missing and print_missing: print_log("Constant could not be found: {} (using default: {})" .format(constant_name, default_value), level=logging.WARN) return magpie_value or default_value
def get_constant(constant_name, # type: Str settings_container=None, # type: Optional[AnySettingsContainer] settings_name=None, # type: Optional[Str] default_value=None, # type: Optional[SettingValue] raise_missing=True, # type: bool print_missing=False, # type: bool raise_not_set=True # type: bool ): # type: (...) -> SettingValue """ Search in order for matched value of ``constant_name``: 1. search in settings if specified 2. search alternative setting names 3. search in ``magpie.constants`` definitions 4. search in environment variables Parameter ``constant_name`` is expected to have the format ``MAGPIE_[VARIABLE_NAME]`` although any value can be passed to retrieve generic settings from all above mentioned search locations. If ``settings_name`` is provided as alternative name, it is used as is to search for results if ``constant_name`` was not found. Otherwise, ``magpie.[variable_name]`` is used for additional search when the format ``MAGPIE_[VARIABLE_NAME]`` was used for ``constant_name`` (ie: ``MAGPIE_ADMIN_USER`` will also search for ``magpie.admin_user`` and so on for corresponding constants). :param constant_name: key to search for a value :param settings_container: wsgi app settings container :param settings_name: alternative name for `settings` if specified :param default_value: default value to be returned if not found anywhere, and exception raises are disabled. :param raise_missing: raise exception if key is not found anywhere :param print_missing: print message if key is not found anywhere, return `None` :param raise_not_set: raise an exception if the found key is None, search until last case if previous are `None` :returns: found value or `default_value` :raises: according message based on options (by default raise missing/`None` value) """ from magpie.utils import get_settings, raise_log, print_log missing = True magpie_value = None settings = get_settings(settings_container) if settings_container else None if settings and constant_name in settings: missing = False magpie_value = settings.get(constant_name) if magpie_value is not None: print_log("Constant found in settings with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if not settings_name and constant_name.startswith("MAGPIE_"): settings_name = get_constant_setting_name(constant_name) print_log("Constant alternate search: {}".format(settings_name), level=logging.DEBUG) if settings and settings_name and settings_name in settings: missing = False magpie_value = settings.get(settings_name) if magpie_value is not None: print_log("Constant found in settings with: {}".format(settings_name), level=logging.DEBUG) return magpie_value magpie_globals = globals() if constant_name in magpie_globals: missing = False magpie_value = magpie_globals.get(constant_name) if magpie_value is not None: print_log("Constant found in definitions with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if constant_name in os.environ: missing = False magpie_value = os.environ.get(constant_name) if magpie_value is not None: print_log("Constant found in environment with: {}".format(constant_name), level=logging.DEBUG) return magpie_value if not missing and raise_not_set: raise_log("Constant was found but was not set: {}".format(constant_name), level=logging.ERROR, exception=ValueError) if missing and raise_missing: raise_log("Constant could not be found: {}".format(constant_name), level=logging.ERROR, exception=LookupError) if missing and print_missing: print_log("Constant could not be found: {} (using default: {})" .format(constant_name, default_value), level=logging.WARN) return magpie_value or default_value
def run_database_migration_when_ready(settings, db_session=None): # type: (SettingsType, Optional[Session]) -> None """ Runs db migration if requested by config and need from revisions. """ db_ready = False if asbool( get_constant("MAGPIE_DB_MIGRATION", settings, "magpie.db_migration", default_value=True, raise_missing=False, raise_not_set=False, print_missing=True)): conf_attempts = int( get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS", settings, "magpie.db_migration_attempts", default_value=5, raise_missing=False, raise_not_set=False, print_missing=True)) print_log("Running database migration (as required)...", logger=LOGGER) attempts = max(conf_attempts, 1) if attempts != conf_attempts: print_log( "Database migration attempts updated to {}".format(attempts), logger=LOGGER, level=logging.WARNING) for i in range(1, attempts + 1): try: run_database_migration(db_session=db_session, settings=settings) except ImportError as exc: print_log( "Database migration produced [{!r}] (ignored).".format( exc), logger=LOGGER, level=logging.WARNING, exc_info=exc) except Exception as exc: if i <= attempts: print_log( "Database migration failed [{!r}]. Retrying... ({}/{})" .format(exc, i, attempts), logger=LOGGER, level=logging.WARNING, exc_info=exc) time.sleep(2) continue raise_log("Database migration failed [{!r}]".format(exc), exception=RuntimeError, logger=LOGGER) db_ready = is_database_ready(db_session) if not db_ready: if i <= attempts: print_log("Database not ready. Retrying... ({}/{})".format( i, attempts), logger=LOGGER, level=logging.WARNING) time.sleep(2) continue print_log( "Database not ready. Maximum attempts reached ({})".format( attempts), logger=LOGGER, level=logging.WARNING) break else: print_log( "Database migration skipped as per 'MAGPIE_DB_MIGRATION' requirement...", logger=LOGGER) db_ready = is_database_ready(db_session) if not db_ready: raise_log("Database not ready", exception=RuntimeError, logger=LOGGER)
def magpie_register_services_with_db_session( services_dict, db_session, push_to_phoenix=False, force_update=False, update_getcapabilities_permissions=False): db_session.begin(subtransactions=True) existing_services_names = [ n[0] for n in db_session.query(models.Service.resource_name) ] magpie_anonymous_user = get_constant("MAGPIE_ANONYMOUS_USER") anonymous_user = UserService.by_user_name(magpie_anonymous_user, db_session=db_session) for svc_name, svc_values in services_dict.items(): svc_new_url = os.path.expandvars(svc_values["url"]) svc_type = svc_values["type"] svc_sync_type = svc_values.get("sync_type") if force_update and svc_name in existing_services_names: svc = models.Service.by_service_name(svc_name, db_session=db_session) if svc.url == svc_new_url: print_log( "Service URL already properly set [{url}] ({svc})".format( url=svc.url, svc=svc_name), logger=LOGGER) else: print_log( "Service URL update [{url_old}] => [{url_new}] ({svc})". format(url_old=svc.url, url_new=svc_new_url, svc=svc_name), logger=LOGGER) svc.url = svc_new_url svc.sync_type = svc_sync_type elif not force_update and svc_name in existing_services_names: print_log( "Skipping service [{svc}] (conflict)".format(svc=svc_name), logger=LOGGER) else: print_log("Adding service [{svc}]".format(svc=svc_name), logger=LOGGER) # noinspection PyArgumentList svc = models.Service( resource_name=svc_name, resource_type=models.Service.resource_type_name, url=svc_new_url, type=svc_type, sync_type=svc_sync_type) db_session.add(svc) getcap_perm = Permission.GET_CAPABILITIES if update_getcapabilities_permissions and anonymous_user is None: print_log( "Cannot update 'getcapabilities' permission of non existing anonymous user", level=logging.WARN, logger=LOGGER) elif update_getcapabilities_permissions and getcap_perm in SERVICE_TYPE_DICT[ svc_type].permissions: svc = db_session.query(models.Service.resource_id).filter_by( resource_name=svc_name).first() svc_perm_getcapabilities = UserResourcePermissionService.by_resource_user_and_perm( user_id=anonymous_user.id, perm_name=getcap_perm.value, resource_id=svc.resource_id, db_session=db_session) if svc_perm_getcapabilities is None: print_log("Adding '{}' permission to anonymous user.".format( getcap_perm.value), logger=LOGGER) # noinspection PyArgumentList svc_perm_getcapabilities = models.UserResourcePermission( user_id=anonymous_user.id, perm_name=getcap_perm.value, resource_id=svc.resource_id) db_session.add(svc_perm_getcapabilities) transaction.commit() if push_to_phoenix: return phoenix_update_services(services_dict) return True
def magpie_register_services_with_requests(services_dict, push_to_phoenix, username, password, provider, force_update=False, disable_getcapabilities=False): # type: (ConfigDict, bool, Str, Str, Str, bool, bool) -> bool """ Registers magpie services using the provided services configuration. :param services_dict: services configuration definition. :param push_to_phoenix: push registered Magpie services to Phoenix for synced configurations. :param username: login username to use to obtain permissions for services registration. :param password: login password to use to obtain permissions for services registration. :param provider: login provider to use to obtain permissions for services registration. :param force_update: override existing services matched by name :param disable_getcapabilities: do not execute 'GetCapabilities' validation for applicable services. :return: successful operation status """ magpie_url = get_magpie_url() curl_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_magpie") session = requests.Session() success = False try: # Need to login first as admin login_url = magpie_url + SigninAPI.path login_data = { "user_name": username, "password": password, "provider_name": provider } login_loop(login_url, curl_cookies, login_data, "Magpie login response") login_resp = session.post(login_url, data=login_data) if login_resp.status_code != 200: raise_log("Failed login with specified credentials", logger=LOGGER) request_cookies = login_resp.cookies # Register services # Magpie will not overwrite existing services by default, 409 Conflict instead of 201 Created success, statuses_register = register_services( SERVICES_MAGPIE, services_dict, curl_cookies, "Magpie register service") # Service URL update if conflicting and requested if force_update and not success: conflict_services = [ svc_name for svc_name, http_code in statuses_register.items() if http_code == 409 ] statuses_update = magpie_update_services_conflict( conflict_services, services_dict, request_cookies) statuses_register.update( statuses_update) # update previous statuses with new ones # Add 'GetCapabilities' permissions on newly created services to allow 'ping' from Phoenix # Phoenix doesn't register the service if it cannot be checked with this request magpie_add_register_services_perms(services_dict, statuses_register, curl_cookies, request_cookies, disable_getcapabilities) session.get(magpie_url + SignoutAPI.path) # Push updated services to Phoenix if push_to_phoenix: success = phoenix_update_services(services_dict) except Exception as e: print_log( "Exception during magpie register services: [{!r}]".format(e), logger=LOGGER, level=logging.ERROR) finally: session.cookies.clear() if os.path.isfile(curl_cookies): os.remove(curl_cookies) return success
def magpie_add_register_services_perms(services, statuses, curl_cookies, request_cookies, disable_getcapabilities): magpie_url = get_magpie_url() login_usr = get_constant("MAGPIE_ANONYMOUS_USER") for service_name in services: svc_available_perms_url = "{magpie}/services/{svc}/permissions" \ .format(magpie=magpie_url, svc=service_name) resp_available_perms = requests.get(svc_available_perms_url, cookies=request_cookies) if resp_available_perms.status_code == 401: raise_log("Invalid credentials, cannot update service permissions", exception=ValueError, logger=LOGGER) available_perms = get_json(resp_available_perms).get( "permission_names", []) # only applicable to services supporting "GetCapabilities" request if resp_available_perms.status_code and Permission.GET_CAPABILITIES.value in available_perms: # enforce 'getcapabilities' permission if available for service just updated (200) or created (201) # update 'getcapabilities' permission when the service existed and it allowed if (not disable_getcapabilities and statuses[service_name] == 409) \ or statuses[service_name] == 200 or statuses[service_name] == 201: # noqa svc_anonym_add_perms_url = "{magpie}/users/{usr}/services/{svc}/permissions" \ .format(magpie=magpie_url, usr=login_usr, svc=service_name) svc_anonym_perm_data = { "permission_name": Permission.GET_CAPABILITIES.value } requests.post(svc_anonym_add_perms_url, data=svc_anonym_perm_data, cookies=request_cookies) # check service response so Phoenix doesn't refuse registration # try with both the 'direct' URL and the 'GetCapabilities' URL attempt = 0 service_info_url = "{magpie}/services/{svc}".format( magpie=magpie_url, svc=service_name) service_info_resp = requests.get(service_info_url, cookies=request_cookies) service_url = get_json(service_info_resp).get(service_name).get( "service_url") svc_getcap_url = "{svc_url}/wps?service=WPS&version=1.0.0&request=GetCapabilities" \ .format(svc_url=service_url) while True: service_msg_direct = "Service response ({svc})".format( svc=service_name) service_msg_getcap = "Service response ({svc}, GetCapabilities)".format( svc=service_name) err, http = request_curl(service_url, cookies=curl_cookies, msg=service_msg_direct) if not err and http == 200: break err, http = request_curl(svc_getcap_url, cookies=curl_cookies, msg=service_msg_getcap) if not err and http == 200: break print_log( "[{url}] Bad response from service '{svc}' retrying after {sec}s..." .format(svc=service_name, url=service_url, sec=GETCAPABILITIES_INTERVAL), logger=LOGGER) time.sleep(GETCAPABILITIES_INTERVAL) attempt += 1 if attempt >= GETCAPABILITIES_ATTEMPTS: msg = "[{url}] No response from service '{svc}' after {tries} attempts. Skipping..." \ .format(svc=service_name, url=service_url, tries=attempt) print_log(msg, logger=LOGGER) break