def magpie_update_services_conflict(conflict_services, services_dict,
request_cookies):
# type: (List[Str], ConfigDict, Dict[Str, Str]) -> Dict[Str, int]
"""
Resolve conflicting services by name during registration by updating them only if pointing to different URL.
"""
magpie_url = get_magpie_url()
statuses = dict()
for svc_name in conflict_services:
statuses[svc_name] = 409
svc_url_new = services_dict[svc_name]["url"]
svc_url_db = "{magpie}/services/{svc}".format(magpie=magpie_url,
svc=svc_name)
svc_resp = requests.get(svc_url_db, cookies=request_cookies)
svc_info = get_json(svc_resp).get(svc_name)
svc_url_old = svc_info["service_url"]
if svc_url_old != svc_url_new:
svc_info["service_url"] = svc_url_new
res_svc_put = requests.put(svc_url_db,
data=svc_info,
cookies=request_cookies)
statuses[svc_name] = res_svc_put.status_code
print_log(
"[{url_old}] => [{url_new}] Service URL update ({svc}): {resp}"
.format(svc=svc_name,
url_old=svc_url_old,
url_new=svc_url_new,
resp=res_svc_put.status_code),
logger=LOGGER)
return statuses
def phoenix_remove_services():
# type: () -> bool
"""
Removes the Phoenix services using temporary cookies retrieved from login with defined `PHOENIX` constants.
:returns: success status of the procedure.
"""
phoenix_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_phoenix")
error = 0
try:
if not phoenix_login(phoenix_cookies):
print_log("Login unsuccessful from post-login check, aborting...",
logger=LOGGER)
return False
phoenix_url = get_phoenix_url()
remove_services_url = phoenix_url + "/clear_services"
error, http_code = request_curl(remove_services_url,
cookies=phoenix_cookies,
msg="Phoenix remove services")
except Exception as e:
print_log("Exception during phoenix remove services: [{!r}]".format(e),
logger=LOGGER)
finally:
if os.path.isfile(phoenix_cookies):
os.remove(phoenix_cookies)
return error == 0
def register_user_with_group(user_name, group_name, email, password,
db_session):
# type: (Str, Str, Str, Str, Session) -> None
"""
Registers the user if missing and associate him to a group specified by name, also created if missing.
:param user_name: name of the user to create (if missing) and to make part of the group (if specified)
:param group_name: name of the group to create (if missing and specified) and to make the user join (if not already)
:param email: email of the user to be created (if missing)
:param password: password of the user to be created (if missing)
:param db_session: database connexion to apply changes
.. warning::
Should be employed only for **special** users/groups in this module as other expected API behaviour
and operations will not be applied (ex: create additional permissions or user-group references).
"""
if not GroupService.by_group_name(group_name, db_session=db_session):
# noinspection PyArgumentList
new_group = models.Group(group_name=group_name)
db_session.add(new_group)
registered_group = GroupService.by_group_name(group_name=group_name,
db_session=db_session)
registered_user = UserService.by_user_name(user_name,
db_session=db_session)
if not registered_user:
# noinspection PyArgumentList
new_user = models.User(user_name=user_name, email=email)
UserService.set_password(new_user, password)
UserService.regenerate_security_code(new_user)
db_session.add(new_user)
if group_name is not None:
registered_user = UserService.by_user_name(user_name,
db_session=db_session)
else:
print_log("User '{}' already exist".format(user_name),
level=logging.DEBUG)
# noinspection PyBroadException
try:
# ensure the reference between user/group exists (user joined the group)
user_group_refs = BaseService.all(models.UserGroup,
db_session=db_session)
user_group_refs_tup = [(ref.group_id, ref.user_id)
for ref in user_group_refs]
if (registered_group.id,
registered_user.id) not in user_group_refs_tup:
# noinspection PyArgumentList
group_entry = models.UserGroup(group_id=registered_group.id,
user_id=registered_user.id)
db_session.add(group_entry)
except Exception: # in case reference already exists, avoid duplicate error
db_session.rollback()
def phoenix_update_services(services_dict):
if not phoenix_remove_services():
print_log(
"Could not remove services, aborting register sync services to Phoenix",
logger=LOGGER)
return False
if not phoenix_register_services(services_dict):
print_log(
"Failed services registration from Magpie to Phoenix\n" +
"[warning: services could have been removed but could not be re-added]",
logger=LOGGER)
return False
return True
def init_users_group(db_session, settings=None):
# type: (Session, Optional[AnySettingsContainer]) -> None
"""
Registers into database the group matching :py:data:`magpie.constants.MAGPIE_USERS_GROUP` if not defined.
"""
usr_grp_name = get_constant("MAGPIE_USERS_GROUP",
settings_container=settings)
if not GroupService.by_group_name(usr_grp_name, db_session=db_session):
user_group = models.Group(group_name=usr_grp_name) # noqa
db_session.add(user_group)
else:
print_log("MAGPIE_USERS_GROUP already initialized",
level=logging.DEBUG)
def init_users_group(db_session, settings=None):
# type: (Session, Optional[AnySettingsContainer]) -> None
"""
Registers in db the group matching ``MAGPIE_USERS_GROUP`` if not defined.
"""
usr_grp_name = get_constant("MAGPIE_USERS_GROUP",
settings_container=settings)
if not GroupService.by_group_name(usr_grp_name, db_session=db_session):
# noinspection PyArgumentList
user_group = models.Group(group_name=usr_grp_name)
db_session.add(user_group)
else:
print_log("MAGPIE_USERS_GROUP already initialized",
level=logging.DEBUG)
def get_version(request):
"""
Version information of the API.
"""
version_db = None
try:
version_db = get_database_revision(request.db)
except Exception as exc:
print_log("Failed to retrieve database revision: [{!r}]".format(exc),
LOGGER, logging.WARNING)
version = {"version": __meta__.__version__, "db_version": version_db}
return ax.valid_http(http_success=HTTPOk,
content=version,
content_type=CONTENT_TYPE_JSON,
detail=s.Version_GET_OkResponseSchema.description)
def is_database_ready(db_session=None, container=None):
# type: (Optional[Session], Optional[AnySettingsContainer]) -> bool
"""
Obtains the database status against expected table names to ensure it is ready for use.
"""
if isinstance(db_session, Session):
engine = db_session.bind
else:
engine = get_engine(container=container)
inspector = Inspector.from_engine(engine)
table_names = inspector.get_table_names()
for _, obj in inspect.getmembers(models):
if inspect.isclass(obj) and hasattr(obj, "__tablename__"):
if obj.__tablename__ not in table_names:
print_log(
"Database table (or its associated parent) is missing for '{}' object"
.format(obj),
logger=LOGGER,
level=logging.ERROR)
return False
return True
def request_curl(url,
cookie_jar=None,
cookies=None,
form_params=None,
msg="Response"):
# type: (Str, Optional[Str], Optional[Str], Optional[Str], Optional[Str]) -> Tuple[int, int]
"""
Executes a request using cURL.
:returns: tuple of the returned system command code and the response http code
"""
# arg -k allows to ignore insecure SSL errors, ie: access 'https' page not configured for it
# curl_cmd = 'curl -k -L -s -o /dev/null -w "{msg_out} : %{{http_code}}\\n" {params} {url}'
# curl_cmd = curl_cmd.format(msg_out=msg, params=params, url=url)
msg_sep = msg + ": "
params = [
"curl", "-k", "-L", "-s", "-o", "/dev/null", "-w",
msg_sep + "%{http_code}"
]
if cookie_jar is not None and cookies is not None:
raise ValueError(
"CookiesType and Cookie_Jar cannot be both set simultaneously")
if cookie_jar is not None:
params.extend(["--cookie-jar", cookie_jar]) # save cookies
if cookies is not None:
params.extend(["--cookie", cookies]) # use cookies
if form_params is not None:
params.extend(["--data", form_params])
params.extend([url])
curl_out = subprocess.Popen(params, stdout=subprocess.PIPE)
curl_msg = curl_out.communicate()[0] # type: Str
curl_err = curl_out.returncode # type: int
http_code = int(curl_msg.split(msg_sep)[1])
print_log("[{url}] {response}".format(response=curl_msg, url=url),
logger=LOGGER)
return curl_err, http_code
def phoenix_register_services(services_dict, allowed_service_types=None):
phoenix_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_phoenix")
success = False
statuses = dict()
try:
allowed_service_types = SERVICES_PHOENIX_ALLOWED if allowed_service_types is None else allowed_service_types
allowed_service_types = [svc.upper() for svc in allowed_service_types]
if not phoenix_login(phoenix_cookies):
print_log("Login unsuccessful from post-login check, aborting...",
logger=LOGGER,
level=logging.WARN)
return False
# Filter specific services to push
filtered_services_dict = {}
for svc in services_dict:
if str(services_dict[svc].get(
"type")).upper() in allowed_service_types:
filtered_services_dict[svc] = services_dict[svc]
filtered_services_dict[svc]["type"] = filtered_services_dict[
svc]["type"].upper()
# Register services
success, statuses = register_services(SERVICES_PHOENIX,
filtered_services_dict,
phoenix_cookies,
"Phoenix register service")
except Exception as e:
print_log(
"Exception during phoenix register services: [{!r}]".format(e),
logger=LOGGER,
level=logging.ERROR)
finally:
if os.path.isfile(phoenix_cookies):
os.remove(phoenix_cookies)
return success, statuses
def run_database_migration_when_ready(settings, db_session=None):
# type: (SettingsType, Optional[Session]) -> None
"""
Runs db migration if requested by config and need from revisions.
"""
db_ready = False
if asbool(
get_constant("MAGPIE_DB_MIGRATION",
settings,
"magpie.db_migration",
default_value=True,
raise_missing=False,
raise_not_set=False,
print_missing=True)):
attempts = int(
get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS",
settings,
"magpie.db_migration_attempts",
default_value=5,
raise_missing=False,
raise_not_set=False,
print_missing=True))
print_log("Running database migration (as required)...")
attempts = max(
attempts, 2
) # enforce at least 2 attempts, 1 for db creation and one for actual migration
for i in range(1, attempts + 1):
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
run_database_migration(db_session)
except ImportError as e:
print_log(
"Database migration produced [{!r}] (ignored).".format(e),
level=logging.WARNING)
pass
except Exception as e:
if i <= attempts:
print_log(
"Database migration failed [{!r}]. Retrying... ({}/{})"
.format(e, i, attempts))
time.sleep(2)
continue
else:
raise_log("Database migration failed [{!r}]".format(e),
exception=RuntimeError)
db_ready = is_database_ready(db_session)
if not db_ready:
print_log("Database not ready. Retrying... ({}/{})".format(
i, attempts))
time.sleep(2)
continue
break
else:
db_ready = is_database_ready(db_session)
if not db_ready:
time.sleep(2)
raise_log("Database not ready", exception=RuntimeError)
def main(global_config=None, **settings): # noqa: F811
"""
This function returns a Pyramid WSGI application.
"""
import magpie.constants # pylint: disable=C0415 # avoid circular import
# override magpie ini if provided with --paste to gunicorn, otherwise use environment variable
config_env = get_constant("MAGPIE_INI_FILE_PATH",
settings,
raise_missing=True)
config_ini = (global_config or {}).get("__file__", config_env)
if config_ini != config_env:
magpie.constants.MAGPIE_INI_FILE_PATH = config_ini
settings["magpie.ini_file_path"] = config_ini
print_log("Setting up loggers...", LOGGER)
log_lvl = get_constant("MAGPIE_LOG_LEVEL",
settings,
"magpie.log_level",
default_value="INFO",
raise_missing=False,
raise_not_set=False,
print_missing=True)
# apply proper value in case it was in ini AND env since up until then, only env was check
# we want to prioritize the ini definition
magpie.constants.MAGPIE_LOG_LEVEL = log_lvl
LOGGER.setLevel(log_lvl)
sa_settings = set_sqlalchemy_log_level(log_lvl)
print_log("Looking for db migration requirement...", LOGGER)
run_database_migration_when_ready(
settings) # cannot pass db session as it might not even exist yet!
# NOTE:
# migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set
# to 'echo=False' because engines are re-created as needed... (ie: missing db)
# apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`"""
set_sqlalchemy_log_level(log_lvl)
# fetch db session here, otherwise, any following db engine connection will re-initialize
# with a new engine class and logging settings don't get re-evaluated/applied
db_session = get_db_session_from_config_ini(config_ini,
settings_override=sa_settings)
print_log("Validate settings that require explicit definitions...", LOGGER)
for req_config in [
"MAGPIE_SECRET", "MAGPIE_ADMIN_USER", "MAGPIE_ADMIN_PASSWORD"
]:
get_constant(req_config,
settings_container=settings,
raise_missing=True,
raise_not_set=True)
print_log("Register default users...", LOGGER)
register_defaults(db_session=db_session, settings=settings)
print_log("Register service providers...", logger=LOGGER)
combined_config = get_constant("MAGPIE_CONFIG_PATH",
settings,
default_value=None,
raise_missing=False,
raise_not_set=False,
print_missing=True)
push_phoenix = asbool(
get_constant("PHOENIX_PUSH",
settings,
settings_name="phoenix.push",
default_value=False,
raise_missing=False,
raise_not_set=False,
print_missing=True))
prov_cfg = combined_config or get_constant("MAGPIE_PROVIDERS_CONFIG_PATH",
settings,
default_value="",
raise_missing=False,
raise_not_set=False,
print_missing=True)
magpie_register_services_from_config(prov_cfg,
push_to_phoenix=push_phoenix,
force_update=True,
disable_getcapabilities=False,
db_session=db_session)
print_log("Register configuration permissions...", LOGGER)
perm_cfg = combined_config or get_constant(
"MAGPIE_PERMISSIONS_CONFIG_PATH",
settings,
default_value="",
raise_missing=False,
raise_not_set=False,
print_missing=True)
magpie_register_permissions_from_config(perm_cfg, db_session=db_session)
print_log("Running configurations setup...", LOGGER)
patch_magpie_url(settings)
# avoid cornice conflicting with magpie exception views
settings["handle_exceptions"] = False
config = get_auth_config(settings)
set_cache_regions_from_settings(settings)
# don't use scan otherwise modules like 'magpie.adapter' are
# automatically found and cause import errors on missing packages
print_log("Including Magpie modules...", LOGGER)
config.include("magpie")
# config.scan("magpie")
print_log("Starting Magpie app...", LOGGER)
wsgi_app = config.make_wsgi_app()
return wsgi_app
def main(global_config=None, **settings):
"""
This function returns a Pyramid WSGI application.
"""
config_ini = get_constant("MAGPIE_INI_FILE_PATH", raise_missing=True)
print_log("Setting up loggers...", LOGGER)
log_lvl = get_constant("MAGPIE_LOG_LEVEL", settings, "magpie.log_level", default_value="INFO",
raise_missing=False, raise_not_set=False, print_missing=True)
LOGGER.setLevel(log_lvl)
sa_settings = db.set_sqlalchemy_log_level(log_lvl)
print_log("Looking for db migration requirement...", LOGGER)
db.run_database_migration_when_ready(settings) # cannot pass db session as it might not even exist yet!
# HACK:
# migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set
# to 'echo=False' because engines are re-created as needed... (ie: missing db)
# apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`"""
db.set_sqlalchemy_log_level(log_lvl)
# fetch db session here, otherwise, any following db engine connection will re-initialize
# with a new engine class and logging settings don't get re-evaluated/applied
db_session = db.get_db_session_from_config_ini(config_ini, settings_override=sa_settings)
print_log("Register default users...", LOGGER)
register_default_users(db_session=db_session, settings=settings)
print_log("Register configuration providers...", logger=LOGGER)
push_phoenix = asbool(get_constant("PHOENIX_PUSH", settings, settings_name="magpie.phoenix_push",
raise_missing=False, raise_not_set=False, print_missing=True))
prov_cfg = get_constant("MAGPIE_PROVIDERS_CONFIG_PATH", default_value="",
raise_missing=False, raise_not_set=False, print_missing=True)
if os.path.isfile(prov_cfg):
magpie_register_services_from_config(prov_cfg, push_to_phoenix=push_phoenix,
force_update=True, disable_getcapabilities=False, db_session=db_session)
else:
print_log("No configuration file found for providers registration, skipping...", LOGGER, logging.WARN)
print_log("Register configuration permissions...", LOGGER)
perm_cfg = get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH", default_value="",
raise_missing=False, raise_not_set=False, print_missing=True)
if os.path.isfile(perm_cfg):
magpie_register_permissions_from_config(get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH"), db_session=db_session)
else:
print_log("No configuration file found for permissions registration, skipping...", LOGGER, logging.WARN)
print_log("Running configurations setup...", LOGGER)
patch_magpie_url(settings)
# avoid cornice conflicting with magpie exception views
settings["handle_exceptions"] = False
config = get_auth_config(settings)
set_cache_regions_from_settings(settings)
# Don't use scan otherwise modules like 'magpie.adapter' are
# automatically found and cause import errors on missing packages
config.include("magpie")
# config.scan("magpie")
print_log("Starting Magpie app...", LOGGER)
wsgi_app = config.make_wsgi_app()
return wsgi_app
def get_constant(constant_name, # type: Str
settings_container=None, # type: Optional[AnySettingsContainer]
settings_name=None, # type: Optional[Str]
default_value=None, # type: Optional[SettingValue]
raise_missing=True, # type: bool
print_missing=False, # type: bool
raise_not_set=True # type: bool
): # type: (...) -> SettingValue
"""
Search in order for matched value of :paramref:`constant_name`:
1. search in :py:data:`MAGPIE_CONSTANTS`
2. search in settings if specified
3. search alternative setting names (see below)
4. search in :mod:`magpie.constants` definitions
5. search in environment variables
Parameter :paramref:`constant_name` is expected to have the format ``MAGPIE_[VARIABLE_NAME]`` although any value can
be passed to retrieve generic settings from all above mentioned search locations.
If :paramref:`settings_name` is provided as alternative name, it is used as is to search for results if
:paramref:`constant_name` was not found. Otherwise, ``magpie.[variable_name]`` is used for additional search when
the format ``MAGPIE_[VARIABLE_NAME]`` was used for :paramref:`constant_name`
(i.e.: ``MAGPIE_ADMIN_USER`` will also search for ``magpie.admin_user`` and so on for corresponding constants).
:param constant_name: key to search for a value
:param settings_container: WSGI application settings container (if not provided, uses found one in current thread)
:param settings_name: alternative name for `settings` if specified
:param default_value: default value to be returned if not found anywhere, and exception raises are disabled.
:param raise_missing: raise exception if key is not found anywhere
:param print_missing: print message if key is not found anywhere, return ``None``
:param raise_not_set: raise an exception if the found key is ``None``, search until last case if others are ``None``
:returns: found value or `default_value`
:raises ValueError: if resulting value is invalid based on options (by default raise missing/``None`` value)
:raises LookupError: if no appropriate value could be found from all search locations (according to options)
"""
from magpie.utils import get_settings, print_log, raise_log # pylint: disable=C0415 # avoid circular import error
if constant_name in MAGPIE_CONSTANTS:
return globals()[constant_name]
missing = True
magpie_value = None
if settings_container:
settings = get_settings(settings_container)
else:
# note: this will work only after include of magpie will have triggered configurator setup
print_log("Using settings from local thread.", level=logging.DEBUG)
settings = get_settings(get_current_registry())
if settings and constant_name in settings: # pylint: disable=E1135
missing = False
magpie_value = settings.get(constant_name)
if magpie_value is not None:
print_log("Constant found in settings with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if not settings_name:
settings_name = get_constant_setting_name(constant_name)
print_log("Constant alternate search: {}".format(settings_name), level=logging.DEBUG)
if settings and settings_name and settings_name in settings: # pylint: disable=E1135
missing = False
magpie_value = settings.get(settings_name)
if magpie_value is not None:
print_log("Constant found in settings with: {}".format(settings_name), level=logging.DEBUG)
return magpie_value
magpie_globals = globals()
if constant_name in magpie_globals:
missing = False
magpie_value = magpie_globals.get(constant_name)
if magpie_value is not None:
print_log("Constant found in definitions with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if constant_name in os.environ:
missing = False
magpie_value = os.environ.get(constant_name)
if magpie_value is not None:
print_log("Constant found in environment with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if not missing and raise_not_set:
raise_log("Constant was found but was not set: {}".format(constant_name),
level=logging.ERROR, exception=ValueError)
if missing and raise_missing:
raise_log("Constant could not be found: {}".format(constant_name),
level=logging.ERROR, exception=LookupError)
if missing and print_missing:
print_log("Constant could not be found: {} (using default: {})"
.format(constant_name, default_value), level=logging.WARN)
return magpie_value or default_value
def get_constant(constant_name, # type: Str
settings_container=None, # type: Optional[AnySettingsContainer]
settings_name=None, # type: Optional[Str]
default_value=None, # type: Optional[SettingValue]
raise_missing=True, # type: bool
print_missing=False, # type: bool
raise_not_set=True # type: bool
): # type: (...) -> SettingValue
"""
Search in order for matched value of ``constant_name``:
1. search in settings if specified
2. search alternative setting names
3. search in ``magpie.constants`` definitions
4. search in environment variables
Parameter ``constant_name`` is expected to have the format ``MAGPIE_[VARIABLE_NAME]`` although any value can
be passed to retrieve generic settings from all above mentioned search locations.
If ``settings_name`` is provided as alternative name, it is used as is to search for results if ``constant_name``
was not found. Otherwise, ``magpie.[variable_name]`` is used for additional search when the format
``MAGPIE_[VARIABLE_NAME]`` was used for ``constant_name``
(ie: ``MAGPIE_ADMIN_USER`` will also search for ``magpie.admin_user`` and so on for corresponding constants).
:param constant_name: key to search for a value
:param settings_container: wsgi app settings container
:param settings_name: alternative name for `settings` if specified
:param default_value: default value to be returned if not found anywhere, and exception raises are disabled.
:param raise_missing: raise exception if key is not found anywhere
:param print_missing: print message if key is not found anywhere, return `None`
:param raise_not_set: raise an exception if the found key is None, search until last case if previous are `None`
:returns: found value or `default_value`
:raises: according message based on options (by default raise missing/`None` value)
"""
from magpie.utils import get_settings, raise_log, print_log
missing = True
magpie_value = None
settings = get_settings(settings_container) if settings_container else None
if settings and constant_name in settings:
missing = False
magpie_value = settings.get(constant_name)
if magpie_value is not None:
print_log("Constant found in settings with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if not settings_name and constant_name.startswith("MAGPIE_"):
settings_name = get_constant_setting_name(constant_name)
print_log("Constant alternate search: {}".format(settings_name), level=logging.DEBUG)
if settings and settings_name and settings_name in settings:
missing = False
magpie_value = settings.get(settings_name)
if magpie_value is not None:
print_log("Constant found in settings with: {}".format(settings_name), level=logging.DEBUG)
return magpie_value
magpie_globals = globals()
if constant_name in magpie_globals:
missing = False
magpie_value = magpie_globals.get(constant_name)
if magpie_value is not None:
print_log("Constant found in definitions with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if constant_name in os.environ:
missing = False
magpie_value = os.environ.get(constant_name)
if magpie_value is not None:
print_log("Constant found in environment with: {}".format(constant_name), level=logging.DEBUG)
return magpie_value
if not missing and raise_not_set:
raise_log("Constant was found but was not set: {}".format(constant_name),
level=logging.ERROR, exception=ValueError)
if missing and raise_missing:
raise_log("Constant could not be found: {}".format(constant_name),
level=logging.ERROR, exception=LookupError)
if missing and print_missing:
print_log("Constant could not be found: {} (using default: {})"
.format(constant_name, default_value), level=logging.WARN)
return magpie_value or default_value
def run_database_migration_when_ready(settings, db_session=None):
# type: (SettingsType, Optional[Session]) -> None
"""
Runs db migration if requested by config and need from revisions.
"""
db_ready = False
if asbool(
get_constant("MAGPIE_DB_MIGRATION",
settings,
"magpie.db_migration",
default_value=True,
raise_missing=False,
raise_not_set=False,
print_missing=True)):
conf_attempts = int(
get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS",
settings,
"magpie.db_migration_attempts",
default_value=5,
raise_missing=False,
raise_not_set=False,
print_missing=True))
print_log("Running database migration (as required)...", logger=LOGGER)
attempts = max(conf_attempts, 1)
if attempts != conf_attempts:
print_log(
"Database migration attempts updated to {}".format(attempts),
logger=LOGGER,
level=logging.WARNING)
for i in range(1, attempts + 1):
try:
run_database_migration(db_session=db_session,
settings=settings)
except ImportError as exc:
print_log(
"Database migration produced [{!r}] (ignored).".format(
exc),
logger=LOGGER,
level=logging.WARNING,
exc_info=exc)
except Exception as exc:
if i <= attempts:
print_log(
"Database migration failed [{!r}]. Retrying... ({}/{})"
.format(exc, i, attempts),
logger=LOGGER,
level=logging.WARNING,
exc_info=exc)
time.sleep(2)
continue
raise_log("Database migration failed [{!r}]".format(exc),
exception=RuntimeError,
logger=LOGGER)
db_ready = is_database_ready(db_session)
if not db_ready:
if i <= attempts:
print_log("Database not ready. Retrying... ({}/{})".format(
i, attempts),
logger=LOGGER,
level=logging.WARNING)
time.sleep(2)
continue
print_log(
"Database not ready. Maximum attempts reached ({})".format(
attempts),
logger=LOGGER,
level=logging.WARNING)
break
else:
print_log(
"Database migration skipped as per 'MAGPIE_DB_MIGRATION' requirement...",
logger=LOGGER)
db_ready = is_database_ready(db_session)
if not db_ready:
raise_log("Database not ready", exception=RuntimeError, logger=LOGGER)
def magpie_register_services_with_db_session(
services_dict,
db_session,
push_to_phoenix=False,
force_update=False,
update_getcapabilities_permissions=False):
db_session.begin(subtransactions=True)
existing_services_names = [
n[0] for n in db_session.query(models.Service.resource_name)
]
magpie_anonymous_user = get_constant("MAGPIE_ANONYMOUS_USER")
anonymous_user = UserService.by_user_name(magpie_anonymous_user,
db_session=db_session)
for svc_name, svc_values in services_dict.items():
svc_new_url = os.path.expandvars(svc_values["url"])
svc_type = svc_values["type"]
svc_sync_type = svc_values.get("sync_type")
if force_update and svc_name in existing_services_names:
svc = models.Service.by_service_name(svc_name,
db_session=db_session)
if svc.url == svc_new_url:
print_log(
"Service URL already properly set [{url}] ({svc})".format(
url=svc.url, svc=svc_name),
logger=LOGGER)
else:
print_log(
"Service URL update [{url_old}] => [{url_new}] ({svc})".
format(url_old=svc.url, url_new=svc_new_url, svc=svc_name),
logger=LOGGER)
svc.url = svc_new_url
svc.sync_type = svc_sync_type
elif not force_update and svc_name in existing_services_names:
print_log(
"Skipping service [{svc}] (conflict)".format(svc=svc_name),
logger=LOGGER)
else:
print_log("Adding service [{svc}]".format(svc=svc_name),
logger=LOGGER)
# noinspection PyArgumentList
svc = models.Service(
resource_name=svc_name,
resource_type=models.Service.resource_type_name,
url=svc_new_url,
type=svc_type,
sync_type=svc_sync_type)
db_session.add(svc)
getcap_perm = Permission.GET_CAPABILITIES
if update_getcapabilities_permissions and anonymous_user is None:
print_log(
"Cannot update 'getcapabilities' permission of non existing anonymous user",
level=logging.WARN,
logger=LOGGER)
elif update_getcapabilities_permissions and getcap_perm in SERVICE_TYPE_DICT[
svc_type].permissions:
svc = db_session.query(models.Service.resource_id).filter_by(
resource_name=svc_name).first()
svc_perm_getcapabilities = UserResourcePermissionService.by_resource_user_and_perm(
user_id=anonymous_user.id,
perm_name=getcap_perm.value,
resource_id=svc.resource_id,
db_session=db_session)
if svc_perm_getcapabilities is None:
print_log("Adding '{}' permission to anonymous user.".format(
getcap_perm.value),
logger=LOGGER)
# noinspection PyArgumentList
svc_perm_getcapabilities = models.UserResourcePermission(
user_id=anonymous_user.id,
perm_name=getcap_perm.value,
resource_id=svc.resource_id)
db_session.add(svc_perm_getcapabilities)
transaction.commit()
if push_to_phoenix:
return phoenix_update_services(services_dict)
return True
def magpie_register_services_with_requests(services_dict,
push_to_phoenix,
username,
password,
provider,
force_update=False,
disable_getcapabilities=False):
# type: (ConfigDict, bool, Str, Str, Str, bool, bool) -> bool
"""
Registers magpie services using the provided services configuration.
:param services_dict: services configuration definition.
:param push_to_phoenix: push registered Magpie services to Phoenix for synced configurations.
:param username: login username to use to obtain permissions for services registration.
:param password: login password to use to obtain permissions for services registration.
:param provider: login provider to use to obtain permissions for services registration.
:param force_update: override existing services matched by name
:param disable_getcapabilities: do not execute 'GetCapabilities' validation for applicable services.
:return: successful operation status
"""
magpie_url = get_magpie_url()
curl_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_magpie")
session = requests.Session()
success = False
try:
# Need to login first as admin
login_url = magpie_url + SigninAPI.path
login_data = {
"user_name": username,
"password": password,
"provider_name": provider
}
login_loop(login_url, curl_cookies, login_data,
"Magpie login response")
login_resp = session.post(login_url, data=login_data)
if login_resp.status_code != 200:
raise_log("Failed login with specified credentials", logger=LOGGER)
request_cookies = login_resp.cookies
# Register services
# Magpie will not overwrite existing services by default, 409 Conflict instead of 201 Created
success, statuses_register = register_services(
SERVICES_MAGPIE, services_dict, curl_cookies,
"Magpie register service")
# Service URL update if conflicting and requested
if force_update and not success:
conflict_services = [
svc_name for svc_name, http_code in statuses_register.items()
if http_code == 409
]
statuses_update = magpie_update_services_conflict(
conflict_services, services_dict, request_cookies)
statuses_register.update(
statuses_update) # update previous statuses with new ones
# Add 'GetCapabilities' permissions on newly created services to allow 'ping' from Phoenix
# Phoenix doesn't register the service if it cannot be checked with this request
magpie_add_register_services_perms(services_dict, statuses_register,
curl_cookies, request_cookies,
disable_getcapabilities)
session.get(magpie_url + SignoutAPI.path)
# Push updated services to Phoenix
if push_to_phoenix:
success = phoenix_update_services(services_dict)
except Exception as e:
print_log(
"Exception during magpie register services: [{!r}]".format(e),
logger=LOGGER,
level=logging.ERROR)
finally:
session.cookies.clear()
if os.path.isfile(curl_cookies):
os.remove(curl_cookies)
return success
def magpie_add_register_services_perms(services, statuses, curl_cookies,
request_cookies,
disable_getcapabilities):
magpie_url = get_magpie_url()
login_usr = get_constant("MAGPIE_ANONYMOUS_USER")
for service_name in services:
svc_available_perms_url = "{magpie}/services/{svc}/permissions" \
.format(magpie=magpie_url, svc=service_name)
resp_available_perms = requests.get(svc_available_perms_url,
cookies=request_cookies)
if resp_available_perms.status_code == 401:
raise_log("Invalid credentials, cannot update service permissions",
exception=ValueError,
logger=LOGGER)
available_perms = get_json(resp_available_perms).get(
"permission_names", [])
# only applicable to services supporting "GetCapabilities" request
if resp_available_perms.status_code and Permission.GET_CAPABILITIES.value in available_perms:
# enforce 'getcapabilities' permission if available for service just updated (200) or created (201)
# update 'getcapabilities' permission when the service existed and it allowed
if (not disable_getcapabilities and statuses[service_name] == 409) \
or statuses[service_name] == 200 or statuses[service_name] == 201: # noqa
svc_anonym_add_perms_url = "{magpie}/users/{usr}/services/{svc}/permissions" \
.format(magpie=magpie_url, usr=login_usr, svc=service_name)
svc_anonym_perm_data = {
"permission_name": Permission.GET_CAPABILITIES.value
}
requests.post(svc_anonym_add_perms_url,
data=svc_anonym_perm_data,
cookies=request_cookies)
# check service response so Phoenix doesn't refuse registration
# try with both the 'direct' URL and the 'GetCapabilities' URL
attempt = 0
service_info_url = "{magpie}/services/{svc}".format(
magpie=magpie_url, svc=service_name)
service_info_resp = requests.get(service_info_url,
cookies=request_cookies)
service_url = get_json(service_info_resp).get(service_name).get(
"service_url")
svc_getcap_url = "{svc_url}/wps?service=WPS&version=1.0.0&request=GetCapabilities" \
.format(svc_url=service_url)
while True:
service_msg_direct = "Service response ({svc})".format(
svc=service_name)
service_msg_getcap = "Service response ({svc}, GetCapabilities)".format(
svc=service_name)
err, http = request_curl(service_url,
cookies=curl_cookies,
msg=service_msg_direct)
if not err and http == 200:
break
err, http = request_curl(svc_getcap_url,
cookies=curl_cookies,
msg=service_msg_getcap)
if not err and http == 200:
break
print_log(
"[{url}] Bad response from service '{svc}' retrying after {sec}s..."
.format(svc=service_name,
url=service_url,
sec=GETCAPABILITIES_INTERVAL),
logger=LOGGER)
time.sleep(GETCAPABILITIES_INTERVAL)
attempt += 1
if attempt >= GETCAPABILITIES_ATTEMPTS:
msg = "[{url}] No response from service '{svc}' after {tries} attempts. Skipping..." \
.format(svc=service_name, url=service_url, tries=attempt)
print_log(msg, logger=LOGGER)
break