def test_get_constant_raise_not_set_when_requested():
settings = {"magpie.not_set_but_exists": None}
with pytest.raises(ValueError):
c.get_constant("MAGPIE_NOT_SET_BUT_EXISTS", settings, raise_not_set=True)
with pytest.raises(ValueError):
c.get_constant("magpie.not_set_but_exists", settings, raise_not_set=True)
try:
value = c.get_constant("MAGPIE_NOT_SET_BUT_EXISTS", settings, raise_not_set=False)
assert value is None
except LookupError:
pytest.fail(msg="Should not have raised although constant is not set.")
try:
value = c.get_constant("magpie.not_set_but_exists", settings, raise_not_set=False)
assert value is None
except LookupError:
pytest.fail(msg="Should not have raised although constant is not set.")
def run_migrations_online(connection=None):
"""
Run migrations in 'online' mode.
In this scenario we need to create an Engine and associate a connection with the context.
"""
url = get_db_url()
def connect(c=None):
if isinstance(c, Connection) and not c.closed:
return c
if not isinstance(c, Connectable):
c = create_engine(url, convert_unicode=True, echo=False)
return c.connect()
if not database_exists(url):
db_name = get_constant('MAGPIE_POSTGRES_DB')
LOGGER.warning(
'Database [{}] not found, attempting creation...'.format(db_name))
connection = create_database(url,
encoding='utf8',
template='template0')
# retry connection and run migration
with connect(connection) as migrate_conn:
try:
context.configure(connection=migrate_conn,
target_metadata=target_metadata,
version_table='alembic_version',
transaction_per_migration=True,
render_as_batch=True)
with context.begin_transaction():
context.run_migrations()
finally:
# close the connection only if not given argument
if migrate_conn is not connection:
migrate_conn.close()
def setup_cron_logger(log_level=logging.INFO):
# type: (Union[AnyStr, int]) -> None
if not isinstance(log_level, int):
log_level = logging.getLevelName(log_level)
log_path = constants.get_constant("MAGPIE_CRON_LOG")
log_path = os.path.expandvars(log_path)
log_path = os.path.expanduser(log_path)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(log_level)
file_handler = logging.FileHandler(log_path)
file_handler.setLevel(log_level)
formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s")
file_handler.setFormatter(formatter)
stream_handler.setFormatter(formatter)
LOGGER.addHandler(file_handler)
LOGGER.addHandler(stream_handler)
LOGGER.setLevel(log_level)
def get_magpie_url(container=None):
# type: (Optional[AnySettingsContainer]) -> Str
if container is None:
LOGGER.warning(
"Registry not specified, trying to find Magpie URL from environment"
)
url = get_constant("MAGPIE_URL",
raise_missing=False,
raise_not_set=False,
print_missing=False)
if url:
return url
hostname = (
get_constant("HOSTNAME", raise_not_set=False, raise_missing=False)
or get_constant(
"MAGPIE_HOST", raise_not_set=False, raise_missing=False))
if not hostname:
raise ConfigurationError(
"Missing or unset MAGPIE_HOST or HOSTNAME value.")
magpie_port = get_constant("MAGPIE_PORT", raise_not_set=False)
magpie_scheme = get_constant("MAGPIE_SCHEME",
raise_not_set=False,
raise_missing=False,
default_value="http")
return "{}://{}{}".format(
magpie_scheme, hostname,
":{}".format(magpie_port) if magpie_port else "")
try:
# add "http" scheme to url if omitted from config since further 'requests' calls fail without it
# mostly for testing when only "localhost" is specified
# otherwise config should explicitly define it with 'MAGPIE_URL' env or 'magpie.url' config
settings = get_settings(container)
url_parsed = urlparse(
get_constant("MAGPIE_URL", settings, "magpie.url").strip("/"))
if url_parsed.scheme in ["http", "https"]:
return url_parsed.geturl()
magpie_url = "http://{}".format(url_parsed.geturl())
print_log(
"Missing scheme from settings URL, new value: '{}'".format(
magpie_url), LOGGER, logging.WARNING)
return magpie_url
except AttributeError:
# If magpie.url does not exist, calling strip fct over None will raise this issue
raise ConfigurationError(
"MAGPIE_URL or magpie.url config cannot be found")
def register_default_users(db_session=None, settings=None):
# type: (Optional[Session], Optional[AnySettingsContainer]) -> None
"""
Registers in db every undefined default users and groups matching following variables :
- ``MAGPIE_ANONYMOUS_USER``
- ``MAGPIE_USERS_GROUP``
- ``MAGPIE_ADMIN_GROUP``
- ``MAGPIE_ADMIN_USER``
"""
if not isinstance(db_session, Session):
ini_file_path = get_constant("MAGPIE_INI_FILE_PATH",
settings_container=settings)
db_session = db.get_db_session_from_config_ini(ini_file_path)
if not db.is_database_ready(db_session):
time.sleep(2)
raise_log("Database not ready")
init_admin(db_session, settings)
init_anonymous(db_session, settings)
init_users_group(db_session, settings)
transaction.commit()
db_session.close()
def setUpClass(cls):
cls.usr = get_constant("MAGPIE_TEST_ADMIN_USERNAME")
cls.pwd = get_constant("MAGPIE_TEST_ADMIN_PASSWORD")
cls.url = get_constant("MAGPIE_TEST_REMOTE_SERVER_URL")
cls.headers, cls.cookies = utils.check_or_try_login_user(
cls.url, cls.usr, cls.pwd)
cls.require = "cannot run tests without logged in '{}' user".format(
get_constant("MAGPIE_ADMIN_GROUP"))
cls.json_headers = utils.get_headers(cls.url, {
"Accept": CONTENT_TYPE_JSON,
"Content-Type": CONTENT_TYPE_JSON
})
cls.check_requirements()
cls.version = utils.TestSetup.get_Version(cls)
cls.test_user = get_constant("MAGPIE_ANONYMOUS_USER")
cls.test_group = get_constant("MAGPIE_ANONYMOUS_GROUP")
cls.test_service_type = utils.get_service_types_for_version(
cls.version)[0]
cls.test_service_name = utils.TestSetup.get_AnyServiceOfTestServiceType(
cls)["service_name"]
def setUpClass(cls):
cls.url = get_constant("MAGPIE_TEST_REMOTE_SERVER_URL")
cls.version = utils.TestSetup.get_Version(cls)
# note: admin credentials to setup data on test instance as needed, but not to be used for these tests
cls.grp = get_constant("MAGPIE_ADMIN_GROUP")
cls.usr = get_constant("MAGPIE_TEST_ADMIN_USERNAME",
raise_missing=False,
raise_not_set=False)
cls.pwd = get_constant("MAGPIE_TEST_ADMIN_PASSWORD",
raise_missing=False,
raise_not_set=False)
cls.setup_admin()
cls.test_user_name = get_constant(
"MAGPIE_TEST_USER",
default_value="unittest-no-auth_ui-user-remote",
raise_missing=False,
raise_not_set=False)
cls.test_group_name = get_constant(
"MAGPIE_TEST_GROUP",
default_value="unittest-no-auth_ui-group-remote",
raise_missing=False,
raise_not_set=False)
cls.test_service_type = ServiceWPS.service_type
cls.test_service_name = "magpie-unittest-service-wps"
def setUpClass(cls):
cls.url = get_constant('MAGPIE_TEST_REMOTE_SERVER_URL')
def edit_group_view(request):
"""
Update a group by name.
"""
group = ar.get_group_matchdict_checked(request,
group_name_key="group_name")
special_groups = [
get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=request),
get_constant("MAGPIE_ADMIN_GROUP", settings_container=request),
]
ax.verify_param(
group.group_name,
not_in=True,
param_compare=special_groups,
param_name="group_name",
http_error=HTTPForbidden,
msg_on_fail=s.Group_PATCH_ReservedKeyword_ForbiddenResponseSchema.
description)
new_group_name = ar.get_multiformat_body(request, "group_name")
new_description = ar.get_multiformat_body(request, "description")
new_discoverability = ar.get_multiformat_body(request, "discoverable")
if new_discoverability is not None:
new_discoverability = asbool(new_discoverability)
update_name = group.group_name != new_group_name and new_group_name is not None
update_desc = group.description != new_description and new_description is not None
update_disc = group.discoverable != new_discoverability and new_discoverability is not None
ax.verify_param(
any([update_name, update_desc, update_disc]),
is_true=True,
with_param=False, # params are not useful in response for this case
http_error=HTTPBadRequest,
content={"group_name": group.group_name},
msg_on_fail=s.Group_PATCH_None_BadRequestResponseSchema.description)
if update_name:
ax.verify_param(new_group_name,
not_none=True,
not_empty=True,
http_error=HTTPBadRequest,
msg_on_fail=s.
Group_PATCH_Name_BadRequestResponseSchema.description)
group_name_size_range = range(
1, 1 + get_constant("MAGPIE_GROUP_NAME_MAX_LENGTH",
settings_container=request))
ax.verify_param(len(new_group_name),
is_in=True,
param_compare=group_name_size_range,
http_error=HTTPBadRequest,
msg_on_fail=s.
Group_PATCH_Size_BadRequestResponseSchema.description)
ax.verify_param(
GroupService.by_group_name(new_group_name, db_session=request.db),
is_none=True,
http_error=HTTPConflict,
with_param=False, # don't return group as value
msg_on_fail=s.Group_PATCH_ConflictResponseSchema.description)
group.group_name = new_group_name
if update_desc:
group.description = new_description
if update_disc:
group.discoverable = new_discoverability
return ax.valid_http(http_success=HTTPOk,
detail=s.Group_PATCH_OkResponseSchema.description)
def create_user(user_name, password, email, group_name, db_session):
# type: (Str, Optional[Str], Str, Str, Session) -> HTTPException
"""
Creates a user if it is permitted and not conflicting. Password must be set to `None` if using external identity.
Created user will be part of group matching ``group_name`` (can be ``MAGPIE_ANONYMOUS_GROUP`` for minimal access).
Furthermore, the user will also *always* be associated with ``MAGPIE_ANONYMOUS_GROUP`` (if not already explicitly
requested with ``group_name``) to allow access to resources with public permission. The ``group_name`` **must**
be an existing group.
:returns: valid HTTP response on successful operation.
"""
def _get_group(grp_name):
# type: (Str) -> models.Group
grp = ax.evaluate_call(
lambda: GroupService.by_group_name(grp_name, db_session=db_session
),
httpError=HTTPForbidden,
msgOnFail=s.UserGroup_GET_ForbiddenResponseSchema.description)
ax.verify_param(
grp,
notNone=True,
httpError=HTTPBadRequest,
msgOnFail=s.UserGroup_Check_BadRequestResponseSchema.description)
return grp
# Check that group already exists
group_checked = _get_group(group_name)
# Check if user already exists
user_checked = ax.evaluate_call(
lambda: UserService.by_user_name(user_name=user_name,
db_session=db_session),
httpError=HTTPForbidden,
msgOnFail=s.User_Check_ForbiddenResponseSchema.description)
ax.verify_param(user_checked,
isNone=True,
httpError=HTTPConflict,
msgOnFail=s.User_Check_ConflictResponseSchema.description)
# Create user with specified name and group to assign
# noinspection PyArgumentList
new_user = models.User(user_name=user_name, email=email)
if password:
UserService.set_password(new_user, password)
UserService.regenerate_security_code(new_user)
ax.evaluate_call(
lambda: db_session.add(new_user),
fallback=lambda: db_session.rollback(),
httpError=HTTPForbidden,
msgOnFail=s.Users_POST_ForbiddenResponseSchema.description)
# Fetch user to update fields
new_user = ax.evaluate_call(
lambda: UserService.by_user_name(user_name, db_session=db_session),
httpError=HTTPForbidden,
msgOnFail=s.UserNew_POST_ForbiddenResponseSchema.description)
def _add_to_group(usr, grp):
# type: (models.User, models.Group) -> None
# noinspection PyArgumentList
group_entry = models.UserGroup(group_id=grp.id, user_id=usr.id)
ax.evaluate_call(
lambda: db_session.add(group_entry),
fallback=lambda: db_session.rollback(),
httpError=HTTPForbidden,
msgOnFail=s.UserGroup_GET_ForbiddenResponseSchema.description)
# Assign user to group
new_user_groups = [group_name]
_add_to_group(new_user, group_checked)
# Also add user to anonymous group if not already done
anonym_grp_name = get_constant("MAGPIE_ANONYMOUS_GROUP")
if group_checked.group_name != anonym_grp_name:
_add_to_group(new_user, _get_group(anonym_grp_name))
new_user_groups.append(anonym_grp_name)
return ax.valid_http(
httpSuccess=HTTPCreated,
detail=s.Users_POST_CreatedResponseSchema.description,
content={u"user": uf.format_user(new_user, new_user_groups)})
def authomatic_config(request=None):
defaults_config = {
"popup": True,
}
openid_config = {
"openid": {
"class_": openid.OpenID,
"display_name": "OpenID",
},
}
esgf_config = {
"dkrz": {
"class_": esgfopenid.ESGFOpenID,
"hostname": "esgf-data.dkrz.de",
"provider_url": "https://{hostname}/esgf-idp/openid/{username}",
"display_name": "DKRZ",
},
"ipsl": {
"class_": esgfopenid.ESGFOpenID,
"hostname": "esgf-node.ipsl.upmc.fr",
"display_name": "IPSL",
},
# former "badc"
"ceda": {
"class_": esgfopenid.ESGFOpenID,
"hostname": "esgf-index1.ceda.ac.uk",
"provider_url": "https://{hostname}/openid/{username}",
"display_name": "CEDA",
},
# former "pcmdi"
"llnl": {
"class_": esgfopenid.ESGFOpenID,
"hostname": "esgf-node.llnl.gov",
"display_name": "LLNL",
},
"smhi": {
"class_": esgfopenid.ESGFOpenID,
"hostname": "esg-dn1.nsc.liu.se",
"display_name": "SMHI",
},
}
_get_const_info = dict(raise_missing=False,
raise_not_set=False,
print_missing=True)
oauth2_config = {
"github": {
"class_":
oauth2.GitHub,
"display_name":
"GitHub",
"consumer_key":
get_constant("GITHUB_CLIENT_ID", **_get_const_info),
"consumer_secret":
get_constant("GITHUB_CLIENT_SECRET", **_get_const_info),
"redirect_uri":
request.application_url if request else None,
# "redirect_uri": "{}/providers/github/signin".format(request.application_url) if request else None,
"access_headers": {
"User-Agent": "Magpie"
},
"id":
provider_id(),
"_apis": {
"Get your events":
("GET", "https://api.github.com/users/{user.username}/events"),
"Get your watched repos":
("GET", "https://api.github.com/user/subscriptions"),
},
},
"wso2": {
"class_":
wso2.WSO2,
"display_name":
"WSO2",
"hostname":
get_constant("WSO2_HOSTNAME", **_get_const_info),
"consumer_key":
get_constant("WSO2_CLIENT_ID", **_get_const_info),
"consumer_secret":
get_constant("WSO2_CLIENT_SECRET", **_get_const_info),
"certificate_file":
get_constant("WSO2_CERTIFICATE_FILE", **_get_const_info)
or None, # replace if == ""
"ssl_verify":
asbool(
get_constant("WSO2_SSL_VERIFY",
default_value=True,
**_get_const_info)),
"redirect_uri":
"{}/providers/wso2/signin".format(request.application_url)
if request else None,
"id":
provider_id(),
}
}
# Concatenate the configs.
config = {} # type: JSON
config.update(oauth2_config)
config.update(openid_config)
config.update(esgf_config)
config["__defaults__"] = defaults_config
return config
def update_user_view(request):
"""
Update user information by user name.
"""
user = ar.get_user_matchdict_checked_or_logged(request)
new_user_name = ar.get_multiformat_body(request,
"user_name",
default=user.user_name)
new_email = ar.get_multiformat_body(request, "email", default=user.email)
new_password = ar.get_multiformat_body(request,
"password",
default=user.user_password)
update_username = user.user_name != new_user_name and new_user_name is not None
update_password = user.user_password != new_password and new_password is not None
update_email = user.email != new_email and new_email is not None
ax.verify_param(
any([update_username, update_password, update_email]),
is_true=True,
with_param=False, # params are not useful in response for this case
content={"user_name": user.user_name},
http_error=HTTPBadRequest,
msg_on_fail=s.User_PATCH_BadRequestResponseSchema.description)
# user name change is admin-only operation
if update_username:
ax.verify_param(
get_constant("MAGPIE_ADMIN_GROUP"),
is_in=True,
param_compare=uu.get_user_groups_checked(request.user, request.db),
with_param=False,
http_error=HTTPForbidden,
msg_on_fail=s.User_PATCH_ForbiddenResponseSchema.description)
# logged user updating itself is forbidden if it corresponds to special users
# cannot edit reserved keywords nor apply them to another user
forbidden_user_names = [
get_constant("MAGPIE_ADMIN_USER", request),
get_constant("MAGPIE_ANONYMOUS_USER", request),
get_constant("MAGPIE_LOGGED_USER", request),
]
check_user_name_cases = [user.user_name, new_user_name
] if update_username else [user.user_name]
for check_user_name in check_user_name_cases:
ax.verify_param(
check_user_name,
not_in=True,
param_compare=forbidden_user_names,
param_name="user_name",
http_error=HTTPForbidden,
content={"user_name": str(check_user_name)},
msg_on_fail=s.User_PATCH_ForbiddenResponseSchema.description)
if update_username:
uu.check_user_info(user_name=new_user_name,
check_email=False,
check_password=False,
check_group=False)
existing_user = ax.evaluate_call(
lambda: UserService.by_user_name(new_user_name,
db_session=request.db),
fallback=lambda: request.db.rollback(),
http_error=HTTPForbidden,
msg_on_fail=s.User_PATCH_ForbiddenResponseSchema.description)
ax.verify_param(
existing_user,
is_none=True,
with_param=False,
http_error=HTTPConflict,
msg_on_fail=s.User_PATCH_ConflictResponseSchema.description)
user.user_name = new_user_name
if update_email:
uu.check_user_info(email=new_email,
check_name=False,
check_password=False,
check_group=False)
user.email = new_email
if update_password:
uu.check_user_info(password=new_password,
check_name=False,
check_email=False,
check_group=False)
UserService.set_password(user, new_password)
UserService.regenerate_security_code(user)
return ax.valid_http(http_success=HTTPOk,
detail=s.Users_PATCH_OkResponseSchema.description)
def test_get_constant_alternative_name():
settings = {"magpie.test_some_value": "some-value"}
assert c.get_constant("MAGPIE_TEST_SOME_VALUE", settings) == settings["magpie.test_some_value"]
from magpie.api import requests as ar
from magpie.api import schemas as s
from magpie.api.management.user.user_formats import format_user
from magpie.api.management.user.user_utils import create_user
from magpie.constants import get_constant
from magpie.security import authomatic_setup, get_provider_names
from magpie.utils import CONTENT_TYPE_JSON, convert_response, get_logger, get_magpie_url
if TYPE_CHECKING:
from magpie.typedefs import Session, Str
LOGGER = get_logger(__name__)
# dictionaries of {'provider_id': 'provider_display_name'}
MAGPIE_DEFAULT_PROVIDER = get_constant("MAGPIE_DEFAULT_PROVIDER")
MAGPIE_INTERNAL_PROVIDERS = {MAGPIE_DEFAULT_PROVIDER: MAGPIE_DEFAULT_PROVIDER.capitalize()}
MAGPIE_EXTERNAL_PROVIDERS = get_provider_names()
MAGPIE_PROVIDER_KEYS = list(MAGPIE_INTERNAL_PROVIDERS.keys()) + list(MAGPIE_EXTERNAL_PROVIDERS.keys())
# FIXME: use provider enum
def process_sign_in_external(request, username, provider):
provider_name = provider.lower()
if provider_name == "openid":
query_field = dict(id=username)
elif provider_name == "github":
query_field = None
# query_field = dict(login_field=username)
elif provider_name == "wso2":
query_field = {}
def main(global_config=None, **settings): # noqa: F811
"""
This function returns a Pyramid WSGI application.
"""
import magpie.constants # pylint: disable=C0415 # avoid circular import
# override magpie ini if provided with --paste to gunicorn, otherwise use environment variable
config_env = get_constant("MAGPIE_INI_FILE_PATH",
settings,
raise_missing=True)
config_ini = (global_config or {}).get("__file__", config_env)
if config_ini != config_env:
magpie.constants.MAGPIE_INI_FILE_PATH = config_ini
settings["magpie.ini_file_path"] = config_ini
print_log("Setting up loggers...", LOGGER)
log_lvl = get_constant("MAGPIE_LOG_LEVEL",
settings,
"magpie.log_level",
default_value="INFO",
raise_missing=False,
raise_not_set=False,
print_missing=True)
# apply proper value in case it was in ini AND env since up until then, only env was check
# we want to prioritize the ini definition
magpie.constants.MAGPIE_LOG_LEVEL = log_lvl
LOGGER.setLevel(log_lvl)
sa_settings = set_sqlalchemy_log_level(log_lvl)
print_log("Looking for db migration requirement...", LOGGER)
run_database_migration_when_ready(
settings) # cannot pass db session as it might not even exist yet!
# NOTE:
# migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set
# to 'echo=False' because engines are re-created as needed... (ie: missing db)
# apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`"""
set_sqlalchemy_log_level(log_lvl)
# fetch db session here, otherwise, any following db engine connection will re-initialize
# with a new engine class and logging settings don't get re-evaluated/applied
db_session = get_db_session_from_config_ini(config_ini,
settings_override=sa_settings)
print_log("Validate settings that require explicit definitions...", LOGGER)
for req_config in [
"MAGPIE_SECRET", "MAGPIE_ADMIN_USER", "MAGPIE_ADMIN_PASSWORD"
]:
get_constant(req_config,
settings_container=settings,
raise_missing=True,
raise_not_set=True)
print_log("Register default users...", LOGGER)
register_defaults(db_session=db_session, settings=settings)
print_log("Register service providers...", logger=LOGGER)
combined_config = get_constant("MAGPIE_CONFIG_PATH",
settings,
default_value=None,
raise_missing=False,
raise_not_set=False,
print_missing=True)
push_phoenix = asbool(
get_constant("PHOENIX_PUSH",
settings,
settings_name="phoenix.push",
default_value=False,
raise_missing=False,
raise_not_set=False,
print_missing=True))
prov_cfg = combined_config or get_constant("MAGPIE_PROVIDERS_CONFIG_PATH",
settings,
default_value="",
raise_missing=False,
raise_not_set=False,
print_missing=True)
magpie_register_services_from_config(prov_cfg,
push_to_phoenix=push_phoenix,
force_update=True,
disable_getcapabilities=False,
db_session=db_session)
print_log("Register configuration permissions...", LOGGER)
perm_cfg = combined_config or get_constant(
"MAGPIE_PERMISSIONS_CONFIG_PATH",
settings,
default_value="",
raise_missing=False,
raise_not_set=False,
print_missing=True)
magpie_register_permissions_from_config(perm_cfg, db_session=db_session)
print_log("Running configurations setup...", LOGGER)
patch_magpie_url(settings)
# avoid cornice conflicting with magpie exception views
settings["handle_exceptions"] = False
config = get_auth_config(settings)
set_cache_regions_from_settings(settings)
# don't use scan otherwise modules like 'magpie.adapter' are
# automatically found and cause import errors on missing packages
print_log("Including Magpie modules...", LOGGER)
config.include("magpie")
# config.scan("magpie")
print_log("Starting Magpie app...", LOGGER)
wsgi_app = config.make_wsgi_app()
return wsgi_app
def main(global_config=None, **settings):
"""
This function returns a Pyramid WSGI application.
"""
config_ini = get_constant("MAGPIE_INI_FILE_PATH", raise_missing=True)
print_log("Setting up loggers...", LOGGER)
log_lvl = get_constant("MAGPIE_LOG_LEVEL", settings, "magpie.log_level", default_value="INFO",
raise_missing=False, raise_not_set=False, print_missing=True)
LOGGER.setLevel(log_lvl)
sa_settings = db.set_sqlalchemy_log_level(log_lvl)
print_log("Looking for db migration requirement...", LOGGER)
db.run_database_migration_when_ready(settings) # cannot pass db session as it might not even exist yet!
# HACK:
# migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set
# to 'echo=False' because engines are re-created as needed... (ie: missing db)
# apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`"""
db.set_sqlalchemy_log_level(log_lvl)
# fetch db session here, otherwise, any following db engine connection will re-initialize
# with a new engine class and logging settings don't get re-evaluated/applied
db_session = db.get_db_session_from_config_ini(config_ini, settings_override=sa_settings)
print_log("Register default users...", LOGGER)
register_default_users(db_session=db_session, settings=settings)
print_log("Register configuration providers...", logger=LOGGER)
push_phoenix = asbool(get_constant("PHOENIX_PUSH", settings, settings_name="magpie.phoenix_push",
raise_missing=False, raise_not_set=False, print_missing=True))
prov_cfg = get_constant("MAGPIE_PROVIDERS_CONFIG_PATH", default_value="",
raise_missing=False, raise_not_set=False, print_missing=True)
if os.path.isfile(prov_cfg):
magpie_register_services_from_config(prov_cfg, push_to_phoenix=push_phoenix,
force_update=True, disable_getcapabilities=False, db_session=db_session)
else:
print_log("No configuration file found for providers registration, skipping...", LOGGER, logging.WARN)
print_log("Register configuration permissions...", LOGGER)
perm_cfg = get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH", default_value="",
raise_missing=False, raise_not_set=False, print_missing=True)
if os.path.isfile(perm_cfg):
magpie_register_permissions_from_config(get_constant("MAGPIE_PERMISSIONS_CONFIG_PATH"), db_session=db_session)
else:
print_log("No configuration file found for permissions registration, skipping...", LOGGER, logging.WARN)
print_log("Running configurations setup...", LOGGER)
patch_magpie_url(settings)
# avoid cornice conflicting with magpie exception views
settings["handle_exceptions"] = False
config = get_auth_config(settings)
set_cache_regions_from_settings(settings)
# Don't use scan otherwise modules like 'magpie.adapter' are
# automatically found and cause import errors on missing packages
config.include("magpie")
# config.scan("magpie")
print_log("Starting Magpie app...", LOGGER)
wsgi_app = config.make_wsgi_app()
return wsgi_app
def edit_user(self):
user_name = self.request.matchdict["user_name"]
cur_svc_type = self.request.matchdict["cur_svc_type"]
inherit_grp_perms = self.request.matchdict.get(
"inherit_groups_permissions", False)
own_groups = self.get_user_groups(user_name)
all_groups = self.get_all_groups(
first_default_group=get_constant("MAGPIE_USERS_GROUP"))
# TODO:
# Until the api is modified to make it possible to request from the RemoteResource table,
# we have to access the database directly here
session = self.request.db
try:
# The service type is "default". This function replaces cur_svc_type with the first service type.
svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
except Exception as e:
raise HTTPBadRequest(detail=repr(e))
user_path = schemas.UserAPI.path.format(user_name=user_name)
user_resp = request_api(self.request, user_path, "GET")
check_response(user_resp)
user_info = get_json(user_resp)["user"]
user_info[u"edit_mode"] = u"no_edit"
user_info[u"own_groups"] = own_groups
user_info[u"groups"] = all_groups
user_info[u"inherit_groups_permissions"] = inherit_grp_perms
error_message = ""
# In case of update, changes are not reflected when calling
# get_user_or_group_resources_permissions_dict so we must take care
# of them
res_id = None
removed_perms = None
new_perms = None
if self.request.method == "POST":
res_id = self.request.POST.get(u"resource_id")
is_edit_group_membership = False
is_save_user_info = False
requires_update_name = False
if u"inherit_groups_permissions" in self.request.POST:
inherit_grp_perms = asbool(
self.request.POST[u"inherit_groups_permissions"])
user_info[u"inherit_groups_permissions"] = inherit_grp_perms
if u"delete" in self.request.POST:
resp = request_api(self.request, user_path, "DELETE")
check_response(resp)
return HTTPFound(self.request.route_url("view_users"))
elif u"goto_service" in self.request.POST:
return self.goto_service(res_id)
elif u"clean_resource" in self.request.POST:
# "clean_resource" must be above "edit_permissions" because they"re in the same form.
self.delete_resource(res_id)
elif u"edit_permissions" in self.request.POST:
if not res_id or res_id == "None":
remote_id = int(self.request.POST.get("remote_id"))
services_names = [
s["service_name"] for s in services.values()
]
res_id = self.add_remote_resource(cur_svc_type,
services_names,
user_name,
remote_id,
is_user=True)
removed_perms, new_perms = \
self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True)
elif u"edit_group_membership" in self.request.POST:
is_edit_group_membership = True
elif u"edit_username" in self.request.POST:
user_info[u"edit_mode"] = u"edit_username"
elif u"edit_password" in self.request.POST:
user_info[u"edit_mode"] = u"edit_password"
elif u"edit_email" in self.request.POST:
user_info[u"edit_mode"] = u"edit_email"
elif u"save_username" in self.request.POST:
user_info[u"user_name"] = self.request.POST.get(
u"new_user_name")
is_save_user_info = True
requires_update_name = True
elif u"save_password" in self.request.POST:
user_info[u"password"] = self.request.POST.get(
u"new_user_password")
is_save_user_info = True
elif u"save_email" in self.request.POST:
user_info[u"email"] = self.request.POST.get(u"new_user_email")
is_save_user_info = True
elif u"force_sync" in self.request.POST:
errors = []
for service_info in services.values():
# noinspection PyBroadException
try:
sync_resources.fetch_single_service(
service_info["resource_id"], session)
except Exception:
errors.append(service_info["service_name"])
if errors:
error_message += self.make_sync_error_message(errors)
elif u"clean_all" in self.request.POST:
ids_to_clean = self.request.POST.get("ids_to_clean").split(";")
for id_ in ids_to_clean:
self.delete_resource(id_)
if is_save_user_info:
resp = request_api(self.request,
user_path,
"PUT",
data=user_info)
check_response(resp)
# need to commit updates since we are using the same session
# otherwise, updated user doesn't exist yet in the db for next calls
self.request.tm.commit()
# always remove password from output
user_info.pop(u"password", None)
if requires_update_name:
# re-fetch user groups as current user-group will have changed on new user_name
user_name = user_info[u"user_name"]
user_info[u"own_groups"] = self.get_user_groups(user_name)
# return immediately with updated URL to user with new name
users_url = self.request.route_url("edit_user",
user_name=user_name,
cur_svc_type=cur_svc_type)
return HTTPMovedPermanently(location=users_url)
# edits to groups checkboxes
if is_edit_group_membership:
selected_groups = self.request.POST.getall("member")
removed_groups = list(set(own_groups) - set(selected_groups))
new_groups = list(set(selected_groups) - set(own_groups))
for group in removed_groups:
path = schemas.UserGroupAPI.path.format(
user_name=user_name, group_name=group)
resp = request_api(self.request, path, "DELETE")
check_response(resp)
for group in new_groups:
path = schemas.UserGroupsAPI.path.format(
user_name=user_name)
data = {"group_name": group}
resp = request_api(self.request, path, "POST", data=data)
check_response(resp)
user_info[u"own_groups"] = self.get_user_groups(user_name)
# display resources permissions per service type tab
try:
res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(
user_name,
services,
cur_svc_type,
is_user=True,
is_inherit_groups_permissions=inherit_grp_perms)
except Exception as e:
raise HTTPBadRequest(detail=repr(e))
if res_id and (removed_perms or new_perms):
self.update_user_or_group_resources_permissions_dict(
res_perms, res_id, removed_perms, new_perms)
sync_types = [s["service_sync_type"] for s in services.values()]
sync_implemented = any(s in sync_resources.SYNC_SERVICES_TYPES
for s in sync_types)
info = self.get_remote_resources_info(res_perms, services, session)
res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info
if out_of_sync:
error_message = self.make_sync_error_message(out_of_sync)
user_info[u"error_message"] = error_message
user_info[u"ids_to_clean"] = ";".join(ids_to_clean)
user_info[u"last_sync"] = last_sync_humanized
user_info[u"sync_implemented"] = sync_implemented
user_info[u"out_of_sync"] = out_of_sync
user_info[u"cur_svc_type"] = cur_svc_type
user_info[u"svc_types"] = svc_types
user_info[u"resources"] = res_perms
user_info[u"permissions"] = res_perm_names
return add_template_data(self.request, data=user_info)
def add_user(self):
users_group = get_constant("MAGPIE_USERS_GROUP")
return_data = {
u"conflict_group_name": False,
u"conflict_user_name": False,
u"conflict_user_email": False,
u"invalid_user_name": False,
u"invalid_user_email": False,
u"invalid_password": False,
u"too_long_user_name": False,
u"form_user_name": u"",
u"form_user_email": u"",
u"user_groups":
self.get_all_groups(first_default_group=users_group)
}
check_data = [
u"conflict_group_name", u"conflict_user_name", u"conflict_email",
u"invalid_user_name", u"invalid_email", u"invalid_password"
]
if "create" in self.request.POST:
groups = self.get_all_groups()
user_name = self.request.POST.get("user_name")
group_name = self.request.POST.get("group_name")
user_email = self.request.POST.get("email")
password = self.request.POST.get("password")
return_data[u"form_user_name"] = user_name
return_data[u"form_user_email"] = user_email
if group_name not in groups:
data = {u"group_name": group_name}
resp = request_api(self.request,
schemas.GroupsAPI.path,
"POST",
data=data)
if resp.status_code == HTTPConflict.code:
return_data[u"conflict_group_name"] = True
if user_email in self.get_user_emails():
return_data[u"conflict_user_email"] = True
if user_email == "":
return_data[u"invalid_user_email"] = True
if len(user_name) > get_constant("MAGPIE_USER_NAME_MAX_LENGTH"):
return_data[u"too_long_user_name"] = True
if user_name in self.get_user_names():
return_data[u"conflict_user_name"] = True
if user_name == "":
return_data[u"invalid_user_name"] = True
if password == "":
return_data[u"invalid_password"] = True
for check_fail in check_data:
if return_data.get(check_fail, False):
return add_template_data(self.request, return_data)
data = {
u"user_name": user_name,
u"email": user_email,
u"password": password,
u"group_name": group_name
}
resp = request_api(self.request,
schemas.UsersAPI.path,
"POST",
data=data)
check_response(resp)
return HTTPFound(self.request.route_url("view_users"))
return add_template_data(self.request, return_data)
def run_database_migration_when_ready(settings, db_session=None):
# type: (SettingsType, Optional[Session]) -> None
"""
Runs db migration if requested by config and need from revisions.
"""
db_ready = False
if asbool(
get_constant("MAGPIE_DB_MIGRATION",
settings,
"magpie.db_migration",
default_value=True,
raise_missing=False,
raise_not_set=False,
print_missing=True)):
conf_attempts = int(
get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS",
settings,
"magpie.db_migration_attempts",
default_value=5,
raise_missing=False,
raise_not_set=False,
print_missing=True))
print_log("Running database migration (as required)...", logger=LOGGER)
attempts = max(conf_attempts, 1)
if attempts != conf_attempts:
print_log(
"Database migration attempts updated to {}".format(attempts),
logger=LOGGER,
level=logging.WARNING)
for i in range(1, attempts + 1):
try:
run_database_migration(db_session=db_session,
settings=settings)
except ImportError as exc:
print_log(
"Database migration produced [{!r}] (ignored).".format(
exc),
logger=LOGGER,
level=logging.WARNING,
exc_info=exc)
except Exception as exc:
if i <= attempts:
print_log(
"Database migration failed [{!r}]. Retrying... ({}/{})"
.format(exc, i, attempts),
logger=LOGGER,
level=logging.WARNING,
exc_info=exc)
time.sleep(2)
continue
raise_log("Database migration failed [{!r}]".format(exc),
exception=RuntimeError,
logger=LOGGER)
db_ready = is_database_ready(db_session)
if not db_ready:
if i <= attempts:
print_log("Database not ready. Retrying... ({}/{})".format(
i, attempts),
logger=LOGGER,
level=logging.WARNING)
time.sleep(2)
continue
print_log(
"Database not ready. Maximum attempts reached ({})".format(
attempts),
logger=LOGGER,
level=logging.WARNING)
break
else:
print_log(
"Database migration skipped as per 'MAGPIE_DB_MIGRATION' requirement...",
logger=LOGGER)
db_ready = is_database_ready(db_session)
if not db_ready:
raise_log("Database not ready", exception=RuntimeError, logger=LOGGER)
def includeme(config):
from magpie.ui.user.views import UserViews
LOGGER.info("Adding UI user...")
path = "/ui/users/{}".format(get_constant("MAGPIE_LOGGED_USER"))
config.add_route(UserViews.edit_current_user.__name__, path)
config.scan()
def test_get_constant_with_same_name():
test_value = "test-constant"
c.MAGPIE_CRON_LOG = test_value
value = c.get_constant("MAGPIE_CRON_LOG")
assert value == test_value
def effective_permissions(self,
user,
resource,
permissions=None,
allow_match=True):
# type: (models.User, ServiceOrResourceType, Optional[Collection[Permission]], bool) -> List[PermissionSet]
"""
Obtains the effective permissions the user has over the specified resource.
Recursively rewinds the resource tree from the specified resource up to the top-most parent service the resource
resides under (or directly if the resource is the service) and retrieve permissions along the way that should be
applied to children when using scoped-resource inheritance. Rewinding of the tree can terminate earlier when
permissions can be immediately resolved such as when more restrictive conditions enforce denied access.
Both user and group permission inheritance is resolved simultaneously to tree hierarchy with corresponding
allow and deny conditions. User :term:`Direct Permissions` have priority over all its groups
:term:`Inherited Permissions`, and denied permissions have priority over allowed access ones.
All applicable permissions on the resource (as defined by :meth:`allowed_permissions`) will have their
resolution (Allow/Deny) provided as output, unless a specific subset of permissions is requested using
:paramref:`permissions`. Other permissions are ignored in this case to only resolve requested ones.
For example, this parameter can be used to request only ACL resolution from specific permissions applicable
for a given request, as obtained by :meth:`permission_requested`.
Permissions scoped as `match` can be ignored using :paramref:`allow_match`, such as when the targeted resource
does not exist.
.. seealso::
- :meth:`ServiceInterface.resource_requested`
"""
if not permissions:
permissions = self.allowed_permissions(resource)
requested_perms = set(permissions) # type: Set[Permission]
effective_perms = dict() # type: Dict[Permission, PermissionSet]
# immediately return all permissions if user is an admin
db_session = self.request.db
admin_group = get_constant("MAGPIE_ADMIN_GROUP", self.request)
admin_group = GroupService.by_group_name(admin_group,
db_session=db_session)
if admin_group in user.groups: # noqa
return [
PermissionSet(perm,
access=Access.ALLOW,
scope=Scope.MATCH,
typ=PermissionType.EFFECTIVE,
reason=PERMISSION_REASON_ADMIN)
for perm in permissions
]
# level at which last permission was found, -1 if not found
# employed to resolve with *closest* scope and for applicable 'reason' combination on same level
effective_level = dict() # type: Dict[Permission, Optional[int]]
current_level = 1 # one-based to avoid ``if level:`` check failing with zero
full_break = False
# current and parent resource(s) recursive-scope
while resource is not None and not full_break: # bottom-up until service is reached
# include both permissions set in database as well as defined directly on resource
cur_res_perms = ResourceService.perms_for_user(
resource, user, db_session=db_session)
cur_res_perms.extend(permission_to_pyramid_acls(resource.__acl__))
for perm_name in requested_perms:
if full_break:
break
for perm_tup in cur_res_perms:
perm_set = PermissionSet(perm_tup)
# if user is owner (directly or via groups), all permissions are set,
# but continue processing this resource until end in case user explicit deny reverts it
if perm_tup.perm_name == ALL_PERMISSIONS:
# FIXME:
# This block needs to be validated if support of ownership rules are added.
# Conditions must be revised according to wanted behaviour...
# General idea for now is that explict user/group deny should be prioritized over resource
# ownership permissions since these can be attributed to *any user* while explicit deny are
# definitely set by an admin-level user.
for perm in requested_perms:
if perm_set.access == Access.DENY:
all_perm = PermissionSet(
perm, perm_set.access, perm.scope,
PermissionType.OWNED)
effective_perms[perm] = all_perm
else:
all_perm = PermissionSet(
perm, perm_set.access, perm.scope,
PermissionType.OWNED)
effective_perms.setdefault(perm, all_perm)
full_break = True
break
# skip if the current permission must not be processed (at all or for the moment until next 'name')
if perm_set.name not in requested_perms or perm_set.name != perm_name:
continue
# only first resource can use match (if even enabled with found one), parents are recursive-only
if not allow_match and perm_set.scope == Scope.MATCH:
continue
# pick the first permission if none was found up to this point
prev_perm = effective_perms.get(perm_name)
scope_level = effective_level.get(perm_name)
if not prev_perm:
effective_perms[perm_name] = perm_set
effective_level[perm_name] = current_level
continue
# user direct permissions have priority over inherited ones from groups
# if inherited permission was found during previous iteration, override it with direct permission
if perm_set.type == PermissionType.DIRECT:
# - reset resolution scope of previous permission attributed to group as it takes precedence
# - since there can't be more than one user permission-name per resource on a given level,
# scope resolution is done after applying this *closest* permission, ignore higher level ones
if prev_perm.type == PermissionType.INHERITED or not scope_level:
effective_perms[perm_name] = perm_set
effective_level[perm_name] = current_level
continue # final decision for this user, skip any group permissions
# resolve prioritized permission according to ALLOW/DENY, scope and group priority
# (see 'PermissionSet.resolve' method for extensive details)
# skip if last permission is not on group to avoid redundant USER > GROUP check processed before
if prev_perm.type == PermissionType.INHERITED:
# - If new permission to process is done against the previous permission from *same* tree-level,
# there is a possibility to combine equal priority groups. In such case, reason is 'MULTIPLE'.
# - If not of equal priority, the appropriate permission is selected and reason is overridden
# accordingly by the new higher priority permission.
# - If no permission was defined at all (first occurrence), also set it using current permission
if scope_level in [None, current_level]:
resolved_perm = PermissionSet.resolve(
perm_set,
prev_perm,
context=PermissionType.EFFECTIVE)
effective_perms[perm_name] = resolved_perm
effective_level[perm_name] = current_level
# - If new permission is at *different* tree-level, it applies only if the group has higher
# priority than the previous one, to respect the *closest* scope to the target resource.
# Same priorities are ignored as they were already resolved by *closest* scope above.
# - Reset scope level with new permission such that another permission of same group priority as
# that could be processed in next iteration can be compared against it, to resolve 'access'
# priority between them.
elif perm_set.group_priority > prev_perm.group_priority:
effective_perms[perm_name] = perm_set
effective_level[perm_name] = current_level
# don't bother moving to parent if everything is resolved already
# can only assume nothing left to resolve if all permissions are direct on user (highest priority)
# if any found permission is group inherited, higher level user permission could still override it
if (len(effective_perms) == len(requested_perms)
and all(perm.type == PermissionType.DIRECT
for perm in effective_perms.values())):
break
# otherwise, move to parent if any available, since we are not done rewinding the resource tree
allow_match = False # reset match not applicable anymore for following parent resources
current_level += 1
if resource.parent_id:
resource = ResourceService.by_resource_id(
resource.parent_id, db_session=db_session)
else:
resource = None
# set deny for all still unresolved permissions from requested ones
resolved_perms = set(effective_perms)
missing_perms = set(permissions) - resolved_perms
final_perms = set(effective_perms.values()) # type: Set[PermissionSet]
for perm_name in missing_perms:
perm = PermissionSet(perm_name,
access=Access.DENY,
scope=Scope.MATCH,
typ=PermissionType.EFFECTIVE,
reason=PERMISSION_REASON_DEFAULT)
final_perms.add(perm)
# enforce type and scope (use MATCH to make it explicit that it applies specifically for this resource)
for perm in final_perms:
perm.type = PermissionType.EFFECTIVE
perm.scope = Scope.MATCH
return list(final_perms)
def authomatic_config(request=None):
defaults_config = {
'popup': True,
}
openid_config = {
'openid': {
'class_': openid.OpenID,
'display_name': 'OpenID',
},
}
esgf_config = {
'dkrz': {
'class_': esgfopenid.ESGFOpenID,
'hostname': 'esgf-data.dkrz.de',
'provider_url': 'https://{hostname}/esgf-idp/openid/{username}',
'display_name': 'DKRZ',
},
'ipsl': {
'class_': esgfopenid.ESGFOpenID,
'hostname': 'providers-node.ipsl.fr',
'display_name': 'IPSL',
},
'badc': {
'class_': esgfopenid.ESGFOpenID,
'hostname': 'ceda.ac.uk',
'provider_url': 'https://{hostname}/openid/{username}',
'display_name': 'BADC',
},
'pcmdi': {
'class_': esgfopenid.ESGFOpenID,
'hostname': 'providers-node.llnl.gov',
'display_name': 'PCMDI',
},
'smhi': {
'class_': esgfopenid.ESGFOpenID,
'hostname': 'esg-dn1.nsc.liu.se',
'display_name': 'SMHI',
},
}
_get_const_info = dict(raise_missing=False,
raise_not_set=False,
print_missing=True)
oauth2_config = {
'github': {
'class_':
oauth2.GitHub,
'display_name':
'GitHub',
'consumer_key':
get_constant('GITHUB_CLIENT_ID', **_get_const_info),
'consumer_secret':
get_constant('GITHUB_CLIENT_SECRET', **_get_const_info),
'redirect_uri':
request.application_url if request else None,
# 'redirect_uri': '{}/providers/github/signin'.format(request.application_url) if request else None,
'access_headers': {
'User-Agent': 'Magpie'
},
'id':
provider_id(),
'_apis': {
'Get your events':
('GET', 'https://api.github.com/users/{user.username}/events'),
'Get your watched repos':
('GET', 'https://api.github.com/user/subscriptions'),
},
},
'wso2': {
'class_':
wso2.WSO2,
'display_name':
'WSO2',
'hostname':
get_constant('WSO2_HOSTNAME', **_get_const_info),
'consumer_key':
get_constant('WSO2_CLIENT_ID', **_get_const_info),
'consumer_secret':
get_constant('WSO2_CLIENT_SECRET', **_get_const_info),
'certificate_file':
get_constant('WSO2_CERTIFICATE_FILE', **_get_const_info)
or None, # replace if == ''
'ssl_verify':
asbool(
get_constant('WSO2_SSL_VERIFY',
default_value=True,
**_get_const_info)),
'redirect_uri':
'{}/providers/wso2/signin'.format(request.application_url)
if request else None,
'id':
provider_id(),
}
}
# Concatenate the configs.
config = {} # type: JSON
config.update(oauth2_config)
config.update(openid_config)
config.update(esgf_config)
config['__defaults__'] = defaults_config
return config
def test_constant_protected_no_override():
for const_name in c.MAGPIE_CONSTANTS:
with mock.patch.dict("os.environ", {const_name: "override-value"}):
const = c.get_constant(const_name)
assert const != "override-value"
def run_batch_update_user_command(test_app, expected_users,
create_command_xargs, delete_command_xargs):
"""
Tests batch user creation and deletion of the CLI utility.
Because CLI utility employs requests that cannot be mocked if executed through sub-process, we call it directly.
"""
test_admin_usr = get_constant("MAGPIE_TEST_ADMIN_USERNAME",
raise_not_set=False,
raise_missing=False)
test_admin_pwd = get_constant("MAGPIE_TEST_ADMIN_PASSWORD",
raise_not_set=False,
raise_missing=False)
if not test_admin_usr or not test_admin_pwd:
test_admin_usr = get_constant("MAGPIE_ADMIN_USER")
test_admin_pwd = get_constant("MAGPIE_ADMIN_PASSWORD")
test_url = "http://localhost"
def mock_request(*args, **kwargs):
method, url, args = args[0], args[1], args[2:]
path = url.replace(test_url, "")
# because CLI utility does multiple login tests, we must force TestApp logout to forget session
# otherwise, error is raised because of user session mismatch between previous login and new one requested
if path.startswith("/signin"):
utils.check_or_try_logout_user(test_app)
return utils.test_request(test_app, method, path, *args, **kwargs)
def run_command(operation_name, operation_args):
with tempfile2.TemporaryDirectory() as tmpdir:
with mock.patch("requests.Session.request",
side_effect=mock_request):
with mock.patch("requests.request", side_effect=mock_request):
cmd = [
test_url, test_admin_usr, test_admin_pwd, "-o", tmpdir
] + operation_args
assert batch_update_users.main(
cmd) == 0, "failed execution due to invalid arguments"
assert len(
os.listdir(tmpdir)
) == 1, "utility should have produced 1 output file"
file = os.path.join(tmpdir, os.listdir(tmpdir)[0])
utils.check_val_is_in(operation_name, file)
assert os.path.isfile(file)
with open(file, "r") as fd:
file_text = fd.read()
assert all([test_user in file_text for test_user in expected_users]), \
"all users should have been processed and logged in output result file"
# cleanup in case of previous failure
_, test_admin_cookies = utils.check_or_try_login_user(
test_app, username=test_admin_usr, password=test_admin_pwd)
for user in expected_users:
utils.TestSetup.delete_TestUser(
test_app,
override_user_name=user, # noqa
override_headers={},
override_cookies=test_admin_cookies)
utils.check_or_try_logout_user(test_app)
# test user creation and validate that users were all created
run_command("create", create_command_xargs)
utils.check_or_try_logout_user(test_app)
_, test_admin_cookies = utils.check_or_try_login_user(
test_app, username=test_admin_usr, password=test_admin_pwd)
resp = utils.test_request(test_app,
"GET",
"/users",
cookies=test_admin_cookies)
body = utils.check_response_basic_info(resp)
for user in expected_users:
utils.check_val_is_in(user, body["user_names"])
# test user deletion and validate users are all deleted
run_command("delete", ["-d"] + delete_command_xargs)
utils.check_or_try_logout_user(test_app)
_, test_admin_cookies = utils.check_or_try_login_user(
test_app, username=test_admin_usr, password=test_admin_pwd)
resp = utils.test_request(test_app,
"GET",
"/users",
cookies=test_admin_cookies)
body = utils.check_response_basic_info(resp)
for user in expected_users:
utils.check_val_not_in(user, body["user_names"])
def create_user(user_name, password, email, group_name, db_session):
# type: (Str, Optional[Str], Str, Optional[Str], Session) -> HTTPException
"""
Creates a user if it is permitted and not conflicting. Password must be set to ``None`` if using external identity.
Created user will immediately assigned membership to the group matching :paramref:`group_name`
(can be :py:data:`MAGPIE_ANONYMOUS_GROUP` for minimal access). If no group is provided, this anonymous group will
be applied by default, creating a user effectively without any permissions other than ones set directly for him.
Furthermore, the user will also *always* be associated with :py:data:`MAGPIE_ANONYMOUS_GROUP` (if not already
explicitly or implicitly requested with :paramref:`group_name`) to allow access to resources with public permission.
Argument :paramref:`group_name` **MUST** be an existing group if provided.
:returns: valid HTTP response on successful operation.
"""
def _get_group(grp_name):
# type: (Str) -> models.Group
ax.verify_param(grp_name, not_none=True, not_empty=True, matches=True,
param_compare=ax.PARAM_REGEX, param_name="group_name",
http_error=HTTPBadRequest, msg_on_fail=s.UserGroup_Check_BadRequestResponseSchema.description)
grp = ax.evaluate_call(lambda: GroupService.by_group_name(grp_name, db_session=db_session),
http_error=HTTPForbidden,
msg_on_fail=s.UserGroup_GET_ForbiddenResponseSchema.description)
ax.verify_param(grp, not_none=True, http_error=HTTPNotFound, with_param=False,
msg_on_fail=s.UserGroup_Check_NotFoundResponseSchema.description)
return grp
# Check that group already exists
if group_name is None:
group_name = get_constant("MAGPIE_ANONYMOUS_GROUP")
is_internal = password is not None
check_user_info(user_name, email, password, group_name, check_password=is_internal)
group_checked = _get_group(group_name)
# Check if user already exists
user_checked = ax.evaluate_call(lambda: UserService.by_user_name(user_name=user_name, db_session=db_session),
http_error=HTTPForbidden,
msg_on_fail=s.User_Check_ForbiddenResponseSchema.description)
ax.verify_param(user_checked, is_none=True, with_param=False, http_error=HTTPConflict,
msg_on_fail=s.User_Check_ConflictResponseSchema.description)
# Create user with specified name and group to assign
new_user = models.User(user_name=user_name, email=email) # noqa
if is_internal:
UserService.set_password(new_user, password)
UserService.regenerate_security_code(new_user)
ax.evaluate_call(lambda: db_session.add(new_user), fallback=lambda: db_session.rollback(),
http_error=HTTPForbidden, msg_on_fail=s.Users_POST_ForbiddenResponseSchema.description)
# Fetch user to update fields
new_user = ax.evaluate_call(lambda: UserService.by_user_name(user_name, db_session=db_session),
http_error=HTTPForbidden,
msg_on_fail=s.UserNew_POST_ForbiddenResponseSchema.description)
def _add_to_group(usr, grp):
# type: (models.User, models.Group) -> None
group_entry = models.UserGroup(group_id=grp.id, user_id=usr.id) # noqa
ax.evaluate_call(lambda: db_session.add(group_entry), fallback=lambda: db_session.rollback(),
http_error=HTTPForbidden, msg_on_fail=s.UserGroup_GET_ForbiddenResponseSchema.description)
# Assign user to group
new_user_groups = [group_name]
_add_to_group(new_user, group_checked)
# Also add user to anonymous group if not already done
anonym_grp_name = get_constant("MAGPIE_ANONYMOUS_GROUP")
if group_checked.group_name != anonym_grp_name:
_add_to_group(new_user, _get_group(anonym_grp_name))
new_user_groups.append(anonym_grp_name)
return ax.valid_http(http_success=HTTPCreated, detail=s.Users_POST_CreatedResponseSchema.description,
content={"user": uf.format_user(new_user, new_user_groups)})
def get_user_resources_view(request):
"""
List all resources a user has permissions on.
"""
inherit_groups_perms = asbool(
ar.get_query_param(request, ["inherit", "inherited"]))
resolve_groups_perms = asbool(
ar.get_query_param(request, ["resolve", "resolved"]))
filtered_perms = asbool(ar.get_query_param(request,
["filter", "filtered"]))
user = ar.get_user_matchdict_checked_or_logged(request)
db = request.db
# skip admin-only full listing of resources if filtered view is requested
is_admin = False
if not filtered_perms and request.user is not None:
admin_group = get_constant("MAGPIE_ADMIN_GROUP",
settings_container=request)
is_admin = admin_group in [
group.group_name for group in request.user.groups
]
def build_json_user_resource_tree(usr):
json_res = {}
perm_type = PermissionType.INHERITED if inherit_groups_perms else PermissionType.DIRECT
services = ResourceService.all(models.Service, db_session=db)
# add service-types so they are ordered and listed if no service of that type was defined
for svc_type in sorted(SERVICE_TYPE_DICT):
json_res[svc_type] = {}
for svc in services:
svc_perms = uu.get_user_service_permissions(
user=usr,
service=svc,
request=request,
inherit_groups_permissions=inherit_groups_perms,
resolve_groups_permissions=resolve_groups_perms)
res_perms_dict = uu.get_user_service_resources_permissions_dict(
user=usr,
service=svc,
request=request,
inherit_groups_permissions=inherit_groups_perms,
resolve_groups_permissions=resolve_groups_perms)
# always allow admin to view full resource tree, unless explicitly requested to be filtered
# otherwise (non-admin), only add details if there is at least one resource permission (any level)
if (is_admin and not filtered_perms) or (svc_perms
or res_perms_dict):
json_res[svc.type][
svc.resource_name] = format_service_resources(
svc,
db_session=db,
service_perms=svc_perms,
resources_perms_dict=res_perms_dict,
permission_type=perm_type,
show_all_children=False,
show_private_url=False,
)
return json_res
usr_res_dict = ax.evaluate_call(
lambda: build_json_user_resource_tree(user),
fallback=lambda: db.rollback(),
http_error=HTTPNotFound,
msg_on_fail=s.UserResources_GET_NotFoundResponseSchema.description,
content={
"user_name": user.user_name,
"resource_types": [models.Service.resource_type_name]
})
return ax.valid_http(
http_success=HTTPOk,
content={"resources": usr_res_dict},
detail=s.UserResources_GET_OkResponseSchema.description)
def run_database_migration_when_ready(settings, db_session=None):
# type: (SettingsType, Optional[Session]) -> None
"""
Runs db migration if requested by config and need from revisions.
"""
db_ready = False
if asbool(
get_constant("MAGPIE_DB_MIGRATION",
settings,
"magpie.db_migration",
default_value=True,
raise_missing=False,
raise_not_set=False,
print_missing=True)):
attempts = int(
get_constant("MAGPIE_DB_MIGRATION_ATTEMPTS",
settings,
"magpie.db_migration_attempts",
default_value=5,
raise_missing=False,
raise_not_set=False,
print_missing=True))
print_log("Running database migration (as required)...")
attempts = max(
attempts, 2
) # enforce at least 2 attempts, 1 for db creation and one for actual migration
for i in range(1, attempts + 1):
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
run_database_migration(db_session)
except ImportError as e:
print_log(
"Database migration produced [{!r}] (ignored).".format(e),
level=logging.WARNING)
pass
except Exception as e:
if i <= attempts:
print_log(
"Database migration failed [{!r}]. Retrying... ({}/{})"
.format(e, i, attempts))
time.sleep(2)
continue
else:
raise_log("Database migration failed [{!r}]".format(e),
exception=RuntimeError)
db_ready = is_database_ready(db_session)
if not db_ready:
print_log("Database not ready. Retrying... ({}/{})".format(
i, attempts))
time.sleep(2)
continue
break
else:
db_ready = is_database_ready(db_session)
if not db_ready:
time.sleep(2)
raise_log("Database not ready", exception=RuntimeError)
def check_or_try_login_user(
test_item, # type: AnyMagpieTestType
username=None, # type: Optional[Str]
password=None, # type: Optional[Str]
provider=None, # type: Optional[Str]
headers=None, # type: Optional[Dict[Str, Str]]
use_ui_form_submit=False, # type: bool
version=__meta__.__version__, # type: Str
expect_errors=False, # type: bool
): # type: (...) -> OptionalHeaderCookiesType
"""
Verifies that the required user is already logged in (or none is if username=None), or tries to login him otherwise.
Validates that the logged user (if any), matched the one specified with `username`.
:param test_item: instance of the test application or remote server URL to call
:param username: name of the user to login or None otherwise
:param password: password to use for login if the user was not already logged in
:param provider: provider string to use for login (default: `MAGPIE_DEFAULT_PROVIDER`, ie: magpie's local signin)
:param headers: headers to include in the test request
:param use_ui_form_submit: use Magpie UI login 'form' to obtain cookies
(required for local `WebTest.App` login, ignored by requests using URL)
:param version: server or local app version to evaluate responses with backward compatibility
:param expect_errors: indicate if the login is expected to fail, used only if using UI form & `webtest.TestApp`
:return: headers and cookies of the user session or (None, None)
:raise: Exception on any login failure as required by the caller's specifications (username/password)
"""
app_or_url = get_app_or_url(test_item)
headers = headers or {}
resp = get_session_user(app_or_url, headers)
body = get_json_body(resp)
resp_cookies = None
auth = body.get("authenticated", False)
if auth is False and username is None:
return None, None
if auth is False and username is not None:
provider = provider or get_constant("MAGPIE_DEFAULT_PROVIDER")
data = {
"user_name": username,
"password": password,
"provider_name": provider
}
if isinstance(app_or_url, TestApp):
if use_ui_form_submit:
base_url = app_or_url.app.registry.settings.get("magpie.url")
resp = app_or_url.get(url="{}/ui/login".format(base_url))
form = resp.forms["login_internal"]
form["user_name"] = username
form["password"] = password
form["provider_name"] = provider
resp = form.submit("submit", expect_errors=expect_errors)
resp_cookies = app_or_url.cookies # automatically set by form submit
else:
resp = app_or_url.post_json("/signin", data, headers=headers)
resp_cookies = resp.cookies
else:
resp = requests.post("{}/signin".format(app_or_url),
json=data,
headers=headers)
resp_cookies = resp.cookies
# response OK (200) if directly from API /signin
# response Found (302) if redirected UI /login
if resp.status_code < 400:
return resp.headers, resp_cookies
if auth is True:
if LooseVersion(version) >= LooseVersion("0.6.3"):
logged_user = body.get("user", {}).get("user_name", "")
else:
logged_user = body.get("user_name", "")
if username != logged_user:
raise Exception("invalid user")
if isinstance(app_or_url, TestApp):
resp_cookies = app_or_url.cookies
else:
resp_cookies = resp.cookies
return resp.headers, resp_cookies
def get_phoenix_url():
hostname = get_constant("HOSTNAME")
phoenix_port = get_constant("PHOENIX_PORT", raise_not_set=False)
return "https://{0}{1}".format(
hostname, ":{}".format(phoenix_port) if phoenix_port else "")
