def register_services(
where, # type: Optional[Str]
services_dict, # type: Dict[Str, Dict[Str, Str]]
cookies, # type: Str
message="Register response", # type: Optional[Str]
): # type: (...) -> Tuple[bool, Dict[Str, int]]
"""
Registers services on desired location using provided configurations and access cookies.
:returns: tuple of overall success and individual http response of each service registration.
"""
success = True
svc_url = None
statuses = dict()
register_service_url = None
if where == SERVICES_MAGPIE:
svc_url_tag = "service_url"
get_magpie_url() + ServicesAPI.path
elif where == SERVICES_PHOENIX:
svc_url_tag = "url"
register_service_url = get_phoenix_url() + "/services/register"
else:
raise ValueError("Unknown location for service registration", where)
for service_name in services_dict:
cfg = services_dict[service_name]
cfg["public"] = bool2str(cfg.get("public"))
cfg["c4i"] = bool2str(cfg.get("c4i"))
cfg["url"] = os.path.expandvars(cfg.get("url"))
if where == SERVICES_MAGPIE:
svc_url = cfg["url"]
elif where == SERVICES_PHOENIX:
svc_url = get_twitcher_protected_service_url(service_name)
params = "service_name={name}&" \
"{svc_url_tag}={svc_url}&" \
"service_title={cfg[title]}&" \
"public={cfg[public]}&" \
"c4i={cfg[c4i]}&" \
"service_type={cfg[type]}&" \
"register=register" \
.format(name=service_name, cfg=cfg, svc_url_tag=svc_url_tag, svc_url=svc_url)
service_msg = "{msg} ({svc}) [{url}]".format(msg=message,
svc=service_name,
url=svc_url)
error, http_code = request_curl(register_service_url,
cookies=cookies,
form_params=params,
msg=service_msg)
statuses[service_name] = http_code
success = success and not error and (
(where == SERVICES_PHOENIX and http_code == 200) or
(where == SERVICES_MAGPIE and http_code == 201))
time.sleep(CREATE_SERVICE_INTERVAL)
return success, statuses
def login_success_external(request, external_user_name, external_id, email, provider_name):
# type: (Request, Str, Str, Str, Str) -> HTTPException
"""
Generates the login response in case of successful external provider identification.
"""
# find possibly already registered user by external_id/provider
user = ExternalIdentityService.user_by_external_id_and_provider(external_id, provider_name, request.db)
if user is None:
# create new user with an External Identity
user = new_user_external(external_user_name=external_user_name, external_id=external_id,
email=email, provider_name=provider_name, db_session=request.db)
# set a header to remember user (set-cookie)
headers = remember(request, user.id)
# redirect to 'Homepage-Route' header only if corresponding to Magpie host
if "homepage_route" in request.cookies:
homepage_route = str(request.cookies["homepage_route"])
elif "Homepage-Route" in request.headers:
homepage_route = str(request.headers["Homepage-Route"])
else:
homepage_route = "/"
header_host = urlparse(homepage_route).hostname
magpie_host = get_magpie_url(request)
if header_host and header_host != magpie_host:
ax.raise_http(http_error=HTTPForbidden, detail=s.ProviderSignin_GET_ForbiddenResponseSchema.description)
if not header_host:
homepage_route = magpie_host + ("/" if not homepage_route.startswith("/") else "") + homepage_route
return ax.valid_http(http_success=HTTPFound, detail=s.ProviderSignin_GET_FoundResponseSchema.description,
content={"homepage_route": homepage_route},
http_kwargs={"location": homepage_route, "headers": headers})
def verify_user(request):
# type: (Request) -> HTTPException
"""
Verifies that a valid user authentication on the pointed ``Magpie`` instance (via configuration) also results into a
valid user authentication with the current ``Twitcher`` instance to ensure settings match between them.
:param request: an HTTP request with valid authentication token/cookie credentials.
:return: appropriate HTTP success or error response with details about the result.
"""
magpie_url = get_magpie_url(request)
resp = requests.post(magpie_url + SigninAPI.path,
json=request.json,
headers={
"Content-Type": CONTENT_TYPE_JSON,
"Accept": CONTENT_TYPE_JSON
})
if resp.status_code != HTTPOk.code:
content = {"response": resp.json()}
return raise_http(HTTPForbidden,
detail="Failed Magpie login.",
content=content,
nothrow=True) # noqa
authn_policy = request.registry.queryUtility(IAuthenticationPolicy) # noqa
result = authn_policy.cookie.identify(request)
if result is None:
return raise_http(
HTTPForbidden,
detail="Twitcher login incompatible with Magpie login.",
nothrow=True) # noqa
return valid_http(
HTTPOk,
detail="Twitcher login verified successfully with Magpie login.")
def magpie_update_services_conflict(conflict_services, services_dict,
request_cookies):
# type: (List[Str], ConfigDict, Dict[Str, Str]) -> Dict[Str, int]
"""
Resolve conflicting services by name during registration by updating them only if pointing to different URL.
"""
magpie_url = get_magpie_url()
statuses = dict()
for svc_name in conflict_services:
statuses[svc_name] = 409
svc_url_new = services_dict[svc_name]["url"]
svc_url_db = "{magpie}/services/{svc}".format(magpie=magpie_url,
svc=svc_name)
svc_resp = requests.get(svc_url_db, cookies=request_cookies)
svc_info = get_json(svc_resp).get(svc_name)
svc_url_old = svc_info["service_url"]
if svc_url_old != svc_url_new:
svc_info["service_url"] = svc_url_new
res_svc_put = requests.put(svc_url_db,
data=svc_info,
cookies=request_cookies)
statuses[svc_name] = res_svc_put.status_code
print_log(
"[{url_old}] => [{url_new}] Service URL update ({svc}): {resp}"
.format(svc=svc_name,
url_old=svc_url_old,
url_new=svc_url_new,
resp=res_svc_put.status_code),
logger=LOGGER)
return statuses
def __init__(self, request):
super(MagpieOWSSecurity, self).__init__()
self.magpie_url = get_magpie_url(request)
self.settings = get_settings(request)
self.twitcher_ssl_verify = asbool(
self.settings.get("twitcher.ows_proxy_ssl_verify", True))
self.twitcher_protected_path = self.settings.get(
"twitcher.ows_proxy_protected_path", "/ows")
def api_schema(request):
# type: (Request) -> JSON
"""
Return JSON Swagger specifications of Magpie REST API.
"""
swagger_base_spec = {
"host": get_magpie_url(request.registry),
"schemes": [request.scheme]
}
return s.generate_api_schema(swagger_base_spec)
def __init__(self, request):
# type: (Request) -> None
super(MagpieServiceStore, self).__init__(request)
self.settings = get_settings(request)
self.session_factory = request.registry["dbsession_factory"]
self.magpie_url = get_magpie_url(request)
self.twitcher_ssl_verify = asbool(
self.settings.get("twitcher.ows_proxy_ssl_verify", True))
self.magpie_admin_token = get_admin_cookies(self.settings,
self.twitcher_ssl_verify)
def get_homepage(request):
"""
Magpie API homepage (only if Magpie UI is not enabled).
"""
body = deepcopy(s.InfoAPI)
body.update({
u"title": s.TitleAPI,
u"name": __meta__.__package__,
u"documentation": get_magpie_url() + s.SwaggerAPI.path
})
return ax.valid_http(httpSuccess=HTTPOk,
content=body,
contentType=CONTENT_TYPE_JSON,
detail=s.Version_GET_OkResponseSchema.description)
def __init__(self, request):
self.request = request
self.magpie_url = get_magpie_url(self.request)
self.ui_theme = get_constant("MAGPIE_UI_THEME", self.request)
self.logged_user = get_logged_user(self.request)
anonymous = get_constant("MAGPIE_ANONYMOUS_GROUP",
settings_container=self.request)
admin = get_constant("MAGPIE_ADMIN_GROUP",
settings_container=self.request)
self.__class__.MAGPIE_FIXED_GROUP_MEMBERSHIPS = [
anonymous
] # special groups membership that cannot be edited
self.__class__.MAGPIE_FIXED_GROUP_EDITS = [
anonymous, admin
] # special groups that cannot be edited
# special users that cannot be deleted
self.__class__.MAGPIE_FIXED_USERS = [
get_constant("MAGPIE_ADMIN_USER"),
get_constant("MAGPIE_ANONYMOUS_USER")
]
def verify_user(request):
magpie_url = get_magpie_url(request)
resp = requests.post(magpie_url + SigninAPI.path,
json=request.json,
headers={
"Content-Type": CONTENT_TYPE_JSON,
"Accept": CONTENT_TYPE_JSON
})
if resp.status_code != HTTPOk.code:
content = {"response": resp.json()}
return raise_http(HTTPForbidden,
detail="Failed Magpie login.",
content=content,
nothrow=True)
authn_policy = request.registry.queryUtility(IAuthenticationPolicy)
result = authn_policy.cookie.identify(request)
if result is None:
return raise_http(
HTTPForbidden,
detail="Twitcher login incompatible with Magpie login.",
nothrow=True)
return valid_http(
HTTPOk,
detail="Twitcher login verified successfully with Magpie login.")
def get_hostname(test_item):
# type: (AnyMagpieTestType) -> Str
app_or_url = get_app_or_url(test_item)
if isinstance(app_or_url, TestApp):
app_or_url = get_magpie_url(app_or_url.app.registry)
return urlparse(app_or_url).hostname
def magpie_register_permissions_from_config(permissions_config,
magpie_url=None,
db_session=None):
# type: (Union[Str, ConfigDict], Optional[Str], Optional[Session]) -> None
"""
Applies permissions specified in configuration.
:param permissions_config: file path to 'permissions' config or JSON/YAML equivalent pre-loaded.
:param magpie_url: URL to magpie instance (when using requests; default: `magpie.url` from this app's config).
:param db_session: db session to use instead of requests to directly create/remove permissions with config.
.. seealso::
`magpie/permissions.cfg` for specific parameters and operational details.
"""
permissions = _load_config(permissions_config, "permissions")
if not permissions:
LOGGER.warning("Permissions configuration are empty.")
return
if use_request(db_session):
magpie_url = magpie_url or get_magpie_url()
settings = {'magpie.url': magpie_url}
logging.debug(
"Editing permissions using requests to [{}]...".format(magpie_url))
err_msg = "Invalid credentials to register Magpie permissions."
cookies_or_session = get_admin_cookies(settings, raise_message=err_msg)
else:
logging.debug("Editing permissions using db session...")
cookies_or_session = db_session
logging.info("Found {} permissions to update.".format(len(permissions)))
for i, perm_cfg in enumerate(permissions):
# parameter validation
if not isinstance(perm_cfg, dict) or not all(
f in perm_cfg for f in ["permission", "service"]):
warn_permission(
"Invalid permission format for [{!s}]".format(perm_cfg), i)
continue
if perm_cfg["permission"] not in Permission.values():
warn_permission(
"Unknown permission [{!s}]".format(perm_cfg['permission']), i)
continue
usr_name = perm_cfg.get("user")
grp_name = perm_cfg.get("group")
if not any([usr_name, grp_name]):
warn_permission("Missing required user and/or group field.", i)
continue
if "action" not in perm_cfg:
warn_permission("Unspecified action",
i,
trail="using default (create)...")
perm_cfg["action"] = "create"
if perm_cfg["action"] not in ["create", "remove"]:
warn_permission("Unknown action [{!s}]".format(perm_cfg["action"]),
i)
continue
# retrieve service for permissions validation
svc_name = perm_cfg["service"]
if use_request(cookies_or_session):
svc_path = magpie_url + ServiceAPI.path.format(
service_name=svc_name)
svc_resp = requests.get(svc_path, cookies=cookies_or_session)
if svc_resp.status_code != 200:
warn_permission("Unknown service [{!s}]".format(svc_name), i)
continue
service_info = get_json(svc_resp)[svc_name]
else:
transaction.commit(
) # force any pending transaction to be applied to find possible dependencies
svc = models.Service.by_service_name(svc_name,
db_session=cookies_or_session)
if not svc:
warn_permission(
"Unknown service [{!s}]. Can't edit permissions without service."
.format(svc_name), i)
continue
from magpie.api.management.service.service_formats import format_service
service_info = format_service(svc)
# apply permission config
resource_id, found = parse_resource_path(perm_cfg, i, service_info,
cookies_or_session,
magpie_url)
if found:
if not resource_id:
resource_id = service_info["resource_id"]
apply_permission_entry(perm_cfg, i, resource_id,
cookies_or_session, magpie_url)
if not use_request(cookies_or_session):
transaction.commit()
logging.info("Done processing permissions.")
def parse_resource_path(
permission_config_entry, # type: ConfigItem
entry_index, # type: int
service_info, # type: ConfigItem
cookies_or_session=None, # type: CookiesOrSessionType
magpie_url=None, # type: Optional[Str]
): # type: (...) -> Tuple[Optional[int], bool]
"""
Parses the `resource` field of a permission config entry and retrieves the final resource id. Creates missing
resources as necessary if they can be automatically resolved.
If `cookies` are provided, uses requests to a running `Magpie` instance (with `magpie_url`) to apply permission.
If `session` to db is provided, uses direct db connection instead to apply permission.
:returns: tuple of found id (if any, `None` otherwise), and success status of the parsing operation (error)
"""
if not magpie_url and use_request(cookies_or_session):
raise ValueError(
"cannot use cookies without corresponding request URL")
resource = None
resource_path = permission_config_entry.get("resource", "")
if resource_path.startswith("/"):
resource_path = resource_path[1:]
if resource_path.endswith("/"):
resource_path = resource_path[:-1]
if resource_path:
try:
svc_name = service_info["service_name"]
svc_type = service_info["service_type"]
if use_request(cookies_or_session):
res_path = get_magpie_url() + ServiceResourcesAPI.path.format(
service_name=svc_name)
res_resp = requests.get(res_path, cookies=cookies_or_session)
res_dict = get_json(res_resp)[svc_name]["resources"]
else:
from magpie.api.management.service.service_formats import format_service_resources
svc = models.Service.by_service_name(
svc_name, db_session=cookies_or_session)
res_dict = format_service_resources(
svc, show_all_children=True, db_session=cookies_or_session)
parent = res_dict["resource_id"]
child_resources = res_dict["resources"]
for res in resource_path.split("/"):
# search in existing children resources
if len(child_resources):
# noinspection PyTypeChecker
res_id = list(
filter(
lambda r: res in
[r, child_resources[r]["resource_name"]],
child_resources))
if res_id:
res_info = child_resources[
res_id[0]] # type: Dict[Str, JSON]
child_resources = res_info[
"children"] # update next sub-resource iteration
parent = res_info["resource_id"]
continue
# missing resource, attempt creation
svc_res_types = SERVICE_TYPE_DICT[svc_type].resource_type_names
type_count = len(svc_res_types)
if type_count != 1:
warn_permission(
"Cannot automatically generate resources",
entry_index,
detail=
"Service [{}] of type [{}] allows {} sub-resource types"
.format(svc_name, svc_type, type_count))
raise Exception(
"Missing resource to apply permission.") # fail fast
res_type = svc_res_types[0]
if use_request(cookies_or_session):
body = {
"resource_name": res,
"resource_type": res_type,
"parent_id": parent
}
# noinspection PyUnboundLocalVariable
resp = requests.post(res_path,
json=body,
cookies=cookies_or_session)
else:
from magpie.api.management.resource.resource_utils import create_resource
resp = create_resource(res,
res,
res_type,
parent,
db_session=cookies_or_session)
if resp.status_code != 201:
resp.raise_for_status()
child_resources = {}
parent = get_json(resp)["resource"]["resource_id"]
resource = parent
if not resource:
raise Exception(
"Could not extract child resource from resource path.")
except Exception as ex:
if isinstance(ex, HTTPException):
detail = "{} ({}), {}".format(
type(ex).__name__, ex.status_code, str(ex))
else:
detail = repr(ex)
warn_permission("Failed resources parsing.",
entry_index,
detail=detail)
return None, False
return resource, True
def magpie_register_services_with_requests(services_dict,
push_to_phoenix,
username,
password,
provider,
force_update=False,
disable_getcapabilities=False):
# type: (ConfigDict, bool, Str, Str, Str, bool, bool) -> bool
"""
Registers magpie services using the provided services configuration.
:param services_dict: services configuration definition.
:param push_to_phoenix: push registered Magpie services to Phoenix for synced configurations.
:param username: login username to use to obtain permissions for services registration.
:param password: login password to use to obtain permissions for services registration.
:param provider: login provider to use to obtain permissions for services registration.
:param force_update: override existing services matched by name
:param disable_getcapabilities: do not execute 'GetCapabilities' validation for applicable services.
:return: successful operation status
"""
magpie_url = get_magpie_url()
curl_cookies = os.path.join(LOGIN_TMP_DIR, "login_cookie_magpie")
session = requests.Session()
success = False
try:
# Need to login first as admin
login_url = magpie_url + SigninAPI.path
login_data = {
"user_name": username,
"password": password,
"provider_name": provider
}
login_loop(login_url, curl_cookies, login_data,
"Magpie login response")
login_resp = session.post(login_url, data=login_data)
if login_resp.status_code != 200:
raise_log("Failed login with specified credentials", logger=LOGGER)
request_cookies = login_resp.cookies
# Register services
# Magpie will not overwrite existing services by default, 409 Conflict instead of 201 Created
success, statuses_register = register_services(
SERVICES_MAGPIE, services_dict, curl_cookies,
"Magpie register service")
# Service URL update if conflicting and requested
if force_update and not success:
conflict_services = [
svc_name for svc_name, http_code in statuses_register.items()
if http_code == 409
]
statuses_update = magpie_update_services_conflict(
conflict_services, services_dict, request_cookies)
statuses_register.update(
statuses_update) # update previous statuses with new ones
# Add 'GetCapabilities' permissions on newly created services to allow 'ping' from Phoenix
# Phoenix doesn't register the service if it cannot be checked with this request
magpie_add_register_services_perms(services_dict, statuses_register,
curl_cookies, request_cookies,
disable_getcapabilities)
session.get(magpie_url + SignoutAPI.path)
# Push updated services to Phoenix
if push_to_phoenix:
success = phoenix_update_services(services_dict)
except Exception as e:
print_log(
"Exception during magpie register services: [{!r}]".format(e),
logger=LOGGER,
level=logging.ERROR)
finally:
session.cookies.clear()
if os.path.isfile(curl_cookies):
os.remove(curl_cookies)
return success
def magpie_add_register_services_perms(services, statuses, curl_cookies,
request_cookies,
disable_getcapabilities):
magpie_url = get_magpie_url()
login_usr = get_constant("MAGPIE_ANONYMOUS_USER")
for service_name in services:
svc_available_perms_url = "{magpie}/services/{svc}/permissions" \
.format(magpie=magpie_url, svc=service_name)
resp_available_perms = requests.get(svc_available_perms_url,
cookies=request_cookies)
if resp_available_perms.status_code == 401:
raise_log("Invalid credentials, cannot update service permissions",
exception=ValueError,
logger=LOGGER)
available_perms = get_json(resp_available_perms).get(
"permission_names", [])
# only applicable to services supporting "GetCapabilities" request
if resp_available_perms.status_code and Permission.GET_CAPABILITIES.value in available_perms:
# enforce 'getcapabilities' permission if available for service just updated (200) or created (201)
# update 'getcapabilities' permission when the service existed and it allowed
if (not disable_getcapabilities and statuses[service_name] == 409) \
or statuses[service_name] == 200 or statuses[service_name] == 201: # noqa
svc_anonym_add_perms_url = "{magpie}/users/{usr}/services/{svc}/permissions" \
.format(magpie=magpie_url, usr=login_usr, svc=service_name)
svc_anonym_perm_data = {
"permission_name": Permission.GET_CAPABILITIES.value
}
requests.post(svc_anonym_add_perms_url,
data=svc_anonym_perm_data,
cookies=request_cookies)
# check service response so Phoenix doesn't refuse registration
# try with both the 'direct' URL and the 'GetCapabilities' URL
attempt = 0
service_info_url = "{magpie}/services/{svc}".format(
magpie=magpie_url, svc=service_name)
service_info_resp = requests.get(service_info_url,
cookies=request_cookies)
service_url = get_json(service_info_resp).get(service_name).get(
"service_url")
svc_getcap_url = "{svc_url}/wps?service=WPS&version=1.0.0&request=GetCapabilities" \
.format(svc_url=service_url)
while True:
service_msg_direct = "Service response ({svc})".format(
svc=service_name)
service_msg_getcap = "Service response ({svc}, GetCapabilities)".format(
svc=service_name)
err, http = request_curl(service_url,
cookies=curl_cookies,
msg=service_msg_direct)
if not err and http == 200:
break
err, http = request_curl(svc_getcap_url,
cookies=curl_cookies,
msg=service_msg_getcap)
if not err and http == 200:
break
print_log(
"[{url}] Bad response from service '{svc}' retrying after {sec}s..."
.format(svc=service_name,
url=service_url,
sec=GETCAPABILITIES_INTERVAL),
logger=LOGGER)
time.sleep(GETCAPABILITIES_INTERVAL)
attempt += 1
if attempt >= GETCAPABILITIES_ATTEMPTS:
msg = "[{url}] No response from service '{svc}' after {tries} attempts. Skipping..." \
.format(svc=service_name, url=service_url, tries=attempt)
print_log(msg, logger=LOGGER)
break
def url(self, settings=None):
from magpie.api import schemas as s
return get_magpie_url(settings) + s.TemporaryUrlAPI.path.format(
token=self.token)