def setUp(self):
from unittest import SkipTest
raise SkipTest("Process dispatcher currently not supported")
self._start_container()
self.container.start_rel_from_url('res/deploy/basic.yml')
self.pd_cli = ProcessDispatcherServiceClient()
self.process_definition = IonObject(OT.ProcessDefinition,
name='test_process')
self.process_definition.executable = {
'module': 'ion.services.test.test_process_state_gate',
'class': 'TestProcess'
}
self.process_definition_id = self.pd_cli.create_process_definition(
self.process_definition)
self.event_queue = queue.Queue()
self.process_schedule = IonObject(OT.ProcessSchedule)
self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
self.pid = self.pd_cli.create_process(self.process_definition_id)
self.event_queue = queue.Queue()
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback,
origin=self.pid,
origin_type="DispatchedProcess")
def __init__(self,
read_process_fn=None,
process_id='',
desired_state=None,
*args,
**kwargs):
if not process_id:
raise BadRequest(
"ProcessStateGate trying to wait on invalid process (id = '%s')"
% process_id)
EventSubscriber.__init__(self,
*args,
callback=self._trigger_cb,
event_type=OT.ProcessLifecycleEvent,
origin=process_id,
**kwargs)
self.desired_state = desired_state
self.process_id = process_id
self.read_process_fn = read_process_fn
self.last_chance = None
self.first_chance = None
_ = ProcessStateEnum._str_map[
self.desired_state] # make sure state exists
log.info(
"ProcessStateGate is going to wait on process '%s' for state '%s'",
self.process_id, ProcessStateEnum._str_map[self.desired_state])
def start(self):
# Configure subscriptions for user_cache events
self.event_subscriber = EventSubscriber(
event_type=OT.UserRoleModifiedEvent,
origin_type="Org",
callback=self._event_callback)
self.event_subscriber.add_event_subscription(
event_type=OT.UserRoleCacheResetEvent)
self.process.add_endpoint(self.event_subscriber)
def start(self):
# Configure subscriptions for user_cache events
self.event_subscriber = EventSubscriber(
event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback
)
self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent)
self.process.add_endpoint(self.event_subscriber)
def setUp(self):
from unittest import SkipTest
raise SkipTest("Process dispatcher currently not supported")
self._start_container()
self.container.start_rel_from_url('res/deploy/basic.yml')
self.pd_cli = ProcessDispatcherServiceClient()
self.process_definition = IonObject(OT.ProcessDefinition, name='test_process')
self.process_definition.executable = {'module': 'ion.services.test.test_process_state_gate',
'class': 'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self.event_queue = queue.Queue()
self.process_schedule = IonObject(OT.ProcessSchedule)
self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
self.pid = self.pd_cli.create_process(self.process_definition_id)
self.event_queue = queue.Queue()
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback,
origin=self.pid,
origin_type="DispatchedProcess")
def on_start(self):
self.terminate_loop = Event()
self.has_lock = False
self.lock_expires = None
CoordinatedProcess.evt_count[self.id] = 0
self.bg_loop = gevent.spawn(self._bg_loop)
self.evt_sub = EventSubscriber(event_type=OT.ResourceCommandEvent,
callback=self._on_event)
self.add_endpoint(self.evt_sub)
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg,
"aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (
topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic,
from_name=queue_name,
auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
callback=self._receive_event)
self.evt_sub.add_event_subscription(
event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
class ProcessDispatcherAggregator(object):
""" PD aggregator for heartbeat input from containers etc. """
def __init__(self, pd_core):
self._pd_core = pd_core
self.container = self._pd_core.container
self.registry = self._pd_core.registry
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def stop(self):
# Stop subscribers
self.sub_cont.close()
self.evt_sub.close()
# Wait for subscribers to finish
self.sub_cont_gl.join(timeout=2)
self.sub_cont_gl.kill()
self.sub_cont_gl = None
self.evt_sub_gl.join(timeout=2)
self.evt_sub_gl.kill()
self.evt_sub_gl = None
# -------------------------------------------------------------------------
def _receive_container_info(self, msg, headers, *args):
log.debug("Got container info %s %s %s", msg, headers, args)
if not isinstance(msg, ContainerHeartbeat):
log.warn("Unknown container info format")
return
self.registry.register_container(msg.container_id, msg.ts, EE_STATE_RUNNING, msg.attributes)
def _receive_event(self, event, *args, **kwargs):
log.debug("Got event %s %s %s", event, args, kwargs)
if isinstance(event, ContainerLifecycleEvent):
if event.sub_type == "START":
self.registry.register_container(event.origin, event.ts_created, EE_STATE_RUNNING, {})
elif event.sub_type == "TERMINATE":
self.registry.register_container(event.origin, event.ts_created, EE_STATE_TERMINATED, {})
elif isinstance(event, ProcessLifecycleEvent):
proc_info = dict(state=event.state, proc_type=event.process_type, proc_name=event.process_name,
resource_id=event.process_resource_id, service_name=event.service_name)
self.registry.register_process(event.container_id, event.origin, proc_info)
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
class ServiceGateway(object):
"""
The Service Gateway exports service routes for a web server via a Flask blueprint.
The gateway bridges HTTP requests to ION AMQP RPC calls.
"""
def __init__(self, process, config, response_class):
global sg_instance
sg_instance = self
self.name = "service_gateway"
self.process = process
self.config = config
self.response_class = response_class
self.gateway_base_url = process.gateway_base_url
self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True
self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True
# Optional list of trusted originators can be specified in config.
self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators")
if not self.trusted_originators:
self.trusted_originators = None
log.info("Service Gateway will not check requests against trusted originators since none are configured.")
# Service screening
self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or []
self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or []
self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])
self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True
self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True
# Swagger spec generation support
self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {}
self._swagger_gen = None
if self.swagger_cfg.get("enable", None) is True:
self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)
# Get the user_cache_size
self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)
# Initialize an LRU Cache to keep user roles cached for performance reasons
# maxSize = maximum number of elements to keep in cache
# maxAgeMs = oldest entry to keep
self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)
self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True)
self.rr_client = ResourceRegistryServiceProcessClient(process=self.process)
self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
self.org_client = OrgManagementServiceProcessClient(process=self.process)
# -------------------------------------------------------------------------
# Lifecycle management
def start(self):
# Configure subscriptions for user_cache events
self.event_subscriber = EventSubscriber(
event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback
)
self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent)
self.process.add_endpoint(self.event_subscriber)
def stop(self):
pass
# Stop event subscribers - TODO: This hangs
# self.process.remove_endpoint(self.event_subscriber)
# -------------------------------------------------------------------------
# Event subscriber callbacks
def _event_callback(self, event, *args, **kwargs):
""" Callback function for receiving Events """
if isinstance(event, UserRoleModifiedEvent):
# Event when User Roles are modified
user_role_event = event
org_id = user_role_event.origin
actor_id = user_role_event.actor_id
role_name = user_role_event.role_name
log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name))
# Evict the user and their roles from the cache so that it gets updated with the next call.
if self.user_role_cache and self.user_role_cache.has_key(actor_id):
log.debug("Evicting user from the user_role_cache: %s" % actor_id)
self.user_role_cache.evict(actor_id)
elif isinstance(event, UserRoleCacheResetEvent):
# An event is received to clear the user data cache
self.user_role_cache.clear()
# -------------------------------------------------------------------------
# Routes
def sg_index(self):
return self.gateway_json_response(SG_IDENTIFICATION)
def get_service_spec(self, service_name=None, spec_name=None):
try:
if not self._swagger_gen:
raise NotFound("Spec not available")
if spec_name != "swagger.json":
raise NotFound("Unknown spec format")
swagger_json = self._swagger_gen.get_spec(service_name)
resp = flask.make_response(flask.jsonify(swagger_json))
self._add_cors_headers(resp)
return resp
except Exception as ex:
return self.gateway_error_response(ex)
def process_gateway_request(self, service_name=None, operation=None, id_param=None):
"""
Makes a secure call to a SciON service operation via messaging.
"""
# TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
self._log_request_start("SVC RPC")
try:
result = self._make_service_request(service_name, operation, id_param)
return self.gateway_json_response(result)
except Exception as ex:
return self.gateway_error_response(ex)
finally:
self._log_request_end()
def rest_gateway_request(self, service_name, res_type, id_param=None):
"""
Makes a REST style call to a SciON service operation via messaging.
Get with ID returns the resource, POST without ID creates, PUT with ID updates
and GET without ID returns the collection.
"""
self._log_request_start("SVC REST")
try:
if not service_name:
raise BadRequest("Service name missing")
service_name = str(service_name)
if not res_type:
raise BadRequest("Resource type missing")
res_type = str(res_type)
if request.method == "GET" and id_param:
operation = "read_" + res_type
return self.process_gateway_request(service_name, operation, id_param)
elif request.method == "GET":
ion_res_type = "".join(x.title() for x in res_type.split("_"))
res = self._make_service_request("resource_registry", "find_resources", ion_res_type)
if len(res) == 2:
return self.gateway_json_response(res[0])
raise BadRequest("Unexpected find_resources result")
elif request.method == "PUT":
operation = "update_" + res_type
obj = self._extract_payload_data()
if not obj:
raise BadRequest("Argument object not found")
if id_param:
obj._id = id_param
return self.process_gateway_request(service_name, operation, obj)
elif request.method == "POST":
operation = "create_" + res_type
obj = self._extract_payload_data()
if not obj:
raise BadRequest("Argument object not found")
return self.process_gateway_request(service_name, operation, obj)
else:
raise BadRequest("Bad REST request")
except Exception as ex:
return self.gateway_error_response(ex)
finally:
self._log_request_end()
def _extract_payload_data(self):
request_obj = None
if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
if request.data:
request_obj = json_loads(request.data)
elif request.form:
# Form encoded
if GATEWAY_ARG_JSON in request.form:
payload = request.form[GATEWAY_ARG_JSON]
request_obj = json_loads(str(payload))
if request_obj and is_ion_object_dict(request_obj):
request_obj = self.create_ion_object(request_obj)
return request_obj
def _make_service_request(self, service_name=None, operation=None, id_param=None):
"""
Executes a secure call to a SciON service operation via messaging.
"""
if not service_name:
if self.develop_mode:
# Return a list of available services
result = dict(available_services=get_service_registry().services.keys())
return result
else:
raise BadRequest("Service name missing")
service_name = str(service_name)
if not operation:
if self.develop_mode:
# Return a list of available operations
result = dict(available_operations=[])
return result
else:
raise BadRequest("Service operation missing")
operation = str(operation)
# Apply service white list and black list for initial protection and get service client
service_def = self.get_secure_service_def(service_name)
target_client = service_def.client
# Get service request arguments and operation parameter values request
req_args = self._get_request_args()
param_list = self.create_parameter_list(service_def, operation, req_args, id_param)
# Validate requesting user and expiry and add governance headers
ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
in_login_whitelist = self.in_login_whitelist("request", service_name, operation)
ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist)
param_list["headers"] = self.build_message_headers(ion_actor_id, expiry)
# Make service operation call
client = target_client(process=self.process)
method_call = getattr(client, operation)
result = method_call(**param_list)
return result
def get_resource_schema(self, resource_type):
try:
# Validate requesting user and expiry and add governance headers
ion_actor_id, expiry = self.get_governance_info_from_request()
ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)
return self.gateway_json_response(get_object_schema(resource_type))
except Exception as ex:
return self.gateway_error_response(ex)
def get_attachment(self, attachment_id):
try:
# Create client to interface
attachment = self.rr_client.read_attachment(attachment_id, include_content=True)
return self.response_class(attachment.content, mimetype=attachment.content_type)
except Exception as ex:
return self.gateway_error_response(ex)
def create_attachment(self):
try:
payload = request.form[GATEWAY_ARG_JSON]
json_params = json_loads(str(payload))
actor_id, expiry = self.get_governance_info_from_request(json_params)
actor_id, expiry = self.validate_request(actor_id, expiry)
headers = self.build_message_headers(actor_id, expiry)
data_params = json_params[GATEWAY_ARG_PARAMS]
resource_id = str(data_params.get("resource_id", ""))
fil = request.files["file"]
content = fil.read()
keywords = []
keywords_str = data_params.get("keywords", "")
if keywords_str.strip():
keywords = [str(x.strip()) for x in keywords_str.split(",")]
created_by = data_params.get("attachment_created_by", "unknown user")
modified_by = data_params.get("attachment_modified_by", "unknown user")
# build attachment
attachment = Attachment(
name=str(data_params["attachment_name"]),
description=str(data_params["attachment_description"]),
attachment_type=int(data_params["attachment_type"]),
content_type=str(data_params["attachment_content_type"]),
keywords=keywords,
created_by=created_by,
modified_by=modified_by,
content=content,
)
ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers)
return self.gateway_json_response(ret)
except Exception as ex:
log.exception("Error creating attachment")
return self.gateway_error_response(ex)
def delete_attachment(self, attachment_id):
try:
ret = self.rr_client.delete_attachment(attachment_id)
return self.gateway_json_response(ret)
except Exception as ex:
log.exception("Error deleting attachment")
return self.gateway_error_response(ex)
def get_version_info(self):
import pkg_resources
pkg_list = ["scioncc"]
version = {}
for package in pkg_list:
try:
version["%s-release" % package] = pkg_resources.require(package)[0].version
# @TODO git versions for each?
except pkg_resources.DistributionNotFound:
pass
try:
dir_client = DirectoryServiceProcessClient(process=self.process)
sys_attrs = dir_client.lookup("/System")
if sys_attrs and isinstance(sys_attrs, dict):
version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()})
except Exception as ex:
log.exception("Could not determine system directory attributes")
return self.gateway_json_response(version)
# =========================================================================
# Security and governance helpers
def is_trusted_address(self, requesting_address):
if self.trusted_originators is None:
return True
return requesting_address in self.trusted_originators
def get_governance_info_from_request(self, json_params=None):
# Default values for governance headers.
actor_id = DEFAULT_ACTOR_ID
expiry = DEFAULT_EXPIRY
authtoken = ""
user_session = get_auth()
if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
# Get info from current server session
# NOTE: Actor id may be inside server session
actor_id = user_session["actor_id"]
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry)
# Developer access using api_key
if self.develop_mode and "api_key" in request.args and request.args["api_key"]:
actor_id = str(request.args["api_key"])
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
if 0 < int(expiry) < current_time_millis():
expiry = str(current_time_millis() + 10000)
# flask.session["valid_until"] = int(expiry / 1000)
log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry)
# Check in headers for OAuth2 bearer token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.process.oauth.verify_request([self.process.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
if actor_id:
log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry)
return actor_id, DEFAULT_EXPIRY
# Try to find auth token override
if not authtoken:
if json_params:
if "authtoken" in json_params:
authtoken = json_params["authtoken"]
else:
if "authtoken" in request.args:
authtoken = str(request.args["authtoken"])
# Enable temporary authentication tokens to resolve to actor ids
if authtoken:
try:
token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers())
actor_id = token_info.get("actor_id", actor_id)
expiry = token_info.get("expiry", expiry)
log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
except NotFound:
log.info("Provided authentication token not found: %s", authtoken)
except Unauthorized:
log.info("Authentication token expired or invalid: %s", authtoken)
except Exception as ex:
log.exception("Problem resolving authentication token")
return actor_id, expiry
def in_login_whitelist(self, category, svc, op):
"""Returns True if service op is whitelisted for anonymous access"""
entry = "%s/%s/%s" % (category, svc, op)
return entry in self.no_login_whitelist
def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
# Since this is an anonymous request, there really is no expiry associated with it
if not in_whitelist and self.require_login:
raise Unauthorized("Anonymous access not permitted")
else:
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
try:
user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers())
except NotFound as e:
if not in_whitelist and self.require_login:
# This could be a restart of the system with a new preload.
# TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
user_session = get_auth()
if user_session.get("actor_id", None) == ion_actor_id:
clear_auth()
raise Unauthorized("Invalid identity", exc_id="01.10")
else:
# If the user isn't found default to anonymous
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
# Need to convert to int first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as ex:
raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
if not in_whitelist and self.require_login:
raise Unauthorized("User authentication expired")
else:
log.warn("User authentication expired")
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
return ion_actor_id, expiry
# -------------------------------------------------------------------------
# Service call (messaging) helpers
def _add_cors_headers(self, resp):
# Set CORS headers so that a Swagger client on a different domain can read spec
resp.headers[
"Access-Control-Allow-Headers"
] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
resp.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
resp.headers["Access-Control-Allow-Origin"] = "*"
resp.headers[
"Access-Control-Request-Headers"
] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
def _log_request_start(self, req_type="SG"):
global req_seqnum
req_seqnum += 1
req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type)
flask.g.req_info = req_info
webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url)
def _log_request_response(self, content_type, result="", content_length=-1, status_code=200):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["resp_content_type"] = content_type
req_info["resp_content_length"] = content_length
req_info["resp_result"] = result
req_info["resp_status"] = req_info.get("resp_status", status_code)
def _log_request_error(self, result, status_code):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["resp_error"] = True
req_info["resp_status"] = status_code
webapi_log.warn(
"%s REQUEST (%s) ERROR (%s%s) - %s: %s",
req_info["req_type"],
req_info["request_id"],
status_code,
"/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
result[GATEWAY_ERROR_EXCEPTION],
result[GATEWAY_ERROR_MESSAGE],
)
else:
webapi_log.warn(
"REQUEST ERROR (%s%s) - %s: %s",
status_code,
"/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
result[GATEWAY_ERROR_EXCEPTION],
result[GATEWAY_ERROR_MESSAGE],
)
def _log_request_end(self):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["end_time"] = time.time()
webapi_log.info(
"%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
req_info["req_type"],
req_info["request_id"],
req_info.get("resp_status", ""),
req_info["end_time"] - req_info["start_time"],
req_info.get("resp_content_length", ""),
req_info.get("resp_content_type", ""),
)
else:
webapi_log.warn("REQUEST END - missing start info")
def _get_request_args(self):
"""Extracts service request arguments from HTTP request. Supports various
methods and forms of encoding. Separates arguments for special parameters
from service operation parameters.
Returns a dict with the service request arguments, containing key params
with the actual values for the service operation parameters.
"""
str_args = False
request_args = {}
if request.method == "POST" or request.method == "PUT":
# Use only body args and ignore any args from query string
if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
# JSON body request
if request.data:
request_args = json_loads(request.data)
if GATEWAY_ARG_PARAMS not in request_args:
# Magic fallback: Directly use JSON first level as args if params key not present
request_args = {GATEWAY_ARG_PARAMS: request_args}
elif request.form:
# Form encoded payload
if GATEWAY_ARG_JSON in request.form:
payload = request.form[GATEWAY_ARG_JSON]
request_args = json_loads(str(payload))
if GATEWAY_ARG_PARAMS not in request_args:
# Magic fallback: Directly use JSON first level as args if params key not present
request_args = {GATEWAY_ARG_PARAMS: request_args}
else:
# Fallback: Directly use form values
str_args = True
request_args = {GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)}
else:
# No args found in body
request_args = {GATEWAY_ARG_PARAMS: {}}
elif request.method == "GET":
str_args = True
REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
args_dict = request.args.to_dict(flat=True)
request_args = {k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL}
req_params = {k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL}
request_args[GATEWAY_ARG_PARAMS] = req_params
request_args["str_args"] = str_args # Indicate downstream that args are str (GET or form encoded)
# log.info("Request args: %s" % request_args)
return request_args
def _get_typed_arg_value(self, given_value, param_def, strict):
"""Returns a service operation argument value, based on a given value and param schema definition.
"""
param_type = param_def["type"]
if isinstance(given_value, unicode):
# Convert all unicode to str in UTF-8
given_value = given_value.encode("utf8") # Make all unicode into str
if isinstance(given_value, IonObjectBase) and (
given_value._get_type() == param_type or param_type in given_value._get_extends()
):
return given_value
elif is_ion_object_dict(given_value) and (param_type == "NoneType" or hasattr(objects, param_type)):
return self.create_ion_object(given_value)
elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"):
arg_val = get_typed_value(given_value, targettype=param_type, strict=strict)
return arg_val
else:
raise BadRequest("Cannot convert param value to type %s" % param_type)
def create_parameter_list(self, service_def, operation, request_args, id_param=None):
"""Build service call parameter list dynamically from service operation definition
"""
service_schema = service_def.schema
service_op_schema = service_schema["operations"][operation]
svc_op_param_list = service_op_schema["in_list"]
svc_params = {}
if id_param:
# Magic shorthand: if one argument is given, fill the first service argument
if svc_op_param_list:
fill_par = svc_op_param_list[0]
fill_par_def = service_op_schema["in"][fill_par]
arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False)
svc_params[fill_par] = arg_val
return svc_params
request_args = request_args or {}
# Cannot be strict for a URL string query arguments or directly form encoded
strict_types = False if request_args.get("str_args", False) else self.strict_types
req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
for param_name in svc_op_param_list:
param_def = service_op_schema["in"][param_name]
if param_name in req_op_args:
arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types)
svc_params[param_name] = arg_val
if "timeout" in request_args:
svc_params["timeout"] = float(request_args["timeout"])
optional_args = [param for param in req_op_args if param not in svc_params and param != "timeout"]
if optional_args and "optional_args" in svc_op_param_list:
# Only support basic strings for these optional params for now
svc_params["optional_args"] = {arg: str(req_op_args[arg]) for arg in optional_args}
# log.info("Service params: %s" % svc_params)
return svc_params
def _get_gateway_headers(self):
"""Returns the headers that the service gateway uses to make service calls on behalf of itself
(not a user passing through), e.g. for identity management purposes"""
return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}
def get_secure_service_def(self, service_name):
"""Checks whether the service indicated by given service_name exists and/or
is exposed after white and black listing. Returns service registry entry.
"""
if self.service_whitelist:
if service_name not in self.service_whitelist:
raise Unauthorized("Service access not permitted")
if self.service_blacklist:
if service_name in self.service_blacklist:
raise Unauthorized("Service access not permitted")
# Retrieve service definition
target_service = get_service_registry().get_service_by_name(service_name)
if not target_service:
raise BadRequest("The requested service (%s) is not available" % service_name)
if not target_service.client:
raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name)
if not target_service.schema:
raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name)
return target_service
def build_message_headers(self, actor_id, expiry):
"""Returns the headers that the service gateway uses to make service calls on behalf of a
user, based on the user session or request arguments"""
headers = dict()
headers[MSG_HEADER_ACTOR] = actor_id
headers[MSG_HEADER_VALID] = expiry
req_info = flask.g.get("req_info", None)
if req_info:
headers["request-id"] = str(req_info["request_id"])
# If this is an anonymous requester then there are no roles associated with the request
if actor_id == DEFAULT_ACTOR_ID:
headers[MSG_HEADER_ROLES] = dict()
return headers
try:
# Check to see if the user's roles are cached already - keyed by user id
if self.user_role_cache.has_key(actor_id):
role_header = self.user_role_cache.get(actor_id)
if role_header is not None:
headers[MSG_HEADER_ROLES] = role_header
return headers
# The user's roles were not cached so hit the datastore to find it.
role_list = self.org_client.list_actor_roles(actor_id, headers=self._get_gateway_headers())
org_roles = {}
for role in role_list:
org_roles.setdefault(role.org_governance_name, []).append(role)
role_header = get_role_message_headers(org_roles)
# Cache the roles by user id
self.user_role_cache.put(actor_id, role_header)
except Exception:
role_header = dict() # Default to empty dict if there is a problem finding roles for the user
headers[MSG_HEADER_ROLES] = role_header
return headers
def create_ion_object(self, object_params):
"""Create and initialize an ION object from a dictionary of parameters coming via HTTP,
ready to be passed on to services/messaging. The object is validated after creation.
Note: This is not called for service operation argument signatures
"""
new_obj = IonObject(object_params["type_"])
# Iterate over the parameters to add to object; have to do this instead
# of passing a dict to get around restrictions in object creation on setting _id, _rev params
for param in object_params:
self.set_object_field(new_obj, param, object_params.get(param))
new_obj._validate() # verify that all of the object fields were set with proper types
return new_obj
def set_object_field(self, obj, field, field_val):
"""Recursively set sub object field values.
TODO: This may be an expensive operation. May also be redundant with object code
"""
if isinstance(field_val, dict) and field != "kwargs":
sub_obj = getattr(obj, field)
if isinstance(sub_obj, IonObjectBase):
if "type_" in field_val and field_val["type_"] != sub_obj.type_:
if issubtype(field_val["type_"], sub_obj.type_):
sub_obj = IonObject(field_val["type_"])
setattr(obj, field, sub_obj)
else:
raise Inconsistent(
"Unable to walk the field %s - types don't match: %s %s"
% (field, sub_obj.type_, field_val["type_"])
)
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))
elif isinstance(sub_obj, dict):
setattr(obj, field, field_val)
else:
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))
else:
# type_ already exists in the class.
if field != "type_":
setattr(obj, field, field_val)
# -------------------------------------------------------------------------
# Response content helpers
def json_response(self, response_data):
"""Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
"""
resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2)
resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])):
self._add_cors_headers(resp)
self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
return resp
def gateway_json_response(self, response_data):
"""Returns the normal service gateway response as JSON or as media in case the response
is a media response
"""
if isinstance(response_data, MediaResponse):
log.info("Media response. Content mimetype:%s", response_data.media_mimetype)
content = response_data.body
if response_data.internal_encoding == "base64":
import base64
content = base64.decodestring(content)
elif response_data.internal_encoding == "utf8":
pass
resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype)
self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code)
return resp
if RETURN_MIMETYPE_PARAM in request.args:
return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
return self.response_class(response_data, mimetype=return_mimetype)
result = {GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200}
return self.json_response(result)
def gateway_error_response(self, exc):
"""Forms a service gateway error response.
Can extract multiple stacks from a multi-tier RPC service call exception
"""
if hasattr(exc, "get_stacks"):
# Process potentially multiple stacks.
full_error, exc_stacks = "", exc.get_stacks()
for i in range(len(exc_stacks)):
full_error += exc_stacks[i][0] + "\n"
if i == 0:
full_error += "".join(traceback.format_exception(*sys.exc_info()))
else:
entry = ApplicationException.format_stack(exc_stacks[i][1])
full_error += entry + "\n"
exec_name = exc.__class__.__name__
else:
exc_type, exc_obj, exc_tb = sys.exc_info()
exec_name = exc_type.__name__
full_error = "".join(traceback.format_exception(*sys.exc_info()))
status_code = getattr(exc, "status_code", 400)
if self.log_errors:
if self.develop_mode:
if status_code == 401:
log.warn("%s: %s", exec_name, exc)
else:
log.error(full_error)
else:
if status_code == 401:
log.info("%s: %s", exec_name, exc)
else:
log.info(full_error)
result = {
GATEWAY_ERROR_EXCEPTION: exec_name,
GATEWAY_ERROR_MESSAGE: str(exc.message),
GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "",
}
if self.develop_mode:
result[GATEWAY_ERROR_TRACE] = full_error
if RETURN_MIMETYPE_PARAM in request.args:
return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
return self.response_class(result, mimetype=return_mimetype)
self._log_request_error(result, status_code)
resp = self.json_response({GATEWAY_ERROR: result, GATEWAY_STATUS: status_code})
# Q: Should HTTP status be the error code of the exception?
resp.status_code = status_code
return resp
class ProcessStateGateIntTest(IonIntegrationTestCase):
def setUp(self):
from unittest import SkipTest
raise SkipTest("Process dispatcher currently not supported")
self._start_container()
self.container.start_rel_from_url('res/deploy/basic.yml')
self.pd_cli = ProcessDispatcherServiceClient()
self.process_definition = IonObject(OT.ProcessDefinition, name='test_process')
self.process_definition.executable = {'module': 'ion.services.test.test_process_state_gate',
'class': 'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self.event_queue = queue.Queue()
self.process_schedule = IonObject(OT.ProcessSchedule)
self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
self.pid = self.pd_cli.create_process(self.process_definition_id)
self.event_queue = queue.Queue()
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback,
origin=self.pid,
origin_type="DispatchedProcess")
def tearDown(self):
#stop subscriber if its running
if self.event_sub and self.event_sub._cbthread:
self.event_sub.stop()
self._stop_container()
def _event_callback(self, event, *args, **kwargs):
self.event_queue.put(event)
def latest_event(self, timeout=10):
# get latest event from our local event subscriber
try:
event = self.event_queue.get(timeout=timeout)
except Empty:
event = None
return event
def await_state(self, state, timeout=10):
print "Emptying event queue"
while True:
event = self.latest_event(0)
if event:
print "State %s from event %s" % (event.state, event)
else:
break
self.event_sub.start()
#wait for process state
print "Setting up %s gate" % ProcessStateEnum._str_map[state]
gate = ProcessStateGate(self.pd_cli.read_process,
self.pid,
state)
print "Waiting"
ret = gate.await(timeout)
print "Await got %s" % ret
event = self.latest_event(timeout=1)
# check false positives/negatives
if ret and gate._get_first_chance() is None and event is None:
self.fail("ProcessStateGate got an event that EventSubscriber didnt....")
self.event_sub.stop()
if (not ret) or gate._get_last_chance():
if event and event.state == state:
self.fail("EventSubscriber got state event %s for process %s, ProcessStateGate missed it" %
(ProcessStateEnum._str_map[event.state], self.pid))
return ret
def process_start(self):
print "Scheduling process...",
self.pd_cli.schedule_process(self.process_definition_id,
self.process_schedule,
configuration={},
process_id=self.pid)
print "Done scheduling process."
def process_stop(self):
print "STOPPING process...",
self.pd_cli.cancel_process(self.pid)
print "Done stopping process"
def test_process_state_gate(self):
self.assertFalse(self.await_state(ProcessStateEnum.RUNNING, 1),
"The process was reported as spawned, but we didn't yet")
print "GOING TO ACTUALLY START PROCESS NOW"
spawn_later(1, self.process_start)
self.assertTrue(self.await_state(ProcessStateEnum.RUNNING),
"The process did not spawn")
self.assertFalse(self.await_state(ProcessStateEnum.TERMINATED, 1),
"The process claims to have terminated, but we didn't kill it")
print "communicating with the process to make sure it is really running"
test_client = TestClient()
for i in range(5):
self.assertEqual(i + 1, test_client.count(timeout=10))
spawn_later(1, self.process_stop)
self.assertTrue(self.await_state(ProcessStateEnum.TERMINATED),
"The process failed to be reported as terminated when it was terminated")
self.assertFalse(self.await_state(ProcessStateEnum.RUNNING, 1),
"The process was reported as spawned, but we killed it")
class ProcessStateGateIntTest(IonIntegrationTestCase):
def setUp(self):
from unittest import SkipTest
raise SkipTest("Process dispatcher currently not supported")
self._start_container()
self.container.start_rel_from_url('res/deploy/basic.yml')
self.pd_cli = ProcessDispatcherServiceClient()
self.process_definition = IonObject(OT.ProcessDefinition,
name='test_process')
self.process_definition.executable = {
'module': 'ion.services.test.test_process_state_gate',
'class': 'TestProcess'
}
self.process_definition_id = self.pd_cli.create_process_definition(
self.process_definition)
self.event_queue = queue.Queue()
self.process_schedule = IonObject(OT.ProcessSchedule)
self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
self.pid = self.pd_cli.create_process(self.process_definition_id)
self.event_queue = queue.Queue()
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback,
origin=self.pid,
origin_type="DispatchedProcess")
def tearDown(self):
#stop subscriber if its running
if self.event_sub and self.event_sub._cbthread:
self.event_sub.stop()
self._stop_container()
def _event_callback(self, event, *args, **kwargs):
self.event_queue.put(event)
def latest_event(self, timeout=10):
# get latest event from our local event subscriber
try:
event = self.event_queue.get(timeout=timeout)
except Empty:
event = None
return event
def await_state(self, state, timeout=10):
print "Emptying event queue"
while True:
event = self.latest_event(0)
if event:
print "State %s from event %s" % (event.state, event)
else:
break
self.event_sub.start()
#wait for process state
print "Setting up %s gate" % ProcessStateEnum._str_map[state]
gate = ProcessStateGate(self.pd_cli.read_process, self.pid, state)
print "Waiting"
ret = gate. await (timeout)
print "Await got %s" % ret
event = self.latest_event(timeout=1)
# check false positives/negatives
if ret and gate._get_first_chance() is None and event is None:
self.fail(
"ProcessStateGate got an event that EventSubscriber didnt....")
self.event_sub.stop()
if (not ret) or gate._get_last_chance():
if event and event.state == state:
self.fail(
"EventSubscriber got state event %s for process %s, ProcessStateGate missed it"
% (ProcessStateEnum._str_map[event.state], self.pid))
return ret
def process_start(self):
print "Scheduling process...",
self.pd_cli.schedule_process(self.process_definition_id,
self.process_schedule,
configuration={},
process_id=self.pid)
print "Done scheduling process."
def process_stop(self):
print "STOPPING process...",
self.pd_cli.cancel_process(self.pid)
print "Done stopping process"
def test_process_state_gate(self):
self.assertFalse(
self.await_state(ProcessStateEnum.RUNNING, 1),
"The process was reported as spawned, but we didn't yet")
print "GOING TO ACTUALLY START PROCESS NOW"
spawn_later(1, self.process_start)
self.assertTrue(self.await_state(ProcessStateEnum.RUNNING),
"The process did not spawn")
self.assertFalse(
self.await_state(ProcessStateEnum.TERMINATED, 1),
"The process claims to have terminated, but we didn't kill it")
print "communicating with the process to make sure it is really running"
test_client = TestClient()
for i in range(5):
self.assertEqual(i + 1, test_client.count(timeout=10))
spawn_later(1, self.process_stop)
self.assertTrue(
self.await_state(ProcessStateEnum.TERMINATED),
"The process failed to be reported as terminated when it was terminated"
)
self.assertFalse(
self.await_state(ProcessStateEnum.RUNNING, 1),
"The process was reported as spawned, but we killed it")
class ServiceGateway(object):
"""
The Service Gateway exports service routes for a web server via a Flask blueprint.
The gateway bridges HTTP requests to ION AMQP RPC calls.
"""
def __init__(self, process, config, response_class):
global sg_instance
sg_instance = self
self.name = "service_gateway"
self.process = process
self.config = config
self.response_class = response_class
self.gateway_base_url = process.gateway_base_url
self.develop_mode = self.config.get_safe(CFG_PREFIX +
".develop_mode") is True
self.require_login = self.config.get_safe(CFG_PREFIX +
".require_login") is True
self.token_from_session = self.config.get_safe(
CFG_PREFIX + ".token_from_session") is True
# Optional list of trusted originators can be specified in config.
self.trusted_originators = self.config.get_safe(CFG_PREFIX +
".trusted_originators")
if not self.trusted_originators:
self.trusted_originators = None
log.info(
"Service Gateway will not check requests against trusted originators since none are configured."
)
# Service screening
self.service_blacklist = self.config.get_safe(
CFG_PREFIX + ".service_blacklist") or []
self.service_whitelist = self.config.get_safe(
CFG_PREFIX + ".service_whitelist") or []
self.no_login_whitelist = set(
self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])
self.set_cors_headers = self.config.get_safe(CFG_PREFIX +
".set_cors") is True
self.strict_types = self.config.get_safe(CFG_PREFIX +
".strict_types") is True
# Swagger spec generation support
self.swagger_cfg = self.config.get_safe(CFG_PREFIX +
".swagger_spec") or {}
self._swagger_gen = None
if self.swagger_cfg.get("enable", None) is True:
self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)
# Get the user_cache_size
self.user_cache_size = self.config.get_safe(
CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)
# Initialize an LRU Cache to keep user roles cached for performance reasons
#maxSize = maximum number of elements to keep in cache
#maxAgeMs = oldest entry to keep
self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)
self.request_callback = None
self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors",
True)
self.rr_client = ResourceRegistryServiceProcessClient(
process=self.process)
self.idm_client = IdentityManagementServiceProcessClient(
process=self.process)
self.org_client = OrgManagementServiceProcessClient(
process=self.process)
# -------------------------------------------------------------------------
# Lifecycle management
def start(self):
# Configure subscriptions for user_cache events
self.event_subscriber = EventSubscriber(
event_type=OT.UserRoleModifiedEvent,
origin_type="Org",
callback=self._event_callback)
self.event_subscriber.add_event_subscription(
event_type=OT.UserRoleCacheResetEvent)
self.process.add_endpoint(self.event_subscriber)
def stop(self):
pass
# Stop event subscribers - TODO: This hangs
#self.process.remove_endpoint(self.event_subscriber)
# -------------------------------------------------------------------------
# Event subscriber callbacks
def _event_callback(self, event, *args, **kwargs):
""" Callback function for receiving Events """
if isinstance(event, UserRoleModifiedEvent):
# Event when User Roles are modified
user_role_event = event
org_id = user_role_event.origin
actor_id = user_role_event.actor_id
role_name = user_role_event.role_name
log.debug("User Role modified: %s %s %s" %
(org_id, actor_id, role_name))
# Evict the user and their roles from the cache so that it gets updated with the next call.
if self.user_role_cache and self.user_role_cache.has_key(actor_id):
log.debug("Evicting user from the user_role_cache: %s" %
actor_id)
self.user_role_cache.evict(actor_id)
elif isinstance(event, UserRoleCacheResetEvent):
# An event is received to clear the user data cache
self.user_role_cache.clear()
# -------------------------------------------------------------------------
# Routes
def sg_index(self):
return self.gateway_json_response(SG_IDENTIFICATION)
def get_service_spec(self, service_name=None, spec_name=None):
try:
if not self._swagger_gen:
raise NotFound("Spec not available")
if spec_name != "swagger.json":
raise NotFound("Unknown spec format")
swagger_json = self._swagger_gen.get_spec(service_name)
resp = flask.make_response(flask.jsonify(swagger_json))
self._add_cors_headers(resp)
return resp
except Exception as ex:
return self.gateway_error_response(ex)
def process_gateway_request(self,
service_name=None,
operation=None,
id_param=None):
"""
Makes a secure call to a SciON service operation via messaging.
"""
# TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
self._log_request_start("SVC RPC")
try:
result = self._make_service_request(service_name, operation,
id_param)
return self.gateway_json_response(result)
except Exception as ex:
return self.gateway_error_response(ex)
finally:
self._log_request_end()
def rest_gateway_request(self, service_name, res_type, id_param=None):
"""
Makes a REST style call to a SciON service operation via messaging.
Get with ID returns the resource, POST without ID creates, PUT with ID updates
and GET without ID returns the collection.
"""
self._log_request_start("SVC REST")
try:
if not service_name:
raise BadRequest("Service name missing")
service_name = str(service_name)
if not res_type:
raise BadRequest("Resource type missing")
res_type = str(res_type)
if request.method == "GET" and id_param:
operation = "read_" + res_type
return self.process_gateway_request(service_name, operation,
id_param)
elif request.method == "GET":
ion_res_type = "".join(x.title() for x in res_type.split('_'))
res = self._make_service_request("resource_registry",
"find_resources",
ion_res_type)
if len(res) == 2:
return self.gateway_json_response(res[0])
raise BadRequest("Unexpected find_resources result")
elif request.method == "PUT":
operation = "update_" + res_type
obj = self._extract_payload_data()
if not obj:
raise BadRequest("Argument object not found")
if id_param:
obj._id = id_param
return self.process_gateway_request(service_name, operation,
obj)
elif request.method == "POST":
operation = "create_" + res_type
obj = self._extract_payload_data()
if not obj:
raise BadRequest("Argument object not found")
return self.process_gateway_request(service_name, operation,
obj)
else:
raise BadRequest("Bad REST request")
except Exception as ex:
return self.gateway_error_response(ex)
finally:
self._log_request_end()
def _extract_payload_data(self):
request_obj = None
if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
if request.data:
request_obj = json_loads(request.data)
elif request.form:
# Form encoded
if GATEWAY_ARG_JSON in request.form:
payload = request.form[GATEWAY_ARG_JSON]
request_obj = json_loads(str(payload))
if request_obj and is_ion_object_dict(request_obj):
request_obj = self.create_ion_object(request_obj)
return request_obj
def _make_service_request(self,
service_name=None,
operation=None,
id_param=None):
"""
Executes a secure call to a SciON service operation via messaging.
"""
if not service_name:
if self.develop_mode:
# Return a list of available services
result = dict(
available_services=get_service_registry().services.keys())
return result
else:
raise BadRequest("Service name missing")
service_name = str(service_name)
if not operation:
if self.develop_mode:
# Return a list of available operations
result = dict(available_operations=[])
return result
else:
raise BadRequest("Service operation missing")
operation = str(operation)
# Apply service white list and black list for initial protection and get service client
service_def = self.get_secure_service_def(service_name)
target_client = service_def.client
# Get service request arguments and operation parameter values request
req_args = self._get_request_args()
param_list = self.create_parameter_list(service_def, operation,
req_args, id_param)
# Validate requesting user and expiry and add governance headers
ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
in_login_whitelist = self.in_login_whitelist("request", service_name,
operation)
ion_actor_id, expiry = self.validate_request(
ion_actor_id, expiry, in_whitelist=in_login_whitelist)
param_list["headers"] = self.build_message_headers(
ion_actor_id, expiry)
# Make service operation call
client = target_client(process=self.process)
method_call = getattr(client, operation)
result = method_call(**param_list)
return result
def get_resource_schema(self, resource_type):
try:
# Validate requesting user and expiry and add governance headers
ion_actor_id, expiry = self.get_governance_info_from_request()
ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)
return self.gateway_json_response(get_object_schema(resource_type))
except Exception as ex:
return self.gateway_error_response(ex)
def get_attachment(self, attachment_id):
try:
# Create client to interface
attachment = self.rr_client.read_attachment(attachment_id,
include_content=True)
return self.response_class(attachment.content,
mimetype=attachment.content_type)
except Exception as ex:
return self.gateway_error_response(ex)
def create_attachment(self):
try:
payload = request.form[GATEWAY_ARG_JSON]
json_params = json_loads(str(payload))
actor_id, expiry = self.get_governance_info_from_request(
json_params)
actor_id, expiry = self.validate_request(actor_id, expiry)
headers = self.build_message_headers(actor_id, expiry)
data_params = json_params[GATEWAY_ARG_PARAMS]
resource_id = str(data_params.get("resource_id", ""))
fil = request.files["file"]
content = fil.read()
keywords = []
keywords_str = data_params.get("keywords", "")
if keywords_str.strip():
keywords = [str(x.strip()) for x in keywords_str.split(",")]
created_by = data_params.get("attachment_created_by",
"unknown user")
modified_by = data_params.get("attachment_modified_by",
"unknown user")
# build attachment
attachment = Attachment(
name=str(data_params["attachment_name"]),
description=str(data_params["attachment_description"]),
attachment_type=int(data_params["attachment_type"]),
content_type=str(data_params["attachment_content_type"]),
keywords=keywords,
created_by=created_by,
modified_by=modified_by,
content=content)
ret = self.rr_client.create_attachment(resource_id=resource_id,
attachment=attachment,
headers=headers)
return self.gateway_json_response(ret)
except Exception as ex:
log.exception("Error creating attachment")
return self.gateway_error_response(ex)
def delete_attachment(self, attachment_id):
try:
ret = self.rr_client.delete_attachment(attachment_id)
return self.gateway_json_response(ret)
except Exception as ex:
log.exception("Error deleting attachment")
return self.gateway_error_response(ex)
def get_version_info(self, pack=None):
import pkg_resources
pkg_list = ["scioncc"]
packs = self.config.get_safe(CFG_PREFIX + ".version_packages")
if packs:
pkg_list.extend(packs.split(","))
version = {}
for package in pkg_list:
try:
if pack == "all":
pack_deps = pkg_resources.require(package)
version.update(
{p.project_name: p.version
for p in pack_deps})
else:
version[package] = pkg_resources.require(
package)[0].version
# @TODO git versions for current?
except pkg_resources.DistributionNotFound:
pass
try:
dir_client = DirectoryServiceProcessClient(process=self.process)
sys_attrs = dir_client.lookup("/System")
if sys_attrs and isinstance(sys_attrs, dict):
version.update({
k: v
for (k, v) in sys_attrs.iteritems()
if "version" in k.lower()
})
except Exception as ex:
log.exception("Could not determine system directory attributes")
if pack and pack != "all":
version = {k: v for (k, v) in version.iteritems() if k == pack}
return self.gateway_json_response(version)
# =========================================================================
# Security and governance helpers
def is_trusted_address(self, requesting_address):
if self.trusted_originators is None:
return True
return requesting_address in self.trusted_originators
def get_governance_info_from_request(self, json_params=None):
# Default values for governance headers.
actor_id = DEFAULT_ACTOR_ID
expiry = DEFAULT_EXPIRY
authtoken = ""
user_session = get_auth()
#if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
if user_session.get("actor_id", None):
# Get info from current server session
# NOTE: Actor id may be inside server session
expiry = int(user_session.get("valid_until", 0)) * 1000
if expiry:
# This was a proper non-token server session authentication
expiry = str(expiry)
actor_id = user_session["actor_id"]
log.info(
"Request associated with session actor_id=%s, expiry=%s",
actor_id, expiry)
else:
# We are just taking the user_id out of the session
# TODO: Need to check access token here
expiry = str(expiry)
if self.token_from_session:
actor_id = user_session["actor_id"]
log.info(
"Request associated with actor's token from session; actor_id=%s, expiry=%s",
actor_id, expiry)
# Developer access using api_key
if self.develop_mode and "api_key" in request.args and request.args[
"api_key"]:
actor_id = str(request.args["api_key"])
expiry = str(int(user_session.get("valid_until", 0)) * 1000)
if 0 < int(expiry) < current_time_millis():
expiry = str(current_time_millis() + 10000)
# flask.session["valid_until"] = int(expiry / 1000)
log.info(
"Request associated with actor_id=%s, expiry=%s from developer api_key",
actor_id, expiry)
# Check in headers for OAuth2 bearer token
auth_hdr = request.headers.get("authorization", None)
if auth_hdr:
valid, req = self.process.oauth.verify_request(
[self.process.oauth_scope])
if valid:
actor_id = flask.g.oauth_user.get("actor_id", "")
if actor_id:
log.info(
"Request associated with actor_id=%s, expiry=%s from OAuth token",
actor_id, expiry)
return actor_id, DEFAULT_EXPIRY
# Try to find auth token override
if not authtoken:
if json_params:
if "authtoken" in json_params:
authtoken = json_params["authtoken"]
else:
if "authtoken" in request.args:
authtoken = str(request.args["authtoken"])
# Enable temporary authentication tokens to resolve to actor ids
if authtoken:
try:
token_info = self.idm_client.check_authentication_token(
authtoken, headers=self._get_gateway_headers())
actor_id = token_info.get("actor_id", actor_id)
expiry = token_info.get("expiry", expiry)
log.info("Resolved token %s into actor_id=%s expiry=%s",
authtoken, actor_id, expiry)
except NotFound:
log.info("Provided authentication token not found: %s",
authtoken)
except Unauthorized:
log.info("Authentication token expired or invalid: %s",
authtoken)
except Exception as ex:
log.exception("Problem resolving authentication token")
return actor_id, expiry
def in_login_whitelist(self, category, svc, op):
"""Returns True if service op is whitelisted for anonymous access"""
entry = "%s/%s/%s" % (category, svc, op)
return entry in self.no_login_whitelist
def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
# There is no point in looking up an anonymous user - so return default values.
if ion_actor_id == DEFAULT_ACTOR_ID:
# Since this is an anonymous request, there really is no expiry associated with it
if not in_whitelist and self.require_login:
raise Unauthorized("Anonymous access not permitted")
else:
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
try:
user = self.idm_client.read_actor_identity(
actor_id=ion_actor_id, headers=self._get_gateway_headers())
except NotFound as e:
if not in_whitelist and self.require_login:
# This could be a restart of the system with a new preload.
# TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
user_session = get_auth()
if user_session.get("actor_id", None) == ion_actor_id:
clear_auth()
raise Unauthorized("Invalid identity", exc_id="01.10")
else:
# If the user isn't found default to anonymous
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
# Need to convert to int first in order to compare against current time.
try:
int_expiry = int(expiry)
except Exception as ex:
raise Inconsistent(
"Unable to read the expiry value in the request '%s' as an int"
% expiry)
# The user has been validated as being known in the system, so not check the expiry and raise exception if
# the expiry is not set to 0 and less than the current time.
if 0 < int_expiry < current_time_millis():
if not in_whitelist and self.require_login:
raise Unauthorized("User authentication expired")
else:
log.warn("User authentication expired")
return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY
return ion_actor_id, expiry
# -------------------------------------------------------------------------
# Service call (messaging) helpers
def register_request_callback(self, cb_func):
if cb_func is None:
pass
elif self.request_callback:
log.warn("Callback already registered")
self.request_callback = cb_func
def _call_request_callback(self, action, req_info):
if not self.request_callback:
return
try:
self.request_callback(action, req_info)
except Exception:
log.exception("Error calling request callback")
def _add_cors_headers(self, resp):
# Set CORS headers so that a Swagger client on a different domain can read spec
resp.headers[
"Access-Control-Allow-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
resp.headers[
"Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
resp.headers["Access-Control-Allow-Origin"] = "*"
resp.headers[
"Access-Control-Request-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
def _log_request_start(self, req_type="SG"):
global req_seqnum
req_seqnum += 1
req_info = dict(request_id=req_seqnum,
start_time=time.time(),
req_type=req_type,
req_url=request.url)
flask.g.req_info = req_info
webapi_log.info("%s REQUEST (%s) - %s", req_type,
req_info["request_id"], request.url)
self._call_request_callback("start", req_info)
def _log_request_response(self,
content_type,
result="",
content_length=-1,
status_code=200):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["resp_content_type"] = content_type
req_info["resp_content_length"] = content_length
req_info["resp_result"] = result
req_info["resp_status"] = req_info.get("resp_status", status_code)
def _log_request_error(self, result, status_code):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["resp_error"] = True
req_info["resp_status"] = status_code
webapi_log.warn(
"%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"],
req_info["request_id"], status_code,
"/id=" + result[GATEWAY_ERROR_EXCID]
if result[GATEWAY_ERROR_EXCID] else "",
result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])
self._call_request_callback("error", req_info)
else:
webapi_log.warn(
"REQUEST ERROR (%s%s) - %s: %s", status_code,
"/id=" + result[GATEWAY_ERROR_EXCID]
if result[GATEWAY_ERROR_EXCID] else "",
result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])
def _log_request_end(self):
req_info = flask.g.get("req_info", None)
if req_info:
req_info["end_time"] = time.time()
webapi_log.info("%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
req_info["req_type"], req_info["request_id"],
req_info.get("resp_status", ""),
req_info["end_time"] - req_info["start_time"],
req_info.get("resp_content_length", ""),
req_info.get("resp_content_type", ""))
self._call_request_callback("end", req_info)
else:
webapi_log.warn("REQUEST END - missing start info")
def _get_request_args(self):
"""Extracts service request arguments from HTTP request. Supports various
methods and forms of encoding. Separates arguments for special parameters
from service operation parameters.
Returns a dict with the service request arguments, containing key params
with the actual values for the service operation parameters.
"""
str_args = False
request_args = {}
if request.method == "POST" or request.method == "PUT":
# Use only body args and ignore any args from query string
if request.headers.get("content-type",
"").startswith(CONT_TYPE_JSON):
# JSON body request
if request.data:
request_args = json_loads(request.data)
if GATEWAY_ARG_PARAMS not in request_args:
# Magic fallback: Directly use JSON first level as args if params key not present
request_args = {GATEWAY_ARG_PARAMS: request_args}
elif request.form:
# Form encoded payload
if GATEWAY_ARG_JSON in request.form:
payload = request.form[GATEWAY_ARG_JSON]
request_args = json_loads(str(payload))
if GATEWAY_ARG_PARAMS not in request_args:
# Magic fallback: Directly use JSON first level as args if params key not present
request_args = {GATEWAY_ARG_PARAMS: request_args}
else:
# Fallback: Directly use form values
str_args = True
request_args = {
GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)
}
else:
# No args found in body
request_args = {GATEWAY_ARG_PARAMS: {}}
elif request.method == "GET":
str_args = True
REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
args_dict = request.args.to_dict(flat=True)
request_args = {
k: request.args[k]
for k in args_dict if k in REQ_ARGS_SPECIAL
}
req_params = {
k: request.args[k]
for k in args_dict if k not in REQ_ARGS_SPECIAL
}
request_args[GATEWAY_ARG_PARAMS] = req_params
request_args[
"str_args"] = str_args # Indicate downstream that args are str (GET or form encoded)
#log.info("Request args: %s" % request_args)
return request_args
def _get_typed_arg_value(self, given_value, param_def, strict):
"""Returns a service operation argument value, based on a given value and param schema definition.
"""
param_type = param_def["type"]
if isinstance(given_value, unicode):
# Convert all unicode to str in UTF-8
given_value = given_value.encode(
"utf8") # Make all unicode into str
if isinstance(given_value, IonObjectBase) and (
given_value._get_type() == param_type
or param_type in given_value._get_extends()):
return given_value
elif is_ion_object_dict(given_value) and (
param_type == "NoneType" or hasattr(objects, param_type)):
return self.create_ion_object(given_value)
elif param_type in ("str", "bool", "int", "float", "list", "dict",
"NoneType"):
arg_val = get_typed_value(given_value,
targettype=param_type,
strict=strict)
return arg_val
else:
raise BadRequest("Cannot convert param value to type %s" %
param_type)
def create_parameter_list(self,
service_def,
operation,
request_args,
id_param=None):
"""Build service call parameter list dynamically from service operation definition
"""
service_schema = service_def.schema
service_op_schema = service_schema["operations"][operation]
svc_op_param_list = service_op_schema["in_list"]
svc_params = {}
if id_param:
# Magic shorthand: if one argument is given, fill the first service argument
if svc_op_param_list:
fill_par = svc_op_param_list[0]
fill_par_def = service_op_schema["in"][fill_par]
arg_val = self._get_typed_arg_value(id_param,
fill_par_def,
strict=False)
svc_params[fill_par] = arg_val
return svc_params
request_args = request_args or {}
# Cannot be strict for a URL string query arguments or directly form encoded
strict_types = False if request_args.get("str_args",
False) else self.strict_types
req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
for param_name in svc_op_param_list:
param_def = service_op_schema["in"][param_name]
if param_name in req_op_args:
arg_val = self._get_typed_arg_value(req_op_args[param_name],
param_def,
strict=strict_types)
svc_params[param_name] = arg_val
if "timeout" in request_args:
svc_params["timeout"] = float(request_args["timeout"])
optional_args = [
param for param in req_op_args
if param not in svc_params and param != "timeout"
]
if optional_args and "optional_args" in svc_op_param_list:
# Only support basic strings for these optional params for now
svc_params["optional_args"] = {
arg: str(req_op_args[arg])
for arg in optional_args
}
#log.info("Service params: %s" % svc_params)
return svc_params
def _get_gateway_headers(self):
"""Returns the headers that the service gateway uses to make service calls on behalf of itself
(not a user passing through), e.g. for identity management purposes"""
return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}
def get_secure_service_def(self, service_name):
"""Checks whether the service indicated by given service_name exists and/or
is exposed after white and black listing. Returns service registry entry.
"""
if self.service_whitelist:
if service_name not in self.service_whitelist:
raise Unauthorized("Service access not permitted")
if self.service_blacklist:
if service_name in self.service_blacklist:
raise Unauthorized("Service access not permitted")
# Retrieve service definition
target_service = get_service_registry().get_service_by_name(
service_name)
if not target_service:
raise BadRequest("The requested service (%s) is not available" %
service_name)
if not target_service.client:
raise Inconsistent(
"Cannot find a client class for the specified service: %s" %
service_name)
if not target_service.schema:
raise Inconsistent(
"Cannot find a schema for the specified service: %s" %
service_name)
return target_service
def build_message_headers(self, actor_id, expiry):
"""Returns the headers that the service gateway uses to make service calls on behalf of a
user, based on the user session or request arguments"""
headers = dict()
headers[MSG_HEADER_ACTOR] = actor_id
headers[MSG_HEADER_VALID] = expiry
req_info = flask.g.get("req_info", None)
if req_info:
headers["request-id"] = str(req_info["request_id"])
# If this is an anonymous requester then there are no roles associated with the request
if actor_id == DEFAULT_ACTOR_ID:
headers[MSG_HEADER_ROLES] = dict()
return headers
try:
# Check to see if the user's roles are cached already - keyed by user id
if self.user_role_cache.has_key(actor_id):
role_header = self.user_role_cache.get(actor_id)
if role_header is not None:
headers[MSG_HEADER_ROLES] = role_header
return headers
# The user's roles were not cached so hit the datastore to find it.
role_list = self.org_client.list_actor_roles(
actor_id, headers=self._get_gateway_headers())
org_roles = {}
for role in role_list:
org_roles.setdefault(role.org_governance_name, []).append(role)
role_header = get_role_message_headers(org_roles)
# Cache the roles by user id
self.user_role_cache.put(actor_id, role_header)
except Exception:
role_header = dict(
) # Default to empty dict if there is a problem finding roles for the user
headers[MSG_HEADER_ROLES] = role_header
return headers
def create_ion_object(self, object_params):
"""Create and initialize an ION object from a dictionary of parameters coming via HTTP,
ready to be passed on to services/messaging. The object is validated after creation.
Note: This is not called for service operation argument signatures
"""
new_obj = IonObject(object_params["type_"])
# Iterate over the parameters to add to object; have to do this instead
# of passing a dict to get around restrictions in object creation on setting _id, _rev params
for param in object_params:
self.set_object_field(new_obj, param, object_params.get(param))
new_obj._validate(
) # verify that all of the object fields were set with proper types
return new_obj
def set_object_field(self, obj, field, field_val):
"""Recursively set sub object field values.
TODO: This may be an expensive operation. May also be redundant with object code
"""
if isinstance(field_val, dict) and field != "kwargs":
sub_obj = getattr(obj, field)
if isinstance(sub_obj, IonObjectBase):
if "type_" in field_val and field_val["type_"] != sub_obj.type_:
if issubtype(field_val["type_"], sub_obj.type_):
sub_obj = IonObject(field_val["type_"])
setattr(obj, field, sub_obj)
else:
raise Inconsistent(
"Unable to walk the field %s - types don't match: %s %s"
% (field, sub_obj.type_, field_val["type_"]))
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field,
field_val.get(sub_field))
elif isinstance(sub_obj, dict):
setattr(obj, field, field_val)
else:
for sub_field in field_val:
self.set_object_field(sub_obj, sub_field,
field_val.get(sub_field))
else:
# type_ already exists in the class.
if field != "type_":
setattr(obj, field, field_val)
# -------------------------------------------------------------------------
# Response content helpers
def json_response(self, response_data):
"""Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
"""
resp_obj = json_dumps(response_data,
default=encode_ion_object,
indent=None if request.is_xhr else 2)
resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
if self.develop_mode and (self.set_cors_headers or
("api_key" in request.args
and request.args["api_key"])):
self._add_cors_headers(resp)
self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
return resp
def gateway_json_response(self, response_data):
"""Returns the normal service gateway response as JSON or as media in case the response
is a media response
"""
if isinstance(response_data, MediaResponse):
log.info("Media response. Content mimetype:%s",
response_data.media_mimetype)
content = response_data.body
if response_data.internal_encoding == "base64":
import base64
content = base64.decodestring(content)
elif response_data.internal_encoding == "utf8":
pass
resp = self.response_class(content,
response_data.code,
mimetype=response_data.media_mimetype)
self._log_request_response(response_data.media_mimetype, "raw",
len(content), response_data.code)
return resp
if RETURN_MIMETYPE_PARAM in request.args:
return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
return self.response_class(response_data, mimetype=return_mimetype)
result = {
GATEWAY_RESPONSE: response_data,
GATEWAY_STATUS: 200,
}
return self.json_response(result)
def gateway_error_response(self, exc):
"""Forms a service gateway error response.
Can extract multiple stacks from a multi-tier RPC service call exception
"""
if hasattr(exc, "get_stacks"):
# Process potentially multiple stacks.
full_error, exc_stacks = "", exc.get_stacks()
for i in range(len(exc_stacks)):
full_error += exc_stacks[i][0] + "\n"
if i == 0:
full_error += "".join(
traceback.format_exception(*sys.exc_info()))
else:
entry = ApplicationException.format_stack(exc_stacks[i][1])
full_error += entry + "\n"
exec_name = exc.__class__.__name__
else:
exc_type, exc_obj, exc_tb = sys.exc_info()
exec_name = exc_type.__name__
full_error = "".join(traceback.format_exception(*sys.exc_info()))
status_code = getattr(exc, "status_code", 400)
if self.log_errors:
if self.develop_mode:
if status_code == 401:
log.warn("%s: %s", exec_name, exc)
else:
log.error(full_error)
else:
if status_code == 401:
log.info("%s: %s", exec_name, exc)
else:
log.info(full_error)
result = {
GATEWAY_ERROR_EXCEPTION: exec_name,
GATEWAY_ERROR_MESSAGE: str(exc.message),
GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or ""
}
if self.develop_mode:
result[GATEWAY_ERROR_TRACE] = full_error
if RETURN_MIMETYPE_PARAM in request.args:
return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
return self.response_class(result, mimetype=return_mimetype)
self._log_request_error(result, status_code)
resp = self.json_response({
GATEWAY_ERROR: result,
GATEWAY_STATUS: status_code
})
# Q: Should HTTP status be the error code of the exception?
resp.status_code = status_code
return resp
def start(self, process_id=None):
assert self.event_sub is None
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback, origin=process_id, origin_type="DispatchedProcess")
self.event_sub.start()
class ProcessStateWaiter(object):
def __init__(self):
self.event_queue = queue.Queue()
self.event_sub = None
def start(self, process_id=None):
assert self.event_sub is None
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback, origin=process_id, origin_type="DispatchedProcess")
self.event_sub.start()
def stop(self):
if self.event_sub:
self.event_sub.stop()
self.event_sub = None
def _event_callback(self, event, *args, **kwargs):
self.event_queue.put(event)
def await_state_event(self, pid=None, state=None, timeout=30, strict=False):
"""Wait for a state event for a process.
if strict is False, allow intermediary events
"""
start_time = datetime.now()
assert state in ProcessStateEnum._str_map, "process state %s unknown!" % state
state_str = ProcessStateEnum._str_map.get(state)
# stick the pid into a container if it is only one
if pid is not None and not isinstance(pid, (list, tuple)):
pid = (pid,)
while 1:
if datetime.now() - start_time > timedelta(seconds=timeout):
raise AssertionError("Waiter timeout! Waited %s seconds for process %s state %s" % (timeout, pid, state_str))
try:
event = self.event_queue.get(timeout=timeout)
except queue.Empty:
raise AssertionError("Event timeout! Waited %s seconds for process %s state %s" % (timeout, pid, state_str))
log.debug("Got event: %s", event)
if (pid is None or event.origin in pid) and (state is None or event.state == state):
return event
elif strict:
raise AssertionError("Got unexpected event %s. Expected state %s for process %s" % (event, state_str, pid))
def await_many_state_events(self, pids, state=None, timeout=30, strict=False):
pid_set = set(pids)
while pid_set:
event = self.await_state_event(tuple(pid_set), state, timeout=timeout, strict=strict)
pid_set.remove(event.origin)
def await_nothing(self, pid=None, timeout=10):
start_time = datetime.now()
# stick the pid into a container if it is only one
if pid is not None and not isinstance(pid, (list, tuple)):
pid = (pid,)
while 1:
timeleft = timedelta(seconds=timeout) - (datetime.now() - start_time)
timeleft_seconds = timeleft.total_seconds()
if timeleft_seconds <= 0:
return
try:
event = self.event_queue.get(timeout=timeleft_seconds)
if pid is None or event.origin in pid:
state_str = ProcessStateEnum._str_map.get(event.state, str(event.state))
raise AssertionError("Expected no event, but got state %s for process %s" % (state_str, event.origin))
except queue.Empty:
return
class ProcessDispatcherAggregator(object):
""" PD aggregator for heartbeat input from containers etc. """
def __init__(self, pd_core):
self._pd_core = pd_core
self.container = self._pd_core.container
self.registry = self._pd_core.registry
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg,
"aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (
topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic,
from_name=queue_name,
auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
callback=self._receive_event)
self.evt_sub.add_event_subscription(
event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def stop(self):
# Stop subscribers
self.sub_cont.close()
self.evt_sub.close()
# Wait for subscribers to finish
self.sub_cont_gl.join(timeout=2)
self.sub_cont_gl.kill()
self.sub_cont_gl = None
self.evt_sub_gl.join(timeout=2)
self.evt_sub_gl.kill()
self.evt_sub_gl = None
# -------------------------------------------------------------------------
def _receive_container_info(self, msg, headers, *args):
log.debug("Got container info %s %s %s", msg, headers, args)
if not isinstance(msg, ContainerHeartbeat):
log.warn("Unknown container info format")
return
self.registry.register_container(msg.container_id, msg.ts,
EE_STATE_RUNNING, msg.attributes)
def _receive_event(self, event, *args, **kwargs):
log.debug("Got event %s %s %s", event, args, kwargs)
if isinstance(event, ContainerLifecycleEvent):
if event.sub_type == "START":
self.registry.register_container(event.origin,
event.ts_created,
EE_STATE_RUNNING, {})
elif event.sub_type == "TERMINATE":
self.registry.register_container(event.origin,
event.ts_created,
EE_STATE_TERMINATED, {})
elif isinstance(event, ProcessLifecycleEvent):
proc_info = dict(state=event.state,
proc_type=event.process_type,
proc_name=event.process_name,
resource_id=event.process_resource_id,
service_name=event.service_name)
self.registry.register_process(event.container_id, event.origin,
proc_info)