def setUp(self):
        from unittest import SkipTest
        raise SkipTest("Process dispatcher currently not supported")

        self._start_container()
        self.container.start_rel_from_url('res/deploy/basic.yml')

        self.pd_cli = ProcessDispatcherServiceClient()

        self.process_definition = IonObject(OT.ProcessDefinition,
                                            name='test_process')
        self.process_definition.executable = {
            'module': 'ion.services.test.test_process_state_gate',
            'class': 'TestProcess'
        }
        self.process_definition_id = self.pd_cli.create_process_definition(
            self.process_definition)
        self.event_queue = queue.Queue()

        self.process_schedule = IonObject(OT.ProcessSchedule)
        self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS

        self.pid = self.pd_cli.create_process(self.process_definition_id)

        self.event_queue = queue.Queue()

        self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
                                         callback=self._event_callback,
                                         origin=self.pid,
                                         origin_type="DispatchedProcess")
Пример #2
0
    def __init__(self,
                 read_process_fn=None,
                 process_id='',
                 desired_state=None,
                 *args,
                 **kwargs):

        if not process_id:
            raise BadRequest(
                "ProcessStateGate trying to wait on invalid process (id = '%s')"
                % process_id)

        EventSubscriber.__init__(self,
                                 *args,
                                 callback=self._trigger_cb,
                                 event_type=OT.ProcessLifecycleEvent,
                                 origin=process_id,
                                 **kwargs)

        self.desired_state = desired_state
        self.process_id = process_id
        self.read_process_fn = read_process_fn
        self.last_chance = None
        self.first_chance = None

        _ = ProcessStateEnum._str_map[
            self.desired_state]  # make sure state exists
        log.info(
            "ProcessStateGate is going to wait on process '%s' for state '%s'",
            self.process_id, ProcessStateEnum._str_map[self.desired_state])
Пример #3
0
 def start(self):
     # Configure  subscriptions for user_cache events
     self.event_subscriber = EventSubscriber(
         event_type=OT.UserRoleModifiedEvent,
         origin_type="Org",
         callback=self._event_callback)
     self.event_subscriber.add_event_subscription(
         event_type=OT.UserRoleCacheResetEvent)
     self.process.add_endpoint(self.event_subscriber)
Пример #4
0
 def start(self):
     # Configure  subscriptions for user_cache events
     self.event_subscriber = EventSubscriber(
         event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback
     )
     self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent)
     self.process.add_endpoint(self.event_subscriber)
Пример #5
0
    def setUp(self):
        from unittest import SkipTest
        raise SkipTest("Process dispatcher currently not supported")

        self._start_container()
        self.container.start_rel_from_url('res/deploy/basic.yml')

        self.pd_cli = ProcessDispatcherServiceClient()

        self.process_definition = IonObject(OT.ProcessDefinition, name='test_process')
        self.process_definition.executable = {'module': 'ion.services.test.test_process_state_gate',
                                              'class': 'TestProcess'}
        self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
        self.event_queue = queue.Queue()

        self.process_schedule = IonObject(OT.ProcessSchedule)
        self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS

        self.pid = self.pd_cli.create_process(self.process_definition_id)

        self.event_queue = queue.Queue()

        self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
                                         callback=self._event_callback,
                                         origin=self.pid,
                                         origin_type="DispatchedProcess")
    def on_start(self):
        self.terminate_loop = Event()
        self.has_lock = False
        self.lock_expires = None
        CoordinatedProcess.evt_count[self.id] = 0

        self.bg_loop = gevent.spawn(self._bg_loop)

        self.evt_sub = EventSubscriber(event_type=OT.ResourceCommandEvent,
                                       callback=self._on_event)
        self.add_endpoint(self.evt_sub)
Пример #7
0
    def start(self):
        # Create our own queue for container heartbeats and broadcasts
        topic = get_safe(self._pd_core.pd_cfg,
                         "aggregator.container_topic") or "bx_containers"
        queue_name = "pd_aggregator_%s_%s" % (
            topic, create_valid_identifier(self.container.id, dot_sub="_"))
        self.sub_cont = Subscriber(binding=topic,
                                   from_name=queue_name,
                                   auto_delete=True,
                                   callback=self._receive_container_info)
        self.sub_cont_gl = spawn(self.sub_cont.listen)
        self.sub_cont.get_ready_event().wait()

        self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
                                       callback=self._receive_event)
        self.evt_sub.add_event_subscription(
            event_type=OT.ProcessLifecycleEvent)
        self.evt_sub_gl = spawn(self.evt_sub.listen)
        self.evt_sub.get_ready_event().wait()

        log.info("PD Aggregator - event and heartbeat subscribers started")
Пример #8
0
class ProcessDispatcherAggregator(object):
    """ PD aggregator for heartbeat input from containers etc. """

    def __init__(self, pd_core):
        self._pd_core = pd_core
        self.container = self._pd_core.container
        self.registry = self._pd_core.registry

    def start(self):
        # Create our own queue for container heartbeats and broadcasts
        topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
        queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
        self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
                                   callback=self._receive_container_info)
        self.sub_cont_gl = spawn(self.sub_cont.listen)
        self.sub_cont.get_ready_event().wait()

        self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
        self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
        self.evt_sub_gl = spawn(self.evt_sub.listen)
        self.evt_sub.get_ready_event().wait()

        log.info("PD Aggregator - event and heartbeat subscribers started")

    def stop(self):
        # Stop subscribers
        self.sub_cont.close()
        self.evt_sub.close()

        # Wait for subscribers to finish
        self.sub_cont_gl.join(timeout=2)
        self.sub_cont_gl.kill()
        self.sub_cont_gl = None

        self.evt_sub_gl.join(timeout=2)
        self.evt_sub_gl.kill()
        self.evt_sub_gl = None

    # -------------------------------------------------------------------------

    def _receive_container_info(self, msg, headers, *args):
        log.debug("Got container info %s %s %s", msg, headers, args)
        if not isinstance(msg, ContainerHeartbeat):
            log.warn("Unknown container info format")
            return

        self.registry.register_container(msg.container_id, msg.ts, EE_STATE_RUNNING, msg.attributes)

    def _receive_event(self, event, *args, **kwargs):
        log.debug("Got event %s %s %s", event, args, kwargs)
        if isinstance(event, ContainerLifecycleEvent):
            if event.sub_type == "START":
                self.registry.register_container(event.origin, event.ts_created, EE_STATE_RUNNING, {})
            elif event.sub_type == "TERMINATE":
                self.registry.register_container(event.origin, event.ts_created, EE_STATE_TERMINATED, {})

        elif isinstance(event, ProcessLifecycleEvent):
            proc_info = dict(state=event.state, proc_type=event.process_type, proc_name=event.process_name,
                             resource_id=event.process_resource_id, service_name=event.service_name)
            self.registry.register_process(event.container_id, event.origin, proc_info)
Пример #9
0
    def start(self):
        # Create our own queue for container heartbeats and broadcasts
        topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
        queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
        self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
                                   callback=self._receive_container_info)
        self.sub_cont_gl = spawn(self.sub_cont.listen)
        self.sub_cont.get_ready_event().wait()

        self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
        self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
        self.evt_sub_gl = spawn(self.evt_sub.listen)
        self.evt_sub.get_ready_event().wait()

        log.info("PD Aggregator - event and heartbeat subscribers started")
Пример #10
0
class ServiceGateway(object):
    """
    The Service Gateway exports service routes for a web server via a Flask blueprint.
    The gateway bridges HTTP requests to ION AMQP RPC calls.
    """

    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info("Service Gateway will not check requests against trusted originators since none are configured.")

        # Service screening
        self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        # maxSize = maximum number of elements to keep in cache
        # maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True)

        self.rr_client = ResourceRegistryServiceProcessClient(process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
        self.org_client = OrgManagementServiceProcessClient(process=self.process)

    # -------------------------------------------------------------------------
    # Lifecycle management

    def start(self):
        # Configure  subscriptions for user_cache events
        self.event_subscriber = EventSubscriber(
            event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback
        )
        self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent)
        self.process.add_endpoint(self.event_subscriber)

    def stop(self):
        pass
        # Stop event subscribers - TODO: This hangs
        # self.process.remove_endpoint(self.event_subscriber)

    # -------------------------------------------------------------------------
    # Event subscriber callbacks

    def _event_callback(self, event, *args, **kwargs):
        """ Callback function for receiving Events """
        if isinstance(event, UserRoleModifiedEvent):
            # Event when User Roles are modified
            user_role_event = event
            org_id = user_role_event.origin
            actor_id = user_role_event.actor_id
            role_name = user_role_event.role_name
            log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name))

            # Evict the user and their roles from the cache so that it gets updated with the next call.
            if self.user_role_cache and self.user_role_cache.has_key(actor_id):
                log.debug("Evicting user from the user_role_cache: %s" % actor_id)
                self.user_role_cache.evict(actor_id)
        elif isinstance(event, UserRoleCacheResetEvent):
            # An event is received to clear the user data cache
            self.user_role_cache.clear()

    # -------------------------------------------------------------------------
    # Routes

    def sg_index(self):
        return self.gateway_json_response(SG_IDENTIFICATION)

    def get_service_spec(self, service_name=None, spec_name=None):
        try:
            if not self._swagger_gen:
                raise NotFound("Spec not available")
            if spec_name != "swagger.json":
                raise NotFound("Unknown spec format")

            swagger_json = self._swagger_gen.get_spec(service_name)

            resp = flask.make_response(flask.jsonify(swagger_json))
            self._add_cors_headers(resp)
            return resp

        except Exception as ex:
            return self.gateway_error_response(ex)

    def process_gateway_request(self, service_name=None, operation=None, id_param=None):
        """
        Makes a secure call to a SciON service operation via messaging.
        """
        # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
        self._log_request_start("SVC RPC")
        try:
            result = self._make_service_request(service_name, operation, id_param)
            return self.gateway_json_response(result)

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def rest_gateway_request(self, service_name, res_type, id_param=None):
        """
        Makes a REST style call to a SciON service operation via messaging.
        Get with ID returns the resource, POST without ID creates, PUT with ID updates
        and GET without ID returns the collection.
        """
        self._log_request_start("SVC REST")
        try:
            if not service_name:
                raise BadRequest("Service name missing")
            service_name = str(service_name)
            if not res_type:
                raise BadRequest("Resource type missing")
            res_type = str(res_type)

            if request.method == "GET" and id_param:
                operation = "read_" + res_type
                return self.process_gateway_request(service_name, operation, id_param)
            elif request.method == "GET":
                ion_res_type = "".join(x.title() for x in res_type.split("_"))
                res = self._make_service_request("resource_registry", "find_resources", ion_res_type)
                if len(res) == 2:
                    return self.gateway_json_response(res[0])
                raise BadRequest("Unexpected find_resources result")
            elif request.method == "PUT":
                operation = "update_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                if id_param:
                    obj._id = id_param
                return self.process_gateway_request(service_name, operation, obj)
            elif request.method == "POST":
                operation = "create_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                return self.process_gateway_request(service_name, operation, obj)
            else:
                raise BadRequest("Bad REST request")

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def _extract_payload_data(self):
        request_obj = None
        if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
            if request.data:
                request_obj = json_loads(request.data)
        elif request.form:
            # Form encoded
            if GATEWAY_ARG_JSON in request.form:
                payload = request.form[GATEWAY_ARG_JSON]
                request_obj = json_loads(str(payload))

        if request_obj and is_ion_object_dict(request_obj):
            request_obj = self.create_ion_object(request_obj)

        return request_obj

    def _make_service_request(self, service_name=None, operation=None, id_param=None):
        """
        Executes a secure call to a SciON service operation via messaging.
        """
        if not service_name:
            if self.develop_mode:
                # Return a list of available services
                result = dict(available_services=get_service_registry().services.keys())
                return result
            else:
                raise BadRequest("Service name missing")
        service_name = str(service_name)

        if not operation:
            if self.develop_mode:
                # Return a list of available operations
                result = dict(available_operations=[])
                return result
            else:
                raise BadRequest("Service operation missing")
        operation = str(operation)

        # Apply service white list and black list for initial protection and get service client
        service_def = self.get_secure_service_def(service_name)
        target_client = service_def.client

        # Get service request arguments and operation parameter values request
        req_args = self._get_request_args()

        param_list = self.create_parameter_list(service_def, operation, req_args, id_param)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
        in_login_whitelist = self.in_login_whitelist("request", service_name, operation)
        ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist)
        param_list["headers"] = self.build_message_headers(ion_actor_id, expiry)

        # Make service operation call
        client = target_client(process=self.process)
        method_call = getattr(client, operation)
        result = method_call(**param_list)

        return result

    def get_resource_schema(self, resource_type):
        try:
            # Validate requesting user and expiry and add governance headers
            ion_actor_id, expiry = self.get_governance_info_from_request()
            ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)

            return self.gateway_json_response(get_object_schema(resource_type))

        except Exception as ex:
            return self.gateway_error_response(ex)

    def get_attachment(self, attachment_id):
        try:
            # Create client to interface
            attachment = self.rr_client.read_attachment(attachment_id, include_content=True)

            return self.response_class(attachment.content, mimetype=attachment.content_type)

        except Exception as ex:
            return self.gateway_error_response(ex)

    def create_attachment(self):
        try:
            payload = request.form[GATEWAY_ARG_JSON]
            json_params = json_loads(str(payload))

            actor_id, expiry = self.get_governance_info_from_request(json_params)
            actor_id, expiry = self.validate_request(actor_id, expiry)
            headers = self.build_message_headers(actor_id, expiry)

            data_params = json_params[GATEWAY_ARG_PARAMS]
            resource_id = str(data_params.get("resource_id", ""))
            fil = request.files["file"]
            content = fil.read()

            keywords = []
            keywords_str = data_params.get("keywords", "")
            if keywords_str.strip():
                keywords = [str(x.strip()) for x in keywords_str.split(",")]

            created_by = data_params.get("attachment_created_by", "unknown user")
            modified_by = data_params.get("attachment_modified_by", "unknown user")

            # build attachment
            attachment = Attachment(
                name=str(data_params["attachment_name"]),
                description=str(data_params["attachment_description"]),
                attachment_type=int(data_params["attachment_type"]),
                content_type=str(data_params["attachment_content_type"]),
                keywords=keywords,
                created_by=created_by,
                modified_by=modified_by,
                content=content,
            )

            ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers)

            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error creating attachment")
            return self.gateway_error_response(ex)

    def delete_attachment(self, attachment_id):
        try:
            ret = self.rr_client.delete_attachment(attachment_id)
            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error deleting attachment")
            return self.gateway_error_response(ex)

    def get_version_info(self):
        import pkg_resources

        pkg_list = ["scioncc"]

        version = {}
        for package in pkg_list:
            try:
                version["%s-release" % package] = pkg_resources.require(package)[0].version
                # @TODO git versions for each?
            except pkg_resources.DistributionNotFound:
                pass

        try:
            dir_client = DirectoryServiceProcessClient(process=self.process)
            sys_attrs = dir_client.lookup("/System")
            if sys_attrs and isinstance(sys_attrs, dict):
                version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()})
        except Exception as ex:
            log.exception("Could not determine system directory attributes")

        return self.gateway_json_response(version)

    # =========================================================================
    # Security and governance helpers

    def is_trusted_address(self, requesting_address):
        if self.trusted_originators is None:
            return True

        return requesting_address in self.trusted_originators

    def get_governance_info_from_request(self, json_params=None):
        # Default values for governance headers.
        actor_id = DEFAULT_ACTOR_ID
        expiry = DEFAULT_EXPIRY
        authtoken = ""
        user_session = get_auth()
        if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
            # Get info from current server session
            # NOTE: Actor id may be inside server session
            actor_id = user_session["actor_id"]
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry)

        # Developer access using api_key
        if self.develop_mode and "api_key" in request.args and request.args["api_key"]:
            actor_id = str(request.args["api_key"])
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            if 0 < int(expiry) < current_time_millis():
                expiry = str(current_time_millis() + 10000)
                # flask.session["valid_until"] = int(expiry / 1000)
            log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry)

        # Check in headers for OAuth2 bearer token
        auth_hdr = request.headers.get("authorization", None)
        if auth_hdr:
            valid, req = self.process.oauth.verify_request([self.process.oauth_scope])
            if valid:
                actor_id = flask.g.oauth_user.get("actor_id", "")
                if actor_id:
                    log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry)
                    return actor_id, DEFAULT_EXPIRY

        # Try to find auth token override
        if not authtoken:
            if json_params:
                if "authtoken" in json_params:
                    authtoken = json_params["authtoken"]
            else:
                if "authtoken" in request.args:
                    authtoken = str(request.args["authtoken"])

        # Enable temporary authentication tokens to resolve to actor ids
        if authtoken:
            try:
                token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers())
                actor_id = token_info.get("actor_id", actor_id)
                expiry = token_info.get("expiry", expiry)
                log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
            except NotFound:
                log.info("Provided authentication token not found: %s", authtoken)
            except Unauthorized:
                log.info("Authentication token expired or invalid: %s", authtoken)
            except Exception as ex:
                log.exception("Problem resolving authentication token")

        return actor_id, expiry

    def in_login_whitelist(self, category, svc, op):
        """Returns True if service op is whitelisted for anonymous access"""
        entry = "%s/%s/%s" % (category, svc, op)
        return entry in self.no_login_whitelist

    def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
        # There is no point in looking up an anonymous user - so return default values.
        if ion_actor_id == DEFAULT_ACTOR_ID:
            # Since this is an anonymous request, there really is no expiry associated with it
            if not in_whitelist and self.require_login:
                raise Unauthorized("Anonymous access not permitted")
            else:
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        try:
            user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers())
        except NotFound as e:
            if not in_whitelist and self.require_login:
                # This could be a restart of the system with a new preload.
                # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
                user_session = get_auth()
                if user_session.get("actor_id", None) == ion_actor_id:
                    clear_auth()
                raise Unauthorized("Invalid identity", exc_id="01.10")
            else:
                # If the user isn't found default to anonymous
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        # Need to convert to int first in order to compare against current time.
        try:
            int_expiry = int(expiry)
        except Exception as ex:
            raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)

        # The user has been validated as being known in the system, so not check the expiry and raise exception if
        # the expiry is not set to 0 and less than the current time.
        if 0 < int_expiry < current_time_millis():
            if not in_whitelist and self.require_login:
                raise Unauthorized("User authentication expired")
            else:
                log.warn("User authentication expired")
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        return ion_actor_id, expiry

    # -------------------------------------------------------------------------
    # Service call (messaging) helpers

    def _add_cors_headers(self, resp):
        # Set CORS headers so that a Swagger client on a different domain can read spec
        resp.headers[
            "Access-Control-Allow-Headers"
        ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
        resp.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
        resp.headers["Access-Control-Allow-Origin"] = "*"
        resp.headers[
            "Access-Control-Request-Headers"
        ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport,  *"

    def _log_request_start(self, req_type="SG"):
        global req_seqnum
        req_seqnum += 1
        req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type)
        flask.g.req_info = req_info
        webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url)

    def _log_request_response(self, content_type, result="", content_length=-1, status_code=200):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_content_type"] = content_type
            req_info["resp_content_length"] = content_length
            req_info["resp_result"] = result
            req_info["resp_status"] = req_info.get("resp_status", status_code)

    def _log_request_error(self, result, status_code):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_error"] = True
            req_info["resp_status"] = status_code
            webapi_log.warn(
                "%s REQUEST (%s) ERROR (%s%s) - %s: %s",
                req_info["req_type"],
                req_info["request_id"],
                status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION],
                result[GATEWAY_ERROR_MESSAGE],
            )
        else:
            webapi_log.warn(
                "REQUEST ERROR (%s%s) - %s: %s",
                status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION],
                result[GATEWAY_ERROR_MESSAGE],
            )

    def _log_request_end(self):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["end_time"] = time.time()
            webapi_log.info(
                "%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
                req_info["req_type"],
                req_info["request_id"],
                req_info.get("resp_status", ""),
                req_info["end_time"] - req_info["start_time"],
                req_info.get("resp_content_length", ""),
                req_info.get("resp_content_type", ""),
            )
        else:
            webapi_log.warn("REQUEST END - missing start info")

    def _get_request_args(self):
        """Extracts service request arguments from HTTP request. Supports various
        methods and forms of encoding. Separates arguments for special parameters
        from service operation parameters.
        Returns a dict with the service request arguments, containing key params
        with the actual values for the service operation parameters.
        """
        str_args = False
        request_args = {}
        if request.method == "POST" or request.method == "PUT":
            # Use only body args and ignore any args from query string
            if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
                # JSON body request
                if request.data:
                    request_args = json_loads(request.data)
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
            elif request.form:
                # Form encoded payload
                if GATEWAY_ARG_JSON in request.form:
                    payload = request.form[GATEWAY_ARG_JSON]
                    request_args = json_loads(str(payload))
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
                else:
                    # Fallback: Directly use form values
                    str_args = True
                    request_args = {GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)}
            else:
                # No args found in body
                request_args = {GATEWAY_ARG_PARAMS: {}}

        elif request.method == "GET":
            str_args = True
            REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
            args_dict = request.args.to_dict(flat=True)
            request_args = {k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL}
            req_params = {k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL}
            request_args[GATEWAY_ARG_PARAMS] = req_params

        request_args["str_args"] = str_args  # Indicate downstream that args are str (GET or form encoded)
        # log.info("Request args: %s" % request_args)
        return request_args

    def _get_typed_arg_value(self, given_value, param_def, strict):
        """Returns a service operation argument value, based on a given value and param schema definition.
        """
        param_type = param_def["type"]
        if isinstance(given_value, unicode):
            # Convert all unicode to str in UTF-8
            given_value = given_value.encode("utf8")  # Make all unicode into str

        if isinstance(given_value, IonObjectBase) and (
            given_value._get_type() == param_type or param_type in given_value._get_extends()
        ):
            return given_value
        elif is_ion_object_dict(given_value) and (param_type == "NoneType" or hasattr(objects, param_type)):
            return self.create_ion_object(given_value)
        elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"):
            arg_val = get_typed_value(given_value, targettype=param_type, strict=strict)
            return arg_val
        else:
            raise BadRequest("Cannot convert param value to type %s" % param_type)

    def create_parameter_list(self, service_def, operation, request_args, id_param=None):
        """Build service call parameter list dynamically from service operation definition
        """
        service_schema = service_def.schema
        service_op_schema = service_schema["operations"][operation]
        svc_op_param_list = service_op_schema["in_list"]
        svc_params = {}

        if id_param:
            # Magic shorthand: if one argument is given, fill the first service argument
            if svc_op_param_list:
                fill_par = svc_op_param_list[0]
                fill_par_def = service_op_schema["in"][fill_par]
                arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False)
                svc_params[fill_par] = arg_val
            return svc_params

        request_args = request_args or {}
        # Cannot be strict for a URL string query arguments or directly form encoded
        strict_types = False if request_args.get("str_args", False) else self.strict_types
        req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
        for param_name in svc_op_param_list:
            param_def = service_op_schema["in"][param_name]
            if param_name in req_op_args:
                arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types)
                svc_params[param_name] = arg_val
        if "timeout" in request_args:
            svc_params["timeout"] = float(request_args["timeout"])

        optional_args = [param for param in req_op_args if param not in svc_params and param != "timeout"]
        if optional_args and "optional_args" in svc_op_param_list:
            # Only support basic strings for these optional params for now
            svc_params["optional_args"] = {arg: str(req_op_args[arg]) for arg in optional_args}

        # log.info("Service params: %s" % svc_params)
        return svc_params

    def _get_gateway_headers(self):
        """Returns the headers that the service gateway uses to make service calls on behalf of itself
         (not a user passing through), e.g. for identity management purposes"""
        return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}

    def get_secure_service_def(self, service_name):
        """Checks whether the service indicated by given service_name exists and/or
        is exposed after white and black listing. Returns service registry entry.
        """
        if self.service_whitelist:
            if service_name not in self.service_whitelist:
                raise Unauthorized("Service access not permitted")
        if self.service_blacklist:
            if service_name in self.service_blacklist:
                raise Unauthorized("Service access not permitted")

        # Retrieve service definition
        target_service = get_service_registry().get_service_by_name(service_name)
        if not target_service:
            raise BadRequest("The requested service (%s) is not available" % service_name)
        if not target_service.client:
            raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name)
        if not target_service.schema:
            raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name)

        return target_service

    def build_message_headers(self, actor_id, expiry):
        """Returns the headers that the service gateway uses to make service calls on behalf of a
        user, based on the user session or request arguments"""
        headers = dict()
        headers[MSG_HEADER_ACTOR] = actor_id
        headers[MSG_HEADER_VALID] = expiry
        req_info = flask.g.get("req_info", None)
        if req_info:
            headers["request-id"] = str(req_info["request_id"])

        # If this is an anonymous requester then there are no roles associated with the request
        if actor_id == DEFAULT_ACTOR_ID:
            headers[MSG_HEADER_ROLES] = dict()
            return headers

        try:
            # Check to see if the user's roles are cached already - keyed by user id
            if self.user_role_cache.has_key(actor_id):
                role_header = self.user_role_cache.get(actor_id)
                if role_header is not None:
                    headers[MSG_HEADER_ROLES] = role_header
                    return headers

            # The user's roles were not cached so hit the datastore to find it.
            role_list = self.org_client.list_actor_roles(actor_id, headers=self._get_gateway_headers())
            org_roles = {}
            for role in role_list:
                org_roles.setdefault(role.org_governance_name, []).append(role)

            role_header = get_role_message_headers(org_roles)

            # Cache the roles by user id
            self.user_role_cache.put(actor_id, role_header)

        except Exception:
            role_header = dict()  # Default to empty dict if there is a problem finding roles for the user

        headers[MSG_HEADER_ROLES] = role_header

        return headers

    def create_ion_object(self, object_params):
        """Create and initialize an ION object from a dictionary of parameters coming via HTTP,
        ready to be passed on to services/messaging. The object is validated after creation.
        Note: This is not called for service operation argument signatures
        """
        new_obj = IonObject(object_params["type_"])

        # Iterate over the parameters to add to object; have to do this instead
        # of passing a dict to get around restrictions in object creation on setting _id, _rev params
        for param in object_params:
            self.set_object_field(new_obj, param, object_params.get(param))

        new_obj._validate()  # verify that all of the object fields were set with proper types
        return new_obj

    def set_object_field(self, obj, field, field_val):
        """Recursively set sub object field values.
        TODO: This may be an expensive operation. May also be redundant with object code
        """
        if isinstance(field_val, dict) and field != "kwargs":
            sub_obj = getattr(obj, field)

            if isinstance(sub_obj, IonObjectBase):

                if "type_" in field_val and field_val["type_"] != sub_obj.type_:
                    if issubtype(field_val["type_"], sub_obj.type_):
                        sub_obj = IonObject(field_val["type_"])
                        setattr(obj, field, sub_obj)
                    else:
                        raise Inconsistent(
                            "Unable to walk the field %s - types don't match: %s %s"
                            % (field, sub_obj.type_, field_val["type_"])
                        )

                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))

            elif isinstance(sub_obj, dict):
                setattr(obj, field, field_val)

            else:
                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))
        else:
            # type_ already exists in the class.
            if field != "type_":
                setattr(obj, field, field_val)

    # -------------------------------------------------------------------------
    # Response content helpers

    def json_response(self, response_data):
        """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
        """
        resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2)
        resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
        if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])):
            self._add_cors_headers(resp)
        self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
        return resp

    def gateway_json_response(self, response_data):
        """Returns the normal service gateway response as JSON or as media in case the response
        is a media response
        """
        if isinstance(response_data, MediaResponse):
            log.info("Media response. Content mimetype:%s", response_data.media_mimetype)
            content = response_data.body
            if response_data.internal_encoding == "base64":
                import base64

                content = base64.decodestring(content)
            elif response_data.internal_encoding == "utf8":
                pass
            resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype)
            self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code)
            return resp

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(response_data, mimetype=return_mimetype)

        result = {GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200}
        return self.json_response(result)

    def gateway_error_response(self, exc):
        """Forms a service gateway error response.
        Can extract multiple stacks from a multi-tier RPC service call exception
        """
        if hasattr(exc, "get_stacks"):
            # Process potentially multiple stacks.
            full_error, exc_stacks = "", exc.get_stacks()
            for i in range(len(exc_stacks)):
                full_error += exc_stacks[i][0] + "\n"
                if i == 0:
                    full_error += "".join(traceback.format_exception(*sys.exc_info()))
                else:
                    entry = ApplicationException.format_stack(exc_stacks[i][1])
                    full_error += entry + "\n"

            exec_name = exc.__class__.__name__
        else:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            exec_name = exc_type.__name__
            full_error = "".join(traceback.format_exception(*sys.exc_info()))

        status_code = getattr(exc, "status_code", 400)
        if self.log_errors:
            if self.develop_mode:
                if status_code == 401:
                    log.warn("%s: %s", exec_name, exc)
                else:
                    log.error(full_error)
            else:
                if status_code == 401:
                    log.info("%s: %s", exec_name, exc)
                else:
                    log.info(full_error)

        result = {
            GATEWAY_ERROR_EXCEPTION: exec_name,
            GATEWAY_ERROR_MESSAGE: str(exc.message),
            GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "",
        }
        if self.develop_mode:
            result[GATEWAY_ERROR_TRACE] = full_error

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(result, mimetype=return_mimetype)

        self._log_request_error(result, status_code)

        resp = self.json_response({GATEWAY_ERROR: result, GATEWAY_STATUS: status_code})
        # Q: Should HTTP status be the error code of the exception?
        resp.status_code = status_code
        return resp
Пример #11
0
class ProcessStateGateIntTest(IonIntegrationTestCase):

    def setUp(self):
        from unittest import SkipTest
        raise SkipTest("Process dispatcher currently not supported")

        self._start_container()
        self.container.start_rel_from_url('res/deploy/basic.yml')

        self.pd_cli = ProcessDispatcherServiceClient()

        self.process_definition = IonObject(OT.ProcessDefinition, name='test_process')
        self.process_definition.executable = {'module': 'ion.services.test.test_process_state_gate',
                                              'class': 'TestProcess'}
        self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
        self.event_queue = queue.Queue()

        self.process_schedule = IonObject(OT.ProcessSchedule)
        self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS

        self.pid = self.pd_cli.create_process(self.process_definition_id)

        self.event_queue = queue.Queue()

        self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
                                         callback=self._event_callback,
                                         origin=self.pid,
                                         origin_type="DispatchedProcess")

    def tearDown(self):
        #stop subscriber if its running
        if self.event_sub and self.event_sub._cbthread:
            self.event_sub.stop()
        self._stop_container()


    def _event_callback(self, event, *args, **kwargs):
        self.event_queue.put(event)


    def latest_event(self, timeout=10):
        # get latest event from our local event subscriber
        try:
            event = self.event_queue.get(timeout=timeout)
        except Empty:
            event = None
        return event


    def await_state(self, state, timeout=10):

        print "Emptying event queue"
        while True:
            event = self.latest_event(0)
            if event:
                print "State %s from event %s" % (event.state, event)
            else:
                break
        self.event_sub.start()

        #wait for process state
        print "Setting up %s gate" % ProcessStateEnum._str_map[state]
        gate = ProcessStateGate(self.pd_cli.read_process,
                                self.pid,
                                state)
        print "Waiting"
        ret = gate.await(timeout)

        print "Await got %s" % ret
        event = self.latest_event(timeout=1)

        # check false positives/negatives
        if ret and gate._get_first_chance() is None and event is None:
            self.fail("ProcessStateGate got an event that EventSubscriber didnt....")

        self.event_sub.stop()

        if (not ret) or gate._get_last_chance():
            if event and event.state == state:
                self.fail("EventSubscriber got state event %s for process %s, ProcessStateGate missed it" %
                          (ProcessStateEnum._str_map[event.state], self.pid))


        return ret

    def process_start(self):
        print "Scheduling process...",
        self.pd_cli.schedule_process(self.process_definition_id,
                                     self.process_schedule,
                                     configuration={},
                                     process_id=self.pid)
        print "Done scheduling process."


    def process_stop(self):
        print "STOPPING process...",
        self.pd_cli.cancel_process(self.pid)
        print "Done stopping process"


    def test_process_state_gate(self):

        self.assertFalse(self.await_state(ProcessStateEnum.RUNNING, 1),
                         "The process was reported as spawned, but we didn't yet")

        print "GOING TO ACTUALLY START PROCESS NOW"
        spawn_later(1, self.process_start)

        self.assertTrue(self.await_state(ProcessStateEnum.RUNNING),
                        "The process did not spawn")

        self.assertFalse(self.await_state(ProcessStateEnum.TERMINATED, 1),
                        "The process claims to have terminated, but we didn't kill it")


        print "communicating with the process to make sure it is really running"
        test_client = TestClient()
        for i in range(5):
            self.assertEqual(i + 1, test_client.count(timeout=10))

        spawn_later(1, self.process_stop)

        self.assertTrue(self.await_state(ProcessStateEnum.TERMINATED),
                        "The process failed to be reported as terminated when it was terminated")

        self.assertFalse(self.await_state(ProcessStateEnum.RUNNING, 1),
                         "The process was reported as spawned, but we killed it")
class ProcessStateGateIntTest(IonIntegrationTestCase):
    def setUp(self):
        from unittest import SkipTest
        raise SkipTest("Process dispatcher currently not supported")

        self._start_container()
        self.container.start_rel_from_url('res/deploy/basic.yml')

        self.pd_cli = ProcessDispatcherServiceClient()

        self.process_definition = IonObject(OT.ProcessDefinition,
                                            name='test_process')
        self.process_definition.executable = {
            'module': 'ion.services.test.test_process_state_gate',
            'class': 'TestProcess'
        }
        self.process_definition_id = self.pd_cli.create_process_definition(
            self.process_definition)
        self.event_queue = queue.Queue()

        self.process_schedule = IonObject(OT.ProcessSchedule)
        self.process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS

        self.pid = self.pd_cli.create_process(self.process_definition_id)

        self.event_queue = queue.Queue()

        self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
                                         callback=self._event_callback,
                                         origin=self.pid,
                                         origin_type="DispatchedProcess")

    def tearDown(self):
        #stop subscriber if its running
        if self.event_sub and self.event_sub._cbthread:
            self.event_sub.stop()
        self._stop_container()

    def _event_callback(self, event, *args, **kwargs):
        self.event_queue.put(event)

    def latest_event(self, timeout=10):
        # get latest event from our local event subscriber
        try:
            event = self.event_queue.get(timeout=timeout)
        except Empty:
            event = None
        return event

    def await_state(self, state, timeout=10):

        print "Emptying event queue"
        while True:
            event = self.latest_event(0)
            if event:
                print "State %s from event %s" % (event.state, event)
            else:
                break
        self.event_sub.start()

        #wait for process state
        print "Setting up %s gate" % ProcessStateEnum._str_map[state]
        gate = ProcessStateGate(self.pd_cli.read_process, self.pid, state)
        print "Waiting"
        ret = gate. await (timeout)

        print "Await got %s" % ret
        event = self.latest_event(timeout=1)

        # check false positives/negatives
        if ret and gate._get_first_chance() is None and event is None:
            self.fail(
                "ProcessStateGate got an event that EventSubscriber didnt....")

        self.event_sub.stop()

        if (not ret) or gate._get_last_chance():
            if event and event.state == state:
                self.fail(
                    "EventSubscriber got state event %s for process %s, ProcessStateGate missed it"
                    % (ProcessStateEnum._str_map[event.state], self.pid))

        return ret

    def process_start(self):
        print "Scheduling process...",
        self.pd_cli.schedule_process(self.process_definition_id,
                                     self.process_schedule,
                                     configuration={},
                                     process_id=self.pid)
        print "Done scheduling process."

    def process_stop(self):
        print "STOPPING process...",
        self.pd_cli.cancel_process(self.pid)
        print "Done stopping process"

    def test_process_state_gate(self):

        self.assertFalse(
            self.await_state(ProcessStateEnum.RUNNING, 1),
            "The process was reported as spawned, but we didn't yet")

        print "GOING TO ACTUALLY START PROCESS NOW"
        spawn_later(1, self.process_start)

        self.assertTrue(self.await_state(ProcessStateEnum.RUNNING),
                        "The process did not spawn")

        self.assertFalse(
            self.await_state(ProcessStateEnum.TERMINATED, 1),
            "The process claims to have terminated, but we didn't kill it")

        print "communicating with the process to make sure it is really running"
        test_client = TestClient()
        for i in range(5):
            self.assertEqual(i + 1, test_client.count(timeout=10))

        spawn_later(1, self.process_stop)

        self.assertTrue(
            self.await_state(ProcessStateEnum.TERMINATED),
            "The process failed to be reported as terminated when it was terminated"
        )

        self.assertFalse(
            self.await_state(ProcessStateEnum.RUNNING, 1),
            "The process was reported as spawned, but we killed it")
Пример #13
0
class ServiceGateway(object):
    """
    The Service Gateway exports service routes for a web server via a Flask blueprint.
    The gateway bridges HTTP requests to ION AMQP RPC calls.
    """
    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX +
                                                 ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX +
                                                  ".require_login") is True
        self.token_from_session = self.config.get_safe(
            CFG_PREFIX + ".token_from_session") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX +
                                                        ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info(
                "Service Gateway will not check requests against trusted originators since none are configured."
            )

        # Service screening
        self.service_blacklist = self.config.get_safe(
            CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(
            CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(
            self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX +
                                                     ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX +
                                                 ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX +
                                                ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(
            CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        #maxSize = maximum number of elements to keep in cache
        #maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.request_callback = None
        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors",
                                               True)

        self.rr_client = ResourceRegistryServiceProcessClient(
            process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(
            process=self.process)
        self.org_client = OrgManagementServiceProcessClient(
            process=self.process)

    # -------------------------------------------------------------------------
    # Lifecycle management

    def start(self):
        # Configure  subscriptions for user_cache events
        self.event_subscriber = EventSubscriber(
            event_type=OT.UserRoleModifiedEvent,
            origin_type="Org",
            callback=self._event_callback)
        self.event_subscriber.add_event_subscription(
            event_type=OT.UserRoleCacheResetEvent)
        self.process.add_endpoint(self.event_subscriber)

    def stop(self):
        pass
        # Stop event subscribers - TODO: This hangs
        #self.process.remove_endpoint(self.event_subscriber)

    # -------------------------------------------------------------------------
    # Event subscriber callbacks

    def _event_callback(self, event, *args, **kwargs):
        """ Callback function for receiving Events """
        if isinstance(event, UserRoleModifiedEvent):
            # Event when User Roles are modified
            user_role_event = event
            org_id = user_role_event.origin
            actor_id = user_role_event.actor_id
            role_name = user_role_event.role_name
            log.debug("User Role modified: %s %s %s" %
                      (org_id, actor_id, role_name))

            # Evict the user and their roles from the cache so that it gets updated with the next call.
            if self.user_role_cache and self.user_role_cache.has_key(actor_id):
                log.debug("Evicting user from the user_role_cache: %s" %
                          actor_id)
                self.user_role_cache.evict(actor_id)
        elif isinstance(event, UserRoleCacheResetEvent):
            # An event is received to clear the user data cache
            self.user_role_cache.clear()

    # -------------------------------------------------------------------------
    # Routes

    def sg_index(self):
        return self.gateway_json_response(SG_IDENTIFICATION)

    def get_service_spec(self, service_name=None, spec_name=None):
        try:
            if not self._swagger_gen:
                raise NotFound("Spec not available")
            if spec_name != "swagger.json":
                raise NotFound("Unknown spec format")

            swagger_json = self._swagger_gen.get_spec(service_name)

            resp = flask.make_response(flask.jsonify(swagger_json))
            self._add_cors_headers(resp)
            return resp

        except Exception as ex:
            return self.gateway_error_response(ex)

    def process_gateway_request(self,
                                service_name=None,
                                operation=None,
                                id_param=None):
        """
        Makes a secure call to a SciON service operation via messaging.
        """
        # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
        self._log_request_start("SVC RPC")
        try:
            result = self._make_service_request(service_name, operation,
                                                id_param)
            return self.gateway_json_response(result)

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def rest_gateway_request(self, service_name, res_type, id_param=None):
        """
        Makes a REST style call to a SciON service operation via messaging.
        Get with ID returns the resource, POST without ID creates, PUT with ID updates
        and GET without ID returns the collection.
        """
        self._log_request_start("SVC REST")
        try:
            if not service_name:
                raise BadRequest("Service name missing")
            service_name = str(service_name)
            if not res_type:
                raise BadRequest("Resource type missing")
            res_type = str(res_type)

            if request.method == "GET" and id_param:
                operation = "read_" + res_type
                return self.process_gateway_request(service_name, operation,
                                                    id_param)
            elif request.method == "GET":
                ion_res_type = "".join(x.title() for x in res_type.split('_'))
                res = self._make_service_request("resource_registry",
                                                 "find_resources",
                                                 ion_res_type)
                if len(res) == 2:
                    return self.gateway_json_response(res[0])
                raise BadRequest("Unexpected find_resources result")
            elif request.method == "PUT":
                operation = "update_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                if id_param:
                    obj._id = id_param
                return self.process_gateway_request(service_name, operation,
                                                    obj)
            elif request.method == "POST":
                operation = "create_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                return self.process_gateway_request(service_name, operation,
                                                    obj)
            else:
                raise BadRequest("Bad REST request")

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def _extract_payload_data(self):
        request_obj = None
        if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
            if request.data:
                request_obj = json_loads(request.data)
        elif request.form:
            # Form encoded
            if GATEWAY_ARG_JSON in request.form:
                payload = request.form[GATEWAY_ARG_JSON]
                request_obj = json_loads(str(payload))

        if request_obj and is_ion_object_dict(request_obj):
            request_obj = self.create_ion_object(request_obj)

        return request_obj

    def _make_service_request(self,
                              service_name=None,
                              operation=None,
                              id_param=None):
        """
        Executes a secure call to a SciON service operation via messaging.
        """
        if not service_name:
            if self.develop_mode:
                # Return a list of available services
                result = dict(
                    available_services=get_service_registry().services.keys())
                return result
            else:
                raise BadRequest("Service name missing")
        service_name = str(service_name)

        if not operation:
            if self.develop_mode:
                # Return a list of available operations
                result = dict(available_operations=[])
                return result
            else:
                raise BadRequest("Service operation missing")
        operation = str(operation)

        # Apply service white list and black list for initial protection and get service client
        service_def = self.get_secure_service_def(service_name)
        target_client = service_def.client

        # Get service request arguments and operation parameter values request
        req_args = self._get_request_args()

        param_list = self.create_parameter_list(service_def, operation,
                                                req_args, id_param)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
        in_login_whitelist = self.in_login_whitelist("request", service_name,
                                                     operation)
        ion_actor_id, expiry = self.validate_request(
            ion_actor_id, expiry, in_whitelist=in_login_whitelist)
        param_list["headers"] = self.build_message_headers(
            ion_actor_id, expiry)

        # Make service operation call
        client = target_client(process=self.process)
        method_call = getattr(client, operation)
        result = method_call(**param_list)

        return result

    def get_resource_schema(self, resource_type):
        try:
            # Validate requesting user and expiry and add governance headers
            ion_actor_id, expiry = self.get_governance_info_from_request()
            ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)

            return self.gateway_json_response(get_object_schema(resource_type))

        except Exception as ex:
            return self.gateway_error_response(ex)

    def get_attachment(self, attachment_id):
        try:
            # Create client to interface
            attachment = self.rr_client.read_attachment(attachment_id,
                                                        include_content=True)

            return self.response_class(attachment.content,
                                       mimetype=attachment.content_type)

        except Exception as ex:
            return self.gateway_error_response(ex)

    def create_attachment(self):
        try:
            payload = request.form[GATEWAY_ARG_JSON]
            json_params = json_loads(str(payload))

            actor_id, expiry = self.get_governance_info_from_request(
                json_params)
            actor_id, expiry = self.validate_request(actor_id, expiry)
            headers = self.build_message_headers(actor_id, expiry)

            data_params = json_params[GATEWAY_ARG_PARAMS]
            resource_id = str(data_params.get("resource_id", ""))
            fil = request.files["file"]
            content = fil.read()

            keywords = []
            keywords_str = data_params.get("keywords", "")
            if keywords_str.strip():
                keywords = [str(x.strip()) for x in keywords_str.split(",")]

            created_by = data_params.get("attachment_created_by",
                                         "unknown user")
            modified_by = data_params.get("attachment_modified_by",
                                          "unknown user")

            # build attachment
            attachment = Attachment(
                name=str(data_params["attachment_name"]),
                description=str(data_params["attachment_description"]),
                attachment_type=int(data_params["attachment_type"]),
                content_type=str(data_params["attachment_content_type"]),
                keywords=keywords,
                created_by=created_by,
                modified_by=modified_by,
                content=content)

            ret = self.rr_client.create_attachment(resource_id=resource_id,
                                                   attachment=attachment,
                                                   headers=headers)

            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error creating attachment")
            return self.gateway_error_response(ex)

    def delete_attachment(self, attachment_id):
        try:
            ret = self.rr_client.delete_attachment(attachment_id)
            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error deleting attachment")
            return self.gateway_error_response(ex)

    def get_version_info(self, pack=None):
        import pkg_resources
        pkg_list = ["scioncc"]

        packs = self.config.get_safe(CFG_PREFIX + ".version_packages")
        if packs:
            pkg_list.extend(packs.split(","))

        version = {}
        for package in pkg_list:
            try:
                if pack == "all":
                    pack_deps = pkg_resources.require(package)
                    version.update(
                        {p.project_name: p.version
                         for p in pack_deps})
                else:
                    version[package] = pkg_resources.require(
                        package)[0].version
                # @TODO git versions for current?
            except pkg_resources.DistributionNotFound:
                pass

        try:
            dir_client = DirectoryServiceProcessClient(process=self.process)
            sys_attrs = dir_client.lookup("/System")
            if sys_attrs and isinstance(sys_attrs, dict):
                version.update({
                    k: v
                    for (k, v) in sys_attrs.iteritems()
                    if "version" in k.lower()
                })
        except Exception as ex:
            log.exception("Could not determine system directory attributes")

        if pack and pack != "all":
            version = {k: v for (k, v) in version.iteritems() if k == pack}

        return self.gateway_json_response(version)

    # =========================================================================
    # Security and governance helpers

    def is_trusted_address(self, requesting_address):
        if self.trusted_originators is None:
            return True

        return requesting_address in self.trusted_originators

    def get_governance_info_from_request(self, json_params=None):
        # Default values for governance headers.
        actor_id = DEFAULT_ACTOR_ID
        expiry = DEFAULT_EXPIRY
        authtoken = ""
        user_session = get_auth()
        #if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
        if user_session.get("actor_id", None):
            # Get info from current server session
            # NOTE: Actor id may be inside server session
            expiry = int(user_session.get("valid_until", 0)) * 1000
            if expiry:
                # This was a proper non-token server session authentication
                expiry = str(expiry)
                actor_id = user_session["actor_id"]
                log.info(
                    "Request associated with session actor_id=%s, expiry=%s",
                    actor_id, expiry)
            else:
                # We are just taking the user_id out of the session
                # TODO: Need to check access token here
                expiry = str(expiry)
                if self.token_from_session:
                    actor_id = user_session["actor_id"]
                    log.info(
                        "Request associated with actor's token from session; actor_id=%s, expiry=%s",
                        actor_id, expiry)

        # Developer access using api_key
        if self.develop_mode and "api_key" in request.args and request.args[
                "api_key"]:
            actor_id = str(request.args["api_key"])
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            if 0 < int(expiry) < current_time_millis():
                expiry = str(current_time_millis() + 10000)
                # flask.session["valid_until"] = int(expiry / 1000)
            log.info(
                "Request associated with actor_id=%s, expiry=%s from developer api_key",
                actor_id, expiry)

        # Check in headers for OAuth2 bearer token
        auth_hdr = request.headers.get("authorization", None)
        if auth_hdr:
            valid, req = self.process.oauth.verify_request(
                [self.process.oauth_scope])
            if valid:
                actor_id = flask.g.oauth_user.get("actor_id", "")
                if actor_id:
                    log.info(
                        "Request associated with actor_id=%s, expiry=%s from OAuth token",
                        actor_id, expiry)
                    return actor_id, DEFAULT_EXPIRY

        # Try to find auth token override
        if not authtoken:
            if json_params:
                if "authtoken" in json_params:
                    authtoken = json_params["authtoken"]
            else:
                if "authtoken" in request.args:
                    authtoken = str(request.args["authtoken"])

        # Enable temporary authentication tokens to resolve to actor ids
        if authtoken:
            try:
                token_info = self.idm_client.check_authentication_token(
                    authtoken, headers=self._get_gateway_headers())
                actor_id = token_info.get("actor_id", actor_id)
                expiry = token_info.get("expiry", expiry)
                log.info("Resolved token %s into actor_id=%s expiry=%s",
                         authtoken, actor_id, expiry)
            except NotFound:
                log.info("Provided authentication token not found: %s",
                         authtoken)
            except Unauthorized:
                log.info("Authentication token expired or invalid: %s",
                         authtoken)
            except Exception as ex:
                log.exception("Problem resolving authentication token")

        return actor_id, expiry

    def in_login_whitelist(self, category, svc, op):
        """Returns True if service op is whitelisted for anonymous access"""
        entry = "%s/%s/%s" % (category, svc, op)
        return entry in self.no_login_whitelist

    def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
        # There is no point in looking up an anonymous user - so return default values.
        if ion_actor_id == DEFAULT_ACTOR_ID:
            # Since this is an anonymous request, there really is no expiry associated with it
            if not in_whitelist and self.require_login:
                raise Unauthorized("Anonymous access not permitted")
            else:
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        try:
            user = self.idm_client.read_actor_identity(
                actor_id=ion_actor_id, headers=self._get_gateway_headers())
        except NotFound as e:
            if not in_whitelist and self.require_login:
                # This could be a restart of the system with a new preload.
                # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
                user_session = get_auth()
                if user_session.get("actor_id", None) == ion_actor_id:
                    clear_auth()
                raise Unauthorized("Invalid identity", exc_id="01.10")
            else:
                # If the user isn't found default to anonymous
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        # Need to convert to int first in order to compare against current time.
        try:
            int_expiry = int(expiry)
        except Exception as ex:
            raise Inconsistent(
                "Unable to read the expiry value in the request '%s' as an int"
                % expiry)

        # The user has been validated as being known in the system, so not check the expiry and raise exception if
        # the expiry is not set to 0 and less than the current time.
        if 0 < int_expiry < current_time_millis():
            if not in_whitelist and self.require_login:
                raise Unauthorized("User authentication expired")
            else:
                log.warn("User authentication expired")
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        return ion_actor_id, expiry

    # -------------------------------------------------------------------------
    # Service call (messaging) helpers

    def register_request_callback(self, cb_func):
        if cb_func is None:
            pass
        elif self.request_callback:
            log.warn("Callback already registered")
        self.request_callback = cb_func

    def _call_request_callback(self, action, req_info):
        if not self.request_callback:
            return
        try:
            self.request_callback(action, req_info)
        except Exception:
            log.exception("Error calling request callback")

    def _add_cors_headers(self, resp):
        # Set CORS headers so that a Swagger client on a different domain can read spec
        resp.headers[
            "Access-Control-Allow-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
        resp.headers[
            "Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
        resp.headers["Access-Control-Allow-Origin"] = "*"
        resp.headers[
            "Access-Control-Request-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport,  *"

    def _log_request_start(self, req_type="SG"):
        global req_seqnum
        req_seqnum += 1
        req_info = dict(request_id=req_seqnum,
                        start_time=time.time(),
                        req_type=req_type,
                        req_url=request.url)
        flask.g.req_info = req_info
        webapi_log.info("%s REQUEST (%s) - %s", req_type,
                        req_info["request_id"], request.url)
        self._call_request_callback("start", req_info)

    def _log_request_response(self,
                              content_type,
                              result="",
                              content_length=-1,
                              status_code=200):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_content_type"] = content_type
            req_info["resp_content_length"] = content_length
            req_info["resp_result"] = result
            req_info["resp_status"] = req_info.get("resp_status", status_code)

    def _log_request_error(self, result, status_code):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_error"] = True
            req_info["resp_status"] = status_code
            webapi_log.warn(
                "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"],
                req_info["request_id"], status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID]
                if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])
            self._call_request_callback("error", req_info)
        else:
            webapi_log.warn(
                "REQUEST ERROR (%s%s) - %s: %s", status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID]
                if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])

    def _log_request_end(self):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["end_time"] = time.time()
            webapi_log.info("%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
                            req_info["req_type"], req_info["request_id"],
                            req_info.get("resp_status", ""),
                            req_info["end_time"] - req_info["start_time"],
                            req_info.get("resp_content_length", ""),
                            req_info.get("resp_content_type", ""))
            self._call_request_callback("end", req_info)
        else:
            webapi_log.warn("REQUEST END - missing start info")

    def _get_request_args(self):
        """Extracts service request arguments from HTTP request. Supports various
        methods and forms of encoding. Separates arguments for special parameters
        from service operation parameters.
        Returns a dict with the service request arguments, containing key params
        with the actual values for the service operation parameters.
        """
        str_args = False
        request_args = {}
        if request.method == "POST" or request.method == "PUT":
            # Use only body args and ignore any args from query string
            if request.headers.get("content-type",
                                   "").startswith(CONT_TYPE_JSON):
                # JSON body request
                if request.data:
                    request_args = json_loads(request.data)
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
            elif request.form:
                # Form encoded payload
                if GATEWAY_ARG_JSON in request.form:
                    payload = request.form[GATEWAY_ARG_JSON]
                    request_args = json_loads(str(payload))
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
                else:
                    # Fallback: Directly use form values
                    str_args = True
                    request_args = {
                        GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)
                    }
            else:
                # No args found in body
                request_args = {GATEWAY_ARG_PARAMS: {}}

        elif request.method == "GET":
            str_args = True
            REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
            args_dict = request.args.to_dict(flat=True)
            request_args = {
                k: request.args[k]
                for k in args_dict if k in REQ_ARGS_SPECIAL
            }
            req_params = {
                k: request.args[k]
                for k in args_dict if k not in REQ_ARGS_SPECIAL
            }
            request_args[GATEWAY_ARG_PARAMS] = req_params

        request_args[
            "str_args"] = str_args  # Indicate downstream that args are str (GET or form encoded)
        #log.info("Request args: %s" % request_args)
        return request_args

    def _get_typed_arg_value(self, given_value, param_def, strict):
        """Returns a service operation argument value, based on a given value and param schema definition.
        """
        param_type = param_def["type"]
        if isinstance(given_value, unicode):
            # Convert all unicode to str in UTF-8
            given_value = given_value.encode(
                "utf8")  # Make all unicode into str

        if isinstance(given_value, IonObjectBase) and (
                given_value._get_type() == param_type
                or param_type in given_value._get_extends()):
            return given_value
        elif is_ion_object_dict(given_value) and (
                param_type == "NoneType" or hasattr(objects, param_type)):
            return self.create_ion_object(given_value)
        elif param_type in ("str", "bool", "int", "float", "list", "dict",
                            "NoneType"):
            arg_val = get_typed_value(given_value,
                                      targettype=param_type,
                                      strict=strict)
            return arg_val
        else:
            raise BadRequest("Cannot convert param value to type %s" %
                             param_type)

    def create_parameter_list(self,
                              service_def,
                              operation,
                              request_args,
                              id_param=None):
        """Build service call parameter list dynamically from service operation definition
        """
        service_schema = service_def.schema
        service_op_schema = service_schema["operations"][operation]
        svc_op_param_list = service_op_schema["in_list"]
        svc_params = {}

        if id_param:
            # Magic shorthand: if one argument is given, fill the first service argument
            if svc_op_param_list:
                fill_par = svc_op_param_list[0]
                fill_par_def = service_op_schema["in"][fill_par]
                arg_val = self._get_typed_arg_value(id_param,
                                                    fill_par_def,
                                                    strict=False)
                svc_params[fill_par] = arg_val
            return svc_params

        request_args = request_args or {}
        # Cannot be strict for a URL string query arguments or directly form encoded
        strict_types = False if request_args.get("str_args",
                                                 False) else self.strict_types
        req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
        for param_name in svc_op_param_list:
            param_def = service_op_schema["in"][param_name]
            if param_name in req_op_args:
                arg_val = self._get_typed_arg_value(req_op_args[param_name],
                                                    param_def,
                                                    strict=strict_types)
                svc_params[param_name] = arg_val
        if "timeout" in request_args:
            svc_params["timeout"] = float(request_args["timeout"])

        optional_args = [
            param for param in req_op_args
            if param not in svc_params and param != "timeout"
        ]
        if optional_args and "optional_args" in svc_op_param_list:
            # Only support basic strings for these optional params for now
            svc_params["optional_args"] = {
                arg: str(req_op_args[arg])
                for arg in optional_args
            }

        #log.info("Service params: %s" % svc_params)
        return svc_params

    def _get_gateway_headers(self):
        """Returns the headers that the service gateway uses to make service calls on behalf of itself
         (not a user passing through), e.g. for identity management purposes"""
        return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}

    def get_secure_service_def(self, service_name):
        """Checks whether the service indicated by given service_name exists and/or
        is exposed after white and black listing. Returns service registry entry.
        """
        if self.service_whitelist:
            if service_name not in self.service_whitelist:
                raise Unauthorized("Service access not permitted")
        if self.service_blacklist:
            if service_name in self.service_blacklist:
                raise Unauthorized("Service access not permitted")

        # Retrieve service definition
        target_service = get_service_registry().get_service_by_name(
            service_name)
        if not target_service:
            raise BadRequest("The requested service (%s) is not available" %
                             service_name)
        if not target_service.client:
            raise Inconsistent(
                "Cannot find a client class for the specified service: %s" %
                service_name)
        if not target_service.schema:
            raise Inconsistent(
                "Cannot find a schema for the specified service: %s" %
                service_name)

        return target_service

    def build_message_headers(self, actor_id, expiry):
        """Returns the headers that the service gateway uses to make service calls on behalf of a
        user, based on the user session or request arguments"""
        headers = dict()
        headers[MSG_HEADER_ACTOR] = actor_id
        headers[MSG_HEADER_VALID] = expiry
        req_info = flask.g.get("req_info", None)
        if req_info:
            headers["request-id"] = str(req_info["request_id"])

        # If this is an anonymous requester then there are no roles associated with the request
        if actor_id == DEFAULT_ACTOR_ID:
            headers[MSG_HEADER_ROLES] = dict()
            return headers

        try:
            # Check to see if the user's roles are cached already - keyed by user id
            if self.user_role_cache.has_key(actor_id):
                role_header = self.user_role_cache.get(actor_id)
                if role_header is not None:
                    headers[MSG_HEADER_ROLES] = role_header
                    return headers

            # The user's roles were not cached so hit the datastore to find it.
            role_list = self.org_client.list_actor_roles(
                actor_id, headers=self._get_gateway_headers())
            org_roles = {}
            for role in role_list:
                org_roles.setdefault(role.org_governance_name, []).append(role)

            role_header = get_role_message_headers(org_roles)

            # Cache the roles by user id
            self.user_role_cache.put(actor_id, role_header)

        except Exception:
            role_header = dict(
            )  # Default to empty dict if there is a problem finding roles for the user

        headers[MSG_HEADER_ROLES] = role_header

        return headers

    def create_ion_object(self, object_params):
        """Create and initialize an ION object from a dictionary of parameters coming via HTTP,
        ready to be passed on to services/messaging. The object is validated after creation.
        Note: This is not called for service operation argument signatures
        """
        new_obj = IonObject(object_params["type_"])

        # Iterate over the parameters to add to object; have to do this instead
        # of passing a dict to get around restrictions in object creation on setting _id, _rev params
        for param in object_params:
            self.set_object_field(new_obj, param, object_params.get(param))

        new_obj._validate(
        )  # verify that all of the object fields were set with proper types
        return new_obj

    def set_object_field(self, obj, field, field_val):
        """Recursively set sub object field values.
        TODO: This may be an expensive operation. May also be redundant with object code
        """
        if isinstance(field_val, dict) and field != "kwargs":
            sub_obj = getattr(obj, field)

            if isinstance(sub_obj, IonObjectBase):

                if "type_" in field_val and field_val["type_"] != sub_obj.type_:
                    if issubtype(field_val["type_"], sub_obj.type_):
                        sub_obj = IonObject(field_val["type_"])
                        setattr(obj, field, sub_obj)
                    else:
                        raise Inconsistent(
                            "Unable to walk the field %s - types don't match: %s %s"
                            % (field, sub_obj.type_, field_val["type_"]))

                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field,
                                          field_val.get(sub_field))

            elif isinstance(sub_obj, dict):
                setattr(obj, field, field_val)

            else:
                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field,
                                          field_val.get(sub_field))
        else:
            # type_ already exists in the class.
            if field != "type_":
                setattr(obj, field, field_val)

    # -------------------------------------------------------------------------
    # Response content helpers

    def json_response(self, response_data):
        """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
        """
        resp_obj = json_dumps(response_data,
                              default=encode_ion_object,
                              indent=None if request.is_xhr else 2)
        resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
        if self.develop_mode and (self.set_cors_headers or
                                  ("api_key" in request.args
                                   and request.args["api_key"])):
            self._add_cors_headers(resp)
        self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
        return resp

    def gateway_json_response(self, response_data):
        """Returns the normal service gateway response as JSON or as media in case the response
        is a media response
        """
        if isinstance(response_data, MediaResponse):
            log.info("Media response. Content mimetype:%s",
                     response_data.media_mimetype)
            content = response_data.body
            if response_data.internal_encoding == "base64":
                import base64
                content = base64.decodestring(content)
            elif response_data.internal_encoding == "utf8":
                pass
            resp = self.response_class(content,
                                       response_data.code,
                                       mimetype=response_data.media_mimetype)
            self._log_request_response(response_data.media_mimetype, "raw",
                                       len(content), response_data.code)
            return resp

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(response_data, mimetype=return_mimetype)

        result = {
            GATEWAY_RESPONSE: response_data,
            GATEWAY_STATUS: 200,
        }
        return self.json_response(result)

    def gateway_error_response(self, exc):
        """Forms a service gateway error response.
        Can extract multiple stacks from a multi-tier RPC service call exception
        """
        if hasattr(exc, "get_stacks"):
            # Process potentially multiple stacks.
            full_error, exc_stacks = "", exc.get_stacks()
            for i in range(len(exc_stacks)):
                full_error += exc_stacks[i][0] + "\n"
                if i == 0:
                    full_error += "".join(
                        traceback.format_exception(*sys.exc_info()))
                else:
                    entry = ApplicationException.format_stack(exc_stacks[i][1])
                    full_error += entry + "\n"

            exec_name = exc.__class__.__name__
        else:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            exec_name = exc_type.__name__
            full_error = "".join(traceback.format_exception(*sys.exc_info()))

        status_code = getattr(exc, "status_code", 400)
        if self.log_errors:
            if self.develop_mode:
                if status_code == 401:
                    log.warn("%s: %s", exec_name, exc)
                else:
                    log.error(full_error)
            else:
                if status_code == 401:
                    log.info("%s: %s", exec_name, exc)
                else:
                    log.info(full_error)

        result = {
            GATEWAY_ERROR_EXCEPTION: exec_name,
            GATEWAY_ERROR_MESSAGE: str(exc.message),
            GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or ""
        }
        if self.develop_mode:
            result[GATEWAY_ERROR_TRACE] = full_error

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(result, mimetype=return_mimetype)

        self._log_request_error(result, status_code)

        resp = self.json_response({
            GATEWAY_ERROR: result,
            GATEWAY_STATUS: status_code
        })
        # Q: Should HTTP status be the error code of the exception?
        resp.status_code = status_code
        return resp
Пример #14
0
 def start(self, process_id=None):
     assert self.event_sub is None
     self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
         callback=self._event_callback, origin=process_id, origin_type="DispatchedProcess")
     self.event_sub.start()
Пример #15
0
class ProcessStateWaiter(object):
    def __init__(self):
        self.event_queue = queue.Queue()
        self.event_sub = None

    def start(self, process_id=None):
        assert self.event_sub is None
        self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
            callback=self._event_callback, origin=process_id, origin_type="DispatchedProcess")
        self.event_sub.start()

    def stop(self):
        if self.event_sub:
            self.event_sub.stop()
            self.event_sub = None

    def _event_callback(self, event, *args, **kwargs):
        self.event_queue.put(event)

    def await_state_event(self, pid=None, state=None, timeout=30, strict=False):
        """Wait for a state event for a process.
        if strict is False, allow intermediary events
        """

        start_time = datetime.now()

        assert state in ProcessStateEnum._str_map, "process state %s unknown!" % state
        state_str = ProcessStateEnum._str_map.get(state)

        # stick the pid into a container if it is only one
        if pid is not None and not isinstance(pid, (list, tuple)):
            pid = (pid,)

        while 1:
            if datetime.now() - start_time > timedelta(seconds=timeout):
                raise AssertionError("Waiter timeout! Waited %s seconds for process %s state %s" % (timeout, pid, state_str))
            try:
                event = self.event_queue.get(timeout=timeout)
            except queue.Empty:
                raise AssertionError("Event timeout! Waited %s seconds for process %s state %s" % (timeout, pid, state_str))
            log.debug("Got event: %s", event)

            if (pid is None or event.origin in pid) and (state is None or event.state == state):
                return event

            elif strict:
                raise AssertionError("Got unexpected event %s. Expected state %s for process %s" % (event, state_str, pid))

    def await_many_state_events(self, pids, state=None, timeout=30, strict=False):
        pid_set = set(pids)
        while pid_set:
            event = self.await_state_event(tuple(pid_set), state, timeout=timeout, strict=strict)
            pid_set.remove(event.origin)

    def await_nothing(self, pid=None, timeout=10):
        start_time = datetime.now()

        # stick the pid into a container if it is only one
        if pid is not None and not isinstance(pid, (list, tuple)):
            pid = (pid,)

        while 1:
            timeleft = timedelta(seconds=timeout) - (datetime.now() - start_time)
            timeleft_seconds = timeleft.total_seconds()
            if timeleft_seconds <= 0:
                return
            try:
                event = self.event_queue.get(timeout=timeleft_seconds)
                if pid is None or event.origin in pid:
                    state_str = ProcessStateEnum._str_map.get(event.state, str(event.state))
                    raise AssertionError("Expected no event, but got state %s for process %s" % (state_str, event.origin))

            except queue.Empty:
                return
Пример #16
0
class ProcessDispatcherAggregator(object):
    """ PD aggregator for heartbeat input from containers etc. """
    def __init__(self, pd_core):
        self._pd_core = pd_core
        self.container = self._pd_core.container
        self.registry = self._pd_core.registry

    def start(self):
        # Create our own queue for container heartbeats and broadcasts
        topic = get_safe(self._pd_core.pd_cfg,
                         "aggregator.container_topic") or "bx_containers"
        queue_name = "pd_aggregator_%s_%s" % (
            topic, create_valid_identifier(self.container.id, dot_sub="_"))
        self.sub_cont = Subscriber(binding=topic,
                                   from_name=queue_name,
                                   auto_delete=True,
                                   callback=self._receive_container_info)
        self.sub_cont_gl = spawn(self.sub_cont.listen)
        self.sub_cont.get_ready_event().wait()

        self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
                                       callback=self._receive_event)
        self.evt_sub.add_event_subscription(
            event_type=OT.ProcessLifecycleEvent)
        self.evt_sub_gl = spawn(self.evt_sub.listen)
        self.evt_sub.get_ready_event().wait()

        log.info("PD Aggregator - event and heartbeat subscribers started")

    def stop(self):
        # Stop subscribers
        self.sub_cont.close()
        self.evt_sub.close()

        # Wait for subscribers to finish
        self.sub_cont_gl.join(timeout=2)
        self.sub_cont_gl.kill()
        self.sub_cont_gl = None

        self.evt_sub_gl.join(timeout=2)
        self.evt_sub_gl.kill()
        self.evt_sub_gl = None

    # -------------------------------------------------------------------------

    def _receive_container_info(self, msg, headers, *args):
        log.debug("Got container info %s %s %s", msg, headers, args)
        if not isinstance(msg, ContainerHeartbeat):
            log.warn("Unknown container info format")
            return

        self.registry.register_container(msg.container_id, msg.ts,
                                         EE_STATE_RUNNING, msg.attributes)

    def _receive_event(self, event, *args, **kwargs):
        log.debug("Got event %s %s %s", event, args, kwargs)
        if isinstance(event, ContainerLifecycleEvent):
            if event.sub_type == "START":
                self.registry.register_container(event.origin,
                                                 event.ts_created,
                                                 EE_STATE_RUNNING, {})
            elif event.sub_type == "TERMINATE":
                self.registry.register_container(event.origin,
                                                 event.ts_created,
                                                 EE_STATE_TERMINATED, {})

        elif isinstance(event, ProcessLifecycleEvent):
            proc_info = dict(state=event.state,
                             proc_type=event.process_type,
                             proc_name=event.process_name,
                             resource_id=event.process_resource_id,
                             service_name=event.service_name)
            self.registry.register_process(event.container_id, event.origin,
                                           proc_info)