def op_load_system_policies(cls, calling_process):
        """
        Create the initial set of policy rules for the system.
        To establish clear rule precedence, denying all anonymous access to Org services first
        and then add rules which Permit access to specific operations based on conditions.
        """
        orgms_client = OrgManagementServiceProcessClient(process=calling_process)
        policyms_client = PolicyManagementServiceProcessClient(process=calling_process)

        ion_org = orgms_client.find_org()
        system_actor = get_system_actor()
        log.info('System actor: %s', system_actor._id)

        sa_user_header = get_system_actor_header(system_actor)

        policy_rules_filename = calling_process.CFG.get_safe("bootstrap.initial_policy_rules")
        if not policy_rules_filename:
            raise ContainerConfigError("Policy rules file not configured")
        if not os.path.exists(policy_rules_filename):
            raise ContainerConfigError("Policy rules file does not exist")

        with open(policy_rules_filename, "r") as f:
            policy_rules_yml = f.read()
        policy_rules_cfg = yaml.safe_load(policy_rules_yml)
        if "type" not in policy_rules_cfg or policy_rules_cfg["type"] != "scioncc_policy_rules":
            raise ContainerConfigError("Invalid policy rules file content")

        log.info("Loading %s policy rules", len(policy_rules_cfg["rules"]))
        for rule_cfg in policy_rules_cfg["rules"]:
            rule_name, policy_type, rule_desc = rule_cfg["name"], rule_cfg["policy_type"], rule_cfg.get("description", "")
            if rule_cfg.get("enable") is False:
                log.info("Policy rule %s disabled", rule_name)
                continue
            log.info("Loading policy rule %s (%s)", rule_name, policy_type)
            rule_filename = rule_cfg["rule_def"]
            if not os.path.isabs(rule_filename):
                rule_filename = os.path.join(os.path.dirname(policy_rules_filename), rule_filename)
            with open(rule_filename, "r") as f:
                rule_def = f.read()
            ordinal = rule_cfg.get("ordinal", 0)

            # Create the policy
            if policy_type == "common_service_access":
                policyms_client.create_common_service_access_policy(rule_name, rule_desc, rule_def, ordinal=ordinal,
                                                                    headers=sa_user_header)
            elif policy_type == "service_access":
                service_name = rule_cfg["service_name"]
                policyms_client.create_service_access_policy(service_name, rule_name, rule_desc, rule_def,
                                                             ordinal=ordinal, headers=sa_user_header)
            elif policy_type == "resource_access":
                resource_type, resource_name = rule_cfg["resource_type"], rule_cfg["resource_name"]
                res_ids, _ = calling_process.container.resource_registry.find_resources(
                        restype=resource_type, name=resource_name, id_only=True)
                if res_ids:
                    resource_id = res_ids[0]
                    policyms_client.create_resource_access_policy(resource_id, rule_name, rule_desc, rule_def,
                                                                   ordinal=ordinal, headers=sa_user_header)
            else:
                raise ContainerConfigError("Rule %s has invalid policy type: %s" % (rule_name, policy_type))
Example #2
0
    def on_initial_bootstrap(self, process, config, **kwargs):
        org_ms_client = OrgManagementServiceProcessClient(process=process)
        ex_ms_client = ExchangeManagementServiceProcessClient(process=process)

        system_actor = get_system_actor()
        if not system_actor:
            raise AbortBootstrap("Cannot find system actor")
        system_actor_id = system_actor._id

        # Create root Org: ION
        root_orgname = config.system.root_org
        org = Org(name=root_orgname, description="ION Root Org")
        self.org_id = org_ms_client.create_org(org)

        # Instantiate initial set of User Roles for this Org
        superuser_role = UserRole(
            governance_name=SUPERUSER_ROLE,
            name='Superuser role',
            description='Has all permissions system wide')
        org_ms_client.add_org_role(self.org_id, superuser_role)
        org_ms_client.grant_role(self.org_id, system_actor_id, SUPERUSER_ROLE)

        # Make the ION system agent a manager for the ION Org
        org_ms_client.grant_role(self.org_id, system_actor_id, MODERATOR_ROLE)

        # Create root ExchangeSpace
        system_xs_name = process.container.ex_manager.system_xs_name
        xs = ExchangeSpace(name=system_xs_name, description="ION service XS")
        self.xs_id = ex_ms_client.create_exchange_space(xs, self.org_id)
Example #3
0
    def on_initial_bootstrap(self, process, config, **kwargs):
        org_ms_client = OrgManagementServiceProcessClient(process=process)
        ex_ms_client = ExchangeManagementServiceProcessClient(process=process)

        system_actor = get_system_actor()
        if not system_actor:
            raise AbortBootstrap("Cannot find system actor")
        system_actor_id = system_actor._id

        # Create root Org: ION
        root_orgname = config.system.root_org
        org = Org(name=root_orgname, description="ION Root Org")
        self.org_id = org_ms_client.create_org(org)

        # Instantiate initial set of User Roles for this Org
        superuser_role = UserRole(governance_name=SUPERUSER_ROLE, name='Superuser role',
                                  description='Has all permissions system wide')
        org_ms_client.add_org_role(self.org_id, superuser_role)
        org_ms_client.grant_role(self.org_id, system_actor_id, SUPERUSER_ROLE)

        # Make the ION system agent a manager for the ION Org
        org_ms_client.grant_role(self.org_id, system_actor_id, MODERATOR_ROLE)

        # Create root ExchangeSpace
        system_xs_name = process.container.ex_manager.system_xs_name
        xs = ExchangeSpace(name=system_xs_name, description="ION service XS")
        self.xs_id = ex_ms_client.create_exchange_space(xs, self.org_id)
Example #4
0
    def on_init(self):
        self.process = getattr(self, "process", None) or self

        self.rr = self.container.resource_registry
        self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
        self.org_client = OrgManagementServiceProcessClient(process=self.process)
        self.scion_client = ScionManagementProcessClient(process=self.process)

        self.op = self.CFG.get("op", "load")

        if self.op == "auto":
            self.do_auto_preload()
        elif self.op == "init":
            pass  # Initialization only for explicit call
        elif self.op == "load":
            preload_master = CFG.get_safe("scion.preload.default_master")
            if preload_master:
                self.do_preload_master(preload_master)
        else:
            raise BadRequest("Unknown command")
Example #5
0
    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True
        self.token_from_session = self.config.get_safe(CFG_PREFIX + ".token_from_session") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info("Service Gateway will not check requests against trusted originators since none are configured.")

        # Service screening
        self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)
        self.max_content_length = self.config.get_safe(CFG_PREFIX + ".max_content_length")

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        #maxSize = maximum number of elements to keep in cache
        #maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.request_callback = None
        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True)

        self.rr_client = ResourceRegistryServiceProcessClient(process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
        self.org_client = OrgManagementServiceProcessClient(process=self.process)
Example #6
0
class ServiceGateway(object):
    """
    The Service Gateway exports service routes for a web server via a Flask blueprint.
    The gateway bridges HTTP requests to ION AMQP RPC calls.
    """

    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info("Service Gateway will not check requests against trusted originators since none are configured.")

        # Service screening
        self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        # maxSize = maximum number of elements to keep in cache
        # maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True)

        self.rr_client = ResourceRegistryServiceProcessClient(process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
        self.org_client = OrgManagementServiceProcessClient(process=self.process)

    # -------------------------------------------------------------------------
    # Lifecycle management

    def start(self):
        # Configure  subscriptions for user_cache events
        self.event_subscriber = EventSubscriber(
            event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback
        )
        self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent)
        self.process.add_endpoint(self.event_subscriber)

    def stop(self):
        pass
        # Stop event subscribers - TODO: This hangs
        # self.process.remove_endpoint(self.event_subscriber)

    # -------------------------------------------------------------------------
    # Event subscriber callbacks

    def _event_callback(self, event, *args, **kwargs):
        """ Callback function for receiving Events """
        if isinstance(event, UserRoleModifiedEvent):
            # Event when User Roles are modified
            user_role_event = event
            org_id = user_role_event.origin
            actor_id = user_role_event.actor_id
            role_name = user_role_event.role_name
            log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name))

            # Evict the user and their roles from the cache so that it gets updated with the next call.
            if self.user_role_cache and self.user_role_cache.has_key(actor_id):
                log.debug("Evicting user from the user_role_cache: %s" % actor_id)
                self.user_role_cache.evict(actor_id)
        elif isinstance(event, UserRoleCacheResetEvent):
            # An event is received to clear the user data cache
            self.user_role_cache.clear()

    # -------------------------------------------------------------------------
    # Routes

    def sg_index(self):
        return self.gateway_json_response(SG_IDENTIFICATION)

    def get_service_spec(self, service_name=None, spec_name=None):
        try:
            if not self._swagger_gen:
                raise NotFound("Spec not available")
            if spec_name != "swagger.json":
                raise NotFound("Unknown spec format")

            swagger_json = self._swagger_gen.get_spec(service_name)

            resp = flask.make_response(flask.jsonify(swagger_json))
            self._add_cors_headers(resp)
            return resp

        except Exception as ex:
            return self.gateway_error_response(ex)

    def process_gateway_request(self, service_name=None, operation=None, id_param=None):
        """
        Makes a secure call to a SciON service operation via messaging.
        """
        # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
        self._log_request_start("SVC RPC")
        try:
            result = self._make_service_request(service_name, operation, id_param)
            return self.gateway_json_response(result)

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def rest_gateway_request(self, service_name, res_type, id_param=None):
        """
        Makes a REST style call to a SciON service operation via messaging.
        Get with ID returns the resource, POST without ID creates, PUT with ID updates
        and GET without ID returns the collection.
        """
        self._log_request_start("SVC REST")
        try:
            if not service_name:
                raise BadRequest("Service name missing")
            service_name = str(service_name)
            if not res_type:
                raise BadRequest("Resource type missing")
            res_type = str(res_type)

            if request.method == "GET" and id_param:
                operation = "read_" + res_type
                return self.process_gateway_request(service_name, operation, id_param)
            elif request.method == "GET":
                ion_res_type = "".join(x.title() for x in res_type.split("_"))
                res = self._make_service_request("resource_registry", "find_resources", ion_res_type)
                if len(res) == 2:
                    return self.gateway_json_response(res[0])
                raise BadRequest("Unexpected find_resources result")
            elif request.method == "PUT":
                operation = "update_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                if id_param:
                    obj._id = id_param
                return self.process_gateway_request(service_name, operation, obj)
            elif request.method == "POST":
                operation = "create_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                return self.process_gateway_request(service_name, operation, obj)
            else:
                raise BadRequest("Bad REST request")

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def _extract_payload_data(self):
        request_obj = None
        if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
            if request.data:
                request_obj = json_loads(request.data)
        elif request.form:
            # Form encoded
            if GATEWAY_ARG_JSON in request.form:
                payload = request.form[GATEWAY_ARG_JSON]
                request_obj = json_loads(str(payload))

        if request_obj and is_ion_object_dict(request_obj):
            request_obj = self.create_ion_object(request_obj)

        return request_obj

    def _make_service_request(self, service_name=None, operation=None, id_param=None):
        """
        Executes a secure call to a SciON service operation via messaging.
        """
        if not service_name:
            if self.develop_mode:
                # Return a list of available services
                result = dict(available_services=get_service_registry().services.keys())
                return result
            else:
                raise BadRequest("Service name missing")
        service_name = str(service_name)

        if not operation:
            if self.develop_mode:
                # Return a list of available operations
                result = dict(available_operations=[])
                return result
            else:
                raise BadRequest("Service operation missing")
        operation = str(operation)

        # Apply service white list and black list for initial protection and get service client
        service_def = self.get_secure_service_def(service_name)
        target_client = service_def.client

        # Get service request arguments and operation parameter values request
        req_args = self._get_request_args()

        param_list = self.create_parameter_list(service_def, operation, req_args, id_param)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
        in_login_whitelist = self.in_login_whitelist("request", service_name, operation)
        ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist)
        param_list["headers"] = self.build_message_headers(ion_actor_id, expiry)

        # Make service operation call
        client = target_client(process=self.process)
        method_call = getattr(client, operation)
        result = method_call(**param_list)

        return result

    def get_resource_schema(self, resource_type):
        try:
            # Validate requesting user and expiry and add governance headers
            ion_actor_id, expiry = self.get_governance_info_from_request()
            ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)

            return self.gateway_json_response(get_object_schema(resource_type))

        except Exception as ex:
            return self.gateway_error_response(ex)

    def get_attachment(self, attachment_id):
        try:
            # Create client to interface
            attachment = self.rr_client.read_attachment(attachment_id, include_content=True)

            return self.response_class(attachment.content, mimetype=attachment.content_type)

        except Exception as ex:
            return self.gateway_error_response(ex)

    def create_attachment(self):
        try:
            payload = request.form[GATEWAY_ARG_JSON]
            json_params = json_loads(str(payload))

            actor_id, expiry = self.get_governance_info_from_request(json_params)
            actor_id, expiry = self.validate_request(actor_id, expiry)
            headers = self.build_message_headers(actor_id, expiry)

            data_params = json_params[GATEWAY_ARG_PARAMS]
            resource_id = str(data_params.get("resource_id", ""))
            fil = request.files["file"]
            content = fil.read()

            keywords = []
            keywords_str = data_params.get("keywords", "")
            if keywords_str.strip():
                keywords = [str(x.strip()) for x in keywords_str.split(",")]

            created_by = data_params.get("attachment_created_by", "unknown user")
            modified_by = data_params.get("attachment_modified_by", "unknown user")

            # build attachment
            attachment = Attachment(
                name=str(data_params["attachment_name"]),
                description=str(data_params["attachment_description"]),
                attachment_type=int(data_params["attachment_type"]),
                content_type=str(data_params["attachment_content_type"]),
                keywords=keywords,
                created_by=created_by,
                modified_by=modified_by,
                content=content,
            )

            ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers)

            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error creating attachment")
            return self.gateway_error_response(ex)

    def delete_attachment(self, attachment_id):
        try:
            ret = self.rr_client.delete_attachment(attachment_id)
            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error deleting attachment")
            return self.gateway_error_response(ex)

    def get_version_info(self):
        import pkg_resources

        pkg_list = ["scioncc"]

        version = {}
        for package in pkg_list:
            try:
                version["%s-release" % package] = pkg_resources.require(package)[0].version
                # @TODO git versions for each?
            except pkg_resources.DistributionNotFound:
                pass

        try:
            dir_client = DirectoryServiceProcessClient(process=self.process)
            sys_attrs = dir_client.lookup("/System")
            if sys_attrs and isinstance(sys_attrs, dict):
                version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()})
        except Exception as ex:
            log.exception("Could not determine system directory attributes")

        return self.gateway_json_response(version)

    # =========================================================================
    # Security and governance helpers

    def is_trusted_address(self, requesting_address):
        if self.trusted_originators is None:
            return True

        return requesting_address in self.trusted_originators

    def get_governance_info_from_request(self, json_params=None):
        # Default values for governance headers.
        actor_id = DEFAULT_ACTOR_ID
        expiry = DEFAULT_EXPIRY
        authtoken = ""
        user_session = get_auth()
        if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
            # Get info from current server session
            # NOTE: Actor id may be inside server session
            actor_id = user_session["actor_id"]
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry)

        # Developer access using api_key
        if self.develop_mode and "api_key" in request.args and request.args["api_key"]:
            actor_id = str(request.args["api_key"])
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            if 0 < int(expiry) < current_time_millis():
                expiry = str(current_time_millis() + 10000)
                # flask.session["valid_until"] = int(expiry / 1000)
            log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry)

        # Check in headers for OAuth2 bearer token
        auth_hdr = request.headers.get("authorization", None)
        if auth_hdr:
            valid, req = self.process.oauth.verify_request([self.process.oauth_scope])
            if valid:
                actor_id = flask.g.oauth_user.get("actor_id", "")
                if actor_id:
                    log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry)
                    return actor_id, DEFAULT_EXPIRY

        # Try to find auth token override
        if not authtoken:
            if json_params:
                if "authtoken" in json_params:
                    authtoken = json_params["authtoken"]
            else:
                if "authtoken" in request.args:
                    authtoken = str(request.args["authtoken"])

        # Enable temporary authentication tokens to resolve to actor ids
        if authtoken:
            try:
                token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers())
                actor_id = token_info.get("actor_id", actor_id)
                expiry = token_info.get("expiry", expiry)
                log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry)
            except NotFound:
                log.info("Provided authentication token not found: %s", authtoken)
            except Unauthorized:
                log.info("Authentication token expired or invalid: %s", authtoken)
            except Exception as ex:
                log.exception("Problem resolving authentication token")

        return actor_id, expiry

    def in_login_whitelist(self, category, svc, op):
        """Returns True if service op is whitelisted for anonymous access"""
        entry = "%s/%s/%s" % (category, svc, op)
        return entry in self.no_login_whitelist

    def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
        # There is no point in looking up an anonymous user - so return default values.
        if ion_actor_id == DEFAULT_ACTOR_ID:
            # Since this is an anonymous request, there really is no expiry associated with it
            if not in_whitelist and self.require_login:
                raise Unauthorized("Anonymous access not permitted")
            else:
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        try:
            user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers())
        except NotFound as e:
            if not in_whitelist and self.require_login:
                # This could be a restart of the system with a new preload.
                # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
                user_session = get_auth()
                if user_session.get("actor_id", None) == ion_actor_id:
                    clear_auth()
                raise Unauthorized("Invalid identity", exc_id="01.10")
            else:
                # If the user isn't found default to anonymous
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        # Need to convert to int first in order to compare against current time.
        try:
            int_expiry = int(expiry)
        except Exception as ex:
            raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry)

        # The user has been validated as being known in the system, so not check the expiry and raise exception if
        # the expiry is not set to 0 and less than the current time.
        if 0 < int_expiry < current_time_millis():
            if not in_whitelist and self.require_login:
                raise Unauthorized("User authentication expired")
            else:
                log.warn("User authentication expired")
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        return ion_actor_id, expiry

    # -------------------------------------------------------------------------
    # Service call (messaging) helpers

    def _add_cors_headers(self, resp):
        # Set CORS headers so that a Swagger client on a different domain can read spec
        resp.headers[
            "Access-Control-Allow-Headers"
        ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
        resp.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
        resp.headers["Access-Control-Allow-Origin"] = "*"
        resp.headers[
            "Access-Control-Request-Headers"
        ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport,  *"

    def _log_request_start(self, req_type="SG"):
        global req_seqnum
        req_seqnum += 1
        req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type)
        flask.g.req_info = req_info
        webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url)

    def _log_request_response(self, content_type, result="", content_length=-1, status_code=200):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_content_type"] = content_type
            req_info["resp_content_length"] = content_length
            req_info["resp_result"] = result
            req_info["resp_status"] = req_info.get("resp_status", status_code)

    def _log_request_error(self, result, status_code):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_error"] = True
            req_info["resp_status"] = status_code
            webapi_log.warn(
                "%s REQUEST (%s) ERROR (%s%s) - %s: %s",
                req_info["req_type"],
                req_info["request_id"],
                status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION],
                result[GATEWAY_ERROR_MESSAGE],
            )
        else:
            webapi_log.warn(
                "REQUEST ERROR (%s%s) - %s: %s",
                status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION],
                result[GATEWAY_ERROR_MESSAGE],
            )

    def _log_request_end(self):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["end_time"] = time.time()
            webapi_log.info(
                "%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
                req_info["req_type"],
                req_info["request_id"],
                req_info.get("resp_status", ""),
                req_info["end_time"] - req_info["start_time"],
                req_info.get("resp_content_length", ""),
                req_info.get("resp_content_type", ""),
            )
        else:
            webapi_log.warn("REQUEST END - missing start info")

    def _get_request_args(self):
        """Extracts service request arguments from HTTP request. Supports various
        methods and forms of encoding. Separates arguments for special parameters
        from service operation parameters.
        Returns a dict with the service request arguments, containing key params
        with the actual values for the service operation parameters.
        """
        str_args = False
        request_args = {}
        if request.method == "POST" or request.method == "PUT":
            # Use only body args and ignore any args from query string
            if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
                # JSON body request
                if request.data:
                    request_args = json_loads(request.data)
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
            elif request.form:
                # Form encoded payload
                if GATEWAY_ARG_JSON in request.form:
                    payload = request.form[GATEWAY_ARG_JSON]
                    request_args = json_loads(str(payload))
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
                else:
                    # Fallback: Directly use form values
                    str_args = True
                    request_args = {GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)}
            else:
                # No args found in body
                request_args = {GATEWAY_ARG_PARAMS: {}}

        elif request.method == "GET":
            str_args = True
            REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
            args_dict = request.args.to_dict(flat=True)
            request_args = {k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL}
            req_params = {k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL}
            request_args[GATEWAY_ARG_PARAMS] = req_params

        request_args["str_args"] = str_args  # Indicate downstream that args are str (GET or form encoded)
        # log.info("Request args: %s" % request_args)
        return request_args

    def _get_typed_arg_value(self, given_value, param_def, strict):
        """Returns a service operation argument value, based on a given value and param schema definition.
        """
        param_type = param_def["type"]
        if isinstance(given_value, unicode):
            # Convert all unicode to str in UTF-8
            given_value = given_value.encode("utf8")  # Make all unicode into str

        if isinstance(given_value, IonObjectBase) and (
            given_value._get_type() == param_type or param_type in given_value._get_extends()
        ):
            return given_value
        elif is_ion_object_dict(given_value) and (param_type == "NoneType" or hasattr(objects, param_type)):
            return self.create_ion_object(given_value)
        elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"):
            arg_val = get_typed_value(given_value, targettype=param_type, strict=strict)
            return arg_val
        else:
            raise BadRequest("Cannot convert param value to type %s" % param_type)

    def create_parameter_list(self, service_def, operation, request_args, id_param=None):
        """Build service call parameter list dynamically from service operation definition
        """
        service_schema = service_def.schema
        service_op_schema = service_schema["operations"][operation]
        svc_op_param_list = service_op_schema["in_list"]
        svc_params = {}

        if id_param:
            # Magic shorthand: if one argument is given, fill the first service argument
            if svc_op_param_list:
                fill_par = svc_op_param_list[0]
                fill_par_def = service_op_schema["in"][fill_par]
                arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False)
                svc_params[fill_par] = arg_val
            return svc_params

        request_args = request_args or {}
        # Cannot be strict for a URL string query arguments or directly form encoded
        strict_types = False if request_args.get("str_args", False) else self.strict_types
        req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
        for param_name in svc_op_param_list:
            param_def = service_op_schema["in"][param_name]
            if param_name in req_op_args:
                arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types)
                svc_params[param_name] = arg_val
        if "timeout" in request_args:
            svc_params["timeout"] = float(request_args["timeout"])

        optional_args = [param for param in req_op_args if param not in svc_params and param != "timeout"]
        if optional_args and "optional_args" in svc_op_param_list:
            # Only support basic strings for these optional params for now
            svc_params["optional_args"] = {arg: str(req_op_args[arg]) for arg in optional_args}

        # log.info("Service params: %s" % svc_params)
        return svc_params

    def _get_gateway_headers(self):
        """Returns the headers that the service gateway uses to make service calls on behalf of itself
         (not a user passing through), e.g. for identity management purposes"""
        return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}

    def get_secure_service_def(self, service_name):
        """Checks whether the service indicated by given service_name exists and/or
        is exposed after white and black listing. Returns service registry entry.
        """
        if self.service_whitelist:
            if service_name not in self.service_whitelist:
                raise Unauthorized("Service access not permitted")
        if self.service_blacklist:
            if service_name in self.service_blacklist:
                raise Unauthorized("Service access not permitted")

        # Retrieve service definition
        target_service = get_service_registry().get_service_by_name(service_name)
        if not target_service:
            raise BadRequest("The requested service (%s) is not available" % service_name)
        if not target_service.client:
            raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name)
        if not target_service.schema:
            raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name)

        return target_service

    def build_message_headers(self, actor_id, expiry):
        """Returns the headers that the service gateway uses to make service calls on behalf of a
        user, based on the user session or request arguments"""
        headers = dict()
        headers[MSG_HEADER_ACTOR] = actor_id
        headers[MSG_HEADER_VALID] = expiry
        req_info = flask.g.get("req_info", None)
        if req_info:
            headers["request-id"] = str(req_info["request_id"])

        # If this is an anonymous requester then there are no roles associated with the request
        if actor_id == DEFAULT_ACTOR_ID:
            headers[MSG_HEADER_ROLES] = dict()
            return headers

        try:
            # Check to see if the user's roles are cached already - keyed by user id
            if self.user_role_cache.has_key(actor_id):
                role_header = self.user_role_cache.get(actor_id)
                if role_header is not None:
                    headers[MSG_HEADER_ROLES] = role_header
                    return headers

            # The user's roles were not cached so hit the datastore to find it.
            role_list = self.org_client.list_actor_roles(actor_id, headers=self._get_gateway_headers())
            org_roles = {}
            for role in role_list:
                org_roles.setdefault(role.org_governance_name, []).append(role)

            role_header = get_role_message_headers(org_roles)

            # Cache the roles by user id
            self.user_role_cache.put(actor_id, role_header)

        except Exception:
            role_header = dict()  # Default to empty dict if there is a problem finding roles for the user

        headers[MSG_HEADER_ROLES] = role_header

        return headers

    def create_ion_object(self, object_params):
        """Create and initialize an ION object from a dictionary of parameters coming via HTTP,
        ready to be passed on to services/messaging. The object is validated after creation.
        Note: This is not called for service operation argument signatures
        """
        new_obj = IonObject(object_params["type_"])

        # Iterate over the parameters to add to object; have to do this instead
        # of passing a dict to get around restrictions in object creation on setting _id, _rev params
        for param in object_params:
            self.set_object_field(new_obj, param, object_params.get(param))

        new_obj._validate()  # verify that all of the object fields were set with proper types
        return new_obj

    def set_object_field(self, obj, field, field_val):
        """Recursively set sub object field values.
        TODO: This may be an expensive operation. May also be redundant with object code
        """
        if isinstance(field_val, dict) and field != "kwargs":
            sub_obj = getattr(obj, field)

            if isinstance(sub_obj, IonObjectBase):

                if "type_" in field_val and field_val["type_"] != sub_obj.type_:
                    if issubtype(field_val["type_"], sub_obj.type_):
                        sub_obj = IonObject(field_val["type_"])
                        setattr(obj, field, sub_obj)
                    else:
                        raise Inconsistent(
                            "Unable to walk the field %s - types don't match: %s %s"
                            % (field, sub_obj.type_, field_val["type_"])
                        )

                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))

            elif isinstance(sub_obj, dict):
                setattr(obj, field, field_val)

            else:
                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field, field_val.get(sub_field))
        else:
            # type_ already exists in the class.
            if field != "type_":
                setattr(obj, field, field_val)

    # -------------------------------------------------------------------------
    # Response content helpers

    def json_response(self, response_data):
        """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
        """
        resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2)
        resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
        if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])):
            self._add_cors_headers(resp)
        self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
        return resp

    def gateway_json_response(self, response_data):
        """Returns the normal service gateway response as JSON or as media in case the response
        is a media response
        """
        if isinstance(response_data, MediaResponse):
            log.info("Media response. Content mimetype:%s", response_data.media_mimetype)
            content = response_data.body
            if response_data.internal_encoding == "base64":
                import base64

                content = base64.decodestring(content)
            elif response_data.internal_encoding == "utf8":
                pass
            resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype)
            self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code)
            return resp

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(response_data, mimetype=return_mimetype)

        result = {GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200}
        return self.json_response(result)

    def gateway_error_response(self, exc):
        """Forms a service gateway error response.
        Can extract multiple stacks from a multi-tier RPC service call exception
        """
        if hasattr(exc, "get_stacks"):
            # Process potentially multiple stacks.
            full_error, exc_stacks = "", exc.get_stacks()
            for i in range(len(exc_stacks)):
                full_error += exc_stacks[i][0] + "\n"
                if i == 0:
                    full_error += "".join(traceback.format_exception(*sys.exc_info()))
                else:
                    entry = ApplicationException.format_stack(exc_stacks[i][1])
                    full_error += entry + "\n"

            exec_name = exc.__class__.__name__
        else:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            exec_name = exc_type.__name__
            full_error = "".join(traceback.format_exception(*sys.exc_info()))

        status_code = getattr(exc, "status_code", 400)
        if self.log_errors:
            if self.develop_mode:
                if status_code == 401:
                    log.warn("%s: %s", exec_name, exc)
                else:
                    log.error(full_error)
            else:
                if status_code == 401:
                    log.info("%s: %s", exec_name, exc)
                else:
                    log.info(full_error)

        result = {
            GATEWAY_ERROR_EXCEPTION: exec_name,
            GATEWAY_ERROR_MESSAGE: str(exc.message),
            GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "",
        }
        if self.develop_mode:
            result[GATEWAY_ERROR_TRACE] = full_error

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(result, mimetype=return_mimetype)

        self._log_request_error(result, status_code)

        resp = self.json_response({GATEWAY_ERROR: result, GATEWAY_STATUS: status_code})
        # Q: Should HTTP status be the error code of the exception?
        resp.status_code = status_code
        return resp
Example #7
0
class ScionLoader(ImmediateProcess, Preloader):

    def on_init(self):
        self.process = getattr(self, "process", None) or self

        self.rr = self.container.resource_registry
        self.idm_client = IdentityManagementServiceProcessClient(process=self.process)
        self.org_client = OrgManagementServiceProcessClient(process=self.process)
        self.scion_client = ScionManagementProcessClient(process=self.process)

        self.op = self.CFG.get("op", "load")

        if self.op == "auto":
            self.do_auto_preload()
        elif self.op == "init":
            pass  # Initialization only for explicit call
        elif self.op == "load":
            preload_master = CFG.get_safe("scion.preload.default_master")
            if preload_master:
                self.do_preload_master(preload_master)
        else:
            raise BadRequest("Unknown command")

    def do_preload_core(self):
        client_obj = IonObject(RT.ActorIdentity, name="SciON UI Client",
                               details=IonObject(OT.OAuthClientIdentityDetails, default_scopes="scion"))
        client_actor_id = self.idm_client.create_actor_identity(client_obj)
        client_id = "client:scion_ui"
        self.idm_client.set_actor_credentials(client_actor_id, client_id, "client_secret")
        return True

    def do_auto_preload(self):
        """ Load configured preload scenarios and remembers in directory which ones
        have been loaded. """
        log.info("Executing auto preload")
        preload_entries = self.process.container.directory.lookup("/System/Preload")
        preload_changed = False
        if preload_entries is None:
            preload_entries = dict(scenarios={})
            preload_changed = True

        # "HACK" Preload core to get app specifics in
        if "core" not in preload_entries["scenarios"]:
            preload_entries["scenarios"]["core"] = self._get_dir_entry("core")
            self.do_preload_core()
            preload_changed = True

        preload_scenarios = CFG.get_safe("scion.preload.scenarios") or []
        preload_scope = CFG.get_safe("scion.preload.scope")
        for scenario_info in preload_scenarios:
            scope, scenario = scenario_info
            if scope == "all" or scope == preload_scope:
                if scenario not in preload_entries["scenarios"]:
                    preload_entries["scenarios"][scenario] = self._get_dir_entry(scenario)
                    changed = self.do_preload_master(scenario)
                    preload_changed = preload_changed or changed

        if preload_changed:
            self.process.container.directory.register("/System", "Preload", scenarios=preload_entries["scenarios"])

    def _get_dir_entry(self, scenario):
        return dict(ts=get_ion_ts(), name=scenario, sys_name=get_sys_name())

    def do_preload_master(self, master):
        if CFG.get_safe("scion.preload.enabled", False) is not True:
            return False
        log.info("############## PRELOAD SCION RESOURCES (%s) ##############", master)

        skip_steps = CFG.get_safe("skipsteps")
        if skip_steps:
            skip_steps = skip_steps.split(",")
        self.initialize_preloader(self.process, {})

        if os.path.isdir("res/preload/{}".format(master)):
            self.preload_master("res/preload/{}/actions.yml".format(master), skip_steps=skip_steps)
        elif os.path.exists("res/preload/{}".format(master)):
            self.preload_master("res/preload/{}".format(master), skip_steps=skip_steps)
        else:
            raise BadRequest("Cannot find preload master")

        log.info("############## END PRELOAD ##############")
        return True


    # -------------------------------------------------------------------------
    # Action callbacks

    def _load_preload_CommitBulk(self, action_cfg):
        self.commit_bulk()

    def _load_resource_CORE_CreateRole(self, action_cfg):
        org_name = action_cfg["org"]
        role_name = action_cfg["role"]
        org_obj = self.org_client.find_org(org_name)
        role_obj = IonObject(RT.UserRole, name=role_name, governance_name=role_name,
                             description="Role %s.%s" % (org_name, role_name))

        self.org_client.add_org_role(org_obj._id, role_obj, headers=self._get_system_actor_headers())

    def _load_resource_APP_ScionUser(self, action_cfg):
        actor_obj = self.create_object_from_cfg(action_cfg, RT.ActorIdentity)
        username, password = action_cfg["username"], action_cfg.get("password", None)
        user_alias = action_cfg[KEY_ID]
        if user_alias in self.resource_ids:
            log.info("Skipping existing user '%s', alias=%s", username, user_alias)
            return
        log.debug("Preload user '%s', alias=%s", username, user_alias)

        actor_id = self.scion_client.define_user(
                first_name=actor_obj.details.contact.individual_names_given,
                last_name=actor_obj.details.contact.individual_name_family,
                username=username, password=password, email=actor_obj.details.contact.email,
                attributes=actor_obj.details.profile)
        self.scion_client.update_user_contact(actor_id, actor_obj.details.contact)

        res_obj = self.container.resource_registry.read(actor_id)
        res_obj.alt_ids += ['PRE:'+action_cfg[KEY_ID]]
        self.container.resource_registry.update(res_obj)
        self._register_id(action_cfg[KEY_ID], actor_id, res_obj)

        # Roles
        roles = action_cfg.get("roles", [])
        for role in roles:
            org_name, role_name = role.split(".", 1)
            org_obj = self.org_client.find_org(org_name)
            self.org_client.grant_role(org_obj._id, actor_id, role_name, headers=self._get_system_actor_headers())

    def _load_resource_Instrument(self, action_cfg):
        if action_cfg[KEY_ID] in self.resource_ids:
            return
        res_id = self.basic_resource_create(action_cfg, RT.Instrument, "resource_registry", "create", support_bulk=True)
        self.basic_associations_create(action_cfg, action_cfg[KEY_ID], support_bulk=True)

    def _load_resource_Dataset(self, action_cfg):
        if action_cfg[KEY_ID] in self.resource_ids:
            return
        schema_def = {}
        if "schema_def" in action_cfg:
            schema_def = DataSchemaParser.parse_schema_ref(action_cfg["schema_def"])
        if "schema_override" in action_cfg:
            dict_merge(schema_def, action_cfg["schema_override"], inplace=True)
        res_id = self.basic_resource_create(action_cfg, RT.Dataset, "resource_registry", "create", support_bulk=True,
                                            set_attributes=dict(schema_definition=schema_def))
        self.basic_associations_create(action_cfg, action_cfg[KEY_ID], support_bulk=True)

    def _load_action_StartAgent(self, action_cfg):
        asset_id = action_cfg["asset_id"]
        agent_res_id = self.resource_ids[asset_id]
        self.scion_client.start_agent(agent_res_id)

    def _load_action_StopAgent(self, action_cfg):
        asset_id = action_cfg["asset_id"]
        agent_res_id = self.resource_ids[asset_id]
        self.scion_client.stop_agent(agent_res_id)
Example #8
0
    def op_load_system_policies(cls, calling_process):
        """
        Create the initial set of policy rules for the system.
        To establish clear rule precedence, denying all anonymous access to Org services first
        and then add rules which Permit access to specific operations based on conditions.
        """
        orgms_client = OrgManagementServiceProcessClient(
            process=calling_process)
        policyms_client = PolicyManagementServiceProcessClient(
            process=calling_process)

        ion_org = orgms_client.find_org()
        system_actor = get_system_actor()
        log.info('System actor: %s', system_actor._id)

        sa_user_header = get_system_actor_header(system_actor)

        policy_rules_filename = calling_process.CFG.get_safe(
            "bootstrap.initial_policy_rules")
        if not policy_rules_filename:
            raise ContainerConfigError("Policy rules file not configured")
        if not os.path.exists(policy_rules_filename):
            raise ContainerConfigError("Policy rules file does not exist")

        with open(policy_rules_filename, "r") as f:
            policy_rules_yml = f.read()
        policy_rules_cfg = yaml.safe_load(policy_rules_yml)
        if "type" not in policy_rules_cfg or policy_rules_cfg[
                "type"] != "scioncc_policy_rules":
            raise ContainerConfigError("Invalid policy rules file content")

        log.info("Loading %s policy rules", len(policy_rules_cfg["rules"]))
        for rule_cfg in policy_rules_cfg["rules"]:
            rule_name, policy_type, rule_desc = rule_cfg["name"], rule_cfg[
                "policy_type"], rule_cfg.get("description", "")
            if rule_cfg.get("enable") is False:
                log.info("Policy rule %s disabled", rule_name)
                continue
            log.info("Loading policy rule %s (%s)", rule_name, policy_type)
            rule_filename = rule_cfg["rule_def"]
            if not os.path.isabs(rule_filename):
                rule_filename = os.path.join(
                    os.path.dirname(policy_rules_filename), rule_filename)
            with open(rule_filename, "r") as f:
                rule_def = f.read()
            ordinal = rule_cfg.get("ordinal", 0)

            # Create the policy
            if policy_type == "common_service_access":
                policyms_client.create_common_service_access_policy(
                    rule_name,
                    rule_desc,
                    rule_def,
                    ordinal=ordinal,
                    headers=sa_user_header)
            elif policy_type == "service_access":
                service_name = rule_cfg["service_name"]
                policyms_client.create_service_access_policy(
                    service_name,
                    rule_name,
                    rule_desc,
                    rule_def,
                    ordinal=ordinal,
                    headers=sa_user_header)
            elif policy_type == "resource_access":
                resource_type, resource_name = rule_cfg[
                    "resource_type"], rule_cfg["resource_name"]
                res_ids, _ = calling_process.container.resource_registry.find_resources(
                    restype=resource_type, name=resource_name, id_only=True)
                if res_ids:
                    resource_id = res_ids[0]
                    policyms_client.create_resource_access_policy(
                        resource_id,
                        rule_name,
                        rule_desc,
                        rule_def,
                        ordinal=ordinal,
                        headers=sa_user_header)
            else:
                raise ContainerConfigError(
                    "Rule %s has invalid policy type: %s" %
                    (rule_name, policy_type))
Example #9
0
    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX +
                                                 ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX +
                                                  ".require_login") is True
        self.token_from_session = self.config.get_safe(
            CFG_PREFIX + ".token_from_session") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX +
                                                        ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info(
                "Service Gateway will not check requests against trusted originators since none are configured."
            )

        # Service screening
        self.service_blacklist = self.config.get_safe(
            CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(
            CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(
            self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX +
                                                     ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX +
                                                 ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX +
                                                ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(
            CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        #maxSize = maximum number of elements to keep in cache
        #maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.request_callback = None
        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors",
                                               True)

        self.rr_client = ResourceRegistryServiceProcessClient(
            process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(
            process=self.process)
        self.org_client = OrgManagementServiceProcessClient(
            process=self.process)
Example #10
0
class ServiceGateway(object):
    """
    The Service Gateway exports service routes for a web server via a Flask blueprint.
    The gateway bridges HTTP requests to ION AMQP RPC calls.
    """
    def __init__(self, process, config, response_class):
        global sg_instance
        sg_instance = self

        self.name = "service_gateway"
        self.process = process
        self.config = config
        self.response_class = response_class

        self.gateway_base_url = process.gateway_base_url
        self.develop_mode = self.config.get_safe(CFG_PREFIX +
                                                 ".develop_mode") is True
        self.require_login = self.config.get_safe(CFG_PREFIX +
                                                  ".require_login") is True
        self.token_from_session = self.config.get_safe(
            CFG_PREFIX + ".token_from_session") is True

        # Optional list of trusted originators can be specified in config.
        self.trusted_originators = self.config.get_safe(CFG_PREFIX +
                                                        ".trusted_originators")
        if not self.trusted_originators:
            self.trusted_originators = None
            log.info(
                "Service Gateway will not check requests against trusted originators since none are configured."
            )

        # Service screening
        self.service_blacklist = self.config.get_safe(
            CFG_PREFIX + ".service_blacklist") or []
        self.service_whitelist = self.config.get_safe(
            CFG_PREFIX + ".service_whitelist") or []
        self.no_login_whitelist = set(
            self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or [])

        self.set_cors_headers = self.config.get_safe(CFG_PREFIX +
                                                     ".set_cors") is True
        self.strict_types = self.config.get_safe(CFG_PREFIX +
                                                 ".strict_types") is True

        # Swagger spec generation support
        self.swagger_cfg = self.config.get_safe(CFG_PREFIX +
                                                ".swagger_spec") or {}
        self._swagger_gen = None
        if self.swagger_cfg.get("enable", None) is True:
            self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg)

        # Get the user_cache_size
        self.user_cache_size = self.config.get_safe(
            CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE)

        # Initialize an LRU Cache to keep user roles cached for performance reasons
        #maxSize = maximum number of elements to keep in cache
        #maxAgeMs = oldest entry to keep
        self.user_role_cache = LRUCache(self.user_cache_size, 0, 0)

        self.request_callback = None
        self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors",
                                               True)

        self.rr_client = ResourceRegistryServiceProcessClient(
            process=self.process)
        self.idm_client = IdentityManagementServiceProcessClient(
            process=self.process)
        self.org_client = OrgManagementServiceProcessClient(
            process=self.process)

    # -------------------------------------------------------------------------
    # Lifecycle management

    def start(self):
        # Configure  subscriptions for user_cache events
        self.event_subscriber = EventSubscriber(
            event_type=OT.UserRoleModifiedEvent,
            origin_type="Org",
            callback=self._event_callback)
        self.event_subscriber.add_event_subscription(
            event_type=OT.UserRoleCacheResetEvent)
        self.process.add_endpoint(self.event_subscriber)

    def stop(self):
        pass
        # Stop event subscribers - TODO: This hangs
        #self.process.remove_endpoint(self.event_subscriber)

    # -------------------------------------------------------------------------
    # Event subscriber callbacks

    def _event_callback(self, event, *args, **kwargs):
        """ Callback function for receiving Events """
        if isinstance(event, UserRoleModifiedEvent):
            # Event when User Roles are modified
            user_role_event = event
            org_id = user_role_event.origin
            actor_id = user_role_event.actor_id
            role_name = user_role_event.role_name
            log.debug("User Role modified: %s %s %s" %
                      (org_id, actor_id, role_name))

            # Evict the user and their roles from the cache so that it gets updated with the next call.
            if self.user_role_cache and self.user_role_cache.has_key(actor_id):
                log.debug("Evicting user from the user_role_cache: %s" %
                          actor_id)
                self.user_role_cache.evict(actor_id)
        elif isinstance(event, UserRoleCacheResetEvent):
            # An event is received to clear the user data cache
            self.user_role_cache.clear()

    # -------------------------------------------------------------------------
    # Routes

    def sg_index(self):
        return self.gateway_json_response(SG_IDENTIFICATION)

    def get_service_spec(self, service_name=None, spec_name=None):
        try:
            if not self._swagger_gen:
                raise NotFound("Spec not available")
            if spec_name != "swagger.json":
                raise NotFound("Unknown spec format")

            swagger_json = self._swagger_gen.get_spec(service_name)

            resp = flask.make_response(flask.jsonify(swagger_json))
            self._add_cors_headers(resp)
            return resp

        except Exception as ex:
            return self.gateway_error_response(ex)

    def process_gateway_request(self,
                                service_name=None,
                                operation=None,
                                id_param=None):
        """
        Makes a secure call to a SciON service operation via messaging.
        """
        # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text)
        self._log_request_start("SVC RPC")
        try:
            result = self._make_service_request(service_name, operation,
                                                id_param)
            return self.gateway_json_response(result)

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def rest_gateway_request(self, service_name, res_type, id_param=None):
        """
        Makes a REST style call to a SciON service operation via messaging.
        Get with ID returns the resource, POST without ID creates, PUT with ID updates
        and GET without ID returns the collection.
        """
        self._log_request_start("SVC REST")
        try:
            if not service_name:
                raise BadRequest("Service name missing")
            service_name = str(service_name)
            if not res_type:
                raise BadRequest("Resource type missing")
            res_type = str(res_type)

            if request.method == "GET" and id_param:
                operation = "read_" + res_type
                return self.process_gateway_request(service_name, operation,
                                                    id_param)
            elif request.method == "GET":
                ion_res_type = "".join(x.title() for x in res_type.split('_'))
                res = self._make_service_request("resource_registry",
                                                 "find_resources",
                                                 ion_res_type)
                if len(res) == 2:
                    return self.gateway_json_response(res[0])
                raise BadRequest("Unexpected find_resources result")
            elif request.method == "PUT":
                operation = "update_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                if id_param:
                    obj._id = id_param
                return self.process_gateway_request(service_name, operation,
                                                    obj)
            elif request.method == "POST":
                operation = "create_" + res_type
                obj = self._extract_payload_data()
                if not obj:
                    raise BadRequest("Argument object not found")
                return self.process_gateway_request(service_name, operation,
                                                    obj)
            else:
                raise BadRequest("Bad REST request")

        except Exception as ex:
            return self.gateway_error_response(ex)

        finally:
            self._log_request_end()

    def _extract_payload_data(self):
        request_obj = None
        if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON):
            if request.data:
                request_obj = json_loads(request.data)
        elif request.form:
            # Form encoded
            if GATEWAY_ARG_JSON in request.form:
                payload = request.form[GATEWAY_ARG_JSON]
                request_obj = json_loads(str(payload))

        if request_obj and is_ion_object_dict(request_obj):
            request_obj = self.create_ion_object(request_obj)

        return request_obj

    def _make_service_request(self,
                              service_name=None,
                              operation=None,
                              id_param=None):
        """
        Executes a secure call to a SciON service operation via messaging.
        """
        if not service_name:
            if self.develop_mode:
                # Return a list of available services
                result = dict(
                    available_services=get_service_registry().services.keys())
                return result
            else:
                raise BadRequest("Service name missing")
        service_name = str(service_name)

        if not operation:
            if self.develop_mode:
                # Return a list of available operations
                result = dict(available_operations=[])
                return result
            else:
                raise BadRequest("Service operation missing")
        operation = str(operation)

        # Apply service white list and black list for initial protection and get service client
        service_def = self.get_secure_service_def(service_name)
        target_client = service_def.client

        # Get service request arguments and operation parameter values request
        req_args = self._get_request_args()

        param_list = self.create_parameter_list(service_def, operation,
                                                req_args, id_param)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
        in_login_whitelist = self.in_login_whitelist("request", service_name,
                                                     operation)
        ion_actor_id, expiry = self.validate_request(
            ion_actor_id, expiry, in_whitelist=in_login_whitelist)
        param_list["headers"] = self.build_message_headers(
            ion_actor_id, expiry)

        # Make service operation call
        client = target_client(process=self.process)
        method_call = getattr(client, operation)
        result = method_call(**param_list)

        return result

    def get_resource_schema(self, resource_type):
        try:
            # Validate requesting user and expiry and add governance headers
            ion_actor_id, expiry = self.get_governance_info_from_request()
            ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry)

            return self.gateway_json_response(get_object_schema(resource_type))

        except Exception as ex:
            return self.gateway_error_response(ex)

    def get_attachment(self, attachment_id):
        try:
            # Create client to interface
            attachment = self.rr_client.read_attachment(attachment_id,
                                                        include_content=True)

            return self.response_class(attachment.content,
                                       mimetype=attachment.content_type)

        except Exception as ex:
            return self.gateway_error_response(ex)

    def create_attachment(self):
        try:
            payload = request.form[GATEWAY_ARG_JSON]
            json_params = json_loads(str(payload))

            actor_id, expiry = self.get_governance_info_from_request(
                json_params)
            actor_id, expiry = self.validate_request(actor_id, expiry)
            headers = self.build_message_headers(actor_id, expiry)

            data_params = json_params[GATEWAY_ARG_PARAMS]
            resource_id = str(data_params.get("resource_id", ""))
            fil = request.files["file"]
            content = fil.read()

            keywords = []
            keywords_str = data_params.get("keywords", "")
            if keywords_str.strip():
                keywords = [str(x.strip()) for x in keywords_str.split(",")]

            created_by = data_params.get("attachment_created_by",
                                         "unknown user")
            modified_by = data_params.get("attachment_modified_by",
                                          "unknown user")

            # build attachment
            attachment = Attachment(
                name=str(data_params["attachment_name"]),
                description=str(data_params["attachment_description"]),
                attachment_type=int(data_params["attachment_type"]),
                content_type=str(data_params["attachment_content_type"]),
                keywords=keywords,
                created_by=created_by,
                modified_by=modified_by,
                content=content)

            ret = self.rr_client.create_attachment(resource_id=resource_id,
                                                   attachment=attachment,
                                                   headers=headers)

            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error creating attachment")
            return self.gateway_error_response(ex)

    def delete_attachment(self, attachment_id):
        try:
            ret = self.rr_client.delete_attachment(attachment_id)
            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error deleting attachment")
            return self.gateway_error_response(ex)

    def get_version_info(self, pack=None):
        import pkg_resources
        pkg_list = ["scioncc"]

        packs = self.config.get_safe(CFG_PREFIX + ".version_packages")
        if packs:
            pkg_list.extend(packs.split(","))

        version = {}
        for package in pkg_list:
            try:
                if pack == "all":
                    pack_deps = pkg_resources.require(package)
                    version.update(
                        {p.project_name: p.version
                         for p in pack_deps})
                else:
                    version[package] = pkg_resources.require(
                        package)[0].version
                # @TODO git versions for current?
            except pkg_resources.DistributionNotFound:
                pass

        try:
            dir_client = DirectoryServiceProcessClient(process=self.process)
            sys_attrs = dir_client.lookup("/System")
            if sys_attrs and isinstance(sys_attrs, dict):
                version.update({
                    k: v
                    for (k, v) in sys_attrs.iteritems()
                    if "version" in k.lower()
                })
        except Exception as ex:
            log.exception("Could not determine system directory attributes")

        if pack and pack != "all":
            version = {k: v for (k, v) in version.iteritems() if k == pack}

        return self.gateway_json_response(version)

    # =========================================================================
    # Security and governance helpers

    def is_trusted_address(self, requesting_address):
        if self.trusted_originators is None:
            return True

        return requesting_address in self.trusted_originators

    def get_governance_info_from_request(self, json_params=None):
        # Default values for governance headers.
        actor_id = DEFAULT_ACTOR_ID
        expiry = DEFAULT_EXPIRY
        authtoken = ""
        user_session = get_auth()
        #if user_session.get("actor_id", None) and user_session.get("valid_until", 0):
        if user_session.get("actor_id", None):
            # Get info from current server session
            # NOTE: Actor id may be inside server session
            expiry = int(user_session.get("valid_until", 0)) * 1000
            if expiry:
                # This was a proper non-token server session authentication
                expiry = str(expiry)
                actor_id = user_session["actor_id"]
                log.info(
                    "Request associated with session actor_id=%s, expiry=%s",
                    actor_id, expiry)
            else:
                # We are just taking the user_id out of the session
                # TODO: Need to check access token here
                expiry = str(expiry)
                if self.token_from_session:
                    actor_id = user_session["actor_id"]
                    log.info(
                        "Request associated with actor's token from session; actor_id=%s, expiry=%s",
                        actor_id, expiry)

        # Developer access using api_key
        if self.develop_mode and "api_key" in request.args and request.args[
                "api_key"]:
            actor_id = str(request.args["api_key"])
            expiry = str(int(user_session.get("valid_until", 0)) * 1000)
            if 0 < int(expiry) < current_time_millis():
                expiry = str(current_time_millis() + 10000)
                # flask.session["valid_until"] = int(expiry / 1000)
            log.info(
                "Request associated with actor_id=%s, expiry=%s from developer api_key",
                actor_id, expiry)

        # Check in headers for OAuth2 bearer token
        auth_hdr = request.headers.get("authorization", None)
        if auth_hdr:
            valid, req = self.process.oauth.verify_request(
                [self.process.oauth_scope])
            if valid:
                actor_id = flask.g.oauth_user.get("actor_id", "")
                if actor_id:
                    log.info(
                        "Request associated with actor_id=%s, expiry=%s from OAuth token",
                        actor_id, expiry)
                    return actor_id, DEFAULT_EXPIRY

        # Try to find auth token override
        if not authtoken:
            if json_params:
                if "authtoken" in json_params:
                    authtoken = json_params["authtoken"]
            else:
                if "authtoken" in request.args:
                    authtoken = str(request.args["authtoken"])

        # Enable temporary authentication tokens to resolve to actor ids
        if authtoken:
            try:
                token_info = self.idm_client.check_authentication_token(
                    authtoken, headers=self._get_gateway_headers())
                actor_id = token_info.get("actor_id", actor_id)
                expiry = token_info.get("expiry", expiry)
                log.info("Resolved token %s into actor_id=%s expiry=%s",
                         authtoken, actor_id, expiry)
            except NotFound:
                log.info("Provided authentication token not found: %s",
                         authtoken)
            except Unauthorized:
                log.info("Authentication token expired or invalid: %s",
                         authtoken)
            except Exception as ex:
                log.exception("Problem resolving authentication token")

        return actor_id, expiry

    def in_login_whitelist(self, category, svc, op):
        """Returns True if service op is whitelisted for anonymous access"""
        entry = "%s/%s/%s" % (category, svc, op)
        return entry in self.no_login_whitelist

    def validate_request(self, ion_actor_id, expiry, in_whitelist=False):
        # There is no point in looking up an anonymous user - so return default values.
        if ion_actor_id == DEFAULT_ACTOR_ID:
            # Since this is an anonymous request, there really is no expiry associated with it
            if not in_whitelist and self.require_login:
                raise Unauthorized("Anonymous access not permitted")
            else:
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        try:
            user = self.idm_client.read_actor_identity(
                actor_id=ion_actor_id, headers=self._get_gateway_headers())
        except NotFound as e:
            if not in_whitelist and self.require_login:
                # This could be a restart of the system with a new preload.
                # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret
                user_session = get_auth()
                if user_session.get("actor_id", None) == ion_actor_id:
                    clear_auth()
                raise Unauthorized("Invalid identity", exc_id="01.10")
            else:
                # If the user isn't found default to anonymous
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        # Need to convert to int first in order to compare against current time.
        try:
            int_expiry = int(expiry)
        except Exception as ex:
            raise Inconsistent(
                "Unable to read the expiry value in the request '%s' as an int"
                % expiry)

        # The user has been validated as being known in the system, so not check the expiry and raise exception if
        # the expiry is not set to 0 and less than the current time.
        if 0 < int_expiry < current_time_millis():
            if not in_whitelist and self.require_login:
                raise Unauthorized("User authentication expired")
            else:
                log.warn("User authentication expired")
                return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY

        return ion_actor_id, expiry

    # -------------------------------------------------------------------------
    # Service call (messaging) helpers

    def register_request_callback(self, cb_func):
        if cb_func is None:
            pass
        elif self.request_callback:
            log.warn("Callback already registered")
        self.request_callback = cb_func

    def _call_request_callback(self, action, req_info):
        if not self.request_callback:
            return
        try:
            self.request_callback(action, req_info)
        except Exception:
            log.exception("Error calling request callback")

    def _add_cors_headers(self, resp):
        # Set CORS headers so that a Swagger client on a different domain can read spec
        resp.headers[
            "Access-Control-Allow-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *"
        resp.headers[
            "Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT"
        resp.headers["Access-Control-Allow-Origin"] = "*"
        resp.headers[
            "Access-Control-Request-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport,  *"

    def _log_request_start(self, req_type="SG"):
        global req_seqnum
        req_seqnum += 1
        req_info = dict(request_id=req_seqnum,
                        start_time=time.time(),
                        req_type=req_type,
                        req_url=request.url)
        flask.g.req_info = req_info
        webapi_log.info("%s REQUEST (%s) - %s", req_type,
                        req_info["request_id"], request.url)
        self._call_request_callback("start", req_info)

    def _log_request_response(self,
                              content_type,
                              result="",
                              content_length=-1,
                              status_code=200):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_content_type"] = content_type
            req_info["resp_content_length"] = content_length
            req_info["resp_result"] = result
            req_info["resp_status"] = req_info.get("resp_status", status_code)

    def _log_request_error(self, result, status_code):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["resp_error"] = True
            req_info["resp_status"] = status_code
            webapi_log.warn(
                "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"],
                req_info["request_id"], status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID]
                if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])
            self._call_request_callback("error", req_info)
        else:
            webapi_log.warn(
                "REQUEST ERROR (%s%s) - %s: %s", status_code,
                "/id=" + result[GATEWAY_ERROR_EXCID]
                if result[GATEWAY_ERROR_EXCID] else "",
                result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE])

    def _log_request_end(self):
        req_info = flask.g.get("req_info", None)
        if req_info:
            req_info["end_time"] = time.time()
            webapi_log.info("%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s",
                            req_info["req_type"], req_info["request_id"],
                            req_info.get("resp_status", ""),
                            req_info["end_time"] - req_info["start_time"],
                            req_info.get("resp_content_length", ""),
                            req_info.get("resp_content_type", ""))
            self._call_request_callback("end", req_info)
        else:
            webapi_log.warn("REQUEST END - missing start info")

    def _get_request_args(self):
        """Extracts service request arguments from HTTP request. Supports various
        methods and forms of encoding. Separates arguments for special parameters
        from service operation parameters.
        Returns a dict with the service request arguments, containing key params
        with the actual values for the service operation parameters.
        """
        str_args = False
        request_args = {}
        if request.method == "POST" or request.method == "PUT":
            # Use only body args and ignore any args from query string
            if request.headers.get("content-type",
                                   "").startswith(CONT_TYPE_JSON):
                # JSON body request
                if request.data:
                    request_args = json_loads(request.data)
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
            elif request.form:
                # Form encoded payload
                if GATEWAY_ARG_JSON in request.form:
                    payload = request.form[GATEWAY_ARG_JSON]
                    request_args = json_loads(str(payload))
                    if GATEWAY_ARG_PARAMS not in request_args:
                        # Magic fallback: Directly use JSON first level as args if params key not present
                        request_args = {GATEWAY_ARG_PARAMS: request_args}
                else:
                    # Fallback: Directly use form values
                    str_args = True
                    request_args = {
                        GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)
                    }
            else:
                # No args found in body
                request_args = {GATEWAY_ARG_PARAMS: {}}

        elif request.method == "GET":
            str_args = True
            REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"}
            args_dict = request.args.to_dict(flat=True)
            request_args = {
                k: request.args[k]
                for k in args_dict if k in REQ_ARGS_SPECIAL
            }
            req_params = {
                k: request.args[k]
                for k in args_dict if k not in REQ_ARGS_SPECIAL
            }
            request_args[GATEWAY_ARG_PARAMS] = req_params

        request_args[
            "str_args"] = str_args  # Indicate downstream that args are str (GET or form encoded)
        #log.info("Request args: %s" % request_args)
        return request_args

    def _get_typed_arg_value(self, given_value, param_def, strict):
        """Returns a service operation argument value, based on a given value and param schema definition.
        """
        param_type = param_def["type"]
        if isinstance(given_value, unicode):
            # Convert all unicode to str in UTF-8
            given_value = given_value.encode(
                "utf8")  # Make all unicode into str

        if isinstance(given_value, IonObjectBase) and (
                given_value._get_type() == param_type
                or param_type in given_value._get_extends()):
            return given_value
        elif is_ion_object_dict(given_value) and (
                param_type == "NoneType" or hasattr(objects, param_type)):
            return self.create_ion_object(given_value)
        elif param_type in ("str", "bool", "int", "float", "list", "dict",
                            "NoneType"):
            arg_val = get_typed_value(given_value,
                                      targettype=param_type,
                                      strict=strict)
            return arg_val
        else:
            raise BadRequest("Cannot convert param value to type %s" %
                             param_type)

    def create_parameter_list(self,
                              service_def,
                              operation,
                              request_args,
                              id_param=None):
        """Build service call parameter list dynamically from service operation definition
        """
        service_schema = service_def.schema
        service_op_schema = service_schema["operations"][operation]
        svc_op_param_list = service_op_schema["in_list"]
        svc_params = {}

        if id_param:
            # Magic shorthand: if one argument is given, fill the first service argument
            if svc_op_param_list:
                fill_par = svc_op_param_list[0]
                fill_par_def = service_op_schema["in"][fill_par]
                arg_val = self._get_typed_arg_value(id_param,
                                                    fill_par_def,
                                                    strict=False)
                svc_params[fill_par] = arg_val
            return svc_params

        request_args = request_args or {}
        # Cannot be strict for a URL string query arguments or directly form encoded
        strict_types = False if request_args.get("str_args",
                                                 False) else self.strict_types
        req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {}
        for param_name in svc_op_param_list:
            param_def = service_op_schema["in"][param_name]
            if param_name in req_op_args:
                arg_val = self._get_typed_arg_value(req_op_args[param_name],
                                                    param_def,
                                                    strict=strict_types)
                svc_params[param_name] = arg_val
        if "timeout" in request_args:
            svc_params["timeout"] = float(request_args["timeout"])

        optional_args = [
            param for param in req_op_args
            if param not in svc_params and param != "timeout"
        ]
        if optional_args and "optional_args" in svc_op_param_list:
            # Only support basic strings for these optional params for now
            svc_params["optional_args"] = {
                arg: str(req_op_args[arg])
                for arg in optional_args
            }

        #log.info("Service params: %s" % svc_params)
        return svc_params

    def _get_gateway_headers(self):
        """Returns the headers that the service gateway uses to make service calls on behalf of itself
         (not a user passing through), e.g. for identity management purposes"""
        return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY}

    def get_secure_service_def(self, service_name):
        """Checks whether the service indicated by given service_name exists and/or
        is exposed after white and black listing. Returns service registry entry.
        """
        if self.service_whitelist:
            if service_name not in self.service_whitelist:
                raise Unauthorized("Service access not permitted")
        if self.service_blacklist:
            if service_name in self.service_blacklist:
                raise Unauthorized("Service access not permitted")

        # Retrieve service definition
        target_service = get_service_registry().get_service_by_name(
            service_name)
        if not target_service:
            raise BadRequest("The requested service (%s) is not available" %
                             service_name)
        if not target_service.client:
            raise Inconsistent(
                "Cannot find a client class for the specified service: %s" %
                service_name)
        if not target_service.schema:
            raise Inconsistent(
                "Cannot find a schema for the specified service: %s" %
                service_name)

        return target_service

    def build_message_headers(self, actor_id, expiry):
        """Returns the headers that the service gateway uses to make service calls on behalf of a
        user, based on the user session or request arguments"""
        headers = dict()
        headers[MSG_HEADER_ACTOR] = actor_id
        headers[MSG_HEADER_VALID] = expiry
        req_info = flask.g.get("req_info", None)
        if req_info:
            headers["request-id"] = str(req_info["request_id"])

        # If this is an anonymous requester then there are no roles associated with the request
        if actor_id == DEFAULT_ACTOR_ID:
            headers[MSG_HEADER_ROLES] = dict()
            return headers

        try:
            # Check to see if the user's roles are cached already - keyed by user id
            if self.user_role_cache.has_key(actor_id):
                role_header = self.user_role_cache.get(actor_id)
                if role_header is not None:
                    headers[MSG_HEADER_ROLES] = role_header
                    return headers

            # The user's roles were not cached so hit the datastore to find it.
            role_list = self.org_client.list_actor_roles(
                actor_id, headers=self._get_gateway_headers())
            org_roles = {}
            for role in role_list:
                org_roles.setdefault(role.org_governance_name, []).append(role)

            role_header = get_role_message_headers(org_roles)

            # Cache the roles by user id
            self.user_role_cache.put(actor_id, role_header)

        except Exception:
            role_header = dict(
            )  # Default to empty dict if there is a problem finding roles for the user

        headers[MSG_HEADER_ROLES] = role_header

        return headers

    def create_ion_object(self, object_params):
        """Create and initialize an ION object from a dictionary of parameters coming via HTTP,
        ready to be passed on to services/messaging. The object is validated after creation.
        Note: This is not called for service operation argument signatures
        """
        new_obj = IonObject(object_params["type_"])

        # Iterate over the parameters to add to object; have to do this instead
        # of passing a dict to get around restrictions in object creation on setting _id, _rev params
        for param in object_params:
            self.set_object_field(new_obj, param, object_params.get(param))

        new_obj._validate(
        )  # verify that all of the object fields were set with proper types
        return new_obj

    def set_object_field(self, obj, field, field_val):
        """Recursively set sub object field values.
        TODO: This may be an expensive operation. May also be redundant with object code
        """
        if isinstance(field_val, dict) and field != "kwargs":
            sub_obj = getattr(obj, field)

            if isinstance(sub_obj, IonObjectBase):

                if "type_" in field_val and field_val["type_"] != sub_obj.type_:
                    if issubtype(field_val["type_"], sub_obj.type_):
                        sub_obj = IonObject(field_val["type_"])
                        setattr(obj, field, sub_obj)
                    else:
                        raise Inconsistent(
                            "Unable to walk the field %s - types don't match: %s %s"
                            % (field, sub_obj.type_, field_val["type_"]))

                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field,
                                          field_val.get(sub_field))

            elif isinstance(sub_obj, dict):
                setattr(obj, field, field_val)

            else:
                for sub_field in field_val:
                    self.set_object_field(sub_obj, sub_field,
                                          field_val.get(sub_field))
        else:
            # type_ already exists in the class.
            if field != "type_":
                setattr(obj, field, field_val)

    # -------------------------------------------------------------------------
    # Response content helpers

    def json_response(self, response_data):
        """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects
        """
        resp_obj = json_dumps(response_data,
                              default=encode_ion_object,
                              indent=None if request.is_xhr else 2)
        resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON)
        if self.develop_mode and (self.set_cors_headers or
                                  ("api_key" in request.args
                                   and request.args["api_key"])):
            self._add_cors_headers(resp)
        self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj))
        return resp

    def gateway_json_response(self, response_data):
        """Returns the normal service gateway response as JSON or as media in case the response
        is a media response
        """
        if isinstance(response_data, MediaResponse):
            log.info("Media response. Content mimetype:%s",
                     response_data.media_mimetype)
            content = response_data.body
            if response_data.internal_encoding == "base64":
                import base64
                content = base64.decodestring(content)
            elif response_data.internal_encoding == "utf8":
                pass
            resp = self.response_class(content,
                                       response_data.code,
                                       mimetype=response_data.media_mimetype)
            self._log_request_response(response_data.media_mimetype, "raw",
                                       len(content), response_data.code)
            return resp

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(response_data, mimetype=return_mimetype)

        result = {
            GATEWAY_RESPONSE: response_data,
            GATEWAY_STATUS: 200,
        }
        return self.json_response(result)

    def gateway_error_response(self, exc):
        """Forms a service gateway error response.
        Can extract multiple stacks from a multi-tier RPC service call exception
        """
        if hasattr(exc, "get_stacks"):
            # Process potentially multiple stacks.
            full_error, exc_stacks = "", exc.get_stacks()
            for i in range(len(exc_stacks)):
                full_error += exc_stacks[i][0] + "\n"
                if i == 0:
                    full_error += "".join(
                        traceback.format_exception(*sys.exc_info()))
                else:
                    entry = ApplicationException.format_stack(exc_stacks[i][1])
                    full_error += entry + "\n"

            exec_name = exc.__class__.__name__
        else:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            exec_name = exc_type.__name__
            full_error = "".join(traceback.format_exception(*sys.exc_info()))

        status_code = getattr(exc, "status_code", 400)
        if self.log_errors:
            if self.develop_mode:
                if status_code == 401:
                    log.warn("%s: %s", exec_name, exc)
                else:
                    log.error(full_error)
            else:
                if status_code == 401:
                    log.info("%s: %s", exec_name, exc)
                else:
                    log.info(full_error)

        result = {
            GATEWAY_ERROR_EXCEPTION: exec_name,
            GATEWAY_ERROR_MESSAGE: str(exc.message),
            GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or ""
        }
        if self.develop_mode:
            result[GATEWAY_ERROR_TRACE] = full_error

        if RETURN_MIMETYPE_PARAM in request.args:
            return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM])
            return self.response_class(result, mimetype=return_mimetype)

        self._log_request_error(result, status_code)

        resp = self.json_response({
            GATEWAY_ERROR: result,
            GATEWAY_STATUS: status_code
        })
        # Q: Should HTTP status be the error code of the exception?
        resp.status_code = status_code
        return resp