def _rr(self): """ Returns the active resource registry instance or client. Used to directly contact the resource registry via the container if available, otherwise the messaging client to the RR service is returned. """ if self.container.has_capability('RESOURCE_REGISTRY'): return self.container.resource_registry if self._rr_client is None: self._rr_client = ResourceRegistryServiceProcessClient(process=self.container) return self._rr_client
def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True self.token_from_session = self.config.get_safe(CFG_PREFIX + ".token_from_session") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info("Service Gateway will not check requests against trusted originators since none are configured.") # Service screening self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) self.max_content_length = self.config.get_safe(CFG_PREFIX + ".max_content_length") # Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.request_callback = None self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient(process=self.process) self.idm_client = IdentityManagementServiceProcessClient(process=self.process) self.org_client = OrgManagementServiceProcessClient(process=self.process)
def start(self): log.debug("GovernanceController starting ...") self._CFG = CFG self.enabled = CFG.get_safe( 'interceptor.interceptors.governance.config.enabled', False) if not self.enabled: log.warn("GovernanceInterceptor disabled by configuration") self.policy_event_subscriber = None # Containers default to not Org Boundary and ION Root Org self._is_container_org_boundary = CFG.get_safe( 'container.org_boundary', False) self._container_org_name = CFG.get_safe( 'container.org_name', CFG.get_safe('system.root_org', 'ION')) self._container_org_id = None self._system_root_org_name = CFG.get_safe('system.root_org', 'ION') self._is_root_org_container = ( self._container_org_name == self._system_root_org_name) self.system_actor_id = None self.system_actor_user_header = None self.rr_client = ResourceRegistryServiceProcessClient( process=self.container) self.policy_client = PolicyManagementServiceProcessClient( process=self.container) if self.enabled: config = CFG.get_safe('interceptor.interceptors.governance.config') self.initialize_from_config(config) self.policy_event_subscriber = EventSubscriber( event_type=OT.PolicyEvent, callback=self.policy_event_callback) self.policy_event_subscriber.start() self._policy_snapshot = self._get_policy_snapshot() self._log_policy_update("start_governance_ctrl", message="Container start")
class ServiceGateway(object): """ The Service Gateway exports service routes for a web server via a Flask blueprint. The gateway bridges HTTP requests to ION AMQP RPC calls. """ def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info("Service Gateway will not check requests against trusted originators since none are configured.") # Service screening self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) # Initialize an LRU Cache to keep user roles cached for performance reasons # maxSize = maximum number of elements to keep in cache # maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient(process=self.process) self.idm_client = IdentityManagementServiceProcessClient(process=self.process) self.org_client = OrgManagementServiceProcessClient(process=self.process) # ------------------------------------------------------------------------- # Lifecycle management def start(self): # Configure subscriptions for user_cache events self.event_subscriber = EventSubscriber( event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback ) self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent) self.process.add_endpoint(self.event_subscriber) def stop(self): pass # Stop event subscribers - TODO: This hangs # self.process.remove_endpoint(self.event_subscriber) # ------------------------------------------------------------------------- # Event subscriber callbacks def _event_callback(self, event, *args, **kwargs): """ Callback function for receiving Events """ if isinstance(event, UserRoleModifiedEvent): # Event when User Roles are modified user_role_event = event org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) # Evict the user and their roles from the cache so that it gets updated with the next call. if self.user_role_cache and self.user_role_cache.has_key(actor_id): log.debug("Evicting user from the user_role_cache: %s" % actor_id) self.user_role_cache.evict(actor_id) elif isinstance(event, UserRoleCacheResetEvent): # An event is received to clear the user data cache self.user_role_cache.clear() # ------------------------------------------------------------------------- # Routes def sg_index(self): return self.gateway_json_response(SG_IDENTIFICATION) def get_service_spec(self, service_name=None, spec_name=None): try: if not self._swagger_gen: raise NotFound("Spec not available") if spec_name != "swagger.json": raise NotFound("Unknown spec format") swagger_json = self._swagger_gen.get_spec(service_name) resp = flask.make_response(flask.jsonify(swagger_json)) self._add_cors_headers(resp) return resp except Exception as ex: return self.gateway_error_response(ex) def process_gateway_request(self, service_name=None, operation=None, id_param=None): """ Makes a secure call to a SciON service operation via messaging. """ # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text) self._log_request_start("SVC RPC") try: result = self._make_service_request(service_name, operation, id_param) return self.gateway_json_response(result) except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def rest_gateway_request(self, service_name, res_type, id_param=None): """ Makes a REST style call to a SciON service operation via messaging. Get with ID returns the resource, POST without ID creates, PUT with ID updates and GET without ID returns the collection. """ self._log_request_start("SVC REST") try: if not service_name: raise BadRequest("Service name missing") service_name = str(service_name) if not res_type: raise BadRequest("Resource type missing") res_type = str(res_type) if request.method == "GET" and id_param: operation = "read_" + res_type return self.process_gateway_request(service_name, operation, id_param) elif request.method == "GET": ion_res_type = "".join(x.title() for x in res_type.split("_")) res = self._make_service_request("resource_registry", "find_resources", ion_res_type) if len(res) == 2: return self.gateway_json_response(res[0]) raise BadRequest("Unexpected find_resources result") elif request.method == "PUT": operation = "update_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") if id_param: obj._id = id_param return self.process_gateway_request(service_name, operation, obj) elif request.method == "POST": operation = "create_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") return self.process_gateway_request(service_name, operation, obj) else: raise BadRequest("Bad REST request") except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def _extract_payload_data(self): request_obj = None if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): if request.data: request_obj = json_loads(request.data) elif request.form: # Form encoded if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_obj = json_loads(str(payload)) if request_obj and is_ion_object_dict(request_obj): request_obj = self.create_ion_object(request_obj) return request_obj def _make_service_request(self, service_name=None, operation=None, id_param=None): """ Executes a secure call to a SciON service operation via messaging. """ if not service_name: if self.develop_mode: # Return a list of available services result = dict(available_services=get_service_registry().services.keys()) return result else: raise BadRequest("Service name missing") service_name = str(service_name) if not operation: if self.develop_mode: # Return a list of available operations result = dict(available_operations=[]) return result else: raise BadRequest("Service operation missing") operation = str(operation) # Apply service white list and black list for initial protection and get service client service_def = self.get_secure_service_def(service_name) target_client = service_def.client # Get service request arguments and operation parameter values request req_args = self._get_request_args() param_list = self.create_parameter_list(service_def, operation, req_args, id_param) # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request(req_args) in_login_whitelist = self.in_login_whitelist("request", service_name, operation) ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist) param_list["headers"] = self.build_message_headers(ion_actor_id, expiry) # Make service operation call client = target_client(process=self.process) method_call = getattr(client, operation) result = method_call(**param_list) return result def get_resource_schema(self, resource_type): try: # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request() ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry) return self.gateway_json_response(get_object_schema(resource_type)) except Exception as ex: return self.gateway_error_response(ex) def get_attachment(self, attachment_id): try: # Create client to interface attachment = self.rr_client.read_attachment(attachment_id, include_content=True) return self.response_class(attachment.content, mimetype=attachment.content_type) except Exception as ex: return self.gateway_error_response(ex) def create_attachment(self): try: payload = request.form[GATEWAY_ARG_JSON] json_params = json_loads(str(payload)) actor_id, expiry = self.get_governance_info_from_request(json_params) actor_id, expiry = self.validate_request(actor_id, expiry) headers = self.build_message_headers(actor_id, expiry) data_params = json_params[GATEWAY_ARG_PARAMS] resource_id = str(data_params.get("resource_id", "")) fil = request.files["file"] content = fil.read() keywords = [] keywords_str = data_params.get("keywords", "") if keywords_str.strip(): keywords = [str(x.strip()) for x in keywords_str.split(",")] created_by = data_params.get("attachment_created_by", "unknown user") modified_by = data_params.get("attachment_modified_by", "unknown user") # build attachment attachment = Attachment( name=str(data_params["attachment_name"]), description=str(data_params["attachment_description"]), attachment_type=int(data_params["attachment_type"]), content_type=str(data_params["attachment_content_type"]), keywords=keywords, created_by=created_by, modified_by=modified_by, content=content, ) ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error creating attachment") return self.gateway_error_response(ex) def delete_attachment(self, attachment_id): try: ret = self.rr_client.delete_attachment(attachment_id) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error deleting attachment") return self.gateway_error_response(ex) def get_version_info(self): import pkg_resources pkg_list = ["scioncc"] version = {} for package in pkg_list: try: version["%s-release" % package] = pkg_resources.require(package)[0].version # @TODO git versions for each? except pkg_resources.DistributionNotFound: pass try: dir_client = DirectoryServiceProcessClient(process=self.process) sys_attrs = dir_client.lookup("/System") if sys_attrs and isinstance(sys_attrs, dict): version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()}) except Exception as ex: log.exception("Could not determine system directory attributes") return self.gateway_json_response(version) # ========================================================================= # Security and governance helpers def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True return requesting_address in self.trusted_originators def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() if user_session.get("actor_id", None) and user_session.get("valid_until", 0): # Get info from current server session # NOTE: Actor id may be inside server session actor_id = user_session["actor_id"] expiry = str(int(user_session.get("valid_until", 0)) * 1000) log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args["api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request([self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry def in_login_whitelist(self, category, svc, op): """Returns True if service op is whitelisted for anonymous access""" entry = "%s/%s/%s" % (category, svc, op) return entry in self.no_login_whitelist def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry # ------------------------------------------------------------------------- # Service call (messaging) helpers def _add_cors_headers(self, resp): # Set CORS headers so that a Swagger client on a different domain can read spec resp.headers[ "Access-Control-Allow-Headers" ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" resp.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT" resp.headers["Access-Control-Allow-Origin"] = "*" resp.headers[ "Access-Control-Request-Headers" ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" def _log_request_start(self, req_type="SG"): global req_seqnum req_seqnum += 1 req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type) flask.g.req_info = req_info webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url) def _log_request_response(self, content_type, result="", content_length=-1, status_code=200): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_content_type"] = content_type req_info["resp_content_length"] = content_length req_info["resp_result"] = result req_info["resp_status"] = req_info.get("resp_status", status_code) def _log_request_error(self, result, status_code): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_error"] = True req_info["resp_status"] = status_code webapi_log.warn( "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"], req_info["request_id"], status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE], ) else: webapi_log.warn( "REQUEST ERROR (%s%s) - %s: %s", status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE], ) def _log_request_end(self): req_info = flask.g.get("req_info", None) if req_info: req_info["end_time"] = time.time() webapi_log.info( "%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s", req_info["req_type"], req_info["request_id"], req_info.get("resp_status", ""), req_info["end_time"] - req_info["start_time"], req_info.get("resp_content_length", ""), req_info.get("resp_content_type", ""), ) else: webapi_log.warn("REQUEST END - missing start info") def _get_request_args(self): """Extracts service request arguments from HTTP request. Supports various methods and forms of encoding. Separates arguments for special parameters from service operation parameters. Returns a dict with the service request arguments, containing key params with the actual values for the service operation parameters. """ str_args = False request_args = {} if request.method == "POST" or request.method == "PUT": # Use only body args and ignore any args from query string if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): # JSON body request if request.data: request_args = json_loads(request.data) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} elif request.form: # Form encoded payload if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_args = json_loads(str(payload)) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} else: # Fallback: Directly use form values str_args = True request_args = {GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)} else: # No args found in body request_args = {GATEWAY_ARG_PARAMS: {}} elif request.method == "GET": str_args = True REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"} args_dict = request.args.to_dict(flat=True) request_args = {k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL} req_params = {k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL} request_args[GATEWAY_ARG_PARAMS] = req_params request_args["str_args"] = str_args # Indicate downstream that args are str (GET or form encoded) # log.info("Request args: %s" % request_args) return request_args def _get_typed_arg_value(self, given_value, param_def, strict): """Returns a service operation argument value, based on a given value and param schema definition. """ param_type = param_def["type"] if isinstance(given_value, unicode): # Convert all unicode to str in UTF-8 given_value = given_value.encode("utf8") # Make all unicode into str if isinstance(given_value, IonObjectBase) and ( given_value._get_type() == param_type or param_type in given_value._get_extends() ): return given_value elif is_ion_object_dict(given_value) and (param_type == "NoneType" or hasattr(objects, param_type)): return self.create_ion_object(given_value) elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"): arg_val = get_typed_value(given_value, targettype=param_type, strict=strict) return arg_val else: raise BadRequest("Cannot convert param value to type %s" % param_type) def create_parameter_list(self, service_def, operation, request_args, id_param=None): """Build service call parameter list dynamically from service operation definition """ service_schema = service_def.schema service_op_schema = service_schema["operations"][operation] svc_op_param_list = service_op_schema["in_list"] svc_params = {} if id_param: # Magic shorthand: if one argument is given, fill the first service argument if svc_op_param_list: fill_par = svc_op_param_list[0] fill_par_def = service_op_schema["in"][fill_par] arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False) svc_params[fill_par] = arg_val return svc_params request_args = request_args or {} # Cannot be strict for a URL string query arguments or directly form encoded strict_types = False if request_args.get("str_args", False) else self.strict_types req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {} for param_name in svc_op_param_list: param_def = service_op_schema["in"][param_name] if param_name in req_op_args: arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types) svc_params[param_name] = arg_val if "timeout" in request_args: svc_params["timeout"] = float(request_args["timeout"]) optional_args = [param for param in req_op_args if param not in svc_params and param != "timeout"] if optional_args and "optional_args" in svc_op_param_list: # Only support basic strings for these optional params for now svc_params["optional_args"] = {arg: str(req_op_args[arg]) for arg in optional_args} # log.info("Service params: %s" % svc_params) return svc_params def _get_gateway_headers(self): """Returns the headers that the service gateway uses to make service calls on behalf of itself (not a user passing through), e.g. for identity management purposes""" return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY} def get_secure_service_def(self, service_name): """Checks whether the service indicated by given service_name exists and/or is exposed after white and black listing. Returns service registry entry. """ if self.service_whitelist: if service_name not in self.service_whitelist: raise Unauthorized("Service access not permitted") if self.service_blacklist: if service_name in self.service_blacklist: raise Unauthorized("Service access not permitted") # Retrieve service definition target_service = get_service_registry().get_service_by_name(service_name) if not target_service: raise BadRequest("The requested service (%s) is not available" % service_name) if not target_service.client: raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name) if not target_service.schema: raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name) return target_service def build_message_headers(self, actor_id, expiry): """Returns the headers that the service gateway uses to make service calls on behalf of a user, based on the user session or request arguments""" headers = dict() headers[MSG_HEADER_ACTOR] = actor_id headers[MSG_HEADER_VALID] = expiry req_info = flask.g.get("req_info", None) if req_info: headers["request-id"] = str(req_info["request_id"]) # If this is an anonymous requester then there are no roles associated with the request if actor_id == DEFAULT_ACTOR_ID: headers[MSG_HEADER_ROLES] = dict() return headers try: # Check to see if the user's roles are cached already - keyed by user id if self.user_role_cache.has_key(actor_id): role_header = self.user_role_cache.get(actor_id) if role_header is not None: headers[MSG_HEADER_ROLES] = role_header return headers # The user's roles were not cached so hit the datastore to find it. role_list = self.org_client.list_actor_roles(actor_id, headers=self._get_gateway_headers()) org_roles = {} for role in role_list: org_roles.setdefault(role.org_governance_name, []).append(role) role_header = get_role_message_headers(org_roles) # Cache the roles by user id self.user_role_cache.put(actor_id, role_header) except Exception: role_header = dict() # Default to empty dict if there is a problem finding roles for the user headers[MSG_HEADER_ROLES] = role_header return headers def create_ion_object(self, object_params): """Create and initialize an ION object from a dictionary of parameters coming via HTTP, ready to be passed on to services/messaging. The object is validated after creation. Note: This is not called for service operation argument signatures """ new_obj = IonObject(object_params["type_"]) # Iterate over the parameters to add to object; have to do this instead # of passing a dict to get around restrictions in object creation on setting _id, _rev params for param in object_params: self.set_object_field(new_obj, param, object_params.get(param)) new_obj._validate() # verify that all of the object fields were set with proper types return new_obj def set_object_field(self, obj, field, field_val): """Recursively set sub object field values. TODO: This may be an expensive operation. May also be redundant with object code """ if isinstance(field_val, dict) and field != "kwargs": sub_obj = getattr(obj, field) if isinstance(sub_obj, IonObjectBase): if "type_" in field_val and field_val["type_"] != sub_obj.type_: if issubtype(field_val["type_"], sub_obj.type_): sub_obj = IonObject(field_val["type_"]) setattr(obj, field, sub_obj) else: raise Inconsistent( "Unable to walk the field %s - types don't match: %s %s" % (field, sub_obj.type_, field_val["type_"]) ) for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) elif isinstance(sub_obj, dict): setattr(obj, field, field_val) else: for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) else: # type_ already exists in the class. if field != "type_": setattr(obj, field, field_val) # ------------------------------------------------------------------------- # Response content helpers def json_response(self, response_data): """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects """ resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2) resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON) if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])): self._add_cors_headers(resp) self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj)) return resp def gateway_json_response(self, response_data): """Returns the normal service gateway response as JSON or as media in case the response is a media response """ if isinstance(response_data, MediaResponse): log.info("Media response. Content mimetype:%s", response_data.media_mimetype) content = response_data.body if response_data.internal_encoding == "base64": import base64 content = base64.decodestring(content) elif response_data.internal_encoding == "utf8": pass resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype) self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code) return resp if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(response_data, mimetype=return_mimetype) result = {GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200} return self.json_response(result) def gateway_error_response(self, exc): """Forms a service gateway error response. Can extract multiple stacks from a multi-tier RPC service call exception """ if hasattr(exc, "get_stacks"): # Process potentially multiple stacks. full_error, exc_stacks = "", exc.get_stacks() for i in range(len(exc_stacks)): full_error += exc_stacks[i][0] + "\n" if i == 0: full_error += "".join(traceback.format_exception(*sys.exc_info())) else: entry = ApplicationException.format_stack(exc_stacks[i][1]) full_error += entry + "\n" exec_name = exc.__class__.__name__ else: exc_type, exc_obj, exc_tb = sys.exc_info() exec_name = exc_type.__name__ full_error = "".join(traceback.format_exception(*sys.exc_info())) status_code = getattr(exc, "status_code", 400) if self.log_errors: if self.develop_mode: if status_code == 401: log.warn("%s: %s", exec_name, exc) else: log.error(full_error) else: if status_code == 401: log.info("%s: %s", exec_name, exc) else: log.info(full_error) result = { GATEWAY_ERROR_EXCEPTION: exec_name, GATEWAY_ERROR_MESSAGE: str(exc.message), GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "", } if self.develop_mode: result[GATEWAY_ERROR_TRACE] = full_error if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(result, mimetype=return_mimetype) self._log_request_error(result, status_code) resp = self.json_response({GATEWAY_ERROR: result, GATEWAY_STATUS: status_code}) # Q: Should HTTP status be the error code of the exception? resp.status_code = status_code return resp
def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True self.token_from_session = self.config.get_safe( CFG_PREFIX + ".token_from_session") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info( "Service Gateway will not check requests against trusted originators since none are configured." ) # Service screening self.service_blacklist = self.config.get_safe( CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe( CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set( self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe( CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) # Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.request_callback = None self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient( process=self.process) self.idm_client = IdentityManagementServiceProcessClient( process=self.process) self.org_client = OrgManagementServiceProcessClient( process=self.process)
class ServiceGateway(object): """ The Service Gateway exports service routes for a web server via a Flask blueprint. The gateway bridges HTTP requests to ION AMQP RPC calls. """ def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True self.token_from_session = self.config.get_safe( CFG_PREFIX + ".token_from_session") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info( "Service Gateway will not check requests against trusted originators since none are configured." ) # Service screening self.service_blacklist = self.config.get_safe( CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe( CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set( self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe( CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) # Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.request_callback = None self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient( process=self.process) self.idm_client = IdentityManagementServiceProcessClient( process=self.process) self.org_client = OrgManagementServiceProcessClient( process=self.process) # ------------------------------------------------------------------------- # Lifecycle management def start(self): # Configure subscriptions for user_cache events self.event_subscriber = EventSubscriber( event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback) self.event_subscriber.add_event_subscription( event_type=OT.UserRoleCacheResetEvent) self.process.add_endpoint(self.event_subscriber) def stop(self): pass # Stop event subscribers - TODO: This hangs #self.process.remove_endpoint(self.event_subscriber) # ------------------------------------------------------------------------- # Event subscriber callbacks def _event_callback(self, event, *args, **kwargs): """ Callback function for receiving Events """ if isinstance(event, UserRoleModifiedEvent): # Event when User Roles are modified user_role_event = event org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) # Evict the user and their roles from the cache so that it gets updated with the next call. if self.user_role_cache and self.user_role_cache.has_key(actor_id): log.debug("Evicting user from the user_role_cache: %s" % actor_id) self.user_role_cache.evict(actor_id) elif isinstance(event, UserRoleCacheResetEvent): # An event is received to clear the user data cache self.user_role_cache.clear() # ------------------------------------------------------------------------- # Routes def sg_index(self): return self.gateway_json_response(SG_IDENTIFICATION) def get_service_spec(self, service_name=None, spec_name=None): try: if not self._swagger_gen: raise NotFound("Spec not available") if spec_name != "swagger.json": raise NotFound("Unknown spec format") swagger_json = self._swagger_gen.get_spec(service_name) resp = flask.make_response(flask.jsonify(swagger_json)) self._add_cors_headers(resp) return resp except Exception as ex: return self.gateway_error_response(ex) def process_gateway_request(self, service_name=None, operation=None, id_param=None): """ Makes a secure call to a SciON service operation via messaging. """ # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text) self._log_request_start("SVC RPC") try: result = self._make_service_request(service_name, operation, id_param) return self.gateway_json_response(result) except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def rest_gateway_request(self, service_name, res_type, id_param=None): """ Makes a REST style call to a SciON service operation via messaging. Get with ID returns the resource, POST without ID creates, PUT with ID updates and GET without ID returns the collection. """ self._log_request_start("SVC REST") try: if not service_name: raise BadRequest("Service name missing") service_name = str(service_name) if not res_type: raise BadRequest("Resource type missing") res_type = str(res_type) if request.method == "GET" and id_param: operation = "read_" + res_type return self.process_gateway_request(service_name, operation, id_param) elif request.method == "GET": ion_res_type = "".join(x.title() for x in res_type.split('_')) res = self._make_service_request("resource_registry", "find_resources", ion_res_type) if len(res) == 2: return self.gateway_json_response(res[0]) raise BadRequest("Unexpected find_resources result") elif request.method == "PUT": operation = "update_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") if id_param: obj._id = id_param return self.process_gateway_request(service_name, operation, obj) elif request.method == "POST": operation = "create_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") return self.process_gateway_request(service_name, operation, obj) else: raise BadRequest("Bad REST request") except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def _extract_payload_data(self): request_obj = None if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): if request.data: request_obj = json_loads(request.data) elif request.form: # Form encoded if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_obj = json_loads(str(payload)) if request_obj and is_ion_object_dict(request_obj): request_obj = self.create_ion_object(request_obj) return request_obj def _make_service_request(self, service_name=None, operation=None, id_param=None): """ Executes a secure call to a SciON service operation via messaging. """ if not service_name: if self.develop_mode: # Return a list of available services result = dict( available_services=get_service_registry().services.keys()) return result else: raise BadRequest("Service name missing") service_name = str(service_name) if not operation: if self.develop_mode: # Return a list of available operations result = dict(available_operations=[]) return result else: raise BadRequest("Service operation missing") operation = str(operation) # Apply service white list and black list for initial protection and get service client service_def = self.get_secure_service_def(service_name) target_client = service_def.client # Get service request arguments and operation parameter values request req_args = self._get_request_args() param_list = self.create_parameter_list(service_def, operation, req_args, id_param) # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request(req_args) in_login_whitelist = self.in_login_whitelist("request", service_name, operation) ion_actor_id, expiry = self.validate_request( ion_actor_id, expiry, in_whitelist=in_login_whitelist) param_list["headers"] = self.build_message_headers( ion_actor_id, expiry) # Make service operation call client = target_client(process=self.process) method_call = getattr(client, operation) result = method_call(**param_list) return result def get_resource_schema(self, resource_type): try: # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request() ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry) return self.gateway_json_response(get_object_schema(resource_type)) except Exception as ex: return self.gateway_error_response(ex) def get_attachment(self, attachment_id): try: # Create client to interface attachment = self.rr_client.read_attachment(attachment_id, include_content=True) return self.response_class(attachment.content, mimetype=attachment.content_type) except Exception as ex: return self.gateway_error_response(ex) def create_attachment(self): try: payload = request.form[GATEWAY_ARG_JSON] json_params = json_loads(str(payload)) actor_id, expiry = self.get_governance_info_from_request( json_params) actor_id, expiry = self.validate_request(actor_id, expiry) headers = self.build_message_headers(actor_id, expiry) data_params = json_params[GATEWAY_ARG_PARAMS] resource_id = str(data_params.get("resource_id", "")) fil = request.files["file"] content = fil.read() keywords = [] keywords_str = data_params.get("keywords", "") if keywords_str.strip(): keywords = [str(x.strip()) for x in keywords_str.split(",")] created_by = data_params.get("attachment_created_by", "unknown user") modified_by = data_params.get("attachment_modified_by", "unknown user") # build attachment attachment = Attachment( name=str(data_params["attachment_name"]), description=str(data_params["attachment_description"]), attachment_type=int(data_params["attachment_type"]), content_type=str(data_params["attachment_content_type"]), keywords=keywords, created_by=created_by, modified_by=modified_by, content=content) ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error creating attachment") return self.gateway_error_response(ex) def delete_attachment(self, attachment_id): try: ret = self.rr_client.delete_attachment(attachment_id) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error deleting attachment") return self.gateway_error_response(ex) def get_version_info(self, pack=None): import pkg_resources pkg_list = ["scioncc"] packs = self.config.get_safe(CFG_PREFIX + ".version_packages") if packs: pkg_list.extend(packs.split(",")) version = {} for package in pkg_list: try: if pack == "all": pack_deps = pkg_resources.require(package) version.update( {p.project_name: p.version for p in pack_deps}) else: version[package] = pkg_resources.require( package)[0].version # @TODO git versions for current? except pkg_resources.DistributionNotFound: pass try: dir_client = DirectoryServiceProcessClient(process=self.process) sys_attrs = dir_client.lookup("/System") if sys_attrs and isinstance(sys_attrs, dict): version.update({ k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower() }) except Exception as ex: log.exception("Could not determine system directory attributes") if pack and pack != "all": version = {k: v for (k, v) in version.iteritems() if k == pack} return self.gateway_json_response(version) # ========================================================================= # Security and governance helpers def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True return requesting_address in self.trusted_originators def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() #if user_session.get("actor_id", None) and user_session.get("valid_until", 0): if user_session.get("actor_id", None): # Get info from current server session # NOTE: Actor id may be inside server session expiry = int(user_session.get("valid_until", 0)) * 1000 if expiry: # This was a proper non-token server session authentication expiry = str(expiry) actor_id = user_session["actor_id"] log.info( "Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) else: # We are just taking the user_id out of the session # TODO: Need to check access token here expiry = str(expiry) if self.token_from_session: actor_id = user_session["actor_id"] log.info( "Request associated with actor's token from session; actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args[ "api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info( "Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request( [self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info( "Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token( authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry def in_login_whitelist(self, category, svc, op): """Returns True if service op is whitelisted for anonymous access""" entry = "%s/%s/%s" % (category, svc, op) return entry in self.no_login_whitelist def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity( actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent( "Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry # ------------------------------------------------------------------------- # Service call (messaging) helpers def register_request_callback(self, cb_func): if cb_func is None: pass elif self.request_callback: log.warn("Callback already registered") self.request_callback = cb_func def _call_request_callback(self, action, req_info): if not self.request_callback: return try: self.request_callback(action, req_info) except Exception: log.exception("Error calling request callback") def _add_cors_headers(self, resp): # Set CORS headers so that a Swagger client on a different domain can read spec resp.headers[ "Access-Control-Allow-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" resp.headers[ "Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT" resp.headers["Access-Control-Allow-Origin"] = "*" resp.headers[ "Access-Control-Request-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" def _log_request_start(self, req_type="SG"): global req_seqnum req_seqnum += 1 req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type, req_url=request.url) flask.g.req_info = req_info webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url) self._call_request_callback("start", req_info) def _log_request_response(self, content_type, result="", content_length=-1, status_code=200): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_content_type"] = content_type req_info["resp_content_length"] = content_length req_info["resp_result"] = result req_info["resp_status"] = req_info.get("resp_status", status_code) def _log_request_error(self, result, status_code): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_error"] = True req_info["resp_status"] = status_code webapi_log.warn( "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"], req_info["request_id"], status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE]) self._call_request_callback("error", req_info) else: webapi_log.warn( "REQUEST ERROR (%s%s) - %s: %s", status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE]) def _log_request_end(self): req_info = flask.g.get("req_info", None) if req_info: req_info["end_time"] = time.time() webapi_log.info("%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s", req_info["req_type"], req_info["request_id"], req_info.get("resp_status", ""), req_info["end_time"] - req_info["start_time"], req_info.get("resp_content_length", ""), req_info.get("resp_content_type", "")) self._call_request_callback("end", req_info) else: webapi_log.warn("REQUEST END - missing start info") def _get_request_args(self): """Extracts service request arguments from HTTP request. Supports various methods and forms of encoding. Separates arguments for special parameters from service operation parameters. Returns a dict with the service request arguments, containing key params with the actual values for the service operation parameters. """ str_args = False request_args = {} if request.method == "POST" or request.method == "PUT": # Use only body args and ignore any args from query string if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): # JSON body request if request.data: request_args = json_loads(request.data) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} elif request.form: # Form encoded payload if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_args = json_loads(str(payload)) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} else: # Fallback: Directly use form values str_args = True request_args = { GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True) } else: # No args found in body request_args = {GATEWAY_ARG_PARAMS: {}} elif request.method == "GET": str_args = True REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"} args_dict = request.args.to_dict(flat=True) request_args = { k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL } req_params = { k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL } request_args[GATEWAY_ARG_PARAMS] = req_params request_args[ "str_args"] = str_args # Indicate downstream that args are str (GET or form encoded) #log.info("Request args: %s" % request_args) return request_args def _get_typed_arg_value(self, given_value, param_def, strict): """Returns a service operation argument value, based on a given value and param schema definition. """ param_type = param_def["type"] if isinstance(given_value, unicode): # Convert all unicode to str in UTF-8 given_value = given_value.encode( "utf8") # Make all unicode into str if isinstance(given_value, IonObjectBase) and ( given_value._get_type() == param_type or param_type in given_value._get_extends()): return given_value elif is_ion_object_dict(given_value) and ( param_type == "NoneType" or hasattr(objects, param_type)): return self.create_ion_object(given_value) elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"): arg_val = get_typed_value(given_value, targettype=param_type, strict=strict) return arg_val else: raise BadRequest("Cannot convert param value to type %s" % param_type) def create_parameter_list(self, service_def, operation, request_args, id_param=None): """Build service call parameter list dynamically from service operation definition """ service_schema = service_def.schema service_op_schema = service_schema["operations"][operation] svc_op_param_list = service_op_schema["in_list"] svc_params = {} if id_param: # Magic shorthand: if one argument is given, fill the first service argument if svc_op_param_list: fill_par = svc_op_param_list[0] fill_par_def = service_op_schema["in"][fill_par] arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False) svc_params[fill_par] = arg_val return svc_params request_args = request_args or {} # Cannot be strict for a URL string query arguments or directly form encoded strict_types = False if request_args.get("str_args", False) else self.strict_types req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {} for param_name in svc_op_param_list: param_def = service_op_schema["in"][param_name] if param_name in req_op_args: arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types) svc_params[param_name] = arg_val if "timeout" in request_args: svc_params["timeout"] = float(request_args["timeout"]) optional_args = [ param for param in req_op_args if param not in svc_params and param != "timeout" ] if optional_args and "optional_args" in svc_op_param_list: # Only support basic strings for these optional params for now svc_params["optional_args"] = { arg: str(req_op_args[arg]) for arg in optional_args } #log.info("Service params: %s" % svc_params) return svc_params def _get_gateway_headers(self): """Returns the headers that the service gateway uses to make service calls on behalf of itself (not a user passing through), e.g. for identity management purposes""" return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY} def get_secure_service_def(self, service_name): """Checks whether the service indicated by given service_name exists and/or is exposed after white and black listing. Returns service registry entry. """ if self.service_whitelist: if service_name not in self.service_whitelist: raise Unauthorized("Service access not permitted") if self.service_blacklist: if service_name in self.service_blacklist: raise Unauthorized("Service access not permitted") # Retrieve service definition target_service = get_service_registry().get_service_by_name( service_name) if not target_service: raise BadRequest("The requested service (%s) is not available" % service_name) if not target_service.client: raise Inconsistent( "Cannot find a client class for the specified service: %s" % service_name) if not target_service.schema: raise Inconsistent( "Cannot find a schema for the specified service: %s" % service_name) return target_service def build_message_headers(self, actor_id, expiry): """Returns the headers that the service gateway uses to make service calls on behalf of a user, based on the user session or request arguments""" headers = dict() headers[MSG_HEADER_ACTOR] = actor_id headers[MSG_HEADER_VALID] = expiry req_info = flask.g.get("req_info", None) if req_info: headers["request-id"] = str(req_info["request_id"]) # If this is an anonymous requester then there are no roles associated with the request if actor_id == DEFAULT_ACTOR_ID: headers[MSG_HEADER_ROLES] = dict() return headers try: # Check to see if the user's roles are cached already - keyed by user id if self.user_role_cache.has_key(actor_id): role_header = self.user_role_cache.get(actor_id) if role_header is not None: headers[MSG_HEADER_ROLES] = role_header return headers # The user's roles were not cached so hit the datastore to find it. role_list = self.org_client.list_actor_roles( actor_id, headers=self._get_gateway_headers()) org_roles = {} for role in role_list: org_roles.setdefault(role.org_governance_name, []).append(role) role_header = get_role_message_headers(org_roles) # Cache the roles by user id self.user_role_cache.put(actor_id, role_header) except Exception: role_header = dict( ) # Default to empty dict if there is a problem finding roles for the user headers[MSG_HEADER_ROLES] = role_header return headers def create_ion_object(self, object_params): """Create and initialize an ION object from a dictionary of parameters coming via HTTP, ready to be passed on to services/messaging. The object is validated after creation. Note: This is not called for service operation argument signatures """ new_obj = IonObject(object_params["type_"]) # Iterate over the parameters to add to object; have to do this instead # of passing a dict to get around restrictions in object creation on setting _id, _rev params for param in object_params: self.set_object_field(new_obj, param, object_params.get(param)) new_obj._validate( ) # verify that all of the object fields were set with proper types return new_obj def set_object_field(self, obj, field, field_val): """Recursively set sub object field values. TODO: This may be an expensive operation. May also be redundant with object code """ if isinstance(field_val, dict) and field != "kwargs": sub_obj = getattr(obj, field) if isinstance(sub_obj, IonObjectBase): if "type_" in field_val and field_val["type_"] != sub_obj.type_: if issubtype(field_val["type_"], sub_obj.type_): sub_obj = IonObject(field_val["type_"]) setattr(obj, field, sub_obj) else: raise Inconsistent( "Unable to walk the field %s - types don't match: %s %s" % (field, sub_obj.type_, field_val["type_"])) for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) elif isinstance(sub_obj, dict): setattr(obj, field, field_val) else: for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) else: # type_ already exists in the class. if field != "type_": setattr(obj, field, field_val) # ------------------------------------------------------------------------- # Response content helpers def json_response(self, response_data): """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects """ resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2) resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON) if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])): self._add_cors_headers(resp) self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj)) return resp def gateway_json_response(self, response_data): """Returns the normal service gateway response as JSON or as media in case the response is a media response """ if isinstance(response_data, MediaResponse): log.info("Media response. Content mimetype:%s", response_data.media_mimetype) content = response_data.body if response_data.internal_encoding == "base64": import base64 content = base64.decodestring(content) elif response_data.internal_encoding == "utf8": pass resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype) self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code) return resp if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(response_data, mimetype=return_mimetype) result = { GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200, } return self.json_response(result) def gateway_error_response(self, exc): """Forms a service gateway error response. Can extract multiple stacks from a multi-tier RPC service call exception """ if hasattr(exc, "get_stacks"): # Process potentially multiple stacks. full_error, exc_stacks = "", exc.get_stacks() for i in range(len(exc_stacks)): full_error += exc_stacks[i][0] + "\n" if i == 0: full_error += "".join( traceback.format_exception(*sys.exc_info())) else: entry = ApplicationException.format_stack(exc_stacks[i][1]) full_error += entry + "\n" exec_name = exc.__class__.__name__ else: exc_type, exc_obj, exc_tb = sys.exc_info() exec_name = exc_type.__name__ full_error = "".join(traceback.format_exception(*sys.exc_info())) status_code = getattr(exc, "status_code", 400) if self.log_errors: if self.develop_mode: if status_code == 401: log.warn("%s: %s", exec_name, exc) else: log.error(full_error) else: if status_code == 401: log.info("%s: %s", exec_name, exc) else: log.info(full_error) result = { GATEWAY_ERROR_EXCEPTION: exec_name, GATEWAY_ERROR_MESSAGE: str(exc.message), GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "" } if self.develop_mode: result[GATEWAY_ERROR_TRACE] = full_error if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(result, mimetype=return_mimetype) self._log_request_error(result, status_code) resp = self.json_response({ GATEWAY_ERROR: result, GATEWAY_STATUS: status_code }) # Q: Should HTTP status be the error code of the exception? resp.status_code = status_code return resp