class ServiceGatewayService(BaseServiceGatewayService): """ The Service Gateway Service is the service that uses a gevent web server and Flask to bridge HTTP requests to AMQP RPC ION process service calls. """ def on_init(self): #defaults self.http_server = None #retain a pointer to this object for use in ProcessRPC calls global service_gateway_instance ###### # to prevent cascading failure, here's an attempted hack if service_gateway_instance is not None and service_gateway_instance.http_server is not None: service_gateway_instance.http_server.stop() # end hack ###### service_gateway_instance = self self.server_hostname = self.CFG.get_safe('container.service_gateway.web_server.hostname', DEFAULT_WEB_SERVER_HOSTNAME) self.server_port = self.CFG.get_safe('container.service_gateway.web_server.port', DEFAULT_WEB_SERVER_PORT) self.web_server_enabled = self.CFG.get_safe('container.service_gateway.web_server.enabled', True) self.web_logging = self.CFG.get_safe('container.service_gateway.web_server.log') self.log_errors = self.CFG.get_safe('container.service_gateway.log_errors', False) #Optional list of trusted originators can be specified in config. self.trusted_originators = self.CFG.get_safe('container.service_gateway.trusted_originators') if not self.trusted_originators: self.trusted_originators = None log.info("Service Gateway will not check requests against trusted originators since none are configured.") #Get the user_cache_size self.user_cache_size = self.CFG.get_safe('container.service_gateway.user_cache_size', DEFAULT_USER_CACHE_SIZE) #Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_data_cache = LRUCache(self.user_cache_size,0,0) #Start the gevent web server unless disabled if self.web_server_enabled: log.info("Starting service gateway on %s:%s", self.server_hostname, self.server_port) self.start_service(self.server_hostname, self.server_port) #Configure subscriptions for user_cache events self.user_role_event_subscriber = EventSubscriber(event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self.user_role_event_callback) self.user_role_event_subscriber.start() self.user_role_reset_subscriber = EventSubscriber(event_type=OT.UserRoleCacheResetEvent, callback=self.user_role_reset_callback) self.user_role_reset_subscriber.start() def on_quit(self): self.stop_service() if self.user_role_event_subscriber is not None: self.user_role_event_subscriber.stop() if self.user_role_reset_subscriber is not None: self.user_role_reset_subscriber.stop() def start_service(self, hostname=DEFAULT_WEB_SERVER_HOSTNAME, port=DEFAULT_WEB_SERVER_PORT): """Responsible for starting the gevent based web server.""" if self.http_server is not None: self.stop_service() self.http_server = WSGIServer((hostname, port), service_gateway_app, log=self.web_logging) self.http_server.start() return True def stop_service(self): """Responsible for stopping the gevent based web server.""" if self.http_server is not None: self.http_server.stop() return True def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True for addr in self.trusted_originators: if requesting_address == addr: return True return False def user_role_event_callback(self, *args, **kwargs): """ This method is a callback function for receiving Events when User Roles are modified. """ user_role_event = args[0] org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) #Evict the user and their roles from the cache so that it gets updated with the next call. if service_gateway_instance.user_data_cache and service_gateway_instance.user_data_cache.has_key(actor_id): log.debug('Evicting user from the user_data_cache: %s' % actor_id) service_gateway_instance.user_data_cache.evict(actor_id) def user_role_reset_callback(self, *args, **kwargs): ''' This method is a callback function for when an event is received to clear the user data cache ''' self.user_data_cache.clear()
class ServiceGateway(object): """ The Service Gateway exports service routes for a web server via a Flask blueprint. The gateway bridges HTTP requests to ION AMQP RPC calls. """ def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True self.token_from_session = self.config.get_safe( CFG_PREFIX + ".token_from_session") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info( "Service Gateway will not check requests against trusted originators since none are configured." ) # Service screening self.service_blacklist = self.config.get_safe( CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe( CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set( self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe( CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) # Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.request_callback = None self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient( process=self.process) self.idm_client = IdentityManagementServiceProcessClient( process=self.process) self.org_client = OrgManagementServiceProcessClient( process=self.process) # ------------------------------------------------------------------------- # Lifecycle management def start(self): # Configure subscriptions for user_cache events self.event_subscriber = EventSubscriber( event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback) self.event_subscriber.add_event_subscription( event_type=OT.UserRoleCacheResetEvent) self.process.add_endpoint(self.event_subscriber) def stop(self): pass # Stop event subscribers - TODO: This hangs #self.process.remove_endpoint(self.event_subscriber) # ------------------------------------------------------------------------- # Event subscriber callbacks def _event_callback(self, event, *args, **kwargs): """ Callback function for receiving Events """ if isinstance(event, UserRoleModifiedEvent): # Event when User Roles are modified user_role_event = event org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) # Evict the user and their roles from the cache so that it gets updated with the next call. if self.user_role_cache and self.user_role_cache.has_key(actor_id): log.debug("Evicting user from the user_role_cache: %s" % actor_id) self.user_role_cache.evict(actor_id) elif isinstance(event, UserRoleCacheResetEvent): # An event is received to clear the user data cache self.user_role_cache.clear() # ------------------------------------------------------------------------- # Routes def sg_index(self): return self.gateway_json_response(SG_IDENTIFICATION) def get_service_spec(self, service_name=None, spec_name=None): try: if not self._swagger_gen: raise NotFound("Spec not available") if spec_name != "swagger.json": raise NotFound("Unknown spec format") swagger_json = self._swagger_gen.get_spec(service_name) resp = flask.make_response(flask.jsonify(swagger_json)) self._add_cors_headers(resp) return resp except Exception as ex: return self.gateway_error_response(ex) def process_gateway_request(self, service_name=None, operation=None, id_param=None): """ Makes a secure call to a SciON service operation via messaging. """ # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text) self._log_request_start("SVC RPC") try: result = self._make_service_request(service_name, operation, id_param) return self.gateway_json_response(result) except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def rest_gateway_request(self, service_name, res_type, id_param=None): """ Makes a REST style call to a SciON service operation via messaging. Get with ID returns the resource, POST without ID creates, PUT with ID updates and GET without ID returns the collection. """ self._log_request_start("SVC REST") try: if not service_name: raise BadRequest("Service name missing") service_name = str(service_name) if not res_type: raise BadRequest("Resource type missing") res_type = str(res_type) if request.method == "GET" and id_param: operation = "read_" + res_type return self.process_gateway_request(service_name, operation, id_param) elif request.method == "GET": ion_res_type = "".join(x.title() for x in res_type.split('_')) res = self._make_service_request("resource_registry", "find_resources", ion_res_type) if len(res) == 2: return self.gateway_json_response(res[0]) raise BadRequest("Unexpected find_resources result") elif request.method == "PUT": operation = "update_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") if id_param: obj._id = id_param return self.process_gateway_request(service_name, operation, obj) elif request.method == "POST": operation = "create_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") return self.process_gateway_request(service_name, operation, obj) else: raise BadRequest("Bad REST request") except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def _extract_payload_data(self): request_obj = None if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): if request.data: request_obj = json_loads(request.data) elif request.form: # Form encoded if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_obj = json_loads(str(payload)) if request_obj and is_ion_object_dict(request_obj): request_obj = self.create_ion_object(request_obj) return request_obj def _make_service_request(self, service_name=None, operation=None, id_param=None): """ Executes a secure call to a SciON service operation via messaging. """ if not service_name: if self.develop_mode: # Return a list of available services result = dict( available_services=get_service_registry().services.keys()) return result else: raise BadRequest("Service name missing") service_name = str(service_name) if not operation: if self.develop_mode: # Return a list of available operations result = dict(available_operations=[]) return result else: raise BadRequest("Service operation missing") operation = str(operation) # Apply service white list and black list for initial protection and get service client service_def = self.get_secure_service_def(service_name) target_client = service_def.client # Get service request arguments and operation parameter values request req_args = self._get_request_args() param_list = self.create_parameter_list(service_def, operation, req_args, id_param) # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request(req_args) in_login_whitelist = self.in_login_whitelist("request", service_name, operation) ion_actor_id, expiry = self.validate_request( ion_actor_id, expiry, in_whitelist=in_login_whitelist) param_list["headers"] = self.build_message_headers( ion_actor_id, expiry) # Make service operation call client = target_client(process=self.process) method_call = getattr(client, operation) result = method_call(**param_list) return result def get_resource_schema(self, resource_type): try: # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request() ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry) return self.gateway_json_response(get_object_schema(resource_type)) except Exception as ex: return self.gateway_error_response(ex) def get_attachment(self, attachment_id): try: # Create client to interface attachment = self.rr_client.read_attachment(attachment_id, include_content=True) return self.response_class(attachment.content, mimetype=attachment.content_type) except Exception as ex: return self.gateway_error_response(ex) def create_attachment(self): try: payload = request.form[GATEWAY_ARG_JSON] json_params = json_loads(str(payload)) actor_id, expiry = self.get_governance_info_from_request( json_params) actor_id, expiry = self.validate_request(actor_id, expiry) headers = self.build_message_headers(actor_id, expiry) data_params = json_params[GATEWAY_ARG_PARAMS] resource_id = str(data_params.get("resource_id", "")) fil = request.files["file"] content = fil.read() keywords = [] keywords_str = data_params.get("keywords", "") if keywords_str.strip(): keywords = [str(x.strip()) for x in keywords_str.split(",")] created_by = data_params.get("attachment_created_by", "unknown user") modified_by = data_params.get("attachment_modified_by", "unknown user") # build attachment attachment = Attachment( name=str(data_params["attachment_name"]), description=str(data_params["attachment_description"]), attachment_type=int(data_params["attachment_type"]), content_type=str(data_params["attachment_content_type"]), keywords=keywords, created_by=created_by, modified_by=modified_by, content=content) ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error creating attachment") return self.gateway_error_response(ex) def delete_attachment(self, attachment_id): try: ret = self.rr_client.delete_attachment(attachment_id) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error deleting attachment") return self.gateway_error_response(ex) def get_version_info(self, pack=None): import pkg_resources pkg_list = ["scioncc"] packs = self.config.get_safe(CFG_PREFIX + ".version_packages") if packs: pkg_list.extend(packs.split(",")) version = {} for package in pkg_list: try: if pack == "all": pack_deps = pkg_resources.require(package) version.update( {p.project_name: p.version for p in pack_deps}) else: version[package] = pkg_resources.require( package)[0].version # @TODO git versions for current? except pkg_resources.DistributionNotFound: pass try: dir_client = DirectoryServiceProcessClient(process=self.process) sys_attrs = dir_client.lookup("/System") if sys_attrs and isinstance(sys_attrs, dict): version.update({ k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower() }) except Exception as ex: log.exception("Could not determine system directory attributes") if pack and pack != "all": version = {k: v for (k, v) in version.iteritems() if k == pack} return self.gateway_json_response(version) # ========================================================================= # Security and governance helpers def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True return requesting_address in self.trusted_originators def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() #if user_session.get("actor_id", None) and user_session.get("valid_until", 0): if user_session.get("actor_id", None): # Get info from current server session # NOTE: Actor id may be inside server session expiry = int(user_session.get("valid_until", 0)) * 1000 if expiry: # This was a proper non-token server session authentication expiry = str(expiry) actor_id = user_session["actor_id"] log.info( "Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) else: # We are just taking the user_id out of the session # TODO: Need to check access token here expiry = str(expiry) if self.token_from_session: actor_id = user_session["actor_id"] log.info( "Request associated with actor's token from session; actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args[ "api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info( "Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request( [self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info( "Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token( authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry def in_login_whitelist(self, category, svc, op): """Returns True if service op is whitelisted for anonymous access""" entry = "%s/%s/%s" % (category, svc, op) return entry in self.no_login_whitelist def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity( actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent( "Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry # ------------------------------------------------------------------------- # Service call (messaging) helpers def register_request_callback(self, cb_func): if cb_func is None: pass elif self.request_callback: log.warn("Callback already registered") self.request_callback = cb_func def _call_request_callback(self, action, req_info): if not self.request_callback: return try: self.request_callback(action, req_info) except Exception: log.exception("Error calling request callback") def _add_cors_headers(self, resp): # Set CORS headers so that a Swagger client on a different domain can read spec resp.headers[ "Access-Control-Allow-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" resp.headers[ "Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT" resp.headers["Access-Control-Allow-Origin"] = "*" resp.headers[ "Access-Control-Request-Headers"] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" def _log_request_start(self, req_type="SG"): global req_seqnum req_seqnum += 1 req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type, req_url=request.url) flask.g.req_info = req_info webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url) self._call_request_callback("start", req_info) def _log_request_response(self, content_type, result="", content_length=-1, status_code=200): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_content_type"] = content_type req_info["resp_content_length"] = content_length req_info["resp_result"] = result req_info["resp_status"] = req_info.get("resp_status", status_code) def _log_request_error(self, result, status_code): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_error"] = True req_info["resp_status"] = status_code webapi_log.warn( "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"], req_info["request_id"], status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE]) self._call_request_callback("error", req_info) else: webapi_log.warn( "REQUEST ERROR (%s%s) - %s: %s", status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE]) def _log_request_end(self): req_info = flask.g.get("req_info", None) if req_info: req_info["end_time"] = time.time() webapi_log.info("%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s", req_info["req_type"], req_info["request_id"], req_info.get("resp_status", ""), req_info["end_time"] - req_info["start_time"], req_info.get("resp_content_length", ""), req_info.get("resp_content_type", "")) self._call_request_callback("end", req_info) else: webapi_log.warn("REQUEST END - missing start info") def _get_request_args(self): """Extracts service request arguments from HTTP request. Supports various methods and forms of encoding. Separates arguments for special parameters from service operation parameters. Returns a dict with the service request arguments, containing key params with the actual values for the service operation parameters. """ str_args = False request_args = {} if request.method == "POST" or request.method == "PUT": # Use only body args and ignore any args from query string if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): # JSON body request if request.data: request_args = json_loads(request.data) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} elif request.form: # Form encoded payload if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_args = json_loads(str(payload)) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} else: # Fallback: Directly use form values str_args = True request_args = { GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True) } else: # No args found in body request_args = {GATEWAY_ARG_PARAMS: {}} elif request.method == "GET": str_args = True REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"} args_dict = request.args.to_dict(flat=True) request_args = { k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL } req_params = { k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL } request_args[GATEWAY_ARG_PARAMS] = req_params request_args[ "str_args"] = str_args # Indicate downstream that args are str (GET or form encoded) #log.info("Request args: %s" % request_args) return request_args def _get_typed_arg_value(self, given_value, param_def, strict): """Returns a service operation argument value, based on a given value and param schema definition. """ param_type = param_def["type"] if isinstance(given_value, unicode): # Convert all unicode to str in UTF-8 given_value = given_value.encode( "utf8") # Make all unicode into str if isinstance(given_value, IonObjectBase) and ( given_value._get_type() == param_type or param_type in given_value._get_extends()): return given_value elif is_ion_object_dict(given_value) and ( param_type == "NoneType" or hasattr(objects, param_type)): return self.create_ion_object(given_value) elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"): arg_val = get_typed_value(given_value, targettype=param_type, strict=strict) return arg_val else: raise BadRequest("Cannot convert param value to type %s" % param_type) def create_parameter_list(self, service_def, operation, request_args, id_param=None): """Build service call parameter list dynamically from service operation definition """ service_schema = service_def.schema service_op_schema = service_schema["operations"][operation] svc_op_param_list = service_op_schema["in_list"] svc_params = {} if id_param: # Magic shorthand: if one argument is given, fill the first service argument if svc_op_param_list: fill_par = svc_op_param_list[0] fill_par_def = service_op_schema["in"][fill_par] arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False) svc_params[fill_par] = arg_val return svc_params request_args = request_args or {} # Cannot be strict for a URL string query arguments or directly form encoded strict_types = False if request_args.get("str_args", False) else self.strict_types req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {} for param_name in svc_op_param_list: param_def = service_op_schema["in"][param_name] if param_name in req_op_args: arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types) svc_params[param_name] = arg_val if "timeout" in request_args: svc_params["timeout"] = float(request_args["timeout"]) optional_args = [ param for param in req_op_args if param not in svc_params and param != "timeout" ] if optional_args and "optional_args" in svc_op_param_list: # Only support basic strings for these optional params for now svc_params["optional_args"] = { arg: str(req_op_args[arg]) for arg in optional_args } #log.info("Service params: %s" % svc_params) return svc_params def _get_gateway_headers(self): """Returns the headers that the service gateway uses to make service calls on behalf of itself (not a user passing through), e.g. for identity management purposes""" return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY} def get_secure_service_def(self, service_name): """Checks whether the service indicated by given service_name exists and/or is exposed after white and black listing. Returns service registry entry. """ if self.service_whitelist: if service_name not in self.service_whitelist: raise Unauthorized("Service access not permitted") if self.service_blacklist: if service_name in self.service_blacklist: raise Unauthorized("Service access not permitted") # Retrieve service definition target_service = get_service_registry().get_service_by_name( service_name) if not target_service: raise BadRequest("The requested service (%s) is not available" % service_name) if not target_service.client: raise Inconsistent( "Cannot find a client class for the specified service: %s" % service_name) if not target_service.schema: raise Inconsistent( "Cannot find a schema for the specified service: %s" % service_name) return target_service def build_message_headers(self, actor_id, expiry): """Returns the headers that the service gateway uses to make service calls on behalf of a user, based on the user session or request arguments""" headers = dict() headers[MSG_HEADER_ACTOR] = actor_id headers[MSG_HEADER_VALID] = expiry req_info = flask.g.get("req_info", None) if req_info: headers["request-id"] = str(req_info["request_id"]) # If this is an anonymous requester then there are no roles associated with the request if actor_id == DEFAULT_ACTOR_ID: headers[MSG_HEADER_ROLES] = dict() return headers try: # Check to see if the user's roles are cached already - keyed by user id if self.user_role_cache.has_key(actor_id): role_header = self.user_role_cache.get(actor_id) if role_header is not None: headers[MSG_HEADER_ROLES] = role_header return headers # The user's roles were not cached so hit the datastore to find it. role_list = self.org_client.list_actor_roles( actor_id, headers=self._get_gateway_headers()) org_roles = {} for role in role_list: org_roles.setdefault(role.org_governance_name, []).append(role) role_header = get_role_message_headers(org_roles) # Cache the roles by user id self.user_role_cache.put(actor_id, role_header) except Exception: role_header = dict( ) # Default to empty dict if there is a problem finding roles for the user headers[MSG_HEADER_ROLES] = role_header return headers def create_ion_object(self, object_params): """Create and initialize an ION object from a dictionary of parameters coming via HTTP, ready to be passed on to services/messaging. The object is validated after creation. Note: This is not called for service operation argument signatures """ new_obj = IonObject(object_params["type_"]) # Iterate over the parameters to add to object; have to do this instead # of passing a dict to get around restrictions in object creation on setting _id, _rev params for param in object_params: self.set_object_field(new_obj, param, object_params.get(param)) new_obj._validate( ) # verify that all of the object fields were set with proper types return new_obj def set_object_field(self, obj, field, field_val): """Recursively set sub object field values. TODO: This may be an expensive operation. May also be redundant with object code """ if isinstance(field_val, dict) and field != "kwargs": sub_obj = getattr(obj, field) if isinstance(sub_obj, IonObjectBase): if "type_" in field_val and field_val["type_"] != sub_obj.type_: if issubtype(field_val["type_"], sub_obj.type_): sub_obj = IonObject(field_val["type_"]) setattr(obj, field, sub_obj) else: raise Inconsistent( "Unable to walk the field %s - types don't match: %s %s" % (field, sub_obj.type_, field_val["type_"])) for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) elif isinstance(sub_obj, dict): setattr(obj, field, field_val) else: for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) else: # type_ already exists in the class. if field != "type_": setattr(obj, field, field_val) # ------------------------------------------------------------------------- # Response content helpers def json_response(self, response_data): """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects """ resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2) resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON) if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])): self._add_cors_headers(resp) self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj)) return resp def gateway_json_response(self, response_data): """Returns the normal service gateway response as JSON or as media in case the response is a media response """ if isinstance(response_data, MediaResponse): log.info("Media response. Content mimetype:%s", response_data.media_mimetype) content = response_data.body if response_data.internal_encoding == "base64": import base64 content = base64.decodestring(content) elif response_data.internal_encoding == "utf8": pass resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype) self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code) return resp if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(response_data, mimetype=return_mimetype) result = { GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200, } return self.json_response(result) def gateway_error_response(self, exc): """Forms a service gateway error response. Can extract multiple stacks from a multi-tier RPC service call exception """ if hasattr(exc, "get_stacks"): # Process potentially multiple stacks. full_error, exc_stacks = "", exc.get_stacks() for i in range(len(exc_stacks)): full_error += exc_stacks[i][0] + "\n" if i == 0: full_error += "".join( traceback.format_exception(*sys.exc_info())) else: entry = ApplicationException.format_stack(exc_stacks[i][1]) full_error += entry + "\n" exec_name = exc.__class__.__name__ else: exc_type, exc_obj, exc_tb = sys.exc_info() exec_name = exc_type.__name__ full_error = "".join(traceback.format_exception(*sys.exc_info())) status_code = getattr(exc, "status_code", 400) if self.log_errors: if self.develop_mode: if status_code == 401: log.warn("%s: %s", exec_name, exc) else: log.error(full_error) else: if status_code == 401: log.info("%s: %s", exec_name, exc) else: log.info(full_error) result = { GATEWAY_ERROR_EXCEPTION: exec_name, GATEWAY_ERROR_MESSAGE: str(exc.message), GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "" } if self.develop_mode: result[GATEWAY_ERROR_TRACE] = full_error if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(result, mimetype=return_mimetype) self._log_request_error(result, status_code) resp = self.json_response({ GATEWAY_ERROR: result, GATEWAY_STATUS: status_code }) # Q: Should HTTP status be the error code of the exception? resp.status_code = status_code return resp
class ServiceGateway(object): """ The Service Gateway exports service routes for a web server via a Flask blueprint. The gateway bridges HTTP requests to ION AMQP RPC calls. """ def __init__(self, process, config, response_class): global sg_instance sg_instance = self self.name = "service_gateway" self.process = process self.config = config self.response_class = response_class self.gateway_base_url = process.gateway_base_url self.develop_mode = self.config.get_safe(CFG_PREFIX + ".develop_mode") is True self.require_login = self.config.get_safe(CFG_PREFIX + ".require_login") is True # Optional list of trusted originators can be specified in config. self.trusted_originators = self.config.get_safe(CFG_PREFIX + ".trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info("Service Gateway will not check requests against trusted originators since none are configured.") # Service screening self.service_blacklist = self.config.get_safe(CFG_PREFIX + ".service_blacklist") or [] self.service_whitelist = self.config.get_safe(CFG_PREFIX + ".service_whitelist") or [] self.no_login_whitelist = set(self.config.get_safe(CFG_PREFIX + ".no_login_whitelist") or []) self.set_cors_headers = self.config.get_safe(CFG_PREFIX + ".set_cors") is True self.strict_types = self.config.get_safe(CFG_PREFIX + ".strict_types") is True # Swagger spec generation support self.swagger_cfg = self.config.get_safe(CFG_PREFIX + ".swagger_spec") or {} self._swagger_gen = None if self.swagger_cfg.get("enable", None) is True: self._swagger_gen = SwaggerSpecGenerator(config=self.swagger_cfg) # Get the user_cache_size self.user_cache_size = self.config.get_safe(CFG_PREFIX + ".user_cache_size", DEFAULT_USER_CACHE_SIZE) # Initialize an LRU Cache to keep user roles cached for performance reasons # maxSize = maximum number of elements to keep in cache # maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) self.log_errors = self.config.get_safe(CFG_PREFIX + ".log_errors", True) self.rr_client = ResourceRegistryServiceProcessClient(process=self.process) self.idm_client = IdentityManagementServiceProcessClient(process=self.process) self.org_client = OrgManagementServiceProcessClient(process=self.process) # ------------------------------------------------------------------------- # Lifecycle management def start(self): # Configure subscriptions for user_cache events self.event_subscriber = EventSubscriber( event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self._event_callback ) self.event_subscriber.add_event_subscription(event_type=OT.UserRoleCacheResetEvent) self.process.add_endpoint(self.event_subscriber) def stop(self): pass # Stop event subscribers - TODO: This hangs # self.process.remove_endpoint(self.event_subscriber) # ------------------------------------------------------------------------- # Event subscriber callbacks def _event_callback(self, event, *args, **kwargs): """ Callback function for receiving Events """ if isinstance(event, UserRoleModifiedEvent): # Event when User Roles are modified user_role_event = event org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) # Evict the user and their roles from the cache so that it gets updated with the next call. if self.user_role_cache and self.user_role_cache.has_key(actor_id): log.debug("Evicting user from the user_role_cache: %s" % actor_id) self.user_role_cache.evict(actor_id) elif isinstance(event, UserRoleCacheResetEvent): # An event is received to clear the user data cache self.user_role_cache.clear() # ------------------------------------------------------------------------- # Routes def sg_index(self): return self.gateway_json_response(SG_IDENTIFICATION) def get_service_spec(self, service_name=None, spec_name=None): try: if not self._swagger_gen: raise NotFound("Spec not available") if spec_name != "swagger.json": raise NotFound("Unknown spec format") swagger_json = self._swagger_gen.get_spec(service_name) resp = flask.make_response(flask.jsonify(swagger_json)) self._add_cors_headers(resp) return resp except Exception as ex: return self.gateway_error_response(ex) def process_gateway_request(self, service_name=None, operation=None, id_param=None): """ Makes a secure call to a SciON service operation via messaging. """ # TODO make this service smarter to respond to the mime type in the request data (ie. json vs text) self._log_request_start("SVC RPC") try: result = self._make_service_request(service_name, operation, id_param) return self.gateway_json_response(result) except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def rest_gateway_request(self, service_name, res_type, id_param=None): """ Makes a REST style call to a SciON service operation via messaging. Get with ID returns the resource, POST without ID creates, PUT with ID updates and GET without ID returns the collection. """ self._log_request_start("SVC REST") try: if not service_name: raise BadRequest("Service name missing") service_name = str(service_name) if not res_type: raise BadRequest("Resource type missing") res_type = str(res_type) if request.method == "GET" and id_param: operation = "read_" + res_type return self.process_gateway_request(service_name, operation, id_param) elif request.method == "GET": ion_res_type = "".join(x.title() for x in res_type.split("_")) res = self._make_service_request("resource_registry", "find_resources", ion_res_type) if len(res) == 2: return self.gateway_json_response(res[0]) raise BadRequest("Unexpected find_resources result") elif request.method == "PUT": operation = "update_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") if id_param: obj._id = id_param return self.process_gateway_request(service_name, operation, obj) elif request.method == "POST": operation = "create_" + res_type obj = self._extract_payload_data() if not obj: raise BadRequest("Argument object not found") return self.process_gateway_request(service_name, operation, obj) else: raise BadRequest("Bad REST request") except Exception as ex: return self.gateway_error_response(ex) finally: self._log_request_end() def _extract_payload_data(self): request_obj = None if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): if request.data: request_obj = json_loads(request.data) elif request.form: # Form encoded if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_obj = json_loads(str(payload)) if request_obj and is_ion_object_dict(request_obj): request_obj = self.create_ion_object(request_obj) return request_obj def _make_service_request(self, service_name=None, operation=None, id_param=None): """ Executes a secure call to a SciON service operation via messaging. """ if not service_name: if self.develop_mode: # Return a list of available services result = dict(available_services=get_service_registry().services.keys()) return result else: raise BadRequest("Service name missing") service_name = str(service_name) if not operation: if self.develop_mode: # Return a list of available operations result = dict(available_operations=[]) return result else: raise BadRequest("Service operation missing") operation = str(operation) # Apply service white list and black list for initial protection and get service client service_def = self.get_secure_service_def(service_name) target_client = service_def.client # Get service request arguments and operation parameter values request req_args = self._get_request_args() param_list = self.create_parameter_list(service_def, operation, req_args, id_param) # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request(req_args) in_login_whitelist = self.in_login_whitelist("request", service_name, operation) ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist) param_list["headers"] = self.build_message_headers(ion_actor_id, expiry) # Make service operation call client = target_client(process=self.process) method_call = getattr(client, operation) result = method_call(**param_list) return result def get_resource_schema(self, resource_type): try: # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = self.get_governance_info_from_request() ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry) return self.gateway_json_response(get_object_schema(resource_type)) except Exception as ex: return self.gateway_error_response(ex) def get_attachment(self, attachment_id): try: # Create client to interface attachment = self.rr_client.read_attachment(attachment_id, include_content=True) return self.response_class(attachment.content, mimetype=attachment.content_type) except Exception as ex: return self.gateway_error_response(ex) def create_attachment(self): try: payload = request.form[GATEWAY_ARG_JSON] json_params = json_loads(str(payload)) actor_id, expiry = self.get_governance_info_from_request(json_params) actor_id, expiry = self.validate_request(actor_id, expiry) headers = self.build_message_headers(actor_id, expiry) data_params = json_params[GATEWAY_ARG_PARAMS] resource_id = str(data_params.get("resource_id", "")) fil = request.files["file"] content = fil.read() keywords = [] keywords_str = data_params.get("keywords", "") if keywords_str.strip(): keywords = [str(x.strip()) for x in keywords_str.split(",")] created_by = data_params.get("attachment_created_by", "unknown user") modified_by = data_params.get("attachment_modified_by", "unknown user") # build attachment attachment = Attachment( name=str(data_params["attachment_name"]), description=str(data_params["attachment_description"]), attachment_type=int(data_params["attachment_type"]), content_type=str(data_params["attachment_content_type"]), keywords=keywords, created_by=created_by, modified_by=modified_by, content=content, ) ret = self.rr_client.create_attachment(resource_id=resource_id, attachment=attachment, headers=headers) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error creating attachment") return self.gateway_error_response(ex) def delete_attachment(self, attachment_id): try: ret = self.rr_client.delete_attachment(attachment_id) return self.gateway_json_response(ret) except Exception as ex: log.exception("Error deleting attachment") return self.gateway_error_response(ex) def get_version_info(self): import pkg_resources pkg_list = ["scioncc"] version = {} for package in pkg_list: try: version["%s-release" % package] = pkg_resources.require(package)[0].version # @TODO git versions for each? except pkg_resources.DistributionNotFound: pass try: dir_client = DirectoryServiceProcessClient(process=self.process) sys_attrs = dir_client.lookup("/System") if sys_attrs and isinstance(sys_attrs, dict): version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()}) except Exception as ex: log.exception("Could not determine system directory attributes") return self.gateway_json_response(version) # ========================================================================= # Security and governance helpers def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True return requesting_address in self.trusted_originators def get_governance_info_from_request(self, json_params=None): # Default values for governance headers. actor_id = DEFAULT_ACTOR_ID expiry = DEFAULT_EXPIRY authtoken = "" user_session = get_auth() if user_session.get("actor_id", None) and user_session.get("valid_until", 0): # Get info from current server session # NOTE: Actor id may be inside server session actor_id = user_session["actor_id"] expiry = str(int(user_session.get("valid_until", 0)) * 1000) log.info("Request associated with session actor_id=%s, expiry=%s", actor_id, expiry) # Developer access using api_key if self.develop_mode and "api_key" in request.args and request.args["api_key"]: actor_id = str(request.args["api_key"]) expiry = str(int(user_session.get("valid_until", 0)) * 1000) if 0 < int(expiry) < current_time_millis(): expiry = str(current_time_millis() + 10000) # flask.session["valid_until"] = int(expiry / 1000) log.info("Request associated with actor_id=%s, expiry=%s from developer api_key", actor_id, expiry) # Check in headers for OAuth2 bearer token auth_hdr = request.headers.get("authorization", None) if auth_hdr: valid, req = self.process.oauth.verify_request([self.process.oauth_scope]) if valid: actor_id = flask.g.oauth_user.get("actor_id", "") if actor_id: log.info("Request associated with actor_id=%s, expiry=%s from OAuth token", actor_id, expiry) return actor_id, DEFAULT_EXPIRY # Try to find auth token override if not authtoken: if json_params: if "authtoken" in json_params: authtoken = json_params["authtoken"] else: if "authtoken" in request.args: authtoken = str(request.args["authtoken"]) # Enable temporary authentication tokens to resolve to actor ids if authtoken: try: token_info = self.idm_client.check_authentication_token(authtoken, headers=self._get_gateway_headers()) actor_id = token_info.get("actor_id", actor_id) expiry = token_info.get("expiry", expiry) log.info("Resolved token %s into actor_id=%s expiry=%s", authtoken, actor_id, expiry) except NotFound: log.info("Provided authentication token not found: %s", authtoken) except Unauthorized: log.info("Authentication token expired or invalid: %s", authtoken) except Exception as ex: log.exception("Problem resolving authentication token") return actor_id, expiry def in_login_whitelist(self, category, svc, op): """Returns True if service op is whitelisted for anonymous access""" entry = "%s/%s/%s" % (category, svc, op) return entry in self.no_login_whitelist def validate_request(self, ion_actor_id, expiry, in_whitelist=False): # There is no point in looking up an anonymous user - so return default values. if ion_actor_id == DEFAULT_ACTOR_ID: # Since this is an anonymous request, there really is no expiry associated with it if not in_whitelist and self.require_login: raise Unauthorized("Anonymous access not permitted") else: return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY try: user = self.idm_client.read_actor_identity(actor_id=ion_actor_id, headers=self._get_gateway_headers()) except NotFound as e: if not in_whitelist and self.require_login: # This could be a restart of the system with a new preload. # TODO: Invalidate Flask sessions on relaunch/bootstrap with creating new secret user_session = get_auth() if user_session.get("actor_id", None) == ion_actor_id: clear_auth() raise Unauthorized("Invalid identity", exc_id="01.10") else: # If the user isn't found default to anonymous return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY # Need to convert to int first in order to compare against current time. try: int_expiry = int(expiry) except Exception as ex: raise Inconsistent("Unable to read the expiry value in the request '%s' as an int" % expiry) # The user has been validated as being known in the system, so not check the expiry and raise exception if # the expiry is not set to 0 and less than the current time. if 0 < int_expiry < current_time_millis(): if not in_whitelist and self.require_login: raise Unauthorized("User authentication expired") else: log.warn("User authentication expired") return DEFAULT_ACTOR_ID, DEFAULT_EXPIRY return ion_actor_id, expiry # ------------------------------------------------------------------------- # Service call (messaging) helpers def _add_cors_headers(self, resp): # Set CORS headers so that a Swagger client on a different domain can read spec resp.headers[ "Access-Control-Allow-Headers" ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" resp.headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS , PUT" resp.headers["Access-Control-Allow-Origin"] = "*" resp.headers[ "Access-Control-Request-Headers" ] = "Origin, X-Atmosphere-tracking-id, X-Atmosphere-Framework, X-Cache-Date, Content-Type, X-Atmosphere-Transport, *" def _log_request_start(self, req_type="SG"): global req_seqnum req_seqnum += 1 req_info = dict(request_id=req_seqnum, start_time=time.time(), req_type=req_type) flask.g.req_info = req_info webapi_log.info("%s REQUEST (%s) - %s", req_type, req_info["request_id"], request.url) def _log_request_response(self, content_type, result="", content_length=-1, status_code=200): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_content_type"] = content_type req_info["resp_content_length"] = content_length req_info["resp_result"] = result req_info["resp_status"] = req_info.get("resp_status", status_code) def _log_request_error(self, result, status_code): req_info = flask.g.get("req_info", None) if req_info: req_info["resp_error"] = True req_info["resp_status"] = status_code webapi_log.warn( "%s REQUEST (%s) ERROR (%s%s) - %s: %s", req_info["req_type"], req_info["request_id"], status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE], ) else: webapi_log.warn( "REQUEST ERROR (%s%s) - %s: %s", status_code, "/id=" + result[GATEWAY_ERROR_EXCID] if result[GATEWAY_ERROR_EXCID] else "", result[GATEWAY_ERROR_EXCEPTION], result[GATEWAY_ERROR_MESSAGE], ) def _log_request_end(self): req_info = flask.g.get("req_info", None) if req_info: req_info["end_time"] = time.time() webapi_log.info( "%s REQUEST (%s) RESP (%s) - %.3f s, %s bytes, %s", req_info["req_type"], req_info["request_id"], req_info.get("resp_status", ""), req_info["end_time"] - req_info["start_time"], req_info.get("resp_content_length", ""), req_info.get("resp_content_type", ""), ) else: webapi_log.warn("REQUEST END - missing start info") def _get_request_args(self): """Extracts service request arguments from HTTP request. Supports various methods and forms of encoding. Separates arguments for special parameters from service operation parameters. Returns a dict with the service request arguments, containing key params with the actual values for the service operation parameters. """ str_args = False request_args = {} if request.method == "POST" or request.method == "PUT": # Use only body args and ignore any args from query string if request.headers.get("content-type", "").startswith(CONT_TYPE_JSON): # JSON body request if request.data: request_args = json_loads(request.data) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} elif request.form: # Form encoded payload if GATEWAY_ARG_JSON in request.form: payload = request.form[GATEWAY_ARG_JSON] request_args = json_loads(str(payload)) if GATEWAY_ARG_PARAMS not in request_args: # Magic fallback: Directly use JSON first level as args if params key not present request_args = {GATEWAY_ARG_PARAMS: request_args} else: # Fallback: Directly use form values str_args = True request_args = {GATEWAY_ARG_PARAMS: request.form.to_dict(flat=True)} else: # No args found in body request_args = {GATEWAY_ARG_PARAMS: {}} elif request.method == "GET": str_args = True REQ_ARGS_SPECIAL = {"authtoken", "timeout", "headers"} args_dict = request.args.to_dict(flat=True) request_args = {k: request.args[k] for k in args_dict if k in REQ_ARGS_SPECIAL} req_params = {k: request.args[k] for k in args_dict if k not in REQ_ARGS_SPECIAL} request_args[GATEWAY_ARG_PARAMS] = req_params request_args["str_args"] = str_args # Indicate downstream that args are str (GET or form encoded) # log.info("Request args: %s" % request_args) return request_args def _get_typed_arg_value(self, given_value, param_def, strict): """Returns a service operation argument value, based on a given value and param schema definition. """ param_type = param_def["type"] if isinstance(given_value, unicode): # Convert all unicode to str in UTF-8 given_value = given_value.encode("utf8") # Make all unicode into str if isinstance(given_value, IonObjectBase) and ( given_value._get_type() == param_type or param_type in given_value._get_extends() ): return given_value elif is_ion_object_dict(given_value) and (param_type == "NoneType" or hasattr(objects, param_type)): return self.create_ion_object(given_value) elif param_type in ("str", "bool", "int", "float", "list", "dict", "NoneType"): arg_val = get_typed_value(given_value, targettype=param_type, strict=strict) return arg_val else: raise BadRequest("Cannot convert param value to type %s" % param_type) def create_parameter_list(self, service_def, operation, request_args, id_param=None): """Build service call parameter list dynamically from service operation definition """ service_schema = service_def.schema service_op_schema = service_schema["operations"][operation] svc_op_param_list = service_op_schema["in_list"] svc_params = {} if id_param: # Magic shorthand: if one argument is given, fill the first service argument if svc_op_param_list: fill_par = svc_op_param_list[0] fill_par_def = service_op_schema["in"][fill_par] arg_val = self._get_typed_arg_value(id_param, fill_par_def, strict=False) svc_params[fill_par] = arg_val return svc_params request_args = request_args or {} # Cannot be strict for a URL string query arguments or directly form encoded strict_types = False if request_args.get("str_args", False) else self.strict_types req_op_args = request_args.get(GATEWAY_ARG_PARAMS, None) or {} for param_name in svc_op_param_list: param_def = service_op_schema["in"][param_name] if param_name in req_op_args: arg_val = self._get_typed_arg_value(req_op_args[param_name], param_def, strict=strict_types) svc_params[param_name] = arg_val if "timeout" in request_args: svc_params["timeout"] = float(request_args["timeout"]) optional_args = [param for param in req_op_args if param not in svc_params and param != "timeout"] if optional_args and "optional_args" in svc_op_param_list: # Only support basic strings for these optional params for now svc_params["optional_args"] = {arg: str(req_op_args[arg]) for arg in optional_args} # log.info("Service params: %s" % svc_params) return svc_params def _get_gateway_headers(self): """Returns the headers that the service gateway uses to make service calls on behalf of itself (not a user passing through), e.g. for identity management purposes""" return {MSG_HEADER_ACTOR: self.name, MSG_HEADER_VALID: DEFAULT_EXPIRY} def get_secure_service_def(self, service_name): """Checks whether the service indicated by given service_name exists and/or is exposed after white and black listing. Returns service registry entry. """ if self.service_whitelist: if service_name not in self.service_whitelist: raise Unauthorized("Service access not permitted") if self.service_blacklist: if service_name in self.service_blacklist: raise Unauthorized("Service access not permitted") # Retrieve service definition target_service = get_service_registry().get_service_by_name(service_name) if not target_service: raise BadRequest("The requested service (%s) is not available" % service_name) if not target_service.client: raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name) if not target_service.schema: raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name) return target_service def build_message_headers(self, actor_id, expiry): """Returns the headers that the service gateway uses to make service calls on behalf of a user, based on the user session or request arguments""" headers = dict() headers[MSG_HEADER_ACTOR] = actor_id headers[MSG_HEADER_VALID] = expiry req_info = flask.g.get("req_info", None) if req_info: headers["request-id"] = str(req_info["request_id"]) # If this is an anonymous requester then there are no roles associated with the request if actor_id == DEFAULT_ACTOR_ID: headers[MSG_HEADER_ROLES] = dict() return headers try: # Check to see if the user's roles are cached already - keyed by user id if self.user_role_cache.has_key(actor_id): role_header = self.user_role_cache.get(actor_id) if role_header is not None: headers[MSG_HEADER_ROLES] = role_header return headers # The user's roles were not cached so hit the datastore to find it. role_list = self.org_client.list_actor_roles(actor_id, headers=self._get_gateway_headers()) org_roles = {} for role in role_list: org_roles.setdefault(role.org_governance_name, []).append(role) role_header = get_role_message_headers(org_roles) # Cache the roles by user id self.user_role_cache.put(actor_id, role_header) except Exception: role_header = dict() # Default to empty dict if there is a problem finding roles for the user headers[MSG_HEADER_ROLES] = role_header return headers def create_ion_object(self, object_params): """Create and initialize an ION object from a dictionary of parameters coming via HTTP, ready to be passed on to services/messaging. The object is validated after creation. Note: This is not called for service operation argument signatures """ new_obj = IonObject(object_params["type_"]) # Iterate over the parameters to add to object; have to do this instead # of passing a dict to get around restrictions in object creation on setting _id, _rev params for param in object_params: self.set_object_field(new_obj, param, object_params.get(param)) new_obj._validate() # verify that all of the object fields were set with proper types return new_obj def set_object_field(self, obj, field, field_val): """Recursively set sub object field values. TODO: This may be an expensive operation. May also be redundant with object code """ if isinstance(field_val, dict) and field != "kwargs": sub_obj = getattr(obj, field) if isinstance(sub_obj, IonObjectBase): if "type_" in field_val and field_val["type_"] != sub_obj.type_: if issubtype(field_val["type_"], sub_obj.type_): sub_obj = IonObject(field_val["type_"]) setattr(obj, field, sub_obj) else: raise Inconsistent( "Unable to walk the field %s - types don't match: %s %s" % (field, sub_obj.type_, field_val["type_"]) ) for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) elif isinstance(sub_obj, dict): setattr(obj, field, field_val) else: for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) else: # type_ already exists in the class. if field != "type_": setattr(obj, field, field_val) # ------------------------------------------------------------------------- # Response content helpers def json_response(self, response_data): """Private implementation of standard flask jsonify to specify the use of an encoder to walk ION objects """ resp_obj = json_dumps(response_data, default=encode_ion_object, indent=None if request.is_xhr else 2) resp = self.response_class(resp_obj, mimetype=CONT_TYPE_JSON) if self.develop_mode and (self.set_cors_headers or ("api_key" in request.args and request.args["api_key"])): self._add_cors_headers(resp) self._log_request_response(CONT_TYPE_JSON, resp_obj, len(resp_obj)) return resp def gateway_json_response(self, response_data): """Returns the normal service gateway response as JSON or as media in case the response is a media response """ if isinstance(response_data, MediaResponse): log.info("Media response. Content mimetype:%s", response_data.media_mimetype) content = response_data.body if response_data.internal_encoding == "base64": import base64 content = base64.decodestring(content) elif response_data.internal_encoding == "utf8": pass resp = self.response_class(content, response_data.code, mimetype=response_data.media_mimetype) self._log_request_response(response_data.media_mimetype, "raw", len(content), response_data.code) return resp if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(response_data, mimetype=return_mimetype) result = {GATEWAY_RESPONSE: response_data, GATEWAY_STATUS: 200} return self.json_response(result) def gateway_error_response(self, exc): """Forms a service gateway error response. Can extract multiple stacks from a multi-tier RPC service call exception """ if hasattr(exc, "get_stacks"): # Process potentially multiple stacks. full_error, exc_stacks = "", exc.get_stacks() for i in range(len(exc_stacks)): full_error += exc_stacks[i][0] + "\n" if i == 0: full_error += "".join(traceback.format_exception(*sys.exc_info())) else: entry = ApplicationException.format_stack(exc_stacks[i][1]) full_error += entry + "\n" exec_name = exc.__class__.__name__ else: exc_type, exc_obj, exc_tb = sys.exc_info() exec_name = exc_type.__name__ full_error = "".join(traceback.format_exception(*sys.exc_info())) status_code = getattr(exc, "status_code", 400) if self.log_errors: if self.develop_mode: if status_code == 401: log.warn("%s: %s", exec_name, exc) else: log.error(full_error) else: if status_code == 401: log.info("%s: %s", exec_name, exc) else: log.info(full_error) result = { GATEWAY_ERROR_EXCEPTION: exec_name, GATEWAY_ERROR_MESSAGE: str(exc.message), GATEWAY_ERROR_EXCID: getattr(exc, "exc_id", "") or "", } if self.develop_mode: result[GATEWAY_ERROR_TRACE] = full_error if RETURN_MIMETYPE_PARAM in request.args: return_mimetype = str(request.args[RETURN_MIMETYPE_PARAM]) return self.response_class(result, mimetype=return_mimetype) self._log_request_error(result, status_code) resp = self.json_response({GATEWAY_ERROR: result, GATEWAY_STATUS: status_code}) # Q: Should HTTP status be the error code of the exception? resp.status_code = status_code return resp
class ServiceGatewayService(BaseServiceGatewayService): """ The Service Gateway Service is the service that uses a gevent web server and Flask to bridge HTTP requests to AMQP RPC ION process service calls. """ def on_init(self): #defaults self.http_server = None #retain a pointer to this object for use in ProcessRPC calls global service_gateway_instance ###### # to prevent cascading failure, here's an attempted hack if service_gateway_instance is not None and service_gateway_instance.http_server is not None: service_gateway_instance.http_server.stop() # end hack ###### service_gateway_instance = self self.server_hostname = self.CFG.get_safe( 'container.service_gateway.web_server.hostname', DEFAULT_WEB_SERVER_HOSTNAME) self.server_port = self.CFG.get_safe( 'container.service_gateway.web_server.port', DEFAULT_WEB_SERVER_PORT) self.web_server_enabled = self.CFG.get_safe( 'container.service_gateway.web_server.enabled', True) self.web_logging = self.CFG.get_safe( 'container.service_gateway.web_server.log') self.log_errors = self.CFG.get_safe( 'container.service_gateway.log_errors', True) #Optional list of trusted originators can be specified in config. self.trusted_originators = self.CFG.get_safe( 'container.service_gateway.trusted_originators') if not self.trusted_originators: self.trusted_originators = None log.info( "Service Gateway will not check requests against trusted originators since none are configured." ) #Get the user_cache_size self.user_cache_size = self.CFG.get_safe( 'container.service_gateway.user_cache_size', DEFAULT_USER_CACHE_SIZE) #Initialize an LRU Cache to keep user roles cached for performance reasons #maxSize = maximum number of elements to keep in cache #maxAgeMs = oldest entry to keep self.user_role_cache = LRUCache(self.user_cache_size, 0, 0) #Start the gevent web server unless disabled if self.web_server_enabled: log.info("Starting service gateway on %s:%s", self.server_hostname, self.server_port) self.start_service(self.server_hostname, self.server_port) #Configure subscriptions for user_cache events self.user_role_event_subscriber = EventSubscriber( event_type=OT.UserRoleModifiedEvent, origin_type="Org", callback=self.user_role_event_callback) self.add_endpoint(self.user_role_event_subscriber) self.user_role_reset_subscriber = EventSubscriber( event_type=OT.UserRoleCacheResetEvent, callback=self.user_role_reset_callback) self.add_endpoint(self.user_role_reset_subscriber) def on_quit(self): self.stop_service() def start_service(self, hostname=DEFAULT_WEB_SERVER_HOSTNAME, port=DEFAULT_WEB_SERVER_PORT): """Responsible for starting the gevent based web server.""" if self.http_server is not None: self.stop_service() self.http_server = WSGIServer((hostname, port), service_gateway_app, log=self.web_logging) self.http_server.start() return True def stop_service(self): """Responsible for stopping the gevent based web server.""" if self.http_server is not None: self.http_server.stop() return True def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True for addr in self.trusted_originators: if requesting_address == addr: return True return False def user_role_event_callback(self, *args, **kwargs): """ This method is a callback function for receiving Events when User Roles are modified. """ user_role_event = args[0] org_id = user_role_event.origin actor_id = user_role_event.actor_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, actor_id, role_name)) #Evict the user and their roles from the cache so that it gets updated with the next call. if service_gateway_instance.user_role_cache and service_gateway_instance.user_role_cache.has_key( actor_id): log.debug('Evicting user from the user_role_cache: %s' % actor_id) service_gateway_instance.user_role_cache.evict(actor_id) def user_role_reset_callback(self, *args, **kwargs): ''' This method is a callback function for when an event is received to clear the user data cache ''' self.user_role_cache.clear()