def register_view_on_model(rule, endpoint, view_func, **options): # Only pass in the attrs that are included in the rule. # 1. Extract list of variables from the rule rulevars = [v for c, a, v in parse_rule(rule)] if options.get('host'): rulevars.extend(v for c, a, v in parse_rule(options['host'])) if options.get('subdomain'): rulevars.extend( v for c, a, v in parse_rule(options['subdomain'])) # Make a subset of cls.route_model_map with the required variables params = { v: cls.route_model_map[v] for v in rulevars if v in cls.route_model_map } # Hook up is_url_for with the view function's name, endpoint name and parameters. # Register the view for a specific app, unless we're in a Blueprint, # in which case it's not an app. # FIXME: The behaviour of a Blueprint + multi-app combo is unknown and needs tests. if isinstance(app, Blueprint): prefix = app.name + '.' reg_app = None else: prefix = '' reg_app = app cls.model.is_url_for(view_func.__name__, prefix + endpoint, _app=reg_app, **params)(view_func) if callback: # pragma: no cover callback(rule, endpoint, view_func, **options)
def register_view_on_model(rule, endpoint, view_func, **options): # Only pass in the attrs that are included in the rule. # 1. Extract list of variables from the rule rulevars = [v for c, a, v in parse_rule(rule)] if options.get('host'): rulevars.extend(v for c, a, v in parse_rule(options['host'])) if options.get('subdomain'): rulevars.extend(v for c, a, v in parse_rule(options['subdomain'])) # Make a subset of cls.route_model_map with the required variables params = {v: cls.route_model_map[v] for v in rulevars if v in cls.route_model_map} # Hook up is_url_for with the view function's name, endpoint name and parameters. # Register the view for a specific app, unless we're in a Blueprint, # in which case it's not an app. # FIXME: The behaviour of a Blueprint + multi-app combo is unknown and needs tests. if isinstance(app, Blueprint): prefix = app.name + '.' reg_app = None else: prefix = '' reg_app = app cls.model.is_url_for(view_func.__name__, prefix + endpoint, _app=reg_app, **params)(view_func) cls.model.register_view_for( app=reg_app, action=view_func.__name__, classview=cls, attr=view_func.__name__) if callback: # pragma: no cover callback(rule, endpoint, view_func, **options)
def parse_url(path: str): """ Parsing Flask route url to get the normal url path and parameter type. Based on Werkzeug_ builtin converters. .. _werkzeug: https://werkzeug.palletsprojects.com/en/0.15.x/routing/#builtin-converters """ subs = [] parameters = [] for converter, arguments, variable in parse_rule(path): if converter is None: subs.append(variable) continue subs.append(f'{{{variable}}}') args, kwargs = [], {} if arguments: args, kwargs = parse_converter_args(arguments) schema = get_converter(converter, *args, **kwargs) parameters.append({ 'name': variable, 'in': 'path', 'required': True, 'schema': schema, }) return ''.join(subs), parameters
def extract_path_params(path): PATH_TYPES = { 'int': 'integer', 'float': 'number', 'string': 'string', 'default': 'string', } params = [] for converter, arguments, variable in parse_rule(path): if not converter: continue if converter in PATH_TYPES: url_type = PATH_TYPES[converter] else: url_type = 'string' param = { 'name': variable, 'in': 'path', 'required': True, "schema": { "type": url_type } } if converter in PATH_TYPES: param['type'] = PATH_TYPES[converter] else: param['type'] = 'string' params.append(param) return params
def parse_werkzeug_url(url): """ Process a werkzeug URL rule. Args: url (str): The werkzeug URL rule to process. Returns: tuple: A tuple containing the OpenAPI formatted URL and a list of path segment descriptions. """ path = '' parameters = [] for typ, default, segment in parse_rule(url): if not typ: path += segment continue path += '{' + segment + '}' parameters.append({ 'name': segment, 'in': 'path', 'required': True, 'type': WERKZEUG_URL_SWAGGER_TYPE_MAP[typ] }) return path, parameters
def parse_werkzeug_rule(self, rule: str, ctx: dict) -> PathAndParams: """ Convert werkzeug rule to swagger path format and extract parameter info. """ params = {} with io.StringIO() as buf: for converter, arguments, variable in parse_rule(rule): if converter: if arguments is not None: args, kwargs = parse_converter_args(arguments) else: args = () kwargs = {} params[variable] = WerkzeugConverter( converter=converter, args=args, kwargs=kwargs, ) buf.write('{') buf.write(variable) buf.write('}') else: buf.write(variable) return PathAndParams(buf.getvalue(), params)
def _make_schema_for(path, func): """Build a default schema for `func` at `path`. `func` is an api function. `path` is a url path, as recognized by werkzeug's router. `_make_schema_for` will build a schema to validate the body of a request, which will expect the body to be a json object whose keys are (exactly) the set of positional arguments to `func` which cannot be drawn from `path`, and whose values are strings. If all of the arguments to `func` are accounted for by `path`, `_make_schema_for` will return `None` instead. """ path_vars = [ var for (converter, args, var) in parse_rule(path) if converter is not None ] argnames, _, _, _ = inspect.getargspec(func) schema = dict((name, basestring) for name in argnames) for var in path_vars: del schema[var] if schema == {}: return None return Schema(schema)
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] else: if cls.__name__.endswith("View"): route_base = cls.__name__[:-4].lower() else: route_base = cls.__name__.lower() if hasattr(cls, 'subdomain'): sub_rule = parse_rule(cls.subdomain) cls.base_args = [r[2] for r in sub_rule] + getattr(cls,'base_args',[]) return route_base.strip("/")
def _get_base_route__(cls): """Returns the route base to use for the current class.""" base_route = cls.__name__.lower() if cls.base_route is not None: base_route = cls.base_route base_rule = parse_rule(base_route) cls.base_args = [r[2] for r in base_rule] return base_route.strip("/")
def _parse_rule(rule: str) -> str: """parse route""" uri = '' for converter, args, variable in parse_rule(str(rule)): if converter is None: uri += variable continue uri += "{%s}" % variable return uri
def _resolve_method_parameters(self, rule: str, method): """ :param rule: path rule :param method: view function """ params = {} param_names = {} # resolve path variable in route path_variable_type = {} for converter, _, name in parse_rule(rule): if converter is None: continue if converter == 'int': path_variable_type[name] = int elif converter == 'float': path_variable_type[name] = float else: path_variable_type[name] = str args, type_hints = inspect_args(method) for arg, default in args.items(): if default is inspect._empty: required = True default = None else: required = False if inspect.isfunction(default) and getattr( request_annotation, default.__name__, None) is default: default = default() if not isinstance(default, FieldInfo): if arg in path_variable_type: annotation = PathVariable( dtype=None if arg in type_hints else path_variable_type[arg]) else: if arg in type_hints: arg_type = type_hints[arg] arg_ = analyze_arg_type(arg_type) if arg_.is_dict(): annotation = RequestBody() else: annotation = RequestParam() else: annotation = RequestParam() annotation.default = default default = annotation if arg in type_hints and default.dtype is None: default.dtype = type_hints[arg] if default.required is None: default.required = required params[arg] = default if default.name is not None: param_names[default.name] = arg return params, param_names
def _rule_to_url(rule): """ Given a werkzeug rule, return a URITemplate URL. """ url = '' for (converter, arguments, variable) in parse_rule(rule): if converter is None: url += variable else: url += '{' + variable + '}' return url
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] else: route_base = cls.default_route_base() return route_base.strip("/")
def url_builder(self, route): parts = [] for (converter, arguments, variable) in parse_rule(route.rule): parts.append({'variable': converter is not None, 'text': variable}) content = self.render('debug-api/url-builder.html', { 'route': route, 'parts': parts, 'url_for': url_for }) return Markup(content)
def make_resource_param(service, schema, rule, method): if not any(variable == 'resource_id' for _, _, variable in parse_rule(rule.rule)): return [] param = { 'in': 'path', 'name': 'resource_id', 'description': '', 'required': True, } param.update(get_resource_type(service, schema)) return [param]
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.__routebase__ is not None: route_base = cls.__routebase__ base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] elif hasattr(cls, '__tablename__') and cls.__tablename__ is not None: route_base = cls.__tablename__ else: route_base = cls.__name__.lower() return route_base.strip('/')
def get_converter(rule): """ Parse rule will extract the converter from the rule as a generator We iterate through the parse_rule results to find the converter parse_url returns the static rule part in the first iteration parse_url returns the dynamic rule part in the second iteration if its dynamic """ for converter, _, _ in parse_rule(str(rule)): if converter is not None: return converter return None
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] else: if cls.__name__.endswith("View"): route_base = cls.__name__[:-4].lower() else: route_base = cls.__name__.lower() return route_base.strip("/")
def translate_werkzeug_rule(rule): from werkzeug.routing import parse_rule buf = six.StringIO() for conv, arg, var in parse_rule(rule): if conv: buf.write('(') if conv != 'default': buf.write(conv) buf.write(':') buf.write(var) buf.write(')') else: buf.write(var) return buf.getvalue()
def translate_werkzeug_rule(rule): from werkzeug.routing import parse_rule buf = StringIO.StringIO() for conv, arg, var in parse_rule(rule): if conv: buf.write('(') if conv != 'default': buf.write(conv) buf.write(':') buf.write(var) buf.write(')') else: buf.write(var) return buf.getvalue()
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] else: route_base = cls.__name__ for suffix in suffixes: if route_base.endswith(suffix): route_base = route_base[:-len(suffix)] break route_base = route_base.lower() return route_base.strip("/")
def extract_path_params(path): ''' Extract Flask-style parameters from an URL pattern as Swagger ones. ''' params = OrderedDict() for converter, _, variable in parse_rule(path): if not converter: continue param = {'name': variable, 'in': 'path', 'required': True} if converter in PATH_TYPES: param['type'] = PATH_TYPES[converter] else: raise ValueError(f'Unsupported type converter: {converter}') params[variable] = param return params
def get_route_base(cls): """Returns the route base to use for the current class.""" if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) # see: https://github.com/teracyhq/flask-classful/issues/50 if hasattr(cls, 'base_args'): # thanks to: https://github.com/teracyhq/flask-classful/pull/56#issuecomment-328985183 cls.base_args = list( set(cls.base_args).union(r[2] for r in base_rule)) else: cls.base_args = [r[2] for r in base_rule] else: route_base = cls.default_route_base() return route_base.strip("/")
def extract_path_params(path): """ Extract Flask-style parameters from an URL pattern as Swagger ones. """ params = OrderedDict() for converter, arguments, variable in parse_rule(path): if not converter: continue param = {"name": variable, "in": "path", "required": True} if converter in PATH_TYPES: param["type"] = PATH_TYPES[converter] elif converter in current_app.url_map.converters: param["type"] = "string" else: raise ValueError("Unsupported type converter: %s" % converter) params[variable] = param return params
def extract_path_params(path): ''' Extract Flask-style parameters from an URL pattern as Swagger ones. ''' params = OrderedDict() for converter, arguments, variable in parse_rule(path): if not converter: continue param = {'name': variable, 'in': 'path', 'required': True} if converter in PATH_TYPES: param['type'] = PATH_TYPES[converter] elif converter in current_app.url_map.converters: param['type'] = 'string' else: raise ValueError('Unsupported type converter: %s' % converter) params[variable] = param return params
def get_route_base(cls): """Returns the route base to use for the current class.""" first_cap_re = re.compile('(.)([A-Z][a-z]+)') all_cap_re = re.compile('([a-z0-9])([A-Z])') def dashify(name): s1 = first_cap_re.sub(r'\1-\2', name) return all_cap_re.sub(r'\1-\2', s1).lower() if cls.route_base is not None: route_base = cls.route_base base_rule = parse_rule(route_base) cls.base_args = [r[2] for r in base_rule] else: if cls.__name__.endswith("View"): route_base = dashify(cls.__name__[:-4]) else: route_base = dashify(cls.__name__) return route_base.strip("/")
def extract_path_params(path): ''' Extract Flask-style parameters from an URL pattern as Swagger ones. ''' params = OrderedDict() for converter, arguments, variable in parse_rule(path): if not converter: continue param = { 'name': variable, 'in': 'path', 'required': True } if converter in PATH_TYPES: param['type'] = PATH_TYPES[converter] elif converter in current_app.url_map.converters: param['type'] = 'string' else: raise ValueError('Unsupported type converter: %s' % converter) params[variable] = param return params
def _build_regex(rule): for converter, arguments, variable in parse_rule(rule): if converter is None: regex_parts.append(fnmatch.translate(variable)[:-7]) self._trace.append((False, variable)) for part in variable.split('/'): if part: self._weights.append((0, -len(part))) else: if arguments: c_args, c_kwargs = parse_converter_args(arguments) else: c_args = () c_kwargs = {} convobj = self.get_converter(variable, converter, c_args, c_kwargs) regex_parts.append('(?P<%s>%s)' % (variable, convobj.regex)) self._converters[variable] = convobj self._trace.append((True, variable)) self._weights.append((1, convobj.weight)) self.arguments.add(str(variable))
def _make_schema_for(path, func): """Build a default schema for `func` at `path`. `func` is an api function. `path` is a url path, as recognized by werkzeug's router. `_make_schema_for` will build a schema to validate the body of a request, which will expect the body to be a json object whose keys are (exactly) the set of positional arguments to `func` which cannot be drawn from `path`, and whose values are strings. If all of the arguments to `func` are accounted for by `path`, `_make_schema_for` will return `None` instead. """ path_vars = [var for (converter, args, var) in parse_rule(path) if converter is not None] argnames, _, _, _ = inspect.getargspec(func) schema = dict((name, basestring) for name in argnames) for var in path_vars: del schema[var] if schema == {}: return None return Schema(schema)
def before_request(self): if self.recording.is_running() and request.path != self.record_url: Metadata.add_framework('flask', flask.__version__) np = None # See # https://github.com/pallets/werkzeug/blob/2.0.0/src/werkzeug/routing.py#L213 # for a description of parse_rule. if request.url_rule: np = ''.join([ f'{{{p}}}' if c else p for c, _, p in parse_rule(request.url_rule.rule) ]) call_event = HttpServerRequestEvent( request_method=request.method, path_info=request.path, message_parameters=request_params(request), normalized_path_info=np, protocol=request.environ.get('SERVER_PROTOCOL'), headers=request.headers) Recorder().add_event(call_event) appctx = _app_ctx_stack.top appctx.appmap_request_event = call_event appctx.appmap_request_start = time.monotonic()
def parse_path(self, route): from werkzeug.routing import parse_converter_args, parse_rule subs = [] parameters = [] for converter, arguments, variable in parse_rule(str(route)): if converter is None: subs.append(variable) continue subs.append(f"{{{variable}}}") args, kwargs = [], {} if arguments: args, kwargs = parse_converter_args(arguments) schema = None if converter == "any": schema = { "type": "array", "items": { "type": "string", "enum": args, }, } elif converter == "int": schema = { "type": "integer", "format": "int32", } if "max" in kwargs: schema["maximum"] = kwargs["max"] if "min" in kwargs: schema["minimum"] = kwargs["min"] elif converter == "float": schema = { "type": "number", "format": "float", } elif converter == "uuid": schema = { "type": "string", "format": "uuid", } elif converter == "path": schema = { "type": "string", "format": "path", } elif converter == "string": schema = { "type": "string", } for prop in ["length", "maxLength", "minLength"]: if prop in kwargs: schema[prop] = kwargs[prop] elif converter == "default": schema = {"type": "string"} parameters.append({ "name": variable, "in": "path", "required": True, "schema": schema, }) return "".join(subs), parameters
from werkzeug.routing import Map,Rule, parse_rule m = Map([ Rule('/upload<path:f>', endpoint='upload'), ]) c = m.bind('example.com') print(m) url = c.build('upload', dict(f='foo/bar')) print url, 'match', c.test(url) url = c.build('upload', dict(f='/foo/bar')) print url, 'match', c.test(url) print [x for x in parse_rule('/upload<path:f>')]
def parse_path(self, route): from werkzeug.routing import parse_rule, parse_converter_args subs = [] parameters = [] for converter, arguments, variable in parse_rule(str(route)): if converter is None: subs.append(variable) continue subs.append(f'{{{variable}}}') args, kwargs = [], {} if arguments: args, kwargs = parse_converter_args(arguments) schema = None if converter == 'any': schema = { 'type': 'array', 'items': { 'type': 'string', 'enum': args, } } elif converter == 'int': schema = { 'type': 'integer', 'format': 'int32', } if 'max' in kwargs: schema['maximum'] = kwargs['max'] if 'min' in kwargs: schema['minimum'] = kwargs['min'] elif converter == 'float': schema = { 'type': 'number', 'format': 'float', } elif converter == 'uuid': schema = { 'type': 'string', 'format': 'uuid', } elif converter == 'path': schema = { 'type': 'string', 'format': 'path', } elif converter == 'string': schema = { 'type': 'string', } for prop in ['length', 'maxLength', 'minLength']: if prop in kwargs: schema[prop] = kwargs[prop] elif converter == 'default': schema = {'type': 'string'} parameters.append({ 'name': variable, 'in': 'path', 'required': True, 'schema': schema, }) return ''.join(subs), parameters
def swagger(app): output = { "swagger": "2.0", "info": { "title": u"\u00FCtool API Documentation", "description": "Welcome to the EPA's utool interactive RESTful API documentation.", # "termsOfService": "", #"contact": { # "name": u"\u00FCbertool Development Team", # # "url": "", # "email": "*****@*****.**", #}, # "license": { # "name": "", # "url": "" # }, "version": "0.0.1" }, "paths": defaultdict(dict), "definitions": defaultdict(dict), "tags": [] } paths = output['paths'] definitions = output['definitions'] tags = output['tags'] # TODO: Are these needed (from 'flask_swagger') ignore_http_methods = {"HEAD", "OPTIONS"} # technically only responses is non-optional optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security', 'deprecated', 'operationId', 'externalDocs'] # Loop over the Flask-RESTful endpoints being served (called "rules"...e.g. /terrplant/) for rule in app.url_map.iter_rules(): endpoint = app.view_functions[rule.endpoint] print(endpoint) try: class_name = endpoint.view_class() except AttributeError: continue # skip to next iteration in for-loop ("rule" does not contain an ubertool REST endpoint) try: inputs = class_name.get_model_inputs().__dict__ outputs = class_name.get_model_outputs().__dict__ except AttributeError: # This endpoint does not have get_model_inputs() or get_model_outputs() logging.exception(AttributeError.message) continue # skip to next iteration, as this is not an ubertool endpoint # TODO: Logic for UBERTOOL API ENDPOINTS - Move to separate function for code clarity??? methods = {} for http_method in rule.methods.difference(ignore_http_methods): if hasattr(endpoint, 'methods') and http_method in endpoint.methods: http_method = http_method.lower() methods[http_method] = endpoint.view_class.__dict__.get(http_method) else: methods[http_method.lower()] = endpoint # Extract the Rule argument from URL endpoint (e.g. /<jobId>) rule_param = None for converter, arguments, variable in parse_rule(str(rule)): # rule must already be converted to a string if converter: rule_param = variable # Get model name model_name = class_name.name # Instantiate ApiSpec() class for current endpoint and parse YAML for initial class instance properties api_spec = ApiSpec(model_name) # This has to be at the end of the for-loop because it converts the 'rule' object to a string # Rule = endpoint URL relative to hostname; needs to have special characters escaped to be defaultdict key rule = str(rule) for arg in re.findall('(<(.*?\:)?(.*?)>)', rule): rule = rule.replace(arg[0], '{{{0!s}}}'.format(arg[2])) # For each Rule (endpoint) iterate over its HTTP methods (e.g. POST, GET, PUT, etc...) for http_method, handler_method in methods.items(): if http_method == 'post': # Instantiate new Operation class operation = Operation() # Create Operations object from YAML operation.yaml_operation_parse( os.path.join(PROJECT_ROOT, 'REST_UBER', model_name + '_rest', 'apidoc.yaml',), model_name ) api_spec.paths.add_operation(operation) # Append Rule parameter name to parameters list if needed if rule_param: param = { 'in': "path", 'name': rule_param, 'description': "Job ID for model run", 'required': True, "type": "string" } # api_spec.parameters = [param] + api_spec.parameters operation.parameters.insert(0, param) # api_spec.parameters.append(param) # Update the 'path' key in the Swagger JSON with the 'operation' paths[rule].update({'post': operation.__dict__}) # Append the 'tag' (top-level) JSON for each rule/endpoint tag = api_spec.tags.create_tag(model_name, model_name.capitalize() + ' Model') tags.append(tag) # TODO: Definitions JSON; move to separate class definition_template_inputs = { 'type': "object", 'properties': { 'inputs': { "type": "object", "properties": {} }, 'run_type': { "type": 'string', "example": "single" } } } definition_template_outputs = { 'type': "object", 'properties': { 'user_id': { 'type': 'string', }, 'inputs': { # inputs_json 'type': 'object', 'properties': {} }, 'outputs': { # outputs_json 'type': 'object', 'properties': {} }, 'exp_out': { # exp_out_json 'type': 'object', 'properties': {} }, '_id': { 'type': 'string', }, 'run_type': { 'type': 'string', } } } model_def = { model_name.capitalize() + "Inputs": definition_template_inputs, model_name.capitalize() + "Outputs": definition_template_outputs } for k, v in inputs.items(): # Set the inputs to the input and output definition template model_def[model_name.capitalize() + "Inputs"]['properties']['inputs']['properties'][k] = \ model_def[model_name.capitalize() + "Outputs"]['properties']['inputs']['properties'][k] = { "type": "object", "properties": { "0": { # 'type' is JSON data type (e.g. 'number' is a float; 'string' is a string or binary) "type": 'string' if str(v.dtype) == 'object' else 'number', # 'format' is an optional modifier for primitives "format": 'string' if str(v.dtype) == 'object' else 'float' } } } for k, v in outputs.items(): # Set the outputs to the output definition template model_def[model_name.capitalize() + "Outputs"]['properties']['outputs']['properties'][k] = { "type": "object", "properties": { "0": { "type": 'string' if str(v.dtype) == 'object' else 'number', "format": 'string' if str(v.dtype) == 'object' else 'float' } } } definitions.update(model_def) if http_method == 'get': # Instantiate new Operation class operation = Operation( tags=[model_name], summary="Returns " + model_name.capitalize() + " JSON schema", description="Returns the JSON schema needed by the POST method to run " + model_name.capitalize() + " model", parameters=[], produces=['application/json'], responses=OperationResponses( 200, "Returns model input schema required for POST method", schema={ "allOf": [ { "$ref": "#/definitions/" + model_name.capitalize() + "Outputs" }, { "type": "object", "properties": { "notes": { "type": "object", "properties": { "info": {'type': 'string'}, "POST": {'type': 'string'}, "GET": {'type': 'string'}, "www": {'type': 'string'} } }, } } ] } ).get_json() ) paths[rule].update({'get': operation.__dict__}) return output
def swagger(app): output = { "swagger": "2.0", "info": { "title": u"\u00FCbertool API Documentation", "description": "Welcome to the EPA's ubertool interactive RESTful API documentation.", # "termsOfService": "", "contact": { "name": u"\u00FCbertool Development Team", # "url": "", "email": "*****@*****.**", }, # "license": { # "name": "", # "url": "" # }, "version": "0.0.1" }, "paths": defaultdict(dict), "definitions": defaultdict(dict), "tags": [] } paths = output['paths'] definitions = output['definitions'] tags = output['tags'] # TODO: Are these needed (from 'flask_swagger') ignore_http_methods = {"HEAD", "OPTIONS"} # technically only responses is non-optional optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security', 'deprecated', 'operationId', 'externalDocs'] # Loop over the Flask-RESTful endpoints being served (called "rules"...e.g. /terrplant/) for rule in app.url_map.iter_rules(): endpoint = app.view_functions[rule.endpoint] try: class_name = endpoint.view_class() except AttributeError: continue # skip to next iteration in for-loop ("rule" does not contain an ubertool REST endpoint) try: inputs = class_name.get_model_inputs().__dict__ outputs = class_name.get_model_outputs().__dict__ except AttributeError: # This endpoint does not have get_model_inputs() or get_model_outputs() logging.exception(AttributeError.message) continue # skip to next iteration, as this is not an ubertool endpoint # TODO: Logic for UBERTOOL API ENDPOINTS - Move to separate function for code clarity??? methods = {} for http_method in rule.methods.difference(ignore_http_methods): if hasattr(endpoint, 'methods') and http_method in endpoint.methods: http_method = http_method.lower() methods[http_method] = endpoint.view_class.__dict__.get(http_method) else: methods[http_method.lower()] = endpoint # Extract the Rule argument from URL endpoint (e.g. /<jobId>) rule_param = None for converter, arguments, variable in parse_rule(str(rule)): # rule must already be converted to a string if converter: rule_param = variable # Get model name model_name = class_name.name # Instantiate ApiSpec() class for current endpoint and parse YAML for initial class instance properties api_spec = ApiSpec(model_name) # This has to be at the end of the for-loop because it converts the 'rule' object to a string # Rule = endpoint URL relative to hostname; needs to have special characters escaped to be defaultdict key rule = str(rule) for arg in re.findall('(<(.*?\:)?(.*?)>)', rule): rule = rule.replace(arg[0], '{%s}' % arg[2]) # For each Rule (endpoint) iterate over its HTTP methods (e.g. POST, GET, PUT, etc...) for http_method, handler_method in methods.items(): if http_method == 'post': # Instantiate new Operation class operation = Operation() # Create Operations object from YAML operation.yaml_operation_parse( os.path.join(PROJECT_ROOT, 'REST_UBER', model_name + '_rest', 'apidoc.yaml',), model_name ) api_spec.paths.add_operation(operation) # Append Rule parameter name to parameters list if needed if rule_param: param = { 'in': "path", 'name': rule_param, 'description': "Job ID for model run", 'required': True, "type": "string" } # api_spec.parameters = [param] + api_spec.parameters operation.parameters.insert(0, param) # api_spec.parameters.append(param) print "pause" # Update the 'path' key in the Swagger JSON with the 'operation' paths[rule].update({'post': operation.__dict__}) # Append the 'tag' (top-level) JSON for each rule/endpoint tag = api_spec.tags.create_tag(model_name, model_name.capitalize() + ' Model') tags.append(tag) # TODO: Definitions JSON; move to separate class definition_template_inputs = { 'type': "object", 'properties': { 'inputs': { "type": "object", "properties": {} }, 'run_type': { "type": 'string', "example": "single" } } } definition_template_outputs = { 'type': "object", 'properties': { 'user_id': { 'type': 'string', }, 'inputs': { # inputs_json 'type': 'object', 'properties': {} }, 'outputs': { # outputs_json 'type': 'object', 'properties': {} }, 'exp_out': { # exp_out_json 'type': 'object', 'properties': {} }, '_id': { 'type': 'string', }, 'run_type': { 'type': 'string', } } } model_def = { model_name.capitalize() + "Inputs": definition_template_inputs, model_name.capitalize() + "Outputs": definition_template_outputs } for k, v in inputs.items(): # Set the inputs to the input and output definition template model_def[model_name.capitalize() + "Inputs"]['properties']['inputs']['properties'][k] = \ model_def[model_name.capitalize() + "Outputs"]['properties']['inputs']['properties'][k] = { "type": "object", "properties": { "0": { # 'type' is JSON data type (e.g. 'number' is a float; 'string' is a string or binary) "type": 'string' if str(v.dtype) == 'object' else 'number', # 'format' is an optional modifier for primitives "format": 'string' if str(v.dtype) == 'object' else 'float' } } } for k, v in outputs.items(): # Set the outputs to the output definition template model_def[model_name.capitalize() + "Outputs"]['properties']['outputs']['properties'][k] = { "type": "object", "properties": { "0": { "type": 'string' if str(v.dtype) == 'object' else 'number', "format": 'string' if str(v.dtype) == 'object' else 'float' } } } definitions.update(model_def) if http_method == 'get': # Instantiate new Operation class operation = Operation( tags=[model_name], summary="Returns " + model_name.capitalize() + " JSON schema", description="Returns the JSON schema needed by the POST method to run " + model_name.capitalize() + " model", parameters=[], produces=['application/json'], responses=OperationResponses( 200, "Returns model input schema required for POST method", schema={ "allOf": [ { "$ref": "#/definitions/" + model_name.capitalize() + "Outputs" }, { "type": "object", "properties": { "notes": { "type": "object", "properties": { "info": {'type': 'string'}, "POST": {'type': 'string'}, "GET": {'type': 'string'}, "www": {'type': 'string'} } }, } } ] } ).get_json() ) paths[rule].update({'get': operation.__dict__}) return output
def scan_rule(self, rule, generate_urls, inspect_content, inspect_meta_description, inspect_alt_tags, inspect_title): inspect_html = inspect_meta_description or inspect_alt_tags or inspect_title view_function = self.app.view_functions[rule.endpoint] # Count up the number of arguments num_args = 0 for converter, arguments, variable in parse_rule(str(rule)): if converter: num_args += 1 if num_args > 0: # The function has args! args_function = self.endpoint_argument_functions.get(rule.endpoint) if not args_function: raise ConfigurationError('No args function for endpoint "{}". You need to use the ' 'args_function decorator to add a function that yields all of the ' 'combinations of arguments for this function'.format(rule.endpoint)) else: def args_function(): return [{}] results = [] base_url = '{}://{}'.format(self.scheme, self.server_name) for kwargs in args_function(): log.debug(' > args: {}'.format(kwargs)) with self.app.test_request_context(base_url=base_url): result = ScannerResult(rule.endpoint, kwargs) if generate_urls: try: result.url = url_for(rule.endpoint, _external=True, _scheme=self.scheme, **kwargs) except BuildError as e: raise InvalidViewArguments('The arguments {} are not valid for endpoint {}' .format(kwargs, rule.endpoint)) from e if inspect_content: try: response = view_function(**kwargs) except Exception as e: if kwargs: error_message = 'Endpoint {} with args {} raised an exception'.format( rule.endpoint, kwargs ) else: error_message = 'Endpoint {} raised an exception'.format( rule.endpoint ) raise ViewRaisedException(error_message) from e if isinstance(response, str): response_text = response status_code = 200 mimetype = 'text/html' elif isinstance(response, BaseResponse): status_code = response.status_code mimetype = response.mimetype response_text = '' if mimetype.startswith('text/'): for response_part in response.response: response_text += response_part.decode('utf-8') if response.last_modified: result.last_modified = response.last_modified elif isinstance(response, tuple): if len(response) != 2: raise ValueError('Unhandled responstype: tuple with length {}' .format(len(response))) response_text, status_code = response if not isinstance(response_text, str): raise ValueError('Unhandled response_type: tuple with non str as first item') mimetype = 'text/html' else: raise ValueError('Unhandled response type: {}'.format(type(response))) truncated_response_text = response_text.replace('\n', '')[:30] log.debug('{} ({}) {}...'.format( status_code, mimetype, truncated_response_text )) result.status_code = status_code result.mime_type = mimetype if inspect_html and 'html' in result.mime_type: soup = BeautifulSoup(response_text, 'html.parser') if inspect_meta_description: head = soup.find('head') if head: meta_desc_tags = head.find_all('meta', attrs={"name": re.compile(r"^\s*description\s*$", re.I)}) if len(meta_desc_tags) > 1: all_tags = '; '.join([str(t) for t in meta_desc_tags]) raise HtmlParsingError( 'Found multiple description tags for endpoint {}: {}' .format(rule.endpoint, all_tags) ) elif len(meta_desc_tags) == 1: result.meta_description = meta_desc_tags[0]['content'] if inspect_alt_tags: missing_alt_tags = [] all_images = soup.find_all('img') for image in all_images: # Sometimes we may need to skip some images if self.ignore_images_containing: src = image.get('src') skip = False for image_path_search in self.ignore_images_containing: if src and image_path_search in src: skip = True break if skip: continue alt = image.get('alt') if alt is None or not alt.strip(): missing_alt_tags.append(str(image)) result.missing_alt_tags = missing_alt_tags if inspect_title: head = soup.find('head') if head: title_tag = head.find('title') if title_tag: result.title = title_tag.text.strip() results.append(result) return results
def serve_schema(): paths = {} for item in self.m_routes: if len(item['methods']) > 0: rule = str(item['rule']) parameters = [] if item['body']: parameters.append(item['body']) done = '' for converter, arguments, variable in parse_rule(rule): if not converter: done += variable else: done += '{' + variable + '}' typ = '' if converter == 'int': typ = 'number' if converter == 'default' or converter == 'string' or converter == 'path': typ = 'string' if converter == 'float': typ = 'number' parameters.append({ 'name': variable, "in": "path", "required": True, "type": typ, }) dummy = paths[done] = {} for method in item['methods']: dummy[str(method).lower()] = { 'summary': item['summary'], 'consumes': [item['consumes']], 'produces': [item['produces']], 'parameters': parameters, 'security': item['security'], 'tags': item['tags'], "responses": { "200": { "description": "successful operation", } }, } return jsonify({ "openapi": "3.0.0", "servers": [{ 'url': self.url_prefix, }], "paths": paths, "tags": [{ 'name': a } for a in self.tags], "components": { "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "JWT" } } } })