def register_view_on_model(rule, endpoint, view_func, **options):
# Only pass in the attrs that are included in the rule.
# 1. Extract list of variables from the rule
rulevars = [v for c, a, v in parse_rule(rule)]
if options.get('host'):
rulevars.extend(v for c, a, v in parse_rule(options['host']))
if options.get('subdomain'):
rulevars.extend(
v for c, a, v in parse_rule(options['subdomain']))
# Make a subset of cls.route_model_map with the required variables
params = {
v: cls.route_model_map[v]
for v in rulevars if v in cls.route_model_map
}
# Hook up is_url_for with the view function's name, endpoint name and parameters.
# Register the view for a specific app, unless we're in a Blueprint,
# in which case it's not an app.
# FIXME: The behaviour of a Blueprint + multi-app combo is unknown and needs tests.
if isinstance(app, Blueprint):
prefix = app.name + '.'
reg_app = None
else:
prefix = ''
reg_app = app
cls.model.is_url_for(view_func.__name__,
prefix + endpoint,
_app=reg_app,
**params)(view_func)
if callback: # pragma: no cover
callback(rule, endpoint, view_func, **options)
def register_view_on_model(rule, endpoint, view_func, **options):
# Only pass in the attrs that are included in the rule.
# 1. Extract list of variables from the rule
rulevars = [v for c, a, v in parse_rule(rule)]
if options.get('host'):
rulevars.extend(v for c, a, v in parse_rule(options['host']))
if options.get('subdomain'):
rulevars.extend(v for c, a, v in parse_rule(options['subdomain']))
# Make a subset of cls.route_model_map with the required variables
params = {v: cls.route_model_map[v] for v in rulevars if v in cls.route_model_map}
# Hook up is_url_for with the view function's name, endpoint name and parameters.
# Register the view for a specific app, unless we're in a Blueprint,
# in which case it's not an app.
# FIXME: The behaviour of a Blueprint + multi-app combo is unknown and needs tests.
if isinstance(app, Blueprint):
prefix = app.name + '.'
reg_app = None
else:
prefix = ''
reg_app = app
cls.model.is_url_for(view_func.__name__, prefix + endpoint, _app=reg_app, **params)(view_func)
cls.model.register_view_for(
app=reg_app, action=view_func.__name__, classview=cls, attr=view_func.__name__)
if callback: # pragma: no cover
callback(rule, endpoint, view_func, **options)
def parse_url(path: str):
"""
Parsing Flask route url to get the normal url path and parameter type.
Based on Werkzeug_ builtin converters.
.. _werkzeug: https://werkzeug.palletsprojects.com/en/0.15.x/routing/#builtin-converters
"""
subs = []
parameters = []
for converter, arguments, variable in parse_rule(path):
if converter is None:
subs.append(variable)
continue
subs.append(f'{{{variable}}}')
args, kwargs = [], {}
if arguments:
args, kwargs = parse_converter_args(arguments)
schema = get_converter(converter, *args, **kwargs)
parameters.append({
'name': variable,
'in': 'path',
'required': True,
'schema': schema,
})
return ''.join(subs), parameters
def extract_path_params(path):
PATH_TYPES = {
'int': 'integer',
'float': 'number',
'string': 'string',
'default': 'string',
}
params = []
for converter, arguments, variable in parse_rule(path):
if not converter:
continue
if converter in PATH_TYPES:
url_type = PATH_TYPES[converter]
else:
url_type = 'string'
param = {
'name': variable,
'in': 'path',
'required': True,
"schema": {
"type": url_type
}
}
if converter in PATH_TYPES:
param['type'] = PATH_TYPES[converter]
else:
param['type'] = 'string'
params.append(param)
return params
def parse_werkzeug_url(url):
"""
Process a werkzeug URL rule.
Args:
url (str): The werkzeug URL rule to process.
Returns:
tuple: A tuple containing the OpenAPI formatted URL and a list
of path segment descriptions.
"""
path = ''
parameters = []
for typ, default, segment in parse_rule(url):
if not typ:
path += segment
continue
path += '{' + segment + '}'
parameters.append({
'name': segment,
'in': 'path',
'required': True,
'type': WERKZEUG_URL_SWAGGER_TYPE_MAP[typ]
})
return path, parameters
def parse_werkzeug_rule(self, rule: str, ctx: dict) -> PathAndParams:
"""
Convert werkzeug rule to swagger path format and
extract parameter info.
"""
params = {}
with io.StringIO() as buf:
for converter, arguments, variable in parse_rule(rule):
if converter:
if arguments is not None:
args, kwargs = parse_converter_args(arguments)
else:
args = ()
kwargs = {}
params[variable] = WerkzeugConverter(
converter=converter,
args=args,
kwargs=kwargs,
)
buf.write('{')
buf.write(variable)
buf.write('}')
else:
buf.write(variable)
return PathAndParams(buf.getvalue(), params)
def _make_schema_for(path, func):
"""Build a default schema for `func` at `path`.
`func` is an api function.
`path` is a url path, as recognized by werkzeug's router.
`_make_schema_for` will build a schema to validate the body of a request,
which will expect the body to be a json object whose keys are (exactly)
the set of positional arguments to `func` which cannot be drawn from
`path`, and whose values are strings.
If all of the arguments to `func` are accounted for by `path`,
`_make_schema_for` will return `None` instead.
"""
path_vars = [
var for (converter, args, var) in parse_rule(path)
if converter is not None
]
argnames, _, _, _ = inspect.getargspec(func)
schema = dict((name, basestring) for name in argnames)
for var in path_vars:
del schema[var]
if schema == {}:
return None
return Schema(schema)
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
if cls.__name__.endswith("View"):
route_base = cls.__name__[:-4].lower()
else:
route_base = cls.__name__.lower()
if hasattr(cls, 'subdomain'):
sub_rule = parse_rule(cls.subdomain)
cls.base_args = [r[2] for r in sub_rule] + getattr(cls,'base_args',[])
return route_base.strip("/")
def _get_base_route__(cls):
"""Returns the route base to use for the current class."""
base_route = cls.__name__.lower()
if cls.base_route is not None:
base_route = cls.base_route
base_rule = parse_rule(base_route)
cls.base_args = [r[2] for r in base_rule]
return base_route.strip("/")
def _parse_rule(rule: str) -> str:
"""parse route"""
uri = ''
for converter, args, variable in parse_rule(str(rule)):
if converter is None:
uri += variable
continue
uri += "{%s}" % variable
return uri
def _resolve_method_parameters(self, rule: str, method):
"""
:param rule: path rule
:param method: view function
"""
params = {}
param_names = {}
# resolve path variable in route
path_variable_type = {}
for converter, _, name in parse_rule(rule):
if converter is None:
continue
if converter == 'int':
path_variable_type[name] = int
elif converter == 'float':
path_variable_type[name] = float
else:
path_variable_type[name] = str
args, type_hints = inspect_args(method)
for arg, default in args.items():
if default is inspect._empty:
required = True
default = None
else:
required = False
if inspect.isfunction(default) and getattr(
request_annotation, default.__name__, None) is default:
default = default()
if not isinstance(default, FieldInfo):
if arg in path_variable_type:
annotation = PathVariable(
dtype=None if arg in
type_hints else path_variable_type[arg])
else:
if arg in type_hints:
arg_type = type_hints[arg]
arg_ = analyze_arg_type(arg_type)
if arg_.is_dict():
annotation = RequestBody()
else:
annotation = RequestParam()
else:
annotation = RequestParam()
annotation.default = default
default = annotation
if arg in type_hints and default.dtype is None:
default.dtype = type_hints[arg]
if default.required is None:
default.required = required
params[arg] = default
if default.name is not None:
param_names[default.name] = arg
return params, param_names
def _rule_to_url(rule):
"""
Given a werkzeug rule, return a URITemplate URL.
"""
url = ''
for (converter, arguments, variable) in parse_rule(rule):
if converter is None:
url += variable
else:
url += '{' + variable + '}'
return url
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
route_base = cls.default_route_base()
return route_base.strip("/")
def url_builder(self, route):
parts = []
for (converter, arguments, variable) in parse_rule(route.rule):
parts.append({'variable': converter is not None, 'text': variable})
content = self.render('debug-api/url-builder.html', {
'route': route,
'parts': parts,
'url_for': url_for
})
return Markup(content)
def make_resource_param(service, schema, rule, method):
if not any(variable == 'resource_id' for _, _, variable in parse_rule(rule.rule)):
return []
param = {
'in': 'path',
'name': 'resource_id',
'description': '',
'required': True,
}
param.update(get_resource_type(service, schema))
return [param]
def _rule_to_url(rule):
"""
Given a werkzeug rule, return a URITemplate URL.
"""
url = ''
for (converter, arguments, variable) in parse_rule(rule):
if converter is None:
url += variable
else:
url += '{' + variable + '}'
return url
def url_builder(self, route):
parts = []
for (converter, arguments, variable) in parse_rule(route.rule):
parts.append({'variable': converter is not None, 'text': variable})
content = self.render('debug-api/url-builder.html', {
'route': route,
'parts': parts,
'url_for': url_for
})
return Markup(content)
def make_resource_param(service, schema, rule, method):
if not any(variable == 'resource_id'
for _, _, variable in parse_rule(rule.rule)):
return []
param = {
'in': 'path',
'name': 'resource_id',
'description': '',
'required': True,
}
param.update(get_resource_type(service, schema))
return [param]
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.__routebase__ is not None:
route_base = cls.__routebase__
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
elif hasattr(cls, '__tablename__') and cls.__tablename__ is not None:
route_base = cls.__tablename__
else:
route_base = cls.__name__.lower()
return route_base.strip('/')
def get_converter(rule):
"""
Parse rule will extract the converter from the rule as a generator
We iterate through the parse_rule results to find the converter
parse_url returns the static rule part in the first iteration
parse_url returns the dynamic rule part in the second iteration if its dynamic
"""
for converter, _, _ in parse_rule(str(rule)):
if converter is not None:
return converter
return None
def get_converter(rule):
"""
Parse rule will extract the converter from the rule as a generator
We iterate through the parse_rule results to find the converter
parse_url returns the static rule part in the first iteration
parse_url returns the dynamic rule part in the second iteration if its dynamic
"""
for converter, _, _ in parse_rule(str(rule)):
if converter is not None:
return converter
return None
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.__routebase__ is not None:
route_base = cls.__routebase__
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
elif hasattr(cls, '__tablename__') and cls.__tablename__ is not None:
route_base = cls.__tablename__
else:
route_base = cls.__name__.lower()
return route_base.strip('/')
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
if cls.__name__.endswith("View"):
route_base = cls.__name__[:-4].lower()
else:
route_base = cls.__name__.lower()
return route_base.strip("/")
def translate_werkzeug_rule(rule):
from werkzeug.routing import parse_rule
buf = six.StringIO()
for conv, arg, var in parse_rule(rule):
if conv:
buf.write('(')
if conv != 'default':
buf.write(conv)
buf.write(':')
buf.write(var)
buf.write(')')
else:
buf.write(var)
return buf.getvalue()
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
if cls.__name__.endswith("View"):
route_base = cls.__name__[:-4].lower()
else:
route_base = cls.__name__.lower()
return route_base.strip("/")
def translate_werkzeug_rule(rule):
from werkzeug.routing import parse_rule
buf = StringIO.StringIO()
for conv, arg, var in parse_rule(rule):
if conv:
buf.write('(')
if conv != 'default':
buf.write(conv)
buf.write(':')
buf.write(var)
buf.write(')')
else:
buf.write(var)
return buf.getvalue()
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
route_base = cls.__name__
for suffix in suffixes:
if route_base.endswith(suffix):
route_base = route_base[:-len(suffix)]
break
route_base = route_base.lower()
return route_base.strip("/")
def extract_path_params(path):
'''
Extract Flask-style parameters from an URL pattern as Swagger ones.
'''
params = OrderedDict()
for converter, _, variable in parse_rule(path):
if not converter:
continue
param = {'name': variable, 'in': 'path', 'required': True}
if converter in PATH_TYPES:
param['type'] = PATH_TYPES[converter]
else:
raise ValueError(f'Unsupported type converter: {converter}')
params[variable] = param
return params
def get_route_base(cls):
"""Returns the route base to use for the current class."""
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
# see: https://github.com/teracyhq/flask-classful/issues/50
if hasattr(cls, 'base_args'):
# thanks to: https://github.com/teracyhq/flask-classful/pull/56#issuecomment-328985183
cls.base_args = list(
set(cls.base_args).union(r[2] for r in base_rule))
else:
cls.base_args = [r[2] for r in base_rule]
else:
route_base = cls.default_route_base()
return route_base.strip("/")
def extract_path_params(path):
"""
Extract Flask-style parameters from an URL pattern as Swagger ones.
"""
params = OrderedDict()
for converter, arguments, variable in parse_rule(path):
if not converter:
continue
param = {"name": variable, "in": "path", "required": True}
if converter in PATH_TYPES:
param["type"] = PATH_TYPES[converter]
elif converter in current_app.url_map.converters:
param["type"] = "string"
else:
raise ValueError("Unsupported type converter: %s" % converter)
params[variable] = param
return params
def extract_path_params(path):
'''
Extract Flask-style parameters from an URL pattern as Swagger ones.
'''
params = OrderedDict()
for converter, arguments, variable in parse_rule(path):
if not converter:
continue
param = {'name': variable, 'in': 'path', 'required': True}
if converter in PATH_TYPES:
param['type'] = PATH_TYPES[converter]
elif converter in current_app.url_map.converters:
param['type'] = 'string'
else:
raise ValueError('Unsupported type converter: %s' % converter)
params[variable] = param
return params
def get_route_base(cls):
"""Returns the route base to use for the current class."""
first_cap_re = re.compile('(.)([A-Z][a-z]+)')
all_cap_re = re.compile('([a-z0-9])([A-Z])')
def dashify(name):
s1 = first_cap_re.sub(r'\1-\2', name)
return all_cap_re.sub(r'\1-\2', s1).lower()
if cls.route_base is not None:
route_base = cls.route_base
base_rule = parse_rule(route_base)
cls.base_args = [r[2] for r in base_rule]
else:
if cls.__name__.endswith("View"):
route_base = dashify(cls.__name__[:-4])
else:
route_base = dashify(cls.__name__)
return route_base.strip("/")
def extract_path_params(path):
'''
Extract Flask-style parameters from an URL pattern as Swagger ones.
'''
params = OrderedDict()
for converter, arguments, variable in parse_rule(path):
if not converter:
continue
param = {
'name': variable,
'in': 'path',
'required': True
}
if converter in PATH_TYPES:
param['type'] = PATH_TYPES[converter]
elif converter in current_app.url_map.converters:
param['type'] = 'string'
else:
raise ValueError('Unsupported type converter: %s' % converter)
params[variable] = param
return params
def _build_regex(rule):
for converter, arguments, variable in parse_rule(rule):
if converter is None:
regex_parts.append(fnmatch.translate(variable)[:-7])
self._trace.append((False, variable))
for part in variable.split('/'):
if part:
self._weights.append((0, -len(part)))
else:
if arguments:
c_args, c_kwargs = parse_converter_args(arguments)
else:
c_args = ()
c_kwargs = {}
convobj = self.get_converter(variable, converter, c_args,
c_kwargs)
regex_parts.append('(?P<%s>%s)' %
(variable, convobj.regex))
self._converters[variable] = convobj
self._trace.append((True, variable))
self._weights.append((1, convobj.weight))
self.arguments.add(str(variable))
def _make_schema_for(path, func):
"""Build a default schema for `func` at `path`.
`func` is an api function.
`path` is a url path, as recognized by werkzeug's router.
`_make_schema_for` will build a schema to validate the body of a request,
which will expect the body to be a json object whose keys are (exactly)
the set of positional arguments to `func` which cannot be drawn from
`path`, and whose values are strings.
If all of the arguments to `func` are accounted for by `path`,
`_make_schema_for` will return `None` instead.
"""
path_vars = [var for (converter, args, var) in parse_rule(path)
if converter is not None]
argnames, _, _, _ = inspect.getargspec(func)
schema = dict((name, basestring) for name in argnames)
for var in path_vars:
del schema[var]
if schema == {}:
return None
return Schema(schema)
def before_request(self):
if self.recording.is_running() and request.path != self.record_url:
Metadata.add_framework('flask', flask.__version__)
np = None
# See
# https://github.com/pallets/werkzeug/blob/2.0.0/src/werkzeug/routing.py#L213
# for a description of parse_rule.
if request.url_rule:
np = ''.join([
f'{{{p}}}' if c else p
for c, _, p in parse_rule(request.url_rule.rule)
])
call_event = HttpServerRequestEvent(
request_method=request.method,
path_info=request.path,
message_parameters=request_params(request),
normalized_path_info=np,
protocol=request.environ.get('SERVER_PROTOCOL'),
headers=request.headers)
Recorder().add_event(call_event)
appctx = _app_ctx_stack.top
appctx.appmap_request_event = call_event
appctx.appmap_request_start = time.monotonic()
def parse_path(self, route):
from werkzeug.routing import parse_converter_args, parse_rule
subs = []
parameters = []
for converter, arguments, variable in parse_rule(str(route)):
if converter is None:
subs.append(variable)
continue
subs.append(f"{{{variable}}}")
args, kwargs = [], {}
if arguments:
args, kwargs = parse_converter_args(arguments)
schema = None
if converter == "any":
schema = {
"type": "array",
"items": {
"type": "string",
"enum": args,
},
}
elif converter == "int":
schema = {
"type": "integer",
"format": "int32",
}
if "max" in kwargs:
schema["maximum"] = kwargs["max"]
if "min" in kwargs:
schema["minimum"] = kwargs["min"]
elif converter == "float":
schema = {
"type": "number",
"format": "float",
}
elif converter == "uuid":
schema = {
"type": "string",
"format": "uuid",
}
elif converter == "path":
schema = {
"type": "string",
"format": "path",
}
elif converter == "string":
schema = {
"type": "string",
}
for prop in ["length", "maxLength", "minLength"]:
if prop in kwargs:
schema[prop] = kwargs[prop]
elif converter == "default":
schema = {"type": "string"}
parameters.append({
"name": variable,
"in": "path",
"required": True,
"schema": schema,
})
return "".join(subs), parameters
from werkzeug.routing import Map,Rule, parse_rule
m = Map([
Rule('/upload<path:f>', endpoint='upload'),
])
c = m.bind('example.com')
print(m)
url = c.build('upload', dict(f='foo/bar'))
print url, 'match', c.test(url)
url = c.build('upload', dict(f='/foo/bar'))
print url, 'match', c.test(url)
print [x for x in parse_rule('/upload<path:f>')]
def parse_path(self, route):
from werkzeug.routing import parse_rule, parse_converter_args
subs = []
parameters = []
for converter, arguments, variable in parse_rule(str(route)):
if converter is None:
subs.append(variable)
continue
subs.append(f'{{{variable}}}')
args, kwargs = [], {}
if arguments:
args, kwargs = parse_converter_args(arguments)
schema = None
if converter == 'any':
schema = {
'type': 'array',
'items': {
'type': 'string',
'enum': args,
}
}
elif converter == 'int':
schema = {
'type': 'integer',
'format': 'int32',
}
if 'max' in kwargs:
schema['maximum'] = kwargs['max']
if 'min' in kwargs:
schema['minimum'] = kwargs['min']
elif converter == 'float':
schema = {
'type': 'number',
'format': 'float',
}
elif converter == 'uuid':
schema = {
'type': 'string',
'format': 'uuid',
}
elif converter == 'path':
schema = {
'type': 'string',
'format': 'path',
}
elif converter == 'string':
schema = {
'type': 'string',
}
for prop in ['length', 'maxLength', 'minLength']:
if prop in kwargs:
schema[prop] = kwargs[prop]
elif converter == 'default':
schema = {'type': 'string'}
parameters.append({
'name': variable,
'in': 'path',
'required': True,
'schema': schema,
})
return ''.join(subs), parameters
def swagger(app):
output = {
"swagger": "2.0",
"info": {
"title": u"\u00FCtool API Documentation",
"description": "Welcome to the EPA's utool interactive RESTful API documentation.",
# "termsOfService": "",
#"contact": {
# "name": u"\u00FCbertool Development Team",
# # "url": "",
# "email": "*****@*****.**",
#},
# "license": {
# "name": "",
# "url": ""
# },
"version": "0.0.1"
},
"paths": defaultdict(dict),
"definitions": defaultdict(dict),
"tags": []
}
paths = output['paths']
definitions = output['definitions']
tags = output['tags']
# TODO: Are these needed (from 'flask_swagger')
ignore_http_methods = {"HEAD", "OPTIONS"}
# technically only responses is non-optional
optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security',
'deprecated', 'operationId', 'externalDocs']
# Loop over the Flask-RESTful endpoints being served (called "rules"...e.g. /terrplant/)
for rule in app.url_map.iter_rules():
endpoint = app.view_functions[rule.endpoint]
print(endpoint)
try:
class_name = endpoint.view_class()
except AttributeError:
continue # skip to next iteration in for-loop ("rule" does not contain an ubertool REST endpoint)
try:
inputs = class_name.get_model_inputs().__dict__
outputs = class_name.get_model_outputs().__dict__
except AttributeError:
# This endpoint does not have get_model_inputs() or get_model_outputs()
logging.exception(AttributeError.message)
continue # skip to next iteration, as this is not an ubertool endpoint
# TODO: Logic for UBERTOOL API ENDPOINTS - Move to separate function for code clarity???
methods = {}
for http_method in rule.methods.difference(ignore_http_methods):
if hasattr(endpoint, 'methods') and http_method in endpoint.methods:
http_method = http_method.lower()
methods[http_method] = endpoint.view_class.__dict__.get(http_method)
else:
methods[http_method.lower()] = endpoint
# Extract the Rule argument from URL endpoint (e.g. /<jobId>)
rule_param = None
for converter, arguments, variable in parse_rule(str(rule)): # rule must already be converted to a string
if converter:
rule_param = variable
# Get model name
model_name = class_name.name
# Instantiate ApiSpec() class for current endpoint and parse YAML for initial class instance properties
api_spec = ApiSpec(model_name)
# This has to be at the end of the for-loop because it converts the 'rule' object to a string
# Rule = endpoint URL relative to hostname; needs to have special characters escaped to be defaultdict key
rule = str(rule)
for arg in re.findall('(<(.*?\:)?(.*?)>)', rule):
rule = rule.replace(arg[0], '{{{0!s}}}'.format(arg[2]))
# For each Rule (endpoint) iterate over its HTTP methods (e.g. POST, GET, PUT, etc...)
for http_method, handler_method in methods.items():
if http_method == 'post':
# Instantiate new Operation class
operation = Operation()
# Create Operations object from YAML
operation.yaml_operation_parse(
os.path.join(PROJECT_ROOT, 'REST_UBER', model_name + '_rest', 'apidoc.yaml',),
model_name
)
api_spec.paths.add_operation(operation)
# Append Rule parameter name to parameters list if needed
if rule_param:
param = {
'in': "path",
'name': rule_param,
'description': "Job ID for model run",
'required': True,
"type": "string"
}
# api_spec.parameters = [param] + api_spec.parameters
operation.parameters.insert(0, param)
# api_spec.parameters.append(param)
# Update the 'path' key in the Swagger JSON with the 'operation'
paths[rule].update({'post': operation.__dict__})
# Append the 'tag' (top-level) JSON for each rule/endpoint
tag = api_spec.tags.create_tag(model_name, model_name.capitalize() + ' Model')
tags.append(tag)
# TODO: Definitions JSON; move to separate class
definition_template_inputs = {
'type': "object",
'properties': {
'inputs': {
"type": "object",
"properties": {}
},
'run_type': {
"type": 'string',
"example": "single"
}
}
}
definition_template_outputs = {
'type': "object",
'properties': {
'user_id': {
'type': 'string',
},
'inputs': {
# inputs_json
'type': 'object',
'properties': {}
},
'outputs': {
# outputs_json
'type': 'object',
'properties': {}
},
'exp_out': {
# exp_out_json
'type': 'object',
'properties': {}
},
'_id': {
'type': 'string',
},
'run_type': {
'type': 'string',
}
}
}
model_def = {
model_name.capitalize() + "Inputs": definition_template_inputs,
model_name.capitalize() + "Outputs": definition_template_outputs
}
for k, v in inputs.items():
# Set the inputs to the input and output definition template
model_def[model_name.capitalize() + "Inputs"]['properties']['inputs']['properties'][k] = \
model_def[model_name.capitalize() + "Outputs"]['properties']['inputs']['properties'][k] = {
"type": "object",
"properties": {
"0": {
# 'type' is JSON data type (e.g. 'number' is a float; 'string' is a string or binary)
"type": 'string' if str(v.dtype) == 'object' else 'number',
# 'format' is an optional modifier for primitives
"format": 'string' if str(v.dtype) == 'object' else 'float'
}
}
}
for k, v in outputs.items():
# Set the outputs to the output definition template
model_def[model_name.capitalize() + "Outputs"]['properties']['outputs']['properties'][k] = {
"type": "object",
"properties": {
"0": {
"type": 'string' if str(v.dtype) == 'object' else 'number',
"format": 'string' if str(v.dtype) == 'object' else 'float'
}
}
}
definitions.update(model_def)
if http_method == 'get':
# Instantiate new Operation class
operation = Operation(
tags=[model_name],
summary="Returns " + model_name.capitalize() + " JSON schema",
description="Returns the JSON schema needed by the POST method to run " + model_name.capitalize() +
" model",
parameters=[],
produces=['application/json'],
responses=OperationResponses(
200,
"Returns model input schema required for POST method",
schema={
"allOf": [
{
"$ref": "#/definitions/" + model_name.capitalize() + "Outputs"
},
{
"type": "object",
"properties": {
"notes": {
"type": "object",
"properties": {
"info": {'type': 'string'},
"POST": {'type': 'string'},
"GET": {'type': 'string'},
"www": {'type': 'string'}
}
},
}
}
]
}
).get_json()
)
paths[rule].update({'get': operation.__dict__})
return output
def swagger(app):
output = {
"swagger": "2.0",
"info": {
"title": u"\u00FCbertool API Documentation",
"description": "Welcome to the EPA's ubertool interactive RESTful API documentation.",
# "termsOfService": "",
"contact": {
"name": u"\u00FCbertool Development Team",
# "url": "",
"email": "*****@*****.**",
},
# "license": {
# "name": "",
# "url": ""
# },
"version": "0.0.1"
},
"paths": defaultdict(dict),
"definitions": defaultdict(dict),
"tags": []
}
paths = output['paths']
definitions = output['definitions']
tags = output['tags']
# TODO: Are these needed (from 'flask_swagger')
ignore_http_methods = {"HEAD", "OPTIONS"}
# technically only responses is non-optional
optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security',
'deprecated', 'operationId', 'externalDocs']
# Loop over the Flask-RESTful endpoints being served (called "rules"...e.g. /terrplant/)
for rule in app.url_map.iter_rules():
endpoint = app.view_functions[rule.endpoint]
try:
class_name = endpoint.view_class()
except AttributeError:
continue # skip to next iteration in for-loop ("rule" does not contain an ubertool REST endpoint)
try:
inputs = class_name.get_model_inputs().__dict__
outputs = class_name.get_model_outputs().__dict__
except AttributeError:
# This endpoint does not have get_model_inputs() or get_model_outputs()
logging.exception(AttributeError.message)
continue # skip to next iteration, as this is not an ubertool endpoint
# TODO: Logic for UBERTOOL API ENDPOINTS - Move to separate function for code clarity???
methods = {}
for http_method in rule.methods.difference(ignore_http_methods):
if hasattr(endpoint, 'methods') and http_method in endpoint.methods:
http_method = http_method.lower()
methods[http_method] = endpoint.view_class.__dict__.get(http_method)
else:
methods[http_method.lower()] = endpoint
# Extract the Rule argument from URL endpoint (e.g. /<jobId>)
rule_param = None
for converter, arguments, variable in parse_rule(str(rule)): # rule must already be converted to a string
if converter:
rule_param = variable
# Get model name
model_name = class_name.name
# Instantiate ApiSpec() class for current endpoint and parse YAML for initial class instance properties
api_spec = ApiSpec(model_name)
# This has to be at the end of the for-loop because it converts the 'rule' object to a string
# Rule = endpoint URL relative to hostname; needs to have special characters escaped to be defaultdict key
rule = str(rule)
for arg in re.findall('(<(.*?\:)?(.*?)>)', rule):
rule = rule.replace(arg[0], '{%s}' % arg[2])
# For each Rule (endpoint) iterate over its HTTP methods (e.g. POST, GET, PUT, etc...)
for http_method, handler_method in methods.items():
if http_method == 'post':
# Instantiate new Operation class
operation = Operation()
# Create Operations object from YAML
operation.yaml_operation_parse(
os.path.join(PROJECT_ROOT, 'REST_UBER', model_name + '_rest', 'apidoc.yaml',),
model_name
)
api_spec.paths.add_operation(operation)
# Append Rule parameter name to parameters list if needed
if rule_param:
param = {
'in': "path",
'name': rule_param,
'description': "Job ID for model run",
'required': True,
"type": "string"
}
# api_spec.parameters = [param] + api_spec.parameters
operation.parameters.insert(0, param)
# api_spec.parameters.append(param)
print "pause"
# Update the 'path' key in the Swagger JSON with the 'operation'
paths[rule].update({'post': operation.__dict__})
# Append the 'tag' (top-level) JSON for each rule/endpoint
tag = api_spec.tags.create_tag(model_name, model_name.capitalize() + ' Model')
tags.append(tag)
# TODO: Definitions JSON; move to separate class
definition_template_inputs = {
'type': "object",
'properties': {
'inputs': {
"type": "object",
"properties": {}
},
'run_type': {
"type": 'string',
"example": "single"
}
}
}
definition_template_outputs = {
'type': "object",
'properties': {
'user_id': {
'type': 'string',
},
'inputs': {
# inputs_json
'type': 'object',
'properties': {}
},
'outputs': {
# outputs_json
'type': 'object',
'properties': {}
},
'exp_out': {
# exp_out_json
'type': 'object',
'properties': {}
},
'_id': {
'type': 'string',
},
'run_type': {
'type': 'string',
}
}
}
model_def = {
model_name.capitalize() + "Inputs": definition_template_inputs,
model_name.capitalize() + "Outputs": definition_template_outputs
}
for k, v in inputs.items():
# Set the inputs to the input and output definition template
model_def[model_name.capitalize() + "Inputs"]['properties']['inputs']['properties'][k] = \
model_def[model_name.capitalize() + "Outputs"]['properties']['inputs']['properties'][k] = {
"type": "object",
"properties": {
"0": {
# 'type' is JSON data type (e.g. 'number' is a float; 'string' is a string or binary)
"type": 'string' if str(v.dtype) == 'object' else 'number',
# 'format' is an optional modifier for primitives
"format": 'string' if str(v.dtype) == 'object' else 'float'
}
}
}
for k, v in outputs.items():
# Set the outputs to the output definition template
model_def[model_name.capitalize() + "Outputs"]['properties']['outputs']['properties'][k] = {
"type": "object",
"properties": {
"0": {
"type": 'string' if str(v.dtype) == 'object' else 'number',
"format": 'string' if str(v.dtype) == 'object' else 'float'
}
}
}
definitions.update(model_def)
if http_method == 'get':
# Instantiate new Operation class
operation = Operation(
tags=[model_name],
summary="Returns " + model_name.capitalize() + " JSON schema",
description="Returns the JSON schema needed by the POST method to run " + model_name.capitalize() +
" model",
parameters=[],
produces=['application/json'],
responses=OperationResponses(
200,
"Returns model input schema required for POST method",
schema={
"allOf": [
{
"$ref": "#/definitions/" + model_name.capitalize() + "Outputs"
},
{
"type": "object",
"properties": {
"notes": {
"type": "object",
"properties": {
"info": {'type': 'string'},
"POST": {'type': 'string'},
"GET": {'type': 'string'},
"www": {'type': 'string'}
}
},
}
}
]
}
).get_json()
)
paths[rule].update({'get': operation.__dict__})
return output
def scan_rule(self, rule, generate_urls, inspect_content, inspect_meta_description, inspect_alt_tags,
inspect_title):
inspect_html = inspect_meta_description or inspect_alt_tags or inspect_title
view_function = self.app.view_functions[rule.endpoint]
# Count up the number of arguments
num_args = 0
for converter, arguments, variable in parse_rule(str(rule)):
if converter:
num_args += 1
if num_args > 0:
# The function has args!
args_function = self.endpoint_argument_functions.get(rule.endpoint)
if not args_function:
raise ConfigurationError('No args function for endpoint "{}". You need to use the '
'args_function decorator to add a function that yields all of the '
'combinations of arguments for this function'.format(rule.endpoint))
else:
def args_function():
return [{}]
results = []
base_url = '{}://{}'.format(self.scheme, self.server_name)
for kwargs in args_function():
log.debug(' > args: {}'.format(kwargs))
with self.app.test_request_context(base_url=base_url):
result = ScannerResult(rule.endpoint, kwargs)
if generate_urls:
try:
result.url = url_for(rule.endpoint, _external=True, _scheme=self.scheme, **kwargs)
except BuildError as e:
raise InvalidViewArguments('The arguments {} are not valid for endpoint {}'
.format(kwargs, rule.endpoint)) from e
if inspect_content:
try:
response = view_function(**kwargs)
except Exception as e:
if kwargs:
error_message = 'Endpoint {} with args {} raised an exception'.format(
rule.endpoint, kwargs
)
else:
error_message = 'Endpoint {} raised an exception'.format(
rule.endpoint
)
raise ViewRaisedException(error_message) from e
if isinstance(response, str):
response_text = response
status_code = 200
mimetype = 'text/html'
elif isinstance(response, BaseResponse):
status_code = response.status_code
mimetype = response.mimetype
response_text = ''
if mimetype.startswith('text/'):
for response_part in response.response:
response_text += response_part.decode('utf-8')
if response.last_modified:
result.last_modified = response.last_modified
elif isinstance(response, tuple):
if len(response) != 2:
raise ValueError('Unhandled responstype: tuple with length {}'
.format(len(response)))
response_text, status_code = response
if not isinstance(response_text, str):
raise ValueError('Unhandled response_type: tuple with non str as first item')
mimetype = 'text/html'
else:
raise ValueError('Unhandled response type: {}'.format(type(response)))
truncated_response_text = response_text.replace('\n', '')[:30]
log.debug('{} ({}) {}...'.format(
status_code, mimetype, truncated_response_text
))
result.status_code = status_code
result.mime_type = mimetype
if inspect_html and 'html' in result.mime_type:
soup = BeautifulSoup(response_text, 'html.parser')
if inspect_meta_description:
head = soup.find('head')
if head:
meta_desc_tags = head.find_all('meta', attrs={"name": re.compile(r"^\s*description\s*$", re.I)})
if len(meta_desc_tags) > 1:
all_tags = '; '.join([str(t) for t in meta_desc_tags])
raise HtmlParsingError(
'Found multiple description tags for endpoint {}: {}'
.format(rule.endpoint, all_tags)
)
elif len(meta_desc_tags) == 1:
result.meta_description = meta_desc_tags[0]['content']
if inspect_alt_tags:
missing_alt_tags = []
all_images = soup.find_all('img')
for image in all_images:
# Sometimes we may need to skip some images
if self.ignore_images_containing:
src = image.get('src')
skip = False
for image_path_search in self.ignore_images_containing:
if src and image_path_search in src:
skip = True
break
if skip:
continue
alt = image.get('alt')
if alt is None or not alt.strip():
missing_alt_tags.append(str(image))
result.missing_alt_tags = missing_alt_tags
if inspect_title:
head = soup.find('head')
if head:
title_tag = head.find('title')
if title_tag:
result.title = title_tag.text.strip()
results.append(result)
return results
def serve_schema():
paths = {}
for item in self.m_routes:
if len(item['methods']) > 0:
rule = str(item['rule'])
parameters = []
if item['body']:
parameters.append(item['body'])
done = ''
for converter, arguments, variable in parse_rule(rule):
if not converter:
done += variable
else:
done += '{' + variable + '}'
typ = ''
if converter == 'int':
typ = 'number'
if converter == 'default' or converter == 'string' or converter == 'path':
typ = 'string'
if converter == 'float':
typ = 'number'
parameters.append({
'name': variable,
"in": "path",
"required": True,
"type": typ,
})
dummy = paths[done] = {}
for method in item['methods']:
dummy[str(method).lower()] = {
'summary': item['summary'],
'consumes': [item['consumes']],
'produces': [item['produces']],
'parameters': parameters,
'security': item['security'],
'tags': item['tags'],
"responses": {
"200": {
"description": "successful operation",
}
},
}
return jsonify({
"openapi": "3.0.0",
"servers": [{
'url': self.url_prefix,
}],
"paths": paths,
"tags": [{
'name': a
} for a in self.tags],
"components": {
"securitySchemes": {
"bearerAuth": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT"
}
}
}
})
