def check_parameters_match(func, doc=None):
"""Helper to check docstring, returns list of incorrect results"""
from numpydoc import docscrape
incorrect = []
name_ = get_name(func)
if not name_.startswith('mne.') or name_.startswith('mne.externals'):
return incorrect
if inspect.isdatadescriptor(func):
return incorrect
args = _get_args(func)
# drop self
if len(args) > 0 and args[0] == 'self':
args = args[1:]
if doc is None:
with warnings.catch_warnings(record=True) as w:
doc = docscrape.FunctionDoc(func)
if len(w):
raise RuntimeError('Error for %s:\n%s' % (name_, w[0]))
# check set
param_names = [name for name, _, _ in doc['Parameters']]
# clean up some docscrape output:
param_names = [name.split(':')[0].strip('` ') for name in param_names]
param_names = [name for name in param_names if '*' not in name]
if len(param_names) != len(args):
bad = str(sorted(list(set(param_names) - set(args)) +
list(set(args) - set(param_names))))
if not any(d in name_ for d in _docstring_ignores) and \
'deprecation_wrapped' not in func.__code__.co_name:
incorrect += [name_ + ' arg mismatch: ' + bad]
else:
for n1, n2 in zip(param_names, args):
if n1 != n2:
incorrect += [name_ + ' ' + n1 + ' != ' + n2]
return incorrect
def check_parameters_match(func, doc=None, cls=None):
"""Check docstring, return list of incorrect results."""
from numpydoc import docscrape
incorrect = []
name_ = get_name(func, cls=cls)
if not name_.startswith('expyfun.') or \
name_.startswith('expyfun._externals'):
return incorrect
if inspect.isdatadescriptor(func):
return incorrect
args = _get_args(func)
# drop self
if len(args) > 0 and args[0] == 'self':
args = args[1:]
if doc is None:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
try:
doc = docscrape.FunctionDoc(func)
except Exception as exp:
incorrect += [name_ + ' parsing error: ' + str(exp)]
return incorrect
if len(w):
raise RuntimeError('Error for %s:\n%s' % (name_, w[0]))
# check set
parameters = doc['Parameters']
# clean up some docscrape output:
parameters = [[p[0].split(':')[0].strip('` '), p[2]] for p in parameters]
parameters = [p for p in parameters if '*' not in p[0]]
param_names = [p[0] for p in parameters]
if len(param_names) != len(args):
bad = str(
sorted(
list(set(param_names) - set(args)) +
list(set(args) - set(param_names))))
if not any(re.match(d, name_) for d in docstring_ignores) and \
'deprecation_wrapped' not in func.__code__.co_name:
incorrect += [name_ + ' arg mismatch: ' + bad]
else:
for n1, n2 in zip(param_names, args):
if n1 != n2:
incorrect += [name_ + ' ' + n1 + ' != ' + n2]
for param_name, desc in parameters:
desc = '\n'.join(desc)
full_name = name_ + '::' + param_name
if full_name in docstring_length_ignores:
assert len(desc) > char_limit # assert it actually needs to be
elif len(desc) > char_limit:
incorrect += [
'%s too long (%d > %d chars)' %
(full_name, len(desc), char_limit)
]
return incorrect
def getProcessorSpec(self, P):
assert (callable(P))
spec = {"name": P.name, "version": P.version}
npdoc = docscrape.FunctionDoc(P)
#npdoc={"Summary":"","Parameters":[]}
spec["description"] = npdoc["Summary"][0]
params0 = npdoc["Parameters"]
argspec0 = inspect.getfullargspec(
P.__call__ if inspect.isclass(P) else P)
defaults0 = argspec0.kwonlydefaults
if not defaults0:
defaults0 = {}
inputs, outputs, parameters = [], [], []
for j in range(len(params0)):
pp = params0[j]
pname = pp[0]
ptype = pp[1]
pdescr = pp[2][0]
qq = {"name": pname, "description": pdescr}
if pname in defaults0:
qq["optional"] = True
qq["default_value"] = defaults0[pname]
else:
qq["optional"] = False
if (ptype == 'INPUT'):
inputs.append(qq)
elif (ptype == 'OUTPUT'):
outputs.append(qq)
else:
qq["datatype"] = ptype
parameters.append(qq)
spec['inputs'] = inputs
spec['outputs'] = outputs
spec['parameters'] = parameters
if hasattr(P, 'test'):
spec['has_test'] = True
return spec
def check_docstring_parameters(func, doc=None, ignore=None):
"""Helper to check docstring.
Parameters
----------
func : callable
The function object to test.
doc : str, default=None
Docstring if it is passed manually to the test.
ignore : list, default=None
Parameters to ignore.
Returns
-------
incorrect : list
A list of string describing the incorrect results.
"""
from numpydoc import docscrape
incorrect = []
ignore = [] if ignore is None else ignore
func_name = _get_func_name(func)
if not func_name.startswith("sklearn.") or func_name.startswith(
"sklearn.externals"):
return incorrect
# Don't check docstring for property-functions
if inspect.isdatadescriptor(func):
return incorrect
# Don't check docstring for setup / teardown pytest functions
if func_name.split(".")[-1] in ("setup_module", "teardown_module"):
return incorrect
# Dont check estimator_checks module
if func_name.split(".")[2] == "estimator_checks":
return incorrect
# Get the arguments from the function signature
param_signature = list(filter(lambda x: x not in ignore, _get_args(func)))
# drop self
if len(param_signature) > 0 and param_signature[0] == "self":
param_signature.remove("self")
# Analyze function's docstring
if doc is None:
with warnings.catch_warnings(record=True) as w:
try:
doc = docscrape.FunctionDoc(func)
except Exception as exp:
incorrect += [func_name + " parsing error: " + str(exp)]
return incorrect
if len(w):
raise RuntimeError("Error for %s:\n%s" % (func_name, w[0]))
param_docs = []
for name, type_definition, param_doc in doc["Parameters"]:
# Type hints are empty only if parameter name ended with :
if not type_definition.strip():
if ":" in name and name[:name.index(":")][-1:].strip():
incorrect += [
func_name +
" There was no space between the param name and colon (%r)"
% name
]
elif name.rstrip().endswith(":"):
incorrect += [
func_name +
" Parameter %r has an empty type spec. Remove the colon" %
(name.lstrip())
]
# Create a list of parameters to compare with the parameters gotten
# from the func signature
if "*" not in name:
param_docs.append(name.split(":")[0].strip("` "))
# If one of the docstring's parameters had an error then return that
# incorrect message
if len(incorrect) > 0:
return incorrect
# Remove the parameters that should be ignored from list
param_docs = list(filter(lambda x: x not in ignore, param_docs))
# The following is derived from pytest, Copyright (c) 2004-2017 Holger
# Krekel and others, Licensed under MIT License. See
# https://github.com/pytest-dev/pytest
message = []
for i in range(min(len(param_docs), len(param_signature))):
if param_signature[i] != param_docs[i]:
message += [
"There's a parameter name mismatch in function"
" docstring w.r.t. function signature, at index %s"
" diff: %r != %r" % (i, param_signature[i], param_docs[i])
]
break
if len(param_signature) > len(param_docs):
message += [
"Parameters in function docstring have less items w.r.t."
" function signature, first missing item: %s" %
param_signature[len(param_docs)]
]
elif len(param_signature) < len(param_docs):
message += [
"Parameters in function docstring have more items w.r.t."
" function signature, first extra item: %s" %
param_docs[len(param_signature)]
]
# If there wasn't any difference in the parameters themselves between
# docstring and signature including having the same length then return
# empty list
if len(message) == 0:
return []
import difflib
import pprint
param_docs_formatted = pprint.pformat(param_docs).splitlines()
param_signature_formatted = pprint.pformat(param_signature).splitlines()
message += ["Full diff:"]
message.extend(line.strip() for line in difflib.ndiff(
param_signature_formatted, param_docs_formatted))
incorrect.extend(message)
# Prepend function name
incorrect = ["In function: " + func_name] + incorrect
return incorrect
def check_docstring_parameters(func, doc=None, ignore=None, class_name=None):
"""Helper to check docstring
Parameters
----------
func : callable
The function object to test.
doc : str, optional (default: None)
Docstring if it is passed manually to the test.
ignore : None | list
Parameters to ignore.
class_name : string, optional (default: None)
If ``func`` is a class method and the class name is known specify
class_name for the error message.
Returns
-------
incorrect : list
A list of string describing the incorrect results.
"""
from numpydoc import docscrape
incorrect = []
ignore = [] if ignore is None else ignore
func_name = _get_func_name(func, class_name=class_name)
if (not func_name.startswith('sklearn.')
or func_name.startswith('sklearn.externals')):
return incorrect
# Don't check docstring for property-functions
if inspect.isdatadescriptor(func):
return incorrect
args = list(filter(lambda x: x not in ignore, _get_args(func)))
# drop self
if len(args) > 0 and args[0] == 'self':
args.remove('self')
if doc is None:
with warnings.catch_warnings(record=True) as w:
try:
doc = docscrape.FunctionDoc(func)
except Exception as exp:
incorrect += [func_name + ' parsing error: ' + str(exp)]
return incorrect
if len(w):
raise RuntimeError('Error for %s:\n%s' % (func_name, w[0]))
param_names = []
for name, type_definition, param_doc in doc['Parameters']:
if not type_definition.strip():
if ':' in name and name[:name.index(':')][-1:].strip():
incorrect += [
func_name +
' There was no space between the param name and '
'colon (%r)' % name
]
elif name.rstrip().endswith(':'):
incorrect += [
func_name + ' Parameter %r has an empty type spec. '
'Remove the colon' % (name.lstrip())
]
if '*' not in name:
param_names.append(name.split(':')[0].strip('` '))
param_names = list(filter(lambda x: x not in ignore, param_names))
if len(param_names) != len(args):
bad = str(sorted(list(set(param_names) ^ set(args))))
incorrect += [func_name + ' arg mismatch: ' + bad]
else:
for n1, n2 in zip(param_names, args):
if n1 != n2:
incorrect += [func_name + ' ' + n1 + ' != ' + n2]
return incorrect
def generate_function(self, module, function,base_estimator,model_tag):
"""
function to generate open API of python function.
:param f:
:param baseurl:
:param outdir:
:param yaml:
:param write:
:return:
"""
type_table = {
'matrix': 'array',
'array': 'array',
'array-like': 'array',
'numpy array': 'array',
'bool': 'bool',
'int': 'int',
'float': 'float',
'string': 'str',
'dictionary': 'dict',
'dict': 'dict',
'object': 'self',
'self': 'self'
}
module = module
class_name = function
base_estimator = base_estimator
model_tag = model_tag
class_obj = getattr(module, class_name)
parsing_obj = docscrape.FunctionDoc(class_obj)
doc = class_obj.__doc__
paras_dict_func = self.get_parameters(parsing_obj, type_table)
paras_desc = self.get_docstrings(parsing_obj)
description = class_obj.__doc__.strip().split("\n")[0]
#title = class_obj.__name__
parametersfunc,params,docstring = self.populate_parameters_function(function, paras_dict_func, paras_desc)
text1 = '"""'
key = 'return'
returnparam =''
#print(paras_dict_func)
if key in paras_dict_func.keys():
if paras_dict_func['return'] != 'self':
returnparam = paras_dict_func['return']
else:
returnparam
returnparamindex = 0
returnparamindex = parametersfunc.find('return')
if (returnparamindex == -1):
pass
elif (returnparamindex == 0):
parametersfunc = ''
else:
parametersfunc = parametersfunc[:returnparamindex-2]
returnparamindex1 = 0
returnparamindex1 = params.find('return')
if (returnparamindex1 == -1):
pass
elif (returnparamindex1 == 0):
params = ''
else:
params = params[:returnparamindex1 - 2]
functionname = class_obj.__name__
match = re.search(r'X: array', parametersfunc)
X_param_index = parametersfunc.find('X: array')
sample_weight_index = parametersfunc.find('sample_weight: array')
if sample_weight_index != -1:
parametersfunc = parametersfunc[:sample_weight_index - 2]
sample_weight_index_params = params.find('sample_weight')
if sample_weight_index_params != -1:
params = params[:sample_weight_index_params - 2]
if match:
parametersfunc = parametersfunc + "," + " X_shape_x: int," + " X_shape_y: int"
parametersfunc = parametersfunc.replace('array','list')
X_numpyconversion = f"X = np.array(X).reshape(X_shape_x,X_shape_y)"
if returnparam != '':
if returnparam == 'array' and X_param_index == 0 :
returnparam = 'list'
spec = self.functiontemplatearrayandX_param.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_numpyconversion=X_numpyconversion,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
elif X_param_index == 0 :
spec = self.functiontemplatewithX_param.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_numpyconversion=X_numpyconversion,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
else:
spec = self.functiontemplate.format(
functioname=functionname,
description=description,
text1 = text1,
model_tag=model_tag,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
else:
if functionname == 'fit':
spec = self.functiontemplatefit.format(
functioname=functionname,
description=description,
base_estimator=base_estimator,
X_numpyconversion=X_numpyconversion,
model_tag=model_tag,
text1=text1,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
elif functionname =='set_params':
spec = self.functiontemplatesetparams.format(
functioname=functionname,
description=description,
base_estimator=base_estimator,
model_tag=model_tag,
text1=text1,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
else:
spec = self.functiontemplatereturningself.format(
functioname=functionname,
description=description,
text1=text1,
parameters=parametersfunc,
base_estimator=base_estimator,
model_tag=model_tag,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam
)
return spec
def generate_function(self, module, function, base_estimator, model_tag):
"""
:param module: Sklearn module like Linear Regression
:param function: Methods in the class of Linear Regression
:param base_estimator: Sklearn module like Linear Regression
:param model_tag: Tag used to store the name of the model instance
:return:
"""
type_table = {
'matrix': 'array',
'array': 'array',
'array-like': 'array',
'numpy array': 'array',
'bool': 'bool',
'int': 'int',
'float': 'float',
'string': 'str',
'dictionary': 'dict',
'dict': 'dict',
'object': 'self',
'self': 'self'
}
module = module
class_name = function
base_estimator = base_estimator
model_tag = model_tag
class_obj = getattr(module, class_name)
parsing_obj = docscrape.FunctionDoc(class_obj)
doc = class_obj.__doc__
paras_dict_func = self.get_parameters(parsing_obj, type_table)
paras_desc = self.get_docstrings(parsing_obj)
description = class_obj.__doc__.strip().split("\n")[0]
#title = class_obj.__name__
parametersfunc, params, docstring = self.populate_parameters_function(
function, paras_dict_func, paras_desc)
text1 = '"""'
key = 'return'
returnparam = ''
#print(paras_dict_func)
if key in paras_dict_func.keys():
if paras_dict_func['return'] != 'self':
returnparam = paras_dict_func['return']
else:
returnparam
returnparamindex = 0
returnparamindex = parametersfunc.find('return')
if (returnparamindex == -1):
pass
elif (returnparamindex == 0):
parametersfunc = ''
else:
parametersfunc = parametersfunc[:returnparamindex - 2]
returnparamindex1 = 0
returnparamindex1 = params.find('return')
if (returnparamindex1 == -1):
pass
elif (returnparamindex1 == 0):
params = ''
else:
params = params[:returnparamindex1 - 2]
functionname = class_obj.__name__
match = re.search(r'X: array', parametersfunc)
X_param_index = parametersfunc.find('X: array')
y_param_index = parametersfunc.find('y: array')
sample_weight_index = parametersfunc.find('sample_weight: array')
if sample_weight_index != -1:
parametersfunc = parametersfunc[:sample_weight_index - 2]
sample_weight_index_params = params.find('sample_weight')
if sample_weight_index_params != -1:
params = params[:sample_weight_index_params - 2]
if match:
#parametersfunc = parametersfunc + "," + " X_shape_x: int," + " X_shape_y: int"
parametersfunc = parametersfunc
parametersfunc = parametersfunc.replace('array', 'str')
# X_upload = f"X = np.array(X).reshape(X_shape_x,X_shape_y)"
# "~/.cloudmesh/upload-file/" + f"{X}" + ".csv"
X_input = "\"~/.cloudmesh/upload-file/\" + f\"{X}\" + \".csv\""
X_input = "X_input = " + f'{X_input}'
y_input = "\"~/.cloudmesh/upload-file/\" + f\"{y}\" + \".csv\""
y_input = "y_input = " + f'{y_input}'
#X_upload = "X = pd.read_csv(f\"~/.cloudmesh/upload-file/{X}.csv\")"
X_upload = "X = pd.read_csv(X_input)"
y_upload = "y = pd.read_csv(y_input)"
#sample_weight_upload = "sample_weight = pd.read_csv(\"~/.cloudmesh/upload-file/sample_weight.csv\")"
if returnparam != '':
if returnparam == 'array' and (X_param_index == 0
and y_param_index == 10):
returnparam = 'list'
spec = self.functiontemplateretarryXandY.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_input=X_input,
y_input=y_input,
X_upload=X_upload,
y_upload=y_upload,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
elif (X_param_index == 0 and y_param_index == 10):
spec = self.functiontemplateXandY.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_input=X_input,
y_input=y_input,
X_upload=X_upload,
y_upload=y_upload,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
elif (returnparam == 'array'
and (X_param_index == 0 and y_param_index == -1)):
returnparam = 'list'
spec = self.functiontemplateretarryX.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_input=X_input,
X_upload=X_upload,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
elif (X_param_index == 0 and y_param_index == -1):
spec = self.functiontemplatewithonlyX_param.format(
functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
X_input=X_input,
X_upload=X_upload,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
else:
spec = self.functiontemplate.format(functioname=functionname,
description=description,
text1=text1,
model_tag=model_tag,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
else:
if functionname == 'fit':
spec = self.functiontemplatefit.format(
functioname=functionname,
description=description,
base_estimator=base_estimator,
X_input=X_input,
y_input=y_input,
X_upload=X_upload,
y_upload=y_upload,
model_tag=model_tag,
text1=text1,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
elif functionname == 'set_params':
spec = self.functiontemplatesetparams.format(
functioname=functionname,
description=description,
base_estimator=base_estimator,
model_tag=model_tag,
text1=text1,
parameters=parametersfunc,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
else:
spec = self.functiontemplatereturningself.format(
functioname=functionname,
description=description,
text1=text1,
parameters=parametersfunc,
base_estimator=base_estimator,
model_tag=model_tag,
param_wo_type=params,
docstring=docstring,
returnparam1=returnparam)
return spec
def make_InfectionCurveForm():
# here gone all the fields
form_fields = {}
# this gonna store all the model choices
models, methods = [], []
for mname, method in vars(InfectionCurve).items():
if mname.startswith("do_") and callable(method):
label = _(mname.split("_", 1)[-1])
models.append((mname, label))
methods.append(method)
# add the model select to the form
form_fields["model"] = wtf.SelectField(
_("Model"),
choices=models,
description=_("Compartimental model"),
render_kw={"class": "custom-select custom-select-sm"},
default=models[0][0],
)
# now we add the same parameters for all the methods
for method in methods:
# extract all the fields doc from the method documentation
mtd_docs = docscrape.FunctionDoc(method)
docs = {
p.name.split(":")[0]: " ".join(p.desc).strip()
for p in mtd_docs.get("Parameters")
}
# extract all the parameters
params = inspect.signature(method).parameters
for idx, param in enumerate(params.values()):
if idx == 0 or param.name in form_fields:
continue
# extract doc
description = docs.get(param.name)
# extract the label from the name
label = " ".join(param.name.split("_")).title()
# add all the validators
validators = list(DEFAULT_VALIDATORS)
# add the classes to the field element
render_kw = dict(DEFAULT_RENDER_KW)
render_kw["data-ptype"] = "model-param"
# the default
default = param.default
# the type based on the default
Field = PYTYPE_TO_WTF.get(type(default), wtf.StringField)
# create the field
ffield = Field(
_(label),
description=_(description),
default=default,
validators=validators,
render_kw=render_kw,
)
form_fields[param.name] = ffield
# extract all the fields doc from the class documentation
class_docs = docscrape.ClassDoc(InfectionCurve)
docs = {
p.name.split(":")[0]: " ".join(p.desc).strip()
for p in class_docs.get("Parameters")
}
# create one field for attribute
for aname, afield in attr.fields_dict(InfectionCurve).items():
# extract doc
description = docs.get(aname)
# extract the label from the field name
label = " ".join(aname.split("_")).title()
# add all the validators
validators = list(DEFAULT_VALIDATORS)
# add the classes to the field element
render_kw = dict(DEFAULT_RENDER_KW)
render_kw["data-ptype"] = "curve-param"
# determine the field type
Field = PYTYPE_TO_WTF.get(afield.type, wtf.StringField)
# create the field
ffield = Field(
_(label),
description=_(description),
default=afield.default,
validators=validators,
render_kw=render_kw,
)
form_fields[aname] = ffield
# create the form itself
form = type("InfectionCurveForm", (fwtf.FlaskForm, ), form_fields)
return form
