def check_parameters_match(func, doc=None): """Helper to check docstring, returns list of incorrect results""" from numpydoc import docscrape incorrect = [] name_ = get_name(func) if not name_.startswith('mne.') or name_.startswith('mne.externals'): return incorrect if inspect.isdatadescriptor(func): return incorrect args = _get_args(func) # drop self if len(args) > 0 and args[0] == 'self': args = args[1:] if doc is None: with warnings.catch_warnings(record=True) as w: doc = docscrape.FunctionDoc(func) if len(w): raise RuntimeError('Error for %s:\n%s' % (name_, w[0])) # check set param_names = [name for name, _, _ in doc['Parameters']] # clean up some docscrape output: param_names = [name.split(':')[0].strip('` ') for name in param_names] param_names = [name for name in param_names if '*' not in name] if len(param_names) != len(args): bad = str(sorted(list(set(param_names) - set(args)) + list(set(args) - set(param_names)))) if not any(d in name_ for d in _docstring_ignores) and \ 'deprecation_wrapped' not in func.__code__.co_name: incorrect += [name_ + ' arg mismatch: ' + bad] else: for n1, n2 in zip(param_names, args): if n1 != n2: incorrect += [name_ + ' ' + n1 + ' != ' + n2] return incorrect
def check_parameters_match(func, doc=None, cls=None): """Check docstring, return list of incorrect results.""" from numpydoc import docscrape incorrect = [] name_ = get_name(func, cls=cls) if not name_.startswith('expyfun.') or \ name_.startswith('expyfun._externals'): return incorrect if inspect.isdatadescriptor(func): return incorrect args = _get_args(func) # drop self if len(args) > 0 and args[0] == 'self': args = args[1:] if doc is None: with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') try: doc = docscrape.FunctionDoc(func) except Exception as exp: incorrect += [name_ + ' parsing error: ' + str(exp)] return incorrect if len(w): raise RuntimeError('Error for %s:\n%s' % (name_, w[0])) # check set parameters = doc['Parameters'] # clean up some docscrape output: parameters = [[p[0].split(':')[0].strip('` '), p[2]] for p in parameters] parameters = [p for p in parameters if '*' not in p[0]] param_names = [p[0] for p in parameters] if len(param_names) != len(args): bad = str( sorted( list(set(param_names) - set(args)) + list(set(args) - set(param_names)))) if not any(re.match(d, name_) for d in docstring_ignores) and \ 'deprecation_wrapped' not in func.__code__.co_name: incorrect += [name_ + ' arg mismatch: ' + bad] else: for n1, n2 in zip(param_names, args): if n1 != n2: incorrect += [name_ + ' ' + n1 + ' != ' + n2] for param_name, desc in parameters: desc = '\n'.join(desc) full_name = name_ + '::' + param_name if full_name in docstring_length_ignores: assert len(desc) > char_limit # assert it actually needs to be elif len(desc) > char_limit: incorrect += [ '%s too long (%d > %d chars)' % (full_name, len(desc), char_limit) ] return incorrect
def getProcessorSpec(self, P): assert (callable(P)) spec = {"name": P.name, "version": P.version} npdoc = docscrape.FunctionDoc(P) #npdoc={"Summary":"","Parameters":[]} spec["description"] = npdoc["Summary"][0] params0 = npdoc["Parameters"] argspec0 = inspect.getfullargspec( P.__call__ if inspect.isclass(P) else P) defaults0 = argspec0.kwonlydefaults if not defaults0: defaults0 = {} inputs, outputs, parameters = [], [], [] for j in range(len(params0)): pp = params0[j] pname = pp[0] ptype = pp[1] pdescr = pp[2][0] qq = {"name": pname, "description": pdescr} if pname in defaults0: qq["optional"] = True qq["default_value"] = defaults0[pname] else: qq["optional"] = False if (ptype == 'INPUT'): inputs.append(qq) elif (ptype == 'OUTPUT'): outputs.append(qq) else: qq["datatype"] = ptype parameters.append(qq) spec['inputs'] = inputs spec['outputs'] = outputs spec['parameters'] = parameters if hasattr(P, 'test'): spec['has_test'] = True return spec
def check_docstring_parameters(func, doc=None, ignore=None): """Helper to check docstring. Parameters ---------- func : callable The function object to test. doc : str, default=None Docstring if it is passed manually to the test. ignore : list, default=None Parameters to ignore. Returns ------- incorrect : list A list of string describing the incorrect results. """ from numpydoc import docscrape incorrect = [] ignore = [] if ignore is None else ignore func_name = _get_func_name(func) if not func_name.startswith("sklearn.") or func_name.startswith( "sklearn.externals"): return incorrect # Don't check docstring for property-functions if inspect.isdatadescriptor(func): return incorrect # Don't check docstring for setup / teardown pytest functions if func_name.split(".")[-1] in ("setup_module", "teardown_module"): return incorrect # Dont check estimator_checks module if func_name.split(".")[2] == "estimator_checks": return incorrect # Get the arguments from the function signature param_signature = list(filter(lambda x: x not in ignore, _get_args(func))) # drop self if len(param_signature) > 0 and param_signature[0] == "self": param_signature.remove("self") # Analyze function's docstring if doc is None: with warnings.catch_warnings(record=True) as w: try: doc = docscrape.FunctionDoc(func) except Exception as exp: incorrect += [func_name + " parsing error: " + str(exp)] return incorrect if len(w): raise RuntimeError("Error for %s:\n%s" % (func_name, w[0])) param_docs = [] for name, type_definition, param_doc in doc["Parameters"]: # Type hints are empty only if parameter name ended with : if not type_definition.strip(): if ":" in name and name[:name.index(":")][-1:].strip(): incorrect += [ func_name + " There was no space between the param name and colon (%r)" % name ] elif name.rstrip().endswith(":"): incorrect += [ func_name + " Parameter %r has an empty type spec. Remove the colon" % (name.lstrip()) ] # Create a list of parameters to compare with the parameters gotten # from the func signature if "*" not in name: param_docs.append(name.split(":")[0].strip("` ")) # If one of the docstring's parameters had an error then return that # incorrect message if len(incorrect) > 0: return incorrect # Remove the parameters that should be ignored from list param_docs = list(filter(lambda x: x not in ignore, param_docs)) # The following is derived from pytest, Copyright (c) 2004-2017 Holger # Krekel and others, Licensed under MIT License. See # https://github.com/pytest-dev/pytest message = [] for i in range(min(len(param_docs), len(param_signature))): if param_signature[i] != param_docs[i]: message += [ "There's a parameter name mismatch in function" " docstring w.r.t. function signature, at index %s" " diff: %r != %r" % (i, param_signature[i], param_docs[i]) ] break if len(param_signature) > len(param_docs): message += [ "Parameters in function docstring have less items w.r.t." " function signature, first missing item: %s" % param_signature[len(param_docs)] ] elif len(param_signature) < len(param_docs): message += [ "Parameters in function docstring have more items w.r.t." " function signature, first extra item: %s" % param_docs[len(param_signature)] ] # If there wasn't any difference in the parameters themselves between # docstring and signature including having the same length then return # empty list if len(message) == 0: return [] import difflib import pprint param_docs_formatted = pprint.pformat(param_docs).splitlines() param_signature_formatted = pprint.pformat(param_signature).splitlines() message += ["Full diff:"] message.extend(line.strip() for line in difflib.ndiff( param_signature_formatted, param_docs_formatted)) incorrect.extend(message) # Prepend function name incorrect = ["In function: " + func_name] + incorrect return incorrect
def check_docstring_parameters(func, doc=None, ignore=None, class_name=None): """Helper to check docstring Parameters ---------- func : callable The function object to test. doc : str, optional (default: None) Docstring if it is passed manually to the test. ignore : None | list Parameters to ignore. class_name : string, optional (default: None) If ``func`` is a class method and the class name is known specify class_name for the error message. Returns ------- incorrect : list A list of string describing the incorrect results. """ from numpydoc import docscrape incorrect = [] ignore = [] if ignore is None else ignore func_name = _get_func_name(func, class_name=class_name) if (not func_name.startswith('sklearn.') or func_name.startswith('sklearn.externals')): return incorrect # Don't check docstring for property-functions if inspect.isdatadescriptor(func): return incorrect args = list(filter(lambda x: x not in ignore, _get_args(func))) # drop self if len(args) > 0 and args[0] == 'self': args.remove('self') if doc is None: with warnings.catch_warnings(record=True) as w: try: doc = docscrape.FunctionDoc(func) except Exception as exp: incorrect += [func_name + ' parsing error: ' + str(exp)] return incorrect if len(w): raise RuntimeError('Error for %s:\n%s' % (func_name, w[0])) param_names = [] for name, type_definition, param_doc in doc['Parameters']: if not type_definition.strip(): if ':' in name and name[:name.index(':')][-1:].strip(): incorrect += [ func_name + ' There was no space between the param name and ' 'colon (%r)' % name ] elif name.rstrip().endswith(':'): incorrect += [ func_name + ' Parameter %r has an empty type spec. ' 'Remove the colon' % (name.lstrip()) ] if '*' not in name: param_names.append(name.split(':')[0].strip('` ')) param_names = list(filter(lambda x: x not in ignore, param_names)) if len(param_names) != len(args): bad = str(sorted(list(set(param_names) ^ set(args)))) incorrect += [func_name + ' arg mismatch: ' + bad] else: for n1, n2 in zip(param_names, args): if n1 != n2: incorrect += [func_name + ' ' + n1 + ' != ' + n2] return incorrect
def generate_function(self, module, function,base_estimator,model_tag): """ function to generate open API of python function. :param f: :param baseurl: :param outdir: :param yaml: :param write: :return: """ type_table = { 'matrix': 'array', 'array': 'array', 'array-like': 'array', 'numpy array': 'array', 'bool': 'bool', 'int': 'int', 'float': 'float', 'string': 'str', 'dictionary': 'dict', 'dict': 'dict', 'object': 'self', 'self': 'self' } module = module class_name = function base_estimator = base_estimator model_tag = model_tag class_obj = getattr(module, class_name) parsing_obj = docscrape.FunctionDoc(class_obj) doc = class_obj.__doc__ paras_dict_func = self.get_parameters(parsing_obj, type_table) paras_desc = self.get_docstrings(parsing_obj) description = class_obj.__doc__.strip().split("\n")[0] #title = class_obj.__name__ parametersfunc,params,docstring = self.populate_parameters_function(function, paras_dict_func, paras_desc) text1 = '"""' key = 'return' returnparam ='' #print(paras_dict_func) if key in paras_dict_func.keys(): if paras_dict_func['return'] != 'self': returnparam = paras_dict_func['return'] else: returnparam returnparamindex = 0 returnparamindex = parametersfunc.find('return') if (returnparamindex == -1): pass elif (returnparamindex == 0): parametersfunc = '' else: parametersfunc = parametersfunc[:returnparamindex-2] returnparamindex1 = 0 returnparamindex1 = params.find('return') if (returnparamindex1 == -1): pass elif (returnparamindex1 == 0): params = '' else: params = params[:returnparamindex1 - 2] functionname = class_obj.__name__ match = re.search(r'X: array', parametersfunc) X_param_index = parametersfunc.find('X: array') sample_weight_index = parametersfunc.find('sample_weight: array') if sample_weight_index != -1: parametersfunc = parametersfunc[:sample_weight_index - 2] sample_weight_index_params = params.find('sample_weight') if sample_weight_index_params != -1: params = params[:sample_weight_index_params - 2] if match: parametersfunc = parametersfunc + "," + " X_shape_x: int," + " X_shape_y: int" parametersfunc = parametersfunc.replace('array','list') X_numpyconversion = f"X = np.array(X).reshape(X_shape_x,X_shape_y)" if returnparam != '': if returnparam == 'array' and X_param_index == 0 : returnparam = 'list' spec = self.functiontemplatearrayandX_param.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_numpyconversion=X_numpyconversion, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec elif X_param_index == 0 : spec = self.functiontemplatewithX_param.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_numpyconversion=X_numpyconversion, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec else: spec = self.functiontemplate.format( functioname=functionname, description=description, text1 = text1, model_tag=model_tag, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec else: if functionname == 'fit': spec = self.functiontemplatefit.format( functioname=functionname, description=description, base_estimator=base_estimator, X_numpyconversion=X_numpyconversion, model_tag=model_tag, text1=text1, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec elif functionname =='set_params': spec = self.functiontemplatesetparams.format( functioname=functionname, description=description, base_estimator=base_estimator, model_tag=model_tag, text1=text1, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec else: spec = self.functiontemplatereturningself.format( functioname=functionname, description=description, text1=text1, parameters=parametersfunc, base_estimator=base_estimator, model_tag=model_tag, param_wo_type=params, docstring=docstring, returnparam1=returnparam ) return spec
def generate_function(self, module, function, base_estimator, model_tag): """ :param module: Sklearn module like Linear Regression :param function: Methods in the class of Linear Regression :param base_estimator: Sklearn module like Linear Regression :param model_tag: Tag used to store the name of the model instance :return: """ type_table = { 'matrix': 'array', 'array': 'array', 'array-like': 'array', 'numpy array': 'array', 'bool': 'bool', 'int': 'int', 'float': 'float', 'string': 'str', 'dictionary': 'dict', 'dict': 'dict', 'object': 'self', 'self': 'self' } module = module class_name = function base_estimator = base_estimator model_tag = model_tag class_obj = getattr(module, class_name) parsing_obj = docscrape.FunctionDoc(class_obj) doc = class_obj.__doc__ paras_dict_func = self.get_parameters(parsing_obj, type_table) paras_desc = self.get_docstrings(parsing_obj) description = class_obj.__doc__.strip().split("\n")[0] #title = class_obj.__name__ parametersfunc, params, docstring = self.populate_parameters_function( function, paras_dict_func, paras_desc) text1 = '"""' key = 'return' returnparam = '' #print(paras_dict_func) if key in paras_dict_func.keys(): if paras_dict_func['return'] != 'self': returnparam = paras_dict_func['return'] else: returnparam returnparamindex = 0 returnparamindex = parametersfunc.find('return') if (returnparamindex == -1): pass elif (returnparamindex == 0): parametersfunc = '' else: parametersfunc = parametersfunc[:returnparamindex - 2] returnparamindex1 = 0 returnparamindex1 = params.find('return') if (returnparamindex1 == -1): pass elif (returnparamindex1 == 0): params = '' else: params = params[:returnparamindex1 - 2] functionname = class_obj.__name__ match = re.search(r'X: array', parametersfunc) X_param_index = parametersfunc.find('X: array') y_param_index = parametersfunc.find('y: array') sample_weight_index = parametersfunc.find('sample_weight: array') if sample_weight_index != -1: parametersfunc = parametersfunc[:sample_weight_index - 2] sample_weight_index_params = params.find('sample_weight') if sample_weight_index_params != -1: params = params[:sample_weight_index_params - 2] if match: #parametersfunc = parametersfunc + "," + " X_shape_x: int," + " X_shape_y: int" parametersfunc = parametersfunc parametersfunc = parametersfunc.replace('array', 'str') # X_upload = f"X = np.array(X).reshape(X_shape_x,X_shape_y)" # "~/.cloudmesh/upload-file/" + f"{X}" + ".csv" X_input = "\"~/.cloudmesh/upload-file/\" + f\"{X}\" + \".csv\"" X_input = "X_input = " + f'{X_input}' y_input = "\"~/.cloudmesh/upload-file/\" + f\"{y}\" + \".csv\"" y_input = "y_input = " + f'{y_input}' #X_upload = "X = pd.read_csv(f\"~/.cloudmesh/upload-file/{X}.csv\")" X_upload = "X = pd.read_csv(X_input)" y_upload = "y = pd.read_csv(y_input)" #sample_weight_upload = "sample_weight = pd.read_csv(\"~/.cloudmesh/upload-file/sample_weight.csv\")" if returnparam != '': if returnparam == 'array' and (X_param_index == 0 and y_param_index == 10): returnparam = 'list' spec = self.functiontemplateretarryXandY.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_input=X_input, y_input=y_input, X_upload=X_upload, y_upload=y_upload, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec elif (X_param_index == 0 and y_param_index == 10): spec = self.functiontemplateXandY.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_input=X_input, y_input=y_input, X_upload=X_upload, y_upload=y_upload, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec elif (returnparam == 'array' and (X_param_index == 0 and y_param_index == -1)): returnparam = 'list' spec = self.functiontemplateretarryX.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_input=X_input, X_upload=X_upload, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec elif (X_param_index == 0 and y_param_index == -1): spec = self.functiontemplatewithonlyX_param.format( functioname=functionname, description=description, text1=text1, model_tag=model_tag, X_input=X_input, X_upload=X_upload, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec else: spec = self.functiontemplate.format(functioname=functionname, description=description, text1=text1, model_tag=model_tag, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec else: if functionname == 'fit': spec = self.functiontemplatefit.format( functioname=functionname, description=description, base_estimator=base_estimator, X_input=X_input, y_input=y_input, X_upload=X_upload, y_upload=y_upload, model_tag=model_tag, text1=text1, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec elif functionname == 'set_params': spec = self.functiontemplatesetparams.format( functioname=functionname, description=description, base_estimator=base_estimator, model_tag=model_tag, text1=text1, parameters=parametersfunc, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec else: spec = self.functiontemplatereturningself.format( functioname=functionname, description=description, text1=text1, parameters=parametersfunc, base_estimator=base_estimator, model_tag=model_tag, param_wo_type=params, docstring=docstring, returnparam1=returnparam) return spec
def make_InfectionCurveForm(): # here gone all the fields form_fields = {} # this gonna store all the model choices models, methods = [], [] for mname, method in vars(InfectionCurve).items(): if mname.startswith("do_") and callable(method): label = _(mname.split("_", 1)[-1]) models.append((mname, label)) methods.append(method) # add the model select to the form form_fields["model"] = wtf.SelectField( _("Model"), choices=models, description=_("Compartimental model"), render_kw={"class": "custom-select custom-select-sm"}, default=models[0][0], ) # now we add the same parameters for all the methods for method in methods: # extract all the fields doc from the method documentation mtd_docs = docscrape.FunctionDoc(method) docs = { p.name.split(":")[0]: " ".join(p.desc).strip() for p in mtd_docs.get("Parameters") } # extract all the parameters params = inspect.signature(method).parameters for idx, param in enumerate(params.values()): if idx == 0 or param.name in form_fields: continue # extract doc description = docs.get(param.name) # extract the label from the name label = " ".join(param.name.split("_")).title() # add all the validators validators = list(DEFAULT_VALIDATORS) # add the classes to the field element render_kw = dict(DEFAULT_RENDER_KW) render_kw["data-ptype"] = "model-param" # the default default = param.default # the type based on the default Field = PYTYPE_TO_WTF.get(type(default), wtf.StringField) # create the field ffield = Field( _(label), description=_(description), default=default, validators=validators, render_kw=render_kw, ) form_fields[param.name] = ffield # extract all the fields doc from the class documentation class_docs = docscrape.ClassDoc(InfectionCurve) docs = { p.name.split(":")[0]: " ".join(p.desc).strip() for p in class_docs.get("Parameters") } # create one field for attribute for aname, afield in attr.fields_dict(InfectionCurve).items(): # extract doc description = docs.get(aname) # extract the label from the field name label = " ".join(aname.split("_")).title() # add all the validators validators = list(DEFAULT_VALIDATORS) # add the classes to the field element render_kw = dict(DEFAULT_RENDER_KW) render_kw["data-ptype"] = "curve-param" # determine the field type Field = PYTYPE_TO_WTF.get(afield.type, wtf.StringField) # create the field ffield = Field( _(label), description=_(description), default=afield.default, validators=validators, render_kw=render_kw, ) form_fields[aname] = ffield # create the form itself form = type("InfectionCurveForm", (fwtf.FlaskForm, ), form_fields) return form