def param_description( string: Optional[str], param_name: str, errors: Literal['raise', 'ignore'] = 'ignore', ) -> Optional[str]: """Get a param description of a docstring. Args: string: The docstring from which to extract the parameter description. param_name: The name of the parameter. errors: Either 'raise' or 'ignore'. Examples: If a docstring is given, there are a few possibilities. >>> param_description(param_description.__doc__, 'string') 'The docstring from which to extract the parameter description.' >>> param_description(param_description.__doc__, 'foo') >>> param_description(param_description.__doc__, 'foo', errors='raise') Traceback (most recent call last): ... ValueError: Parameter 'foo' not found in docstring. There are cases, when no docstring is assigned to a function. >>> param_description(None, 'foo') >>> param_description(None, 'foo', errors='raise') Traceback (most recent call last): ... ValueError: No docstring was given. Returns: The description of the parameter, if possible. """ if string is None: if errors == 'raise': raise ValueError("No docstring was given.") return None docstring = docstring_parser.parse(string) for param in docstring.params: if param.arg_name == param_name: return param.description if errors == 'raise': raise ValueError(f"Parameter {param_name!r} not found in docstring.") return None
def __init__(self, target): self.target = target self.name = target.__name__ self.short_description: Optional[str] = None self.long_description: Optional[str] = None self.args: List[Attribute] = self.load_args_from_signature(self.target) self.returns: Optional[Attribute] = None self._doc = docstring_parser.parse(target.__doc__) if self.__class__ == Function: self.load_from_doc()
def parse_methods(public_methods: Dict) -> Generator[Method, None, None]: for name, method in public_methods.items(): doc = getdoc(method) if doc is None: print('Docless method: ', name) yield manually_set_method(name) else: parsed_doc = parse(doc.replace('::', '')) yield Method(name, list(parse_arguments(parsed_doc)), doc, parse_return_type(parsed_doc.meta))
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]: if not is_module_available("docstring_parser"): return None from docstring_parser import parse docstring = parse(fn.__doc__) if docstring.returns is None: return None return docstring.returns.description
def from_function(cls, typ: MagicType, func: Callable[..., Any], name: Optional[Text] = None) -> "_MagicSpec": """Creates _MagicSpec from compatible function.""" name = name if name else func.__name__ if not name.isidentifier(): raise MagicParsingError( f"\"{name}\" isn't valid Python identifier, see: " "https://docs.python.org/3/reference/" "lexical_analysis.html#identifiers") if not func.__doc__: raise MagicParsingError("Magics have to have docstring.") spec = getfullargspec(func) if spec.varargs or spec.varkw: raise MagicParsingError( "Magics can't have explicit variadic arguments, " "i.e. `*args` or `**kwargs`") if spec.kwonlyargs and not spec.kwonlydefaults: raise MagicParsingError( "Magics can't have keyword-only arguments without default value." ) args: List[Text] = spec.args args_with_defaults: Dict[Text, Text] = {} args_types: Dict[Text, _MagicArgValueType] = {} if spec.annotations: for arg, typ_ in spec.annotations.items(): if typ_ in _MagicArgValues: args_types[arg] = typ_ else: raise MagicParsingError( f"Magics can only have arguments of type {MagicArgValue}; " f"got {arg}: {typ_}") if spec.defaults: for default in reversed(spec.defaults): args_with_defaults[args.pop()] = default if spec.kwonlydefaults: for arg, default in spec.kwonlydefaults.items(): args_with_defaults[arg] = default docstring = func.__doc__ if func.__doc__ else "" args_descriptions: Dict[Text, Text] = { param.arg_name: param.description # type: ignore for param in parse(docstring).params } return cls(name, docstring, typ, args, args_with_defaults, args_descriptions, args_types)
def method_signature_to_json_schema( m: Callable[..., Any], special_handling: Dict[str, List[JsonParameter]], parameters_to_ignore: Iterable[str]) -> JsonSchema: """Creates a JSON schema representing the parameters to a method""" signature = inspect.signature(m) generated: JsonSchema = {'type': 'object', 'properties': {}} properties: Dict[str, JsonSchemaDocument] = OrderedDict() required: List[str] = [] method_docs = docstring_parser.parse(m.__doc__) def parse_parameter(param_name: str, python_type: PythonType, default: DefaultValue) -> None: if param_name in parameters_to_ignore: return try: param_docs = next(param for param in method_docs.params if param.arg_name == param_name) description = param_docs.description except StopIteration: print(f"Could not find docs for {param_name}") raise if param_name in special_handling: for substitute_parameter in special_handling[param_name]: properties[substitute_parameter. name] = substitute_parameter.json_schema_document if not substitute_parameter.optional: required.append(substitute_parameter.name) if properties[substitute_parameter.name].description is None: properties[ substitute_parameter.name].description = description else: optional = default != inspect.Parameter.empty results = parse_python_parameter_type(name=param_name, python_type=python_type, description=description, optional=optional, default=default) if results is not None: properties[results.name] = results.json_schema_document if not results.optional: required.append(results.name) for param_name, parameter in signature.parameters.items(): python_type = parameter.annotation default = parameter.default parse_parameter(param_name, python_type, default) generated['properties'] = OrderedDict() for name, prop in properties.items(): generated['properties'][name] = prop.to_data() generated['required'] = required return generated
def test_long_description( source: str, expected_short_desc: str, expected_long_desc: str, expected_blank: bool ) -> None: docstring = parse(source) assert docstring.short_description == expected_short_desc assert docstring.long_description == expected_long_desc assert docstring.blank_after_short_description == expected_blank assert docstring.meta == []
def __init__(self, func: Callable[..., Any]) -> None: self._func = func if not isinstance(func, (types.FunctionType, types.MethodType)): raise PedanticTypeCheckException(f'{self.full_name} should be a method or function.') self._full_arg_spec = inspect.getfullargspec(func) self._signature = inspect.signature(func) self._err = f'In function {func.__qualname__}:' + '\n' self._source: str = inspect.getsource(object=func) self._docstring = parse(func.__doc__)
def parse_docstring(doc: Optional[str]): if doc: parsed = parse(doc) params = {arg.arg_name: arg.description for arg in parsed.params} params = {k: val.replace("\n", " ") for k, val in params.items()} return ( "\n".join((desc for desc in (parsed.short_description, parsed.long_description) if desc)), params, ) return "", {}
def _docstring_to_html(docs: str) -> str: """Convert docstring into rich text html.""" from docstring_parser import parse ds = parse(docs) ptemp = "<li><p><strong>{}</strong> (<em>{}</em>) - {}</p></li>" plist = [ptemp.format(p.arg_name, p.type_name, p.description) for p in ds.params] params = "<h3>Parameters</h3><ul>{}</ul>".format("".join(plist)) short = f"<p>{ds.short_description}</p>" if ds.short_description else "" long = f"<p>{ds.long_description}</p>" if ds.long_description else "" return re.sub(r"``?([^`]+)``?", r"<code>\1</code>", f"{short}{long}{params}")
def function_docs_handler(md_template, api): if len(md_template) == 0: md_template += FUNCTION_TEMPLATE md_template += SUBHEADING_TEMPLATE.format(api.__name__) docs = inspect.getdoc(api) docs = parse(docs) short_desc = docs.short_description md_template += SUBHEADING_DESC_TEMPLATE.format(short_desc) md_template = extract_function_docs(md_template, api) return md_template
def build_formats_reference(): TEMPLATE = """ --- title: Formats Reference --- It's a formats reference supported by the main Frictionless package. If you have installed external plugins, there can be more formats available. Below we're listing a format group name (or a parser name) like Excel, which is used, for example, for `xlsx`, `xls` etc formats. Options can be used for creating dialects, for example, `dialect = ExcelDialect(sheet=1)`. {% for format in formats %} ## {{ format.name }} {% if format.options %} {% for option in format.options %} ### {{ option.name }} > Type: {{ option.type }} {{ option.text }} {% endfor %} {% else %} There are no options available. {% endif %} {% endfor %} """ # Input formats = [] modules = [] for item in pkgutil.iter_modules([os.path.dirname(plugins.__file__)]): modules.append(import_module(f"frictionless.plugins.{item.name}")) for module in modules: for name, Dialect in vars(module).items(): match = re.match(r"(.+)Dialect", name) if not match: continue name = match.group(1) data = parse(Dialect.__doc__) format = {"name": name, "options": []} for param in data.params: if param.arg_name.startswith("descriptor"): continue type = param.type_name text = param.description.capitalize() name = stringcase.titlecase(param.arg_name.replace("?", "")) format["options"].append({"name": name, "text": text, "type": type}) formats.append(format) # Ouput template = Template(inspect.cleandoc(TEMPLATE)) document = template.render(formats=formats).strip() write_file(os.path.join("docs", "references", "formats-reference.md"), document) print("Built: Formats Reference")
def build_schemes_reference(): TEMPLATE = """ --- title: Schemes Reference --- It's a schemes reference supported by the main Frictionless package. If you have installed external plugins, there can be more schemes available. Below we're listing a scheme group name (or a loader name) like Remote, which is used, for example, for `http`, `https` etc schemes. Options can be used for creating controls, for example, `control = RemoteControl(http_timeout=1)`. {% for scheme in schemes %} ## {{ scheme.name }} {% if scheme.options %} {% for option in scheme.options %} ### {{ option.name }} > Type: {{ option.type }} {{ option.text }} {% endfor %} {% else %} There are no options available. {% endif %} {% endfor %} """ # Input schemes = [] modules = [] for item in pkgutil.iter_modules([os.path.dirname(plugins.__file__)]): modules.append(import_module(f"frictionless.plugins.{item.name}")) for module in modules: for name, Dialect in vars(module).items(): match = re.match(r"(.+)Control", name) if not match: continue name = match.group(1) data = parse(Dialect.__doc__) scheme = {"name": name, "options": []} for param in data.params: if param.arg_name.startswith("descriptor"): continue type = param.type_name text = param.description.capitalize() name = stringcase.titlecase(param.arg_name.replace("?", "")) scheme["options"].append({"name": name, "text": text, "type": type}) schemes.append(scheme) # Output template = Template(inspect.cleandoc(TEMPLATE)) document = template.render(schemes=schemes).strip() write_file(os.path.join("docs", "references", "schemes-reference.md"), document) print("Built: Schemes Reference")
def annotate_function(function): sig = inspect.signature(function) doc_params = {p.arg_name: p.type_name for p in parse(function.__doc__).params} for k, v in list(doc_params.items()): if "," in k: for split_key in k.split(","): doc_params[split_key.strip()] = v del doc_params[k] hidden = HIDDEN.get(f"{function.__module__}.{function.__name__}", {}) for p in sig.parameters.values(): if p.name in hidden: annotation = bound_default(p) else: annotation = guess_type(p, doc_params.get(p.name)) function.__annotations__[p.name] = annotation doc_returns = parse(function.__doc__).returns # Note skimage.filters.inverse has no return doc string # Note skimage.filters.threshold should be a different type .... if doc_returns is not None: function.__annotations__["return"] = guess_return_type(doc_returns)
def _infer_input_description_from_docstring( fn: Callable) -> Dict[str, Optional[str]]: doc_str = fn.__doc__ if not is_module_available("docstring_parser") or doc_str is None: return {} from docstring_parser import parse try: docstring = parse(doc_str) return {p.arg_name: p.description for p in docstring.params} except Exception: return {}
def test_meta_newlines( source: str, expected_short_desc: T.Optional[str], expected_long_desc: T.Optional[str], expected_blank_short_desc: bool, expected_blank_long_desc: bool ) -> None: docstring = parse(source) assert docstring.short_description == expected_short_desc assert docstring.long_description == expected_long_desc assert docstring.blank_after_short_description == expected_blank_short_desc assert docstring.blank_after_long_description == expected_blank_long_desc assert len(docstring.meta) == 1
def test_function_docstrings_validity( path: Path, node: ast.FunctionDef, docstring: str ) -> None: """Test whether functions have valid docstrings. :param path: path of the file :param node: AST node to check :param docstring: AST node docstring """ with decorated_log(path, node): parsed_docstring = docstring_parser.parse(docstring) verify_function_params(node, parsed_docstring) verify_function_returns(node, parsed_docstring)
def __call__(self, package): for namme, thing in package.__dict__.items(): typ = type(thing) doc_string = thing.__doc__ doc_parse = parse(doc_string) if isinstance(thing, Callable) and doc_parse and (doc_parse.params or doc_parse.returns): fname = str(thing) class T: fname = str(thing) doc = doc_parse def on_post(self, req, resp): try: input = req.bounded_stream.read().decode("utf-8") try: input = {k: eval(v) for k, v in input.items()} except Exception as e: try: input = json_tricks.loads(input) except Exception as e: print( f"could not parse {input}, its empyz now " ) input = {} print(self.doc.__repr__()) result = Import2Rest.functions[self.fname](**input) resp.body = dumps(result) except Exception as e: resp.body = json_tricks.dumps(e.__repr__()) T.__qualname__ = str(fname) self.functions[fname] = thing t = T() path = f"/{namme}" app.add_route(path, t) self.classes.append(T) yield f"localhost:8000{path}", { "method": "post", "params": doc_parse.params, "returns": doc_parse.returns }
def function_definitions(): result = {} for fun_name, fun in OrderedDict(sorted(TEL_FUNCTIONS.items())).items(): docstring = parse(fun.__doc__) expected_args = [] phase = None return_type = None invalid_value = None if issubclass(fun, TelTypedFunction): tfun = cast(TelTypedFunction, fun) expected_args = tfun.expected_arg_types phase = tfun.phase_spec return_type = tfun.return_type_spec invalid_value = tfun.invalid_value_spec description = { 'arguments': [{ 'name': param.arg_name, 'typeName': param.type_name, 'description': param.description, **(_tel_arg_remote_spec(param.arg_name, expected_args) or {}), } for param in docstring.params], 'raises': [{ 'typeName': exc.type_name, 'description': exc.description } for exc in docstring.raises], 'shortDescription': docstring.short_description, 'longDescription': docstring.long_description, 'returns': { 'typeName': docstring.returns.type_name, 'description': docstring.returns.description, **(return_type.to_remote_spec() if return_type else {}), }, **({ 'phase': phase.to_remote_spec() } if phase else {}), **({ 'invalidValue': invalid_value.to_remote_spec() } if invalid_value else {}), } result[fun_name] = description return result
def test_meta_with_multiline_description() -> None: docstring = parse( ''' Short description :param: asd 1 2 3 ''') assert docstring.short_description == 'Short description' assert len(docstring.meta) == 1 assert docstring.meta[0].args == ['param'] assert docstring.meta[0].description == 'asd\n1\n 2\n3'
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]: doc_str = fn.__doc__ if not is_module_available("docstring_parser") or doc_str is None: return None from docstring_parser import parse try: docstring = parse(doc_str) if docstring.returns is None: return None return docstring.returns.description except Exception: return None
def get_help(module, inline=None): results = {} module_path = module.get_path() if inline is not None: doc = parse(module.INLINE_SUBMODULES[inline].__doc__) desc = doc.short_description or "No description provided." cmd_path = " ".join(module_path + [inline]).strip() results[cmd_path] = desc return results # print description of inline submodules for m in module.INLINE_SUBMODULES: doc = parse(module.INLINE_SUBMODULES[m].__doc__) desc = doc.short_description or "No description provided." cmd_path = " ".join(module_path + [m]).strip() results[cmd_path] = desc # print description of submodules for m in module.get_modules(): doc = parse(m.INLINE_SUBMODULES[""].__doc__) desc = doc.short_description or "No description provided." cmd_path = " ".join(m.get_path()).strip() results[cmd_path] = desc return dict(sorted(results.items()))
def extract_result_schema(self, method: Callable) -> Schema: result_schema = Schema(schema={}) if method.__doc__: doc = docstring_parser.parse(method.__doc__) if doc and doc.returns: result_schema = Schema( schema={'type': doc.returns.type_name}, required=True, summary=doc.returns.description.split('.')[0], description=doc.returns.description, ) return result_schema
def parse_methods(public_methods: Dict) -> Generator[Method, None, None]: for name, method in public_methods.items(): doc = getdoc(method) if doc is None: logger.debug(f"Docless method: {name}") yield manually_set_method(name) else: parsed_doc = parse(doc) yield Method( name=name, arguments=list(parse_arguments(parsed_doc)), docstring=clean_doc(doc), return_type=parse_return_type(parsed_doc.meta), )
def test_argument_google_all(): '''Test google full type set.''' name, fn = [ x for x in getmembers(module, isfunction) if x[0] == 'argument_google_all' ][0] sig = signature(fn) document = parse(fn.__doc__) arguments = [] for arg in sig.parameters: docstring = next((d for d in document.params if d.arg_name == arg), None) arguments.append( Argument(parameters=sig.parameters[arg], docstring=docstring)) # print(arguments[0].__dict__) assert arguments[0].help == 'argument string' assert arguments[0].type == str assert arguments[0].default == 'A' # print(arguments[1].__dict__) assert arguments[1].help == 'argument bool' # NOTE: Argparse does not store bool type assert not hasattr(arguments[1], 'type') assert arguments[1].default is False # print(arguments[2].__dict__) assert arguments[2].help == 'argument int' assert arguments[2].type == int assert arguments[2].default == 1 # print(arguments[3].__dict__) assert arguments[3].help == 'argument float' assert arguments[3].type == float assert arguments[3].default == 1.5 # print(arguments[4].__dict__) assert arguments[4].help == 'argument list' assert arguments[4].type == list assert arguments[4].default == ['A'] # print(arguments[5].__dict__) assert arguments[5].help == 'argument set' assert arguments[5].type == set assert arguments[5].default == {'a'} # print(arguments[6].__dict__) assert arguments[6].help == 'argument tuple' assert arguments[6].type == tuple assert arguments[6].default == ('A', )
def merge_super_sigs(cls, exclude=("widget_type", "kwargs", "args", "kwds", "extra")): """Merge the signature and kwarg docs from all superclasses, for clearer docs. Parameters ---------- cls : Type The class being modified exclude : tuple, optional A list of parameter names to excluded from the merged docs/signature, by default ("widget_type", "kwargs", "args", "kwds") Returns ------- cls : Type The modified class (can be used as a decorator) """ params = {} param_docs: list[DocstringParam] = [] for sup in reversed(inspect.getmro(cls)): try: sig = inspect.signature(getattr(sup, "__init__")) # in some environments `object` or `abc.ABC` will raise ValueError here except ValueError: continue for name, param in sig.parameters.items(): if name in exclude: continue params[name] = param param_docs += parse(getattr(sup, "__doc__", "")).params # sphinx_autodoc_typehints isn't removing the type annotations from the signature # so we do it manually when building documentation. if BUILDING_DOCS: params = { k: v.replace(annotation=inspect.Parameter.empty) for k, v in params.items() } cls.__init__.__signature__ = inspect.Signature( sorted(params.values(), key=lambda x: x.kind)) param_docs = [p for p in param_docs if p.arg_name not in exclude] cls.__doc__ = (cls.__doc__ or "").split("Parameters")[0].rstrip() + "\n\n" cls.__doc__ += _param_list_to_str(param_docs) # this makes docs linking work... but requires that all of these be in __init__ cls.__module__ = "magicgui.widgets" return cls
def test_returns() -> None: docstring = parse( ''' Short description ''') assert docstring.returns is None docstring = parse( ''' Short description :returns: description ''') assert docstring.returns is not None assert docstring.returns.type_name is None assert docstring.returns.description == 'description' docstring = parse( ''' Short description :returns int: description ''') assert docstring.returns is not None assert docstring.returns.type_name == 'int' assert docstring.returns.description == 'description'
def test_raises() -> None: docstring = parse( ''' Short description ''') assert len(docstring.raises) == 0 docstring = parse( ''' Short description :raises: description ''') assert len(docstring.raises) == 1 assert docstring.raises[0].type_name is None assert docstring.raises[0].description == 'description' docstring = parse( ''' Short description :raises ValueError: description ''') assert len(docstring.raises) == 1 assert docstring.raises[0].type_name == 'ValueError' assert docstring.raises[0].description == 'description'
def _get_test_case_key(self): docstring = parse(self._testMethodDoc) name = docstring.short_description.strip() pre_condition = docstring.long_description input_data = self._testInputData expected_data = self._testExpectedData description = self._description test_case = JiraTestCase(name=name, pre_condition=pre_condition, input_data=input_data, expected_data=expected_data, folder=self.FOLDER, description=description) test_case_key = test_case.create_test_case(self.ISSUE_KEYS) return test_case_key
def test_argument_type_hints_simple() -> None: '''Test simple argument.''' name, fn = [ x for x in getmembers(module, isfunction) if x[0] == 'argument_type_hints_simple' ][0] sig = signature(fn) docstring = parse(fn.__doc__) arguments = [] for arg in sig.parameters: document = next((d for d in docstring.params if d.arg_name == arg), None) arguments.append( Argument(parameters=sig.parameters[arg], docstring=document)) # print('Arguments: ', arguments[0].__dict__) assert arguments[0].default is None