def param_description(
string: Optional[str],
param_name: str,
errors: Literal['raise', 'ignore'] = 'ignore',
) -> Optional[str]:
"""Get a param description of a docstring.
Args:
string:
The docstring from which to extract the parameter description.
param_name:
The name of the parameter.
errors:
Either 'raise' or 'ignore'.
Examples:
If a docstring is given, there are a few possibilities.
>>> param_description(param_description.__doc__, 'string')
'The docstring from which to extract the parameter description.'
>>> param_description(param_description.__doc__, 'foo')
>>> param_description(param_description.__doc__, 'foo', errors='raise')
Traceback (most recent call last):
...
ValueError: Parameter 'foo' not found in docstring.
There are cases, when no docstring is assigned to a function.
>>> param_description(None, 'foo')
>>> param_description(None, 'foo', errors='raise')
Traceback (most recent call last):
...
ValueError: No docstring was given.
Returns:
The description of the parameter, if possible.
"""
if string is None:
if errors == 'raise':
raise ValueError("No docstring was given.")
return None
docstring = docstring_parser.parse(string)
for param in docstring.params:
if param.arg_name == param_name:
return param.description
if errors == 'raise':
raise ValueError(f"Parameter {param_name!r} not found in docstring.")
return None
def __init__(self, target):
self.target = target
self.name = target.__name__
self.short_description: Optional[str] = None
self.long_description: Optional[str] = None
self.args: List[Attribute] = self.load_args_from_signature(self.target)
self.returns: Optional[Attribute] = None
self._doc = docstring_parser.parse(target.__doc__)
if self.__class__ == Function:
self.load_from_doc()
def parse_methods(public_methods: Dict) -> Generator[Method, None, None]:
for name, method in public_methods.items():
doc = getdoc(method)
if doc is None:
print('Docless method: ', name)
yield manually_set_method(name)
else:
parsed_doc = parse(doc.replace('::', ''))
yield Method(name, list(parse_arguments(parsed_doc)), doc,
parse_return_type(parsed_doc.meta))
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]:
if not is_module_available("docstring_parser"):
return None
from docstring_parser import parse
docstring = parse(fn.__doc__)
if docstring.returns is None:
return None
return docstring.returns.description
def from_function(cls,
typ: MagicType,
func: Callable[..., Any],
name: Optional[Text] = None) -> "_MagicSpec":
"""Creates _MagicSpec from compatible function."""
name = name if name else func.__name__
if not name.isidentifier():
raise MagicParsingError(
f"\"{name}\" isn't valid Python identifier, see: "
"https://docs.python.org/3/reference/"
"lexical_analysis.html#identifiers")
if not func.__doc__:
raise MagicParsingError("Magics have to have docstring.")
spec = getfullargspec(func)
if spec.varargs or spec.varkw:
raise MagicParsingError(
"Magics can't have explicit variadic arguments, "
"i.e. `*args` or `**kwargs`")
if spec.kwonlyargs and not spec.kwonlydefaults:
raise MagicParsingError(
"Magics can't have keyword-only arguments without default value."
)
args: List[Text] = spec.args
args_with_defaults: Dict[Text, Text] = {}
args_types: Dict[Text, _MagicArgValueType] = {}
if spec.annotations:
for arg, typ_ in spec.annotations.items():
if typ_ in _MagicArgValues:
args_types[arg] = typ_
else:
raise MagicParsingError(
f"Magics can only have arguments of type {MagicArgValue}; "
f"got {arg}: {typ_}")
if spec.defaults:
for default in reversed(spec.defaults):
args_with_defaults[args.pop()] = default
if spec.kwonlydefaults:
for arg, default in spec.kwonlydefaults.items():
args_with_defaults[arg] = default
docstring = func.__doc__ if func.__doc__ else ""
args_descriptions: Dict[Text, Text] = {
param.arg_name: param.description # type: ignore
for param in parse(docstring).params
}
return cls(name, docstring, typ, args, args_with_defaults,
args_descriptions, args_types)
def method_signature_to_json_schema(
m: Callable[..., Any], special_handling: Dict[str,
List[JsonParameter]],
parameters_to_ignore: Iterable[str]) -> JsonSchema:
"""Creates a JSON schema representing the parameters to a method"""
signature = inspect.signature(m)
generated: JsonSchema = {'type': 'object', 'properties': {}}
properties: Dict[str, JsonSchemaDocument] = OrderedDict()
required: List[str] = []
method_docs = docstring_parser.parse(m.__doc__)
def parse_parameter(param_name: str, python_type: PythonType,
default: DefaultValue) -> None:
if param_name in parameters_to_ignore:
return
try:
param_docs = next(param for param in method_docs.params
if param.arg_name == param_name)
description = param_docs.description
except StopIteration:
print(f"Could not find docs for {param_name}")
raise
if param_name in special_handling:
for substitute_parameter in special_handling[param_name]:
properties[substitute_parameter.
name] = substitute_parameter.json_schema_document
if not substitute_parameter.optional:
required.append(substitute_parameter.name)
if properties[substitute_parameter.name].description is None:
properties[
substitute_parameter.name].description = description
else:
optional = default != inspect.Parameter.empty
results = parse_python_parameter_type(name=param_name,
python_type=python_type,
description=description,
optional=optional,
default=default)
if results is not None:
properties[results.name] = results.json_schema_document
if not results.optional:
required.append(results.name)
for param_name, parameter in signature.parameters.items():
python_type = parameter.annotation
default = parameter.default
parse_parameter(param_name, python_type, default)
generated['properties'] = OrderedDict()
for name, prop in properties.items():
generated['properties'][name] = prop.to_data()
generated['required'] = required
return generated
def test_long_description(
source: str,
expected_short_desc: str,
expected_long_desc: str,
expected_blank: bool
) -> None:
docstring = parse(source)
assert docstring.short_description == expected_short_desc
assert docstring.long_description == expected_long_desc
assert docstring.blank_after_short_description == expected_blank
assert docstring.meta == []
def __init__(self, func: Callable[..., Any]) -> None:
self._func = func
if not isinstance(func, (types.FunctionType, types.MethodType)):
raise PedanticTypeCheckException(f'{self.full_name} should be a method or function.')
self._full_arg_spec = inspect.getfullargspec(func)
self._signature = inspect.signature(func)
self._err = f'In function {func.__qualname__}:' + '\n'
self._source: str = inspect.getsource(object=func)
self._docstring = parse(func.__doc__)
def parse_docstring(doc: Optional[str]):
if doc:
parsed = parse(doc)
params = {arg.arg_name: arg.description for arg in parsed.params}
params = {k: val.replace("\n", " ") for k, val in params.items()}
return (
"\n".join((desc for desc in (parsed.short_description,
parsed.long_description) if desc)),
params,
)
return "", {}
def _docstring_to_html(docs: str) -> str:
"""Convert docstring into rich text html."""
from docstring_parser import parse
ds = parse(docs)
ptemp = "<li><p><strong>{}</strong> (<em>{}</em>) - {}</p></li>"
plist = [ptemp.format(p.arg_name, p.type_name, p.description) for p in ds.params]
params = "<h3>Parameters</h3><ul>{}</ul>".format("".join(plist))
short = f"<p>{ds.short_description}</p>" if ds.short_description else ""
long = f"<p>{ds.long_description}</p>" if ds.long_description else ""
return re.sub(r"``?([^`]+)``?", r"<code>\1</code>", f"{short}{long}{params}")
def function_docs_handler(md_template, api):
if len(md_template) == 0:
md_template += FUNCTION_TEMPLATE
md_template += SUBHEADING_TEMPLATE.format(api.__name__)
docs = inspect.getdoc(api)
docs = parse(docs)
short_desc = docs.short_description
md_template += SUBHEADING_DESC_TEMPLATE.format(short_desc)
md_template = extract_function_docs(md_template, api)
return md_template
def build_formats_reference():
TEMPLATE = """
---
title: Formats Reference
---
It's a formats reference supported by the main Frictionless package. If you have installed external plugins, there can be more formats available. Below we're listing a format group name (or a parser name) like Excel, which is used, for example, for `xlsx`, `xls` etc formats. Options can be used for creating dialects, for example, `dialect = ExcelDialect(sheet=1)`.
{% for format in formats %}
## {{ format.name }}
{% if format.options %}
{% for option in format.options %}
### {{ option.name }}
> Type: {{ option.type }}
{{ option.text }}
{% endfor %}
{% else %}
There are no options available.
{% endif %}
{% endfor %}
"""
# Input
formats = []
modules = []
for item in pkgutil.iter_modules([os.path.dirname(plugins.__file__)]):
modules.append(import_module(f"frictionless.plugins.{item.name}"))
for module in modules:
for name, Dialect in vars(module).items():
match = re.match(r"(.+)Dialect", name)
if not match:
continue
name = match.group(1)
data = parse(Dialect.__doc__)
format = {"name": name, "options": []}
for param in data.params:
if param.arg_name.startswith("descriptor"):
continue
type = param.type_name
text = param.description.capitalize()
name = stringcase.titlecase(param.arg_name.replace("?", ""))
format["options"].append({"name": name, "text": text, "type": type})
formats.append(format)
# Ouput
template = Template(inspect.cleandoc(TEMPLATE))
document = template.render(formats=formats).strip()
write_file(os.path.join("docs", "references", "formats-reference.md"), document)
print("Built: Formats Reference")
def build_schemes_reference():
TEMPLATE = """
---
title: Schemes Reference
---
It's a schemes reference supported by the main Frictionless package. If you have installed external plugins, there can be more schemes available. Below we're listing a scheme group name (or a loader name) like Remote, which is used, for example, for `http`, `https` etc schemes. Options can be used for creating controls, for example, `control = RemoteControl(http_timeout=1)`.
{% for scheme in schemes %}
## {{ scheme.name }}
{% if scheme.options %}
{% for option in scheme.options %}
### {{ option.name }}
> Type: {{ option.type }}
{{ option.text }}
{% endfor %}
{% else %}
There are no options available.
{% endif %}
{% endfor %}
"""
# Input
schemes = []
modules = []
for item in pkgutil.iter_modules([os.path.dirname(plugins.__file__)]):
modules.append(import_module(f"frictionless.plugins.{item.name}"))
for module in modules:
for name, Dialect in vars(module).items():
match = re.match(r"(.+)Control", name)
if not match:
continue
name = match.group(1)
data = parse(Dialect.__doc__)
scheme = {"name": name, "options": []}
for param in data.params:
if param.arg_name.startswith("descriptor"):
continue
type = param.type_name
text = param.description.capitalize()
name = stringcase.titlecase(param.arg_name.replace("?", ""))
scheme["options"].append({"name": name, "text": text, "type": type})
schemes.append(scheme)
# Output
template = Template(inspect.cleandoc(TEMPLATE))
document = template.render(schemes=schemes).strip()
write_file(os.path.join("docs", "references", "schemes-reference.md"), document)
print("Built: Schemes Reference")
def annotate_function(function):
sig = inspect.signature(function)
doc_params = {p.arg_name: p.type_name for p in parse(function.__doc__).params}
for k, v in list(doc_params.items()):
if "," in k:
for split_key in k.split(","):
doc_params[split_key.strip()] = v
del doc_params[k]
hidden = HIDDEN.get(f"{function.__module__}.{function.__name__}", {})
for p in sig.parameters.values():
if p.name in hidden:
annotation = bound_default(p)
else:
annotation = guess_type(p, doc_params.get(p.name))
function.__annotations__[p.name] = annotation
doc_returns = parse(function.__doc__).returns
# Note skimage.filters.inverse has no return doc string
# Note skimage.filters.threshold should be a different type ....
if doc_returns is not None:
function.__annotations__["return"] = guess_return_type(doc_returns)
def _infer_input_description_from_docstring(
fn: Callable) -> Dict[str, Optional[str]]:
doc_str = fn.__doc__
if not is_module_available("docstring_parser") or doc_str is None:
return {}
from docstring_parser import parse
try:
docstring = parse(doc_str)
return {p.arg_name: p.description for p in docstring.params}
except Exception:
return {}
def test_meta_newlines(
source: str,
expected_short_desc: T.Optional[str],
expected_long_desc: T.Optional[str],
expected_blank_short_desc: bool,
expected_blank_long_desc: bool
) -> None:
docstring = parse(source)
assert docstring.short_description == expected_short_desc
assert docstring.long_description == expected_long_desc
assert docstring.blank_after_short_description == expected_blank_short_desc
assert docstring.blank_after_long_description == expected_blank_long_desc
assert len(docstring.meta) == 1
def test_function_docstrings_validity(
path: Path, node: ast.FunctionDef, docstring: str
) -> None:
"""Test whether functions have valid docstrings.
:param path: path of the file
:param node: AST node to check
:param docstring: AST node docstring
"""
with decorated_log(path, node):
parsed_docstring = docstring_parser.parse(docstring)
verify_function_params(node, parsed_docstring)
verify_function_returns(node, parsed_docstring)
def __call__(self, package):
for namme, thing in package.__dict__.items():
typ = type(thing)
doc_string = thing.__doc__
doc_parse = parse(doc_string)
if isinstance(thing,
Callable) and doc_parse and (doc_parse.params
or doc_parse.returns):
fname = str(thing)
class T:
fname = str(thing)
doc = doc_parse
def on_post(self, req, resp):
try:
input = req.bounded_stream.read().decode("utf-8")
try:
input = {k: eval(v) for k, v in input.items()}
except Exception as e:
try:
input = json_tricks.loads(input)
except Exception as e:
print(
f"could not parse {input}, its empyz now "
)
input = {}
print(self.doc.__repr__())
result = Import2Rest.functions[self.fname](**input)
resp.body = dumps(result)
except Exception as e:
resp.body = json_tricks.dumps(e.__repr__())
T.__qualname__ = str(fname)
self.functions[fname] = thing
t = T()
path = f"/{namme}"
app.add_route(path, t)
self.classes.append(T)
yield f"localhost:8000{path}", {
"method": "post",
"params": doc_parse.params,
"returns": doc_parse.returns
}
def function_definitions():
result = {}
for fun_name, fun in OrderedDict(sorted(TEL_FUNCTIONS.items())).items():
docstring = parse(fun.__doc__)
expected_args = []
phase = None
return_type = None
invalid_value = None
if issubclass(fun, TelTypedFunction):
tfun = cast(TelTypedFunction, fun)
expected_args = tfun.expected_arg_types
phase = tfun.phase_spec
return_type = tfun.return_type_spec
invalid_value = tfun.invalid_value_spec
description = {
'arguments': [{
'name':
param.arg_name,
'typeName':
param.type_name,
'description':
param.description,
**(_tel_arg_remote_spec(param.arg_name, expected_args) or {}),
} for param in docstring.params],
'raises': [{
'typeName': exc.type_name,
'description': exc.description
} for exc in docstring.raises],
'shortDescription':
docstring.short_description,
'longDescription':
docstring.long_description,
'returns': {
'typeName': docstring.returns.type_name,
'description': docstring.returns.description,
**(return_type.to_remote_spec() if return_type else {}),
},
**({
'phase': phase.to_remote_spec()
} if phase else {}),
**({
'invalidValue': invalid_value.to_remote_spec()
} if invalid_value else {}),
}
result[fun_name] = description
return result
def test_meta_with_multiline_description() -> None:
docstring = parse(
'''
Short description
:param: asd
1
2
3
''')
assert docstring.short_description == 'Short description'
assert len(docstring.meta) == 1
assert docstring.meta[0].args == ['param']
assert docstring.meta[0].description == 'asd\n1\n 2\n3'
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]:
doc_str = fn.__doc__
if not is_module_available("docstring_parser") or doc_str is None:
return None
from docstring_parser import parse
try:
docstring = parse(doc_str)
if docstring.returns is None:
return None
return docstring.returns.description
except Exception:
return None
def get_help(module, inline=None):
results = {}
module_path = module.get_path()
if inline is not None:
doc = parse(module.INLINE_SUBMODULES[inline].__doc__)
desc = doc.short_description or "No description provided."
cmd_path = " ".join(module_path + [inline]).strip()
results[cmd_path] = desc
return results
# print description of inline submodules
for m in module.INLINE_SUBMODULES:
doc = parse(module.INLINE_SUBMODULES[m].__doc__)
desc = doc.short_description or "No description provided."
cmd_path = " ".join(module_path + [m]).strip()
results[cmd_path] = desc
# print description of submodules
for m in module.get_modules():
doc = parse(m.INLINE_SUBMODULES[""].__doc__)
desc = doc.short_description or "No description provided."
cmd_path = " ".join(m.get_path()).strip()
results[cmd_path] = desc
return dict(sorted(results.items()))
def extract_result_schema(self, method: Callable) -> Schema:
result_schema = Schema(schema={})
if method.__doc__:
doc = docstring_parser.parse(method.__doc__)
if doc and doc.returns:
result_schema = Schema(
schema={'type': doc.returns.type_name},
required=True,
summary=doc.returns.description.split('.')[0],
description=doc.returns.description,
)
return result_schema
def parse_methods(public_methods: Dict) -> Generator[Method, None, None]:
for name, method in public_methods.items():
doc = getdoc(method)
if doc is None:
logger.debug(f"Docless method: {name}")
yield manually_set_method(name)
else:
parsed_doc = parse(doc)
yield Method(
name=name,
arguments=list(parse_arguments(parsed_doc)),
docstring=clean_doc(doc),
return_type=parse_return_type(parsed_doc.meta),
)
def test_argument_google_all():
'''Test google full type set.'''
name, fn = [
x for x in getmembers(module, isfunction)
if x[0] == 'argument_google_all'
][0]
sig = signature(fn)
document = parse(fn.__doc__)
arguments = []
for arg in sig.parameters:
docstring = next((d for d in document.params if d.arg_name == arg),
None)
arguments.append(
Argument(parameters=sig.parameters[arg], docstring=docstring))
# print(arguments[0].__dict__)
assert arguments[0].help == 'argument string'
assert arguments[0].type == str
assert arguments[0].default == 'A'
# print(arguments[1].__dict__)
assert arguments[1].help == 'argument bool'
# NOTE: Argparse does not store bool type
assert not hasattr(arguments[1], 'type')
assert arguments[1].default is False
# print(arguments[2].__dict__)
assert arguments[2].help == 'argument int'
assert arguments[2].type == int
assert arguments[2].default == 1
# print(arguments[3].__dict__)
assert arguments[3].help == 'argument float'
assert arguments[3].type == float
assert arguments[3].default == 1.5
# print(arguments[4].__dict__)
assert arguments[4].help == 'argument list'
assert arguments[4].type == list
assert arguments[4].default == ['A']
# print(arguments[5].__dict__)
assert arguments[5].help == 'argument set'
assert arguments[5].type == set
assert arguments[5].default == {'a'}
# print(arguments[6].__dict__)
assert arguments[6].help == 'argument tuple'
assert arguments[6].type == tuple
assert arguments[6].default == ('A', )
def merge_super_sigs(cls,
exclude=("widget_type", "kwargs", "args", "kwds",
"extra")):
"""Merge the signature and kwarg docs from all superclasses, for clearer docs.
Parameters
----------
cls : Type
The class being modified
exclude : tuple, optional
A list of parameter names to excluded from the merged docs/signature,
by default ("widget_type", "kwargs", "args", "kwds")
Returns
-------
cls : Type
The modified class (can be used as a decorator)
"""
params = {}
param_docs: list[DocstringParam] = []
for sup in reversed(inspect.getmro(cls)):
try:
sig = inspect.signature(getattr(sup, "__init__"))
# in some environments `object` or `abc.ABC` will raise ValueError here
except ValueError:
continue
for name, param in sig.parameters.items():
if name in exclude:
continue
params[name] = param
param_docs += parse(getattr(sup, "__doc__", "")).params
# sphinx_autodoc_typehints isn't removing the type annotations from the signature
# so we do it manually when building documentation.
if BUILDING_DOCS:
params = {
k: v.replace(annotation=inspect.Parameter.empty)
for k, v in params.items()
}
cls.__init__.__signature__ = inspect.Signature(
sorted(params.values(), key=lambda x: x.kind))
param_docs = [p for p in param_docs if p.arg_name not in exclude]
cls.__doc__ = (cls.__doc__ or "").split("Parameters")[0].rstrip() + "\n\n"
cls.__doc__ += _param_list_to_str(param_docs)
# this makes docs linking work... but requires that all of these be in __init__
cls.__module__ = "magicgui.widgets"
return cls
def test_returns() -> None:
docstring = parse(
'''
Short description
''')
assert docstring.returns is None
docstring = parse(
'''
Short description
:returns: description
''')
assert docstring.returns is not None
assert docstring.returns.type_name is None
assert docstring.returns.description == 'description'
docstring = parse(
'''
Short description
:returns int: description
''')
assert docstring.returns is not None
assert docstring.returns.type_name == 'int'
assert docstring.returns.description == 'description'
def test_raises() -> None:
docstring = parse(
'''
Short description
''')
assert len(docstring.raises) == 0
docstring = parse(
'''
Short description
:raises: description
''')
assert len(docstring.raises) == 1
assert docstring.raises[0].type_name is None
assert docstring.raises[0].description == 'description'
docstring = parse(
'''
Short description
:raises ValueError: description
''')
assert len(docstring.raises) == 1
assert docstring.raises[0].type_name == 'ValueError'
assert docstring.raises[0].description == 'description'
def _get_test_case_key(self):
docstring = parse(self._testMethodDoc)
name = docstring.short_description.strip()
pre_condition = docstring.long_description
input_data = self._testInputData
expected_data = self._testExpectedData
description = self._description
test_case = JiraTestCase(name=name,
pre_condition=pre_condition,
input_data=input_data,
expected_data=expected_data,
folder=self.FOLDER,
description=description)
test_case_key = test_case.create_test_case(self.ISSUE_KEYS)
return test_case_key
def test_argument_type_hints_simple() -> None:
'''Test simple argument.'''
name, fn = [
x for x in getmembers(module, isfunction)
if x[0] == 'argument_type_hints_simple'
][0]
sig = signature(fn)
docstring = parse(fn.__doc__)
arguments = []
for arg in sig.parameters:
document = next((d for d in docstring.params if d.arg_name == arg),
None)
arguments.append(
Argument(parameters=sig.parameters[arg], docstring=document))
# print('Arguments: ', arguments[0].__dict__)
assert arguments[0].default is None
