def test_docstring_parser2():
example = """
Will block thread until `num_tokens` could be consumed from token bucket `key`.
Args:
key (str): identifying a specific token bucket
num_tokens (int): will block without consuming any tokens until
this amount are availabe to be consumed
timeout (int): seconds to block for
retry_interval (Optional[float]): how long to wait between polling
for tokens to be available. `None` means use default interval
which is equal to time needed to replenish `num_tokens`.
Returns:
Tuple[
int,
str,
ClassName,
]
"""
expected = TypeSignature.factory(
arg_types=ArgTypes.factory(
name=ArgsSection.ARGS,
args=OrderedDict(
[
("key", TypeDef.from_tuples((5, 17), ("str", []), (5, 20),)),
("num_tokens", TypeDef.from_tuples((6, 24), ("int", []), (6, 27),)),
("timeout", TypeDef.from_tuples((8, 21), ("int", []), (8, 24),)),
(
"retry_interval",
TypeDef.from_tuples(
(9, 28), ("Optional", [TypeAtom("float", [])]), (9, 43),
),
),
]
),
),
return_type=ReturnType.factory(
name=ReturnsSection.RETURNS,
type_def=TypeDef.from_tuples(
(15, 12),
(
"Tuple",
[
TypeAtom("int", []),
TypeAtom("str", []),
TypeAtom("ClassName", []),
],
),
(19, 13),
),
),
)
result = docstring_parser.parse(example)
assert result == expected
def test_p_arg_list():
example = """
Kwargs:
key (str): identifying a specific token bucket
num_tokens (int) : whitespace around the colon
timeout (int): seconds to block for
retry_interval (Optional[float]): how long to wait between polling
for tokens to be available. `None` means use default interval
which is equal to time needed to replenish `num_tokens`.
*inner_args
**inner_kwargs: passed to inner function
"""
section = p_arg_list.parse(example)
assert section.name == ArgsSection.ARGS # normalised -> "Args" (Enum)
assert section.args == OrderedDict(
[
("key", TypeDef.from_tuples((2, 17), ("str", []), (2, 20),)),
("num_tokens", TypeDef.from_tuples((3, 24), ("int", []), (3, 27),)),
("timeout", TypeDef.from_tuples((4, 21), ("int", []), (4, 24),)),
(
"retry_interval",
TypeDef.from_tuples(
(5, 28), ("Optional", [TypeAtom("float", [])]), (5, 43),
),
),
("*inner_args", None),
("**inner_kwargs", None),
]
)
def generic_typeatom_f(draw, children):
"""
A type var with params (i.e. is 'generic'), without being one of the
special cases such as homogenous tuple, callable (others?)
Args:
draw: provided by @st.composite
children: another hypothesis strategy to draw from
(first arg to function returned by decorator)
"""
return TypeAtom(name=draw(dotted_var_path_f()),
args=draw(small_lists_f(children)))
def test_arg_type(example, start_pos, end_pos):
parser = arg_type << rest_of_line
result = parser.parse(example)
assert result == {
"arg": "key",
"type": TypeDef(start_pos, TypeAtom("str", []), end_pos,),
}
assert result["type"].name == "str"
assert result["type"].args == []
start, _, end = result["type"]
assert slice_by_pos(example, start, end) == "str"
def test_docstring_parser():
example = """
Will block thread until `num_tokens` could be consumed from token bucket `key`.
Keyword Arguments:
key (str): identifying a specific token bucket
num_tokens (int): will block without consuming any tokens until
this amount are availabe to be consumed
timeout (int): seconds to block for
retry_interval (Optional[float]): how long to wait between polling
for tokens to be available. `None` means use default interval
which is equal to time needed to replenish `num_tokens`.
**kwargs (Any)
Returns:
bool: whether we got the requested tokens or not
(False if timed out)
"""
expected = TypeSignature.factory(
arg_types=ArgTypes.factory(
name=ArgsSection.ARGS,
args=OrderedDict(
[
("key", TypeDef.from_tuples((4, 17), ("str", []), (4, 20),)),
("num_tokens", TypeDef.from_tuples((5, 24), ("int", []), (5, 27),)),
("timeout", TypeDef.from_tuples((7, 21), ("int", []), (7, 24),)),
(
"retry_interval",
TypeDef.from_tuples(
(8, 28), ("Optional", [TypeAtom("float", [])]), (8, 43),
),
),
("**kwargs", TypeDef.from_tuples((11, 22), ("Any", []), (11, 25),)),
]
),
),
return_type=ReturnType.factory(
name=ReturnsSection.RETURNS,
type_def=TypeDef.from_tuples((14, 12), ("bool", []), (14, 16),),
),
)
result = docstring_parser.parse(example)
assert result == expected
def _callable() -> parsy.Parser:
"""
Self-referential helper for `type_atom` of Callable type
AFAIK `Callable` is the only type where one of the expected atom positions
(the args of the callable) is a list. Other code is nicer if we treat that
bare list as a TypeAtom name=None.
"""
return (
yield parsy.seq(
parsy.string("Callable"),
between(
parsy.regex(r"\[\s*"),
parsy.regex(r",?\s*\]"),
parsy.seq(
_nested.map(lambda args: TypeAtom(None, args))
<< parsy.regex(r",\s*"),
type_atom,
),
),
).combine(TypeAtom)
)
ReturnsSection,
ReturnType,
TypeAtom,
TypeSignature,
)
@pytest.mark.parametrize(
"example,expected",
[
(
TypeSignature.factory(
arg_types=ArgTypes.factory(
name=ArgsSection.ARGS,
args=OrderedDict([
("key", TypeAtom("str", [])),
("num_tokens", TypeAtom("int", [])),
("timeout", TypeAtom("int", [])),
(
"retry_interval",
TypeAtom("Optional", [TypeAtom("float", [])]),
),
]),
),
return_type=ReturnType.factory(
name=ReturnsSection.RETURNS,
type_def=TypeAtom("bool", []),
),
),
"# type: (str, int, int, Optional[float]) -> bool",
),
def callable_typeatom_f(draw, children):
args_param = draw(small_lists_nonempty_f(children))
returns_param = draw(children)
return TypeAtom(name="Callable",
args=(TypeAtom(None, args_param), returns_param))
def homogenous_tuple_typeatom_f(draw, children):
return TypeAtom(name="Tuple", args=(draw(children), TypeAtom("...", [])))
def noargs_typeatom_f(draw):
return TypeAtom(
name=draw(dotted_var_path_f()),
args=(),
)
from waterloo.types import (
ArgsSection,
ArgTypes,
ImportStrategy,
ReturnsSection,
ReturnType,
TypeAtom,
TypeSignature,
)
@pytest.mark.parametrize(
"expected,example",
[
("str", TypeAtom("str", [])),
("Dict", TypeAtom("Dict", [])),
(
"Dict[int, str]",
TypeAtom("Dict", [TypeAtom("int", []),
TypeAtom("str", [])]),
),
(
"Dict[int, db.models.User]",
TypeAtom("Dict",
[TypeAtom("int", []),
TypeAtom("db.models.User", [])]),
),
(
"my.generic.Container[int]",
TypeAtom("my.generic.Container", [TypeAtom("int", [])]),
def assert_annotation_roundtrip(example: str, result: TypeAtom):
normalised = _normalise_annotation(example)
assert normalised == result.to_annotation(None)
_nested.map(lambda args: TypeAtom(None, args))
<< parsy.regex(r",\s*"),
type_atom,
),
),
).combine(TypeAtom)
)
_type_token = dotted_var_path | parsy.string("...")
# mypy type definition, parsed into its nested components
type_atom = (
_callable
| parsy.seq(_type_token, _nested).combine(TypeAtom)
| _type_token.map(lambda t: TypeAtom(t, []))
| _nested # recurse->
)
# in "Args" section the type def is in parentheses after the var name
# fmt: off
arg_type_def = lexeme(
parsy.string("(") >> scn >> typed_mark(type_atom, TypeDef) << scn << parsy.string(")")
)
# fmt: on
optional_description = parsy.regex(r"[ |\t]*:") | (
parsy.regex(r"[ |\t]*") << parsy.regex(r".+").should_fail("no description expected")
)
assert result == example.rstrip()
@pytest.mark.parametrize(
"example",
["dotted.path", "1name", "no-hyphens", "one two three", "A (A)", "***args"],
)
def test_var_name_invalid(example):
with pytest.raises(parsy.ParseError):
var_name.parse(example)
@pytest.mark.parametrize(
"example,expected",
[
("str", TypeAtom("str", [])),
("Dict", TypeAtom("Dict", [])),
(
"Dict[int, str]",
TypeAtom("Dict", [TypeAtom("int", []), TypeAtom("str", [])]),
),
(
"Dict[int, db.models.User]",
TypeAtom("Dict", [TypeAtom("int", []), TypeAtom("db.models.User", [])]),
),
(
"my.generic.Container[int]",
TypeAtom("my.generic.Container", [TypeAtom("int", [])]),
),
(
"Tuple[int, ...]",