def test_getcallargs_forhints(self):
def func(a, b_c, *d):
return a, b_c, d
self.assertEqual({
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...]
}, getcallargs_forhints(func, *[Any, Any]))
self.assertEqual(
{
'a': Any,
'b_c': Any,
'd': self.relax_for_py2(Tuple[Union[int, str], ...])
}, getcallargs_forhints(func, *[Any, Any, str, int]))
self.assertEqual(
{
'a': int,
'b_c': Tuple[str, Any],
'd': Tuple[Any, ...]
}, getcallargs_forhints(func, *[int, Tuple[str, Any]]))
self.assertEqual(
{
'a': Any,
'b_c': Any,
'd': self.relax_for_py2(Tuple[str, ...])
}, getcallargs_forhints(func, *[Any, Any, Tuple[str, ...]]))
self.assertEqual(
{
'a': Any,
'b_c': Any,
'd': self.relax_for_py2(Tuple[Union[Tuple[str, ...], int],
...])
}, getcallargs_forhints(func, *[Any, Any, Tuple[str, ...], int]))
def type_check_inputs(self, pvalueish):
type_hints = self.get_type_hints().input_types
if type_hints:
args, kwargs = self.raw_side_inputs
def element_type(side_input):
if isinstance(side_input, pvalue.AsSideInput):
return side_input.element_type
return instance_to_type(side_input)
arg_types = [pvalueish.element_type] + [element_type(v) for v in args]
kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()}
argspec_fn = self._process_argspec_fn()
bindings = getcallargs_forhints(argspec_fn, *arg_types, **kwargs_types)
hints = getcallargs_forhints(argspec_fn, *type_hints[0], **type_hints[1])
for arg, hint in hints.items():
if arg.startswith('%unknown%'):
continue
if hint is None:
continue
if not typehints.is_consistent_with(
bindings.get(arg, typehints.Any), hint):
raise TypeCheckError(
'Type hint violation for \'%s\': requires %s but got %s for %s'
% (self.label, hint, bindings[arg], arg))
def type_check_inputs(self, pvalueish):
type_hints = self.get_type_hints().input_types
if type_hints:
args, kwargs = self.raw_side_inputs
def element_type(side_input):
if isinstance(side_input, pvalue.AsSideInput):
return side_input.element_type
return instance_to_type(side_input)
arg_types = [pvalueish.element_type
] + [element_type(v) for v in args]
kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()}
argspec_fn = self._process_argspec_fn()
bindings = getcallargs_forhints(argspec_fn, *arg_types,
**kwargs_types)
hints = getcallargs_forhints(argspec_fn, *type_hints[0],
**type_hints[1])
for arg, hint in hints.items():
if arg.startswith('%unknown%'):
continue
if hint is None:
continue
if not typehints.is_consistent_with(
bindings.get(arg, typehints.Any), hint):
raise TypeCheckError(
'Type hint violation for \'%s\': requires %s but got %s for %s'
% (self.label, hint, bindings[arg], arg))
def type_check_inputs(self, pvalueish):
type_hints = self.get_type_hints()
input_types = type_hints.input_types
if input_types:
args, kwargs = self.raw_side_inputs
def element_type(side_input):
if isinstance(side_input, pvalue.AsSideInput):
return side_input.element_type
return instance_to_type(side_input)
arg_types = [pvalueish.element_type
] + [element_type(v) for v in args]
kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()}
argspec_fn = self._process_argspec_fn()
bindings = getcallargs_forhints(argspec_fn, *arg_types,
**kwargs_types)
hints = getcallargs_forhints(argspec_fn, *input_types[0],
**input_types[1])
for arg, hint in hints.items():
if arg.startswith('__unknown__'):
continue
if hint is None:
continue
if not typehints.is_consistent_with(
bindings.get(arg, typehints.Any), hint):
raise TypeCheckError(
'Type hint violation for \'{label}\': requires {hint} but got '
'{actual_type} for {arg}\nFull type hint:\n{debug_str}'
.format(label=self.label,
hint=hint,
actual_type=bindings[arg],
arg=arg,
debug_str=type_hints.debug_str()))
def test_getcallargs_forhints_varkw(self):
def func(a, b_c, *d, **e):
return a, b_c, d, e
self.assertEqual({
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...],
'e': Dict[str, Union[str, int]]
},
getcallargs_forhints(
func, *[Any, Any], **{
'kw1': str, 'kw2': int
}))
self.assertEqual({
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...],
'e': Dict[str, Union[str, int]]
},
getcallargs_forhints(
func, *[Any, Any], e=Dict[str, Union[int, str]]))
self.assertEqual(
{
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...],
'e': Dict[str, Dict[str, Union[str, int]]]
},
# keyword is not 'e', thus the Dict is considered a value hint.
getcallargs_forhints(func, *[Any, Any], kw1=Dict[str, Union[int, str]]))
def test_getcallargs_forhints_missing_arg(self):
def fn(a, b=None, *args, foo, **kwargs):
return a, b, args, foo, kwargs
with self.assertRaisesRegex(decorators.TypeCheckError, "missing.*'a'"):
decorators.getcallargs_forhints(fn, foo=List[int])
with self.assertRaisesRegex(decorators.TypeCheckError, "missing.*'foo'"):
decorators.getcallargs_forhints(fn, 5)
def test_getcallargs_forhints(self):
func = lambda a, (b, c), *d: None
self.assertEquals(
{'a': Any, 'b': Any, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any]))
self.assertEquals(
{'a': Any, 'b': Any, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any, Any, int]))
self.assertEquals(
{'a': int, 'b': str, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[int, Tuple[str, Any]]))
def test_getcallargs_forhints(self):
func = lambda a, (b, c), *d: None
self.assertEquals(
{'a': Any, 'b': Any, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any]))
self.assertEquals(
{'a': Any, 'b': Any, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any, Any, int]))
self.assertEquals(
{'a': int, 'b': str, 'c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[int, Tuple[str, Any]]))
def test_getcallargs_forhints(self):
def func(a, b_c, *d):
b, c = b_c # pylint: disable=unused-variable
return None
self.assertEquals(
{'a': Any, 'b_c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any]))
self.assertEquals(
{'a': Any, 'b_c': Any, 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[Any, Any, Any, int]))
self.assertEquals(
{'a': int, 'b_c': Tuple[str, Any], 'd': Tuple[Any, ...]},
getcallargs_forhints(func, *[int, Tuple[str, Any]]))
def __init__(self, dofn, type_hints, label=None):
super(TypeCheckWrapperDoFn, self).__init__(dofn)
self._process_fn = self.dofn._process_argspec_fn()
if type_hints.input_types:
input_args, input_kwargs = type_hints.input_types
self._input_hints = getcallargs_forhints(
self._process_fn, *input_args, **input_kwargs)
else:
self._input_hints = None
# TODO(robertwb): Multi-output.
self._output_type_hint = type_hints.simple_output_type(label)
def __init__(self, dofn, type_hints, label=None):
super(TypeCheckWrapperDoFn, self).__init__(dofn)
self._process_fn = self.dofn._process_argspec_fn()
if type_hints.input_types:
input_args, input_kwargs = type_hints.input_types
self._input_hints = getcallargs_forhints(
self._process_fn, *input_args, **input_kwargs)
else:
self._input_hints = None
# TODO(robertwb): Multi-output.
self._output_type_hint = type_hints.simple_output_type(label)
def test_getcallargs_forhints(self):
def fn(a: int, b: str = None, *args: Tuple[T], foo: List[int],
**kwargs: Dict[str, str]) -> Tuple[Any, ...]:
return a, b, args, foo, kwargs
callargs = decorators.getcallargs_forhints(fn, float, foo=List[str])
self.assertDictEqual(callargs,
{'a': float,
'b': str,
'args': Tuple[T],
'foo': List[str],
'kwargs': Dict[str, str]})
def test_getcallargs_forhints_default_arg(self):
# Default args are not necessarily types, so they should be ignored.
def fn(a=List[int], b=None, *args, foo=(), **kwargs) -> Tuple[Any, ...]:
return a, b, args, foo, kwargs
callargs = decorators.getcallargs_forhints(fn)
self.assertDictEqual(callargs,
{'a': Any,
'b': Any,
'args': Tuple[Any, ...],
'foo': Any,
'kwargs': Dict[Any, Any]})
def test_getcallargs_forhints(self):
def func(a, b_c, *d):
b, c = b_c # pylint: disable=unused-variable
return None
self.assertEqual({
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...]
}, getcallargs_forhints(func, *[Any, Any]))
self.assertEqual({
'a': Any,
'b_c': Any,
'd': Tuple[Any, ...]
}, getcallargs_forhints(func, *[Any, Any, Any, int]))
self.assertEqual(
{
'a': int,
'b_c': Tuple[str, Any],
'd': Tuple[Any, ...]
}, getcallargs_forhints(func, *[int, Tuple[str, Any]]))
def test_getcallargs_forhints_builtins(self):
if sys.version_info < (3, 7):
# Signatures for builtins are not supported in 3.5 and 3.6.
self.assertEqual(
{
'_': str,
'__unknown__varargs': Tuple[Any, ...],
'__unknown__keywords': typehints.Dict[Any, Any]
}, getcallargs_forhints(str.upper, str))
self.assertEqual(
{
'_': str,
'__unknown__varargs': Tuple[str, ...],
'__unknown__keywords': typehints.Dict[Any, Any]
}, getcallargs_forhints(str.strip, str, str))
self.assertEqual(
{
'_': str,
'__unknown__varargs': Tuple[typehints.List[int], ...],
'__unknown__keywords': typehints.Dict[Any, Any]
}, getcallargs_forhints(str.join, str, typehints.List[int]))
else:
self.assertEqual({'self': str},
getcallargs_forhints(str.upper, str))
# str.strip has an optional second argument.
self.assertEqual({
'self': str,
'chars': Any
}, getcallargs_forhints(str.strip, str))
self.assertEqual({
'self': str,
'iterable': typehints.List[int]
}, getcallargs_forhints(str.join, str, typehints.List[int]))
def test_monkey_patch_signature(f, args, kwargs):
arg_types = [instance_to_type(v) for v in args]
kwargs_types = {k: instance_to_type(v) for (k, v) in kwargs.items()}
f_temp = _wrap_task_call(f)
try:
getcallargs_forhints(f, *arg_types, **kwargs_types)
except Exception:
print("Failed on {} with parameters {}, {}".format(f, args, kwargs))
raise
try:
getcallargs_forhints(f_temp, *arg_types, **kwargs_types)
except Exception:
print("Failed on {} with parameters {}, {}".format(
f_temp, args, kwargs))
raise
try:
expected_signature = inspect.signature(f)
test_signature = inspect.signature(f_temp)
assert (expected_signature == test_signature
), "Failed on {}, signature {} does not match {}".format(
f, expected_signature, test_signature)
except Exception:
# expected to pass for py2.7
pass
def __init__(self, dofn, type_hints, label=None):
super(TypeCheckWrapperDoFn, self).__init__()
self._dofn = dofn
self._label = label
self._process_fn = self._dofn.process_argspec_fn()
if type_hints.input_types:
input_args, input_kwargs = type_hints.input_types
self._input_hints = getcallargs_forhints(
self._process_fn, *input_args, **input_kwargs)
else:
self._input_hints = None
# TODO(robertwb): Actually extract this.
self.context_var = 'context'
# TODO(robertwb): Multi-output.
self._output_type_hint = type_hints.simple_output_type(label)
def wrapper(*args, **kwargs):
hints = get_type_hints(f)
if hints.input_types:
input_hints = getcallargs_forhints(
f, *hints.input_types[0], **hints.input_types[1])
inputs = inspect.getcallargs(f, *args, **kwargs)
for var, hint in input_hints.items():
value = inputs[var]
new_value = check_or_interleave(hint, value, var)
if new_value is not value:
if var in kwargs:
kwargs[var] = new_value
else:
args = list(args)
for ix, pvar in enumerate(inspect.getargspec(f).args):
if pvar == var:
args[ix] = new_value
break
else:
raise NotImplementedError('Iterable in nested argument %s' % var)
res = f(*args, **kwargs)
return check_or_interleave(hints.simple_output_type('typecheck'), res, None)
def wrapper(*args, **kwargs):
hints = get_type_hints(f)
if hints.input_types: # pylint: disable=too-many-nested-blocks
input_hints = getcallargs_forhints(
f, *hints.input_types[0], **hints.input_types[1])
inputs = get_signature(f).bind(*args, **kwargs).arguments
for var, hint in input_hints.items():
value = inputs[var]
new_value = check_or_interleave(hint, value, var)
if new_value is not value:
if var in kwargs:
kwargs[var] = new_value
else:
args = list(args)
for ix, pvar in enumerate(get_signature(f).parameters):
if pvar == var:
args[ix] = new_value
break
else:
raise NotImplementedError('Iterable in nested argument %s' % var)
res = f(*args, **kwargs)
return check_or_interleave(hints.simple_output_type('typecheck'), res, None)
def wrapper(*args, **kwargs):
hints = get_type_hints(f)
if hints.input_types: # pylint: disable=too-many-nested-blocks
input_hints = getcallargs_forhints(
f, *hints.input_types[0], **hints.input_types[1])
inputs = inspect.getcallargs(f, *args, **kwargs)
for var, hint in input_hints.items():
value = inputs[var]
new_value = check_or_interleave(hint, value, var)
if new_value is not value:
if var in kwargs:
kwargs[var] = new_value
else:
args = list(args)
for ix, pvar in enumerate(getfullargspec(f).args):
if pvar == var:
args[ix] = new_value
break
else:
raise NotImplementedError('Iterable in nested argument %s' % var)
res = f(*args, **kwargs)
return check_or_interleave(hints.simple_output_type('typecheck'), res, None)
