def _type_check(self, type_constraint, datum, is_input): """Typecheck a PTransform related datum according to a type constraint. This function is used to optionally type-check either an input or an output to a PTransform. Args: type_constraint: An instance of a typehints.TypeContraint, one of the white-listed builtin Python types, or a custom user class. datum: An instance of a Python object. is_input: True if 'datum' is an input to a PTransform's DoFn. False otherwise. Raises: TypeError: If 'datum' fails to type-check according to 'type_constraint'. """ datum_type = 'input' if is_input else 'output' try: check_constraint(type_constraint, datum) except CompositeTypeHintError as e: raise_with_traceback(TypeCheckError(e.args[0])) except SimpleTypeHintError: error_msg = ( "According to type-hint expected %s should be of type %s. " "Instead, received '%s', an instance of type %s." % (datum_type, type_constraint, datum, type(datum))) raise_with_traceback(TypeCheckError(error_msg))
def type_check_inputs_or_outputs(self, pvalueish, input_or_output): type_hints = self.get_type_hints() hints = getattr(type_hints, input_or_output + '_types') if hints is None or not any(hints): return arg_hints, kwarg_hints = hints if arg_hints and kwarg_hints: raise TypeCheckError( 'PTransform cannot have both positional and keyword type hints ' 'without overriding %s._type_check_%s()' % (self.__class__, input_or_output)) root_hint = (arg_hints[0] if len(arg_hints) == 1 else arg_hints or kwarg_hints) for context, pvalue_, hint in _ZipPValues().visit( pvalueish, root_hint): if isinstance(pvalue_, DoOutputsTuple): continue if pvalue_.element_type is None: # TODO(robertwb): It's a bug that we ever get here. (typecheck) continue if hint and not typehints.is_consistent_with( pvalue_.element_type, hint): at_context = ' %s %s' % (input_or_output, context) if context else '' raise TypeCheckError( '{type} type hint violation at {label}{context}: expected {hint}, ' 'got {actual_type}\nFull type hint:\n{debug_str}'.format( type=input_or_output.title(), label=self.label, context=at_context, hint=hint, actual_type=pvalue_.element_type, debug_str=type_hints.debug_str()))
def type_check_inputs_or_outputs(self, pvalueish, input_or_output): hints = getattr(self.get_type_hints(), input_or_output + '_types') if not hints: return arg_hints, kwarg_hints = hints if arg_hints and kwarg_hints: raise TypeCheckError( 'PTransform cannot have both positional and keyword type hints ' 'without overriding %s._type_check_%s()' % (self.__class__, input_or_output)) root_hint = (arg_hints[0] if len(arg_hints) == 1 else arg_hints or kwarg_hints) for context, pvalue_, hint in _ZipPValues().visit( pvalueish, root_hint): if pvalue_.element_type is None: # TODO(robertwb): It's a bug that we ever get here. (typecheck) continue if hint and not typehints.is_consistent_with( pvalue_.element_type, hint): at_context = ' %s %s' % (input_or_output, context) if context else '' raise TypeCheckError( '%s type hint violation at %s%s: expected %s, got %s' % (input_or_output.title(), self.label, at_context, hint, pvalue_.element_type))
def _check_type(self, output): if output is None: return output elif isinstance(output, (dict, basestring)): object_type = type(output).__name__ raise TypeCheckError( 'Returning a %s from a ParDo or FlatMap is ' 'discouraged. Please use list("%s") if you really ' 'want this behavior.' % (object_type, output)) elif not isinstance(output, collections.Iterable): raise TypeCheckError('FlatMap and ParDo must return an ' 'iterable. %s was returned instead.' % type(output)) return output
def type_check_inputs(self, pvalueish): type_hints = self.get_type_hints() input_types = type_hints.input_types if input_types: args, kwargs = self.raw_side_inputs def element_type(side_input): if isinstance(side_input, pvalue.AsSideInput): return side_input.element_type return instance_to_type(side_input) arg_types = [pvalueish.element_type ] + [element_type(v) for v in args] kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()} argspec_fn = self._process_argspec_fn() bindings = getcallargs_forhints(argspec_fn, *arg_types, **kwargs_types) hints = getcallargs_forhints(argspec_fn, *input_types[0], **input_types[1]) for arg, hint in hints.items(): if arg.startswith('__unknown__'): continue if hint is None: continue if not typehints.is_consistent_with( bindings.get(arg, typehints.Any), hint): raise TypeCheckError( 'Type hint violation for \'{label}\': requires {hint} but got ' '{actual_type} for {arg}\nFull type hint:\n{debug_str}' .format(label=self.label, hint=hint, actual_type=bindings[arg], arg=arg, debug_str=type_hints.debug_str()))
def type_check_inputs(self, pvalueish): type_hints = self.get_type_hints().input_types if type_hints: args, kwargs = self.raw_side_inputs def element_type(side_input): if isinstance(side_input, pvalue.AsSideInput): return side_input.element_type return instance_to_type(side_input) arg_types = [pvalueish.element_type ] + [element_type(v) for v in args] kwargs_types = {k: element_type(v) for (k, v) in kwargs.items()} argspec_fn = self._process_argspec_fn() bindings = getcallargs_forhints(argspec_fn, *arg_types, **kwargs_types) hints = getcallargs_forhints(argspec_fn, *type_hints[0], **type_hints[1]) for arg, hint in hints.items(): if arg.startswith('%unknown%'): continue if hint is None: continue if not typehints.is_consistent_with( bindings.get(arg, typehints.Any), hint): raise TypeCheckError( 'Type hint violation for \'%s\': requires %s but got %s for %s' % (self.label, hint, bindings[arg], arg))
def wrapper(self, method, args, kwargs): try: result = method(*args, **kwargs) except TypeCheckError as e: error_msg = ('Runtime type violation detected within ParDo(%s): ' '%s' % (self.full_label, e)) raise_with_traceback(TypeCheckError(error_msg)) else: return self._check_type(result)
def wrapper(self, method, args, kwargs): try: result = method(*args, **kwargs) except TypeCheckError as e: error_msg = ('Runtime type violation detected within ParDo(%s): ' '%s' % (self.full_label, e)) six.raise_from(TypeCheckError(error_msg), sys.exc_info()[2]) else: return self._check_type(result)
def wrapper(self, method, args, kwargs): try: result = method(*args, **kwargs) except TypeCheckError as e: # TODO(BEAM-10710): Remove the 'ParDo' prefix for the label name error_msg = ('Runtime type violation detected within ParDo(%s): ' '%s' % (self.full_label, e)) raise_with_traceback(TypeCheckError(error_msg)) else: return self._check_type(result)
def extract_output(self, accumulator, *args, **kwargs): result = self._combinefn.extract_output(accumulator, *args, **kwargs) if self._output_type_hint: try: _check_instance_type(self._output_type_hint.tuple_types[1], result, None, True) except TypeCheckError as e: error_msg = ('Runtime type violation detected within %s: ' '%s' % (self._label, e)) raise_with_traceback(TypeCheckError(error_msg)) return result
def add_input(self, accumulator, element, *args, **kwargs): if self._input_type_hint: try: _check_instance_type( self._input_type_hint[0][0].tuple_types[1], element, 'element', True) except TypeCheckError as e: error_msg = ('Runtime type violation detected within %s: ' '%s' % (self._label, e)) raise_with_traceback(TypeCheckError(error_msg)) return self._combinefn.add_input(accumulator, element, *args, **kwargs)