def _read_side_inputs(self, tags_and_types):
# type: (...) -> Iterator[apache_sideinputs.SideInputMap]
"""Generator reading side inputs in the order prescribed by tags_and_types.
Args:
tags_and_types: List of tuples (tag, type). Each side input has a string
tag that is specified in the worker instruction. The type is actually
a boolean which is True for singleton input (read just first value)
and False for collection input (read all values).
Yields:
With each iteration it yields the result of reading an entire side source
either in singleton or collection mode according to the tags_and_types
argument.
"""
# Only call this on the old path where side_input_maps was not
# provided directly.
assert self.side_input_maps is None
# We will read the side inputs in the order prescribed by the
# tags_and_types argument because this is exactly the order needed to
# replace the ArgumentPlaceholder objects in the args/kwargs of the DoFn
# getting the side inputs.
#
# Note that for each tag there could be several read operations in the
# specification. This can happen for instance if the source has been
# sharded into several files.
for i, (side_tag, view_class,
view_options) in enumerate(tags_and_types):
sources = []
# Using the side_tag in the lambda below will trigger a pylint warning.
# However in this case it is fine because the lambda is used right away
# while the variable has the value assigned by the current iteration of
# the for loop.
# pylint: disable=cell-var-from-loop
for si in filter(lambda o: o.tag == side_tag,
self.spec.side_inputs):
if not isinstance(si, operation_specs.WorkerSideInputSource):
raise NotImplementedError('Unknown side input type: %r' %
si)
sources.append(si.source)
si_counter = opcounters.SideInputReadCounter(
self.counter_factory,
self.state_sampler,
declaring_step=self.name_context.step_name,
# Inputs are 1-indexed, so we add 1 to i in the side input id
input_index=i + 1)
element_counter = opcounters.OperationCounters(
self.counter_factory,
self.name_context.step_name,
view_options['coder'],
i,
suffix='side-input')
iterator_fn = sideinputs.get_iterator_fn_for_sources(
sources,
read_counter=si_counter,
element_counter=element_counter)
yield apache_sideinputs.SideInputMap(
view_class, view_options,
sideinputs.EmulatedIterable(iterator_fn))
def __init__(
self, counter_factory, step_name, output_index, consumers, coder):
self.consumers = consumers
self.opcounter = opcounters.OperationCounters(
counter_factory, step_name, coder, output_index)
# Used in repr.
self.step_name = step_name
self.output_index = output_index
self.coder = coder
def __init__(self,
counter_factory,
step_name, # type: str
output_index,
consumers, # type: List[Operation]
coder,
producer_type_hints
):
self.consumers = consumers
self.opcounter = opcounters.OperationCounters(
counter_factory,
step_name,
coder,
output_index,
producer_type_hints=producer_type_hints)
# Used in repr.
self.step_name = step_name
self.output_index = output_index
self.coder = coder
