def transform_stream(self, stream: Stream):
with closing_if_closable(stream):
stream_estimator = StreamEstimator()
for obj in stream:
with stream_estimator.incoming_object(obj.n_remaining_hint):
if not self.predicate(obj):
continue
obj.n_remaining_hint = stream_estimator.emit()
yield obj
def test_StreamEstimator_non_deterministic_global_estimate():
est = StreamEstimator()
stream_length = 10
n_remaining = []
for i in range(stream_length):
with est.incoming_object(stream_length - i):
local_mult = random.randint(1, 10)
print("local_mult", local_mult)
print("global_estimate", est.global_estimate)
for j in range(local_mult):
n_remaining.append(est.emit())
def transform_stream(self, stream: Stream):
"""Transform a stream."""
with closing_if_closable(stream):
stream_estimator = StreamEstimator()
for obj in stream:
collection = tuple(self.prepare_input(obj, "collection"))
with stream_estimator.incoming_object(obj.n_remaining_hint,
len(collection)):
for value in collection:
yield self.prepare_output(
obj.copy(),
value,
n_remaining_hint=stream_estimator.emit())
def test_StreamEstimator_non_deterministic_full_estimate():
est = StreamEstimator()
stream_length = 10
n_remaining = []
for i in range(stream_length):
local_mult = random.randint(1, 10)
with est.incoming_object(stream_length - i, local_mult):
for j in range(local_mult):
n_remaining.append(est.emit())
# Last estimate is 1
assert n_remaining[-1] == 1
def test_StreamEstimator_no_estimate():
est = StreamEstimator()
stream_length = 10
multiplier = 5
n_remaining = []
for i in range(stream_length):
with est.incoming_object(None):
for j in range(multiplier):
n_remaining.append(est.emit())
# No estimates are available
assert n_remaining == [None for _ in range(stream_length * multiplier)]
def test_StreamEstimator_full_estimate():
est = StreamEstimator()
stream_length = 10
multiplier = 5
n_total = stream_length * multiplier
n_remaining = []
for i in range(stream_length):
with est.incoming_object(stream_length - i, multiplier):
for j in range(multiplier):
n_remaining.append(est.emit())
# Estimates are available
assert n_remaining == [n_total - i for i in range(n_total)]
def test_StreamEstimator_global_estimate():
est = StreamEstimator()
stream_length = 10
multiplier = 5
n_total = stream_length * multiplier
n_remaining = []
for i in range(stream_length):
with est.incoming_object(stream_length - i):
for j in range(multiplier):
n_remaining.append(est.emit())
# Estimates are available after first incoming object is fully processed
assert n_remaining == [
None if i < multiplier else n_total - i
for i in range(stream_length * multiplier)
]
def transform_stream(self, stream: Stream):
stream_estimator = StreamEstimator()
while True:
packed = list(itertools.islice(stream, self.size))
if not packed:
break
with stream_estimator.incoming_object(packed[0].n_remaining_hint,
n_consumed=len(packed)):
packed_values = tuple(
tuple(o[v] for o in packed) for v in self.variables)
yield self.prepare_output(
packed[0],
*packed_values,
n_remaining_hint=stream_estimator.emit())
def test_StreamEstimator_stacked():
length0 = 10
length1 = 5
length2 = 3
n_total = length0 * length1 * length2
est0 = StreamEstimator()
est1 = StreamEstimator()
n_remaining = []
for i in range(length0):
with est0.incoming_object(length0 - i, length1):
for j in range(length1):
with est1.incoming_object(est0.emit(), length2):
for k in range(length2):
n_remaining.append(est1.emit())
# Estimates are available
assert n_remaining == [n_total - i for i in range(n_total)]