def test_mixed_single_items(items_generator_fx):
pipeline = (_pipeline().append_stage(
"reverser0", BatchTextReverser()).append_stage(
"reverser1",
TextReverser()).append_stage("reverser2",
BatchTextReverser()).append_stage(
"duplicator",
TextDuplicator()).build())
item = next(items_generator_fx)
for _ in range(88):
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage_concurrently(
"reverser0", TextReverser, kwargs={
"cycles": 3
}, concurrency=2).append_stage_concurrently(
"reverser1", BatchTextReverser, args=[5],
concurrency=0).append_stage("reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
BatchTextDuplicator(),
concurrency=2).build())
item = next(items_generator_fx)
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage_concurrently(
"reverser0", TextReverser, concurrency=2,
parallel=True).append_stage_concurrently(
"reverser1",
BatchTextReverser,
args=[9],
concurrency=1,
parallel=True).append_stage_concurrently(
"reverser2", TextReverser,
concurrency=0).append_stage_concurrently(
"duplicator",
BatchTextDuplicator,
args=[10],
concurrency=2,
parallel=True,
).build())
item = next(items_generator_fx)
for _ in range(88):
pipeline.process_async(item)
for _ in range(88):
result = pipeline.get_item()
assert result.id == item.id
assert result.payload["text"]
assert result.payload["text"] != item.payload["text"]
pipeline.stop()
def test_run():
pipeline = (_pipeline().set_source(RandomTextSource(10)).append_stage(
"reverser", TextReverser()).append_stage("duplicator",
TextDuplicator()).build())
for item in pipeline.run():
assert len([x for x in item.payload.keys()
if x.startswith("text")]) == 2
assert item.get_timing("reverser")
assert item.get_timing("duplicator")
assert pipeline.count == 10
def test_run():
pipeline = (_pipeline().set_source(RandomTextSource(35)).append_stage(
"reverser",
BatchTextReverser()).append_stage("duplicator",
TextDuplicator()).append_stage(
"batch_duplicator",
BatchTextDuplicator()).build())
for item in pipeline.run():
assert len([x for x in item.payload.keys()
if x.startswith("text")]) == 3
assert item.get_timing("reverser")
assert item.get_timing("duplicator")
assert pipeline.count == 35
pipeline = (_pipeline().set_source(RandomTextSource(40)).append_stage(
"reverser", TextReverser()).append_stage(
"duplicator", TextDuplicator()).append_stage(
"batch_duplicator",
BatchTextDuplicator(check_batch=True)).build())
for _ in pipeline.run():
pass
assert pipeline.count == 40
def test_concurrency_errors():
with pytest.raises(Exception):
pipeline = (_pipeline().set_source(RandomTextSource(10)).append_stage(
"reverser", TextReverser(),
concurrency=1).append_stage("error",
ExceptionStage(),
concurrency=1).build())
try:
for _ in pipeline.run():
pass
except Exception:
tb = traceback.format_exc()
assert 'Exception("test exception")' in tb
assert "CriticalError" in tb
raise
assert pipeline.count == 1
with pytest.raises(Exception):
pipeline = (_pipeline().set_source(RandomTextSource(10)).append_stage(
"reverser", TextReverser(), concurrency=1,
parallel=True).append_stage("error",
ExceptionStage(),
concurrency=1,
parallel=True).build())
try:
for _ in pipeline.run():
pass
except Exception:
tb = traceback.format_exc()
assert 'Exception("test exception")' in tb
assert "CriticalError" in tb
raise
assert pipeline.count == 1
pipeline = (_pipeline().set_source(RandomTextSource(10)).append_stage(
"reverser", TextReverser(), concurrency=1).append_stage(
"error1", ErrorStage(), concurrency=1, parallel=True).append_stage(
"error2", ErrorStage(),
concurrency=2).append_stage("duplicator",
TextDuplicator(),
concurrency=1,
parallel=True).build())
for item in pipeline.run():
assert item.get_timing("reverser")
assert item.get_timing("duplicator")
assert any(k.startswith("text_") for k in item.payload.keys())
assert len(list(item.soft_errors())) == 2
assert item.get_timing("error1")
assert item.get_timing("error2")
assert pipeline.count == 10
def test_mixed_concurrent_run():
pipeline = (_pipeline().set_source(RandomTextSource(96)).append_stage(
"reverser0", TextReverser(),
concurrency=2).append_stage("reverser1",
BatchTextReverser(),
concurrency=0).append_stage(
"reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
BatchTextDuplicator(),
concurrency=2).build())
items = list(pipeline.run())
_check(items, 96, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(81)).append_stage(
"reverser0", BatchTextReverser(), concurrency=2,
parallel=True).append_stage("reverser1",
TextReverser(),
concurrency=1,
parallel=True).append_stage(
"reverser2",
TextReverser(),
concurrency=0).append_stage(
"duplicator",
BatchTextDuplicator(),
concurrency=2,
parallel=True).build())
items = list(pipeline.run())
_check(items, 81, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(92)).append_stage(
"reverser0", BatchTextReverser(), concurrency=0).append_stage(
"reverser1", TextReverser(),
concurrency=1).append_stage("duplicator",
BatchTextDuplicator(),
concurrency=0).build())
items = list(pipeline.run())
_check(items, 92, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(83)).append_stage(
"reverser0", BatchTextReverser(), concurrency=1,
parallel=True).append_stage("reverser1",
BatchTextReverser(),
concurrency=1,
parallel=False).append_stage(
"duplicator",
TextDuplicator(),
concurrency=1,
parallel=True).build())
items = list(pipeline.run())
_check(items, 83, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(100)).append_stage(
"duplicator0", TextDuplicator(), concurrency=0).append_stage(
"reverser", TextReverser(),
concurrency=0).append_stage("duplicator1",
BatchTextDuplicator(check_batch=True),
concurrency=0).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(100)).append_stage(
"duplicator0", TextDuplicator(), concurrency=2).append_stage(
"reverser", TextReverser(),
concurrency=2).append_stage("duplicator1",
BatchTextDuplicator(),
concurrency=2).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
def test_single_items(items_generator_fx):
pipeline = (_pipeline().append_stage(
"reverser0",
TextReverser()).append_stage("reverser1", TextReverser()).append_stage(
"reverser2",
TextReverser()).append_stage("duplicator",
TextDuplicator()).build())
item = next(items_generator_fx)
result = pipeline.process(copy.deepcopy(item))
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage_concurrently(
"reverser0", TextReverser, kwargs={
"cycles": 3
}, concurrency=2).append_stage_concurrently(
"reverser1", TextReverser, args=[5],
concurrency=0).append_stage("reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2).build())
item = next(items_generator_fx)
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage_concurrently(
"reverser0", TextReverser, concurrency=2,
parallel=True).append_stage_concurrently(
"reverser1", TextReverser, args=[9], concurrency=1,
parallel=True).append_stage_concurrently(
"reverser2", TextReverser,
concurrency=0).append_stage_concurrently(
"duplicator",
TextDuplicator,
args=[10],
concurrency=2,
parallel=True).build())
item = next(items_generator_fx)
pipeline.process_async(item)
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage(
"reverser0", TextReverser(),
concurrency=0).append_stage_concurrently("reverser1",
TextReverser,
concurrency=1).append_stage(
"duplicator",
TextDuplicator(10),
concurrency=0).build())
item = next(items_generator_fx)
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] == item.payload["text"]
pipeline = (_pipeline().append_stage_concurrently(
"duplicator0",
TextDuplicator).append_stage("reverser",
TextReverser()).append_stage_concurrently(
"duplicator1",
TextDuplicator).build())
item = next(items_generator_fx)
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline().append_stage(
"reverser", TextReverser(11),
concurrency=0).append_stage_concurrently("duplicator0",
TextDuplicator,
concurrency=0).append_stage(
"duplicator1",
TextDuplicator(),
concurrency=1).build())
item = next(items_generator_fx)
pipeline.process_async(copy.deepcopy(item))
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] != item.payload["text"]
pipeline = (_pipeline(max_init_workers=1).append_stage_concurrently(
"reverser0", TextReverser, args=[20], concurrency=1,
parallel=True).append_stage_concurrently(
"reverser1",
TextReverser,
args=[20],
concurrency=1,
parallel=False).append_stage_concurrently("duplicator",
TextDuplicator,
args=[20],
concurrency=1,
parallel=True).build())
item = next(items_generator_fx)
pipeline.process_async(item)
result = pipeline.get_item()
pipeline.stop()
assert result.id == item.id
assert result.payload["text"] == item.payload["text"]
assert len(result.payload.keys()) > len(item.payload.keys())
def test_huge_run():
pipeline = (_pipeline().set_source(RandomTextSource(200)).append_stage(
"reverser0", TextReverser(15000), concurrency=4,
parallel=True).append_stage("reverser1",
TextReverser(15000),
concurrency=4,
parallel=True).append_stage(
"reverser2",
TextReverser(15000),
concurrency=4,
parallel=True).append_stage(
"duplicator",
TextDuplicator(15000),
concurrency=4,
parallel=True).build())
runner = pipeline.run()
start_time = time.time()
items = list(runner)
elapsed1 = time.time() - start_time
logger.debug("Time for strongly parallel: {}".format(elapsed1))
_check(items, 200)
pipeline = (_pipeline().set_source(RandomTextSource(200)).append_stage(
"reverser0", TextReverser(15000), concurrency=1,
parallel=True).append_stage("reverser1",
TextReverser(15000),
concurrency=2,
parallel=True).append_stage(
"reverser2",
TextReverser(15000),
concurrency=1,
parallel=True).append_stage(
"duplicator",
TextDuplicator(15000),
concurrency=1,
parallel=True).build())
runner = pipeline.run()
start_time = time.time()
items = list(runner)
elapsed2 = time.time() - start_time
logger.debug("Time for mildly parallel: {}".format(elapsed2))
_check(items, 200)
pipeline = (_pipeline().set_source(RandomTextSource(200)).append_stage(
"reverser0", TextReverser(15000),
concurrency=0).append_stage("reverser1",
TextReverser(15000),
concurrency=0).append_stage(
"reverser2",
TextReverser(15000),
concurrency=0).append_stage(
"duplicator",
TextDuplicator(15000),
concurrency=0).build())
runner = pipeline.run()
start_time = time.time()
items = list(runner)
elapsed3 = time.time() - start_time
logger.debug("Time for sequential: {}".format(elapsed3))
_check(items, 200)
assert elapsed3 > elapsed1
assert elapsed3 > elapsed2
assert elapsed2 > elapsed1
def test_errors(caplog):
pipeline = (_pipeline().set_error_manager(ErrorManager()).set_source(
RandomTextSource(10)).append_stage("reverser",
TextReverser()).append_stage(
"error", ErrorStage()).build())
for item in pipeline.run():
assert item.has_errors()
assert item.get_timing("reverser")
assert item.get_timing("error")
error = next(item.soft_errors())
assert error.get_exception() is None
assert str(error) == "test pipeline error"
assert all("stage error has generated an error" in record.msg.lower()
for record in caplog.records if record.levelno == logging.ERROR)
assert pipeline.count == 10
pipeline = (_pipeline().set_error_manager(ErrorManager()).set_source(
RandomTextSource(10)).append_stage("reverser",
TextReverser()).append_stage(
"error",
ErrorStage()).append_stage(
"duplicator",
TextDuplicator()).build())
caplog.clear()
for item in pipeline.run():
assert item.has_errors()
assert item.get_timing("reverser")
assert item.get_timing("duplicator")
assert any(k.startswith("text_") for k in item.payload.keys())
assert item.get_timing("error")
error = next(item.soft_errors())
assert error.get_exception() is None
assert str(error) == "test pipeline error"
assert all("stage error has generated an error" in record.msg.lower()
for record in caplog.records if record.levelno == logging.ERROR)
assert pipeline.count == 10
pipeline = (_pipeline().set_error_manager(ErrorManager()).set_source(
RandomTextSource(10)).append_stage("reverser",
TextReverser()).append_stage(
"error1",
CriticalIOErrorStage()).build())
for item in pipeline.run():
assert item.has_critical_errors()
assert item.get_timing("reverser")
assert item.get_timing("error1")
for error in item.critical_errors():
assert isinstance(error.get_exception(), IOError)
assert str(error) == "test pipeline critical IO error"
pipeline = (_pipeline().set_error_manager(ErrorManager()).set_source(
RandomTextSource(10)).append_stage("reverser",
TextReverser()).append_stage(
"error1",
ExceptionStage()).append_stage(
"error2",
ErrorStage()).build())
caplog.clear()
for item in pipeline.run():
assert item.has_critical_errors()
assert item.get_timing("reverser")
assert item.get_timing("error1")
assert not item.get_timing("error2")
for error in item.critical_errors():
assert isinstance(error.get_exception(), Exception)
assert (str(error) == "test exception"
and str(error.get_exception()) == "test exception"
and str(error) != "test pipeline error")
assert all("stage error1 has generated an error" in record.msg.lower()
for record in caplog.records if record.levelno == logging.ERROR)
assert pipeline.count == 10
with pytest.raises(Exception):
pipeline = (_pipeline().set_source(RandomTextSource(10)).append_stage(
"reverser", TextReverser()).append_stage("error",
ExceptionStage()).build())
try:
for _ in pipeline.run():
pass
except Exception:
assert 'Exception("test exception")' in traceback.format_exc()
raise
assert pipeline.count == 1
pipeline = (_pipeline().set_error_manager(
ErrorManager().no_skip_on_critical_error()).set_source(
RandomTextSource(10)).append_stage(
"reverser1", TextReverser()).append_stage(
"error",
ExceptionStage()).append_stage("reverser2",
TextReverser()).build())
for item in pipeline.run():
assert item.get_timing("reverser1")
assert item.get_timing("error")
assert item.get_timing("reverser2")
assert pipeline.count == 10
def test_concurrent_initialization():
pipeline = (_pipeline().set_source(
RandomTextSource(100)).append_stage_concurrently(
"reverser0", TextReverser, kwargs={
"cycles": 3
}, concurrency=2).append_stage_concurrently(
"reverser1", TextReverser, args=[5],
concurrency=0).append_stage("reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline().set_source(
RandomTextSource(100)).append_stage_concurrently(
"reverser0", TextReverser, concurrency=2,
parallel=True).append_stage_concurrently(
"reverser1",
TextReverser,
args=[10],
concurrency=1,
parallel=True).append_stage_concurrently(
"reverser2", TextReverser,
concurrency=0).append_stage_concurrently(
"duplicator",
TextDuplicator,
args=[10],
concurrency=2,
parallel=True).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(100)).append_stage(
"reverser0", TextReverser(),
concurrency=0).append_stage_concurrently("reverser1",
TextReverser,
concurrency=1).append_stage(
"duplicator",
TextDuplicator(10),
concurrency=0).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline().set_source(
RandomTextSource(100)).append_stage_concurrently(
"duplicator0", TextDuplicator, concurrency=0).append_stage(
"reverser", TextReverser(),
concurrency=0).append_stage_concurrently(
"duplicator1", TextDuplicator, concurrency=0).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline().set_source(RandomTextSource(100)).append_stage(
"reverser", TextReverser(12),
concurrency=0).append_stage_concurrently("duplicator0",
TextDuplicator,
concurrency=0).append_stage(
"duplicator1",
TextDuplicator(),
concurrency=1).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline(max_init_workers=1).set_source(
RandomTextSource(100)).append_stage_concurrently(
"reverser0", TextReverser, args=[20], concurrency=1,
parallel=True).append_stage_concurrently(
"reverser1",
TextReverser,
args=[20],
concurrency=1,
parallel=False).append_stage_concurrently(
"duplicator",
TextDuplicator,
args=[20],
concurrency=1,
parallel=True).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
def test_queue_sizes():
pipeline = (_pipeline(max_queues_size=1).set_source(
RandomTextSource(100)).append_stage(
"reverser0", TextReverser(),
concurrency=2).append_stage("reverser1",
TextReverser(),
concurrency=0).append_stage(
"reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline(max_queues_size=1).set_source(
RandomTextSource(100)).append_stage(
"reverser0", TextReverser(), concurrency=2,
parallel=True).append_stage("reverser1",
TextReverser(),
concurrency=1,
parallel=True).append_stage(
"reverser2",
TextReverser(),
concurrency=0).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2,
parallel=True).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline(max_queues_size=3).set_source(
RandomTextSource(100)).append_stage("waster",
TimeWaster(0.02),
concurrency=2).append_stage(
"reverser",
TimeWaster(0.04),
concurrency=1,
parallel=True).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline(max_queues_size=0).set_source(
RandomTextSource(100)).append_stage(
"reverser0", TextReverser(),
concurrency=2).append_stage("reverser1",
TextReverser(),
concurrency=0).append_stage(
"reverser2",
TextReverser(),
concurrency=1).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2).build())
items = list(pipeline.run())
_check(items, 100, pipeline)
pipeline = (_pipeline(max_queues_size=0).set_source(
RandomTextSource(100)).append_stage(
"reverser0", TextReverser(), concurrency=2,
parallel=True).append_stage("reverser1",
TextReverser(),
concurrency=1,
parallel=True).append_stage(
"reverser2",
TextReverser(),
concurrency=0).append_stage(
"duplicator",
TextDuplicator(),
concurrency=2,
parallel=True).build())
items = list(pipeline.run())
_check(items, 100, pipeline)