def process_batch(stage: BatchStage, items: Sequence[DataItem],
error_manager: ErrorManager) -> List[Optional[DataItem]]:
"""
Execute the :meth:`.stage.BatchStage.process_batch` method of a batch stage for a batch of items
"""
ret: List[Optional[DataItem]] = [None] * len(items)
to_process = {}
for i, item in enumerate(items):
if error_manager.check_critical_errors(item):
ret[i] = item
else:
_logger.debug(f"{stage} is going to process {item}")
to_process[i] = item
time1 = time.time()
try:
_logger.debug(f"{stage} is processing {len(to_process)} items")
processed = stage.process_batch(list(to_process.values()))
_logger.debug(
f"{stage} has finished processing {len(to_process)} items")
except Exception as e:
_logger.debug(
f"{stage} had failures in processing {len(to_process)} items")
spent = (time.time() - time1) / (len(to_process) or 1.0)
for i, item in to_process.items():
item.set_timing(stage.name, spent)
error_manager.handle(e, stage, item)
ret[i] = item
return ret
spent = (time.time() - time1) / (len(to_process) or 1.0)
for n, i in enumerate(to_process.keys()):
item = processed[n]
item.set_timing(stage.name, spent)
ret[i] = item
return ret
def process(stage: Stage, item: DataItem,
error_manager: ErrorManager) -> DataItem:
"""
Execute the :meth:`.stage.Stage.process` method of a stage for an item
"""
if error_manager.check_critical_errors(item):
return item
time1 = time.time()
try:
_logger.debug(f"{stage} is processing {item}")
processed_item = stage.process(item)
_logger.debug(f"{stage} has finished processing {processed_item}")
except Exception as e:
_logger.debug(f"{stage} has failed processing {item}")
item.set_timing(stage.name, time.time() - time1)
error_manager.handle(e, stage, item)
return item
# this can't be in a finally, otherwise it would register the `error_manager.handle` time
processed_item.set_timing(stage.name, time.time() - time1)
return processed_item
def test_critical_errors(caplog):
stage = TextReverser()
manager = ErrorManager()
item = DataItem()
error = CriticalError()
error.with_exception(Exception())
managed_critical_error = manager.handle(error, stage, item)
assert not item.has_errors()
assert item.has_critical_errors()
assert isinstance(
next(item.critical_errors()).get_exception(),
type(managed_critical_error.get_exception()),
)
assert any(caplog.records)
manager = ErrorManager().raise_on_critical_error()
item = DataItem()
with pytest.raises(CriticalError):
manager.handle(CriticalError(), stage, item)
with pytest.raises(Exception):
error = CriticalError().with_exception(Exception())
manager.handle(error, stage, item)
assert any(caplog.records)
assert item.has_critical_errors()
assert not item.has_errors()
manager = ErrorManager().no_skip_on_critical_error()
item = DataItem()
assert manager.handle(CriticalError(), stage, item) is None
assert not item.has_errors()
assert item.has_critical_errors()
assert any(caplog.records)
item = DataItem()
manager.handle(ValueError(), stage, item)
manager.handle(KeyError(), stage, item)
manager.handle(KeyError(), stage, item)
assert not item.has_errors()
assert item.has_critical_errors()
assert len(list(item.critical_errors())) == 3
for record in caplog.records:
assert "has generated an error" in record.message
def test_manager(caplog):
manager = ErrorManager()
stage = TextReverser()
item = DataItem()
manager.handle(SoftError(), stage, item)
assert any(caplog.records)
assert item.has_errors()
assert not item.has_critical_errors()
item = DataItem()
manager.handle(CriticalError(), stage, item)
assert not item.has_errors()
assert item.has_critical_errors()
assert any(caplog.records)
item = DataItem()
manager.handle(ValueError(), stage, item)
manager.handle(KeyError(), stage, item)
manager.handle(KeyError(), stage, item)
assert not item.has_errors()
assert item.has_critical_errors()
assert len(list(item.critical_errors())) == 3
for record in caplog.records:
assert "has generated an error" in record.message