def test_iterstep_failing(self):
# This test is dangerous, as it can stall the whole test-suite.
# The whole sigalarm setting is to protect against that, somehow.
def raise_alarm(*_):
raise Timeout()
def failing_iterator(iterator):
for count, entry in enumerate(iterator):
if count > 1000:
raise FooException
yield entry
pipeline = _threaded.Pipeline([
(_threaded.iterstep(failing_iterator), 1),
],
max_retries=1)
pipeline.put_iterable(range(1000000))
try:
with SigAlarmTimeout(30, handler=raise_alarm):
self.assertRaises(FooException, pipeline.run)
except Timeout:
try:
pipeline.stop()
finally:
raise AssertionError("Timeout exceeded")
def test_pipeline(self):
class Download(_threaded.AbstractStep):
def process(self, entry):
return entry
def process(response):
if response == 17:
raise FooException(response)
return response
class Upload(_threaded.AbstractStep):
def process(self, entry):
logger.debug("Upload %s", entry)
return entry
pipeline = _threaded.Pipeline([
(Download, 10),
(_threaded.funcstep(process), 1),
(_threaded.iterstep(_iterables.chunk(10)), 2),
(_threaded.iterstep(_iterables.flatten), 1),
(Upload, 40),
],
max_retries=1)
pipeline.put_iterable(range(1000))
# with self.assertRaises(FooException):
pipeline.run()
self.assertEquals(pipeline.count, 999, pipeline.count)
pipeline = _threaded.Pipeline([
(Download, 10),
(_threaded.funcstep(process), 1),
(Upload, 40),
],
max_retries=1,
no_log_exceptions=(FooException, ))
pipeline.put_all(range(1000))
pipeline.finish()
pipeline.run()
self.assertEquals(pipeline.count, 999, pipeline.count)
def test_pipeline_crash(self):
class Crasher(_threaded.AbstractStep):
def process(self, entry):
raise ValueError("What are you looking at!?")
pipeline = _threaded.Pipeline([(Crasher, 1)], max_retries=0)
pipeline.put("Hey you!")
pipeline.finish()
import pytest
with pytest.raises(ValueError):
pipeline.run()
def test_meta_processing(self):
class MetaIsDaHotShit(_threaded.AbstractStep):
def process(self, entry):
assert isinstance(entry, str)
return entry
def process_meta(self, entry):
assert isinstance(entry, (int, long))
return entry
pipeline = _threaded.Pipeline([
(MetaIsDaHotShit, 3),
], max_retries=0)
pipeline.put("str")
pipeline.put_meta(123)
pipeline.put_meta(243)
pipeline.put("str")
pipeline.put_meta(325)
pipeline.put("str")
pipeline.finish()
pipeline.run()
self.assertEquals(pipeline.count, 6, pipeline.count)
def test_iterstep_failing_with_iterable_sourcestep(self):
ppln = _threaded.Pipeline([(_threaded.iterstep(_crasher), 1)])
ppln.put_iterable(_itertools.count())
self.assertRaises(Exception, ppln.run)