def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [{
'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]
}, {
'id': 15,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [4, 5, 6]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [1]
}, {
'id': 16,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [7, 8, 9]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [2]
}]
tups_json = '\nend\n'.join(
[json.dumps(tup_dict) for tup_dict in self.tup_dicts] + [''])
self.tups = [
Tuple(tup_dict['id'],
tup_dict['comp'], tup_dict['stream'], tup_dict['task'],
tuple(tup_dict['tuple'])) for tup_dict in self.tup_dicts
]
self.nontick_tups = [
tup for tup in self.tups if tup.stream != '__tick'
]
self.bolt = BatchingBolt(input_stream=BytesIO(
tups_json.encode('utf-8')),
output_stream=BytesIO())
self.bolt.initialize({}, {})
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [
{"id": 14, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [1, 2, 3]},
{"id": 15, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [4, 5, 6]},
{"id": None, "comp": "__system", "stream": "__tick", "task": -1, "tuple": [1]},
{"id": 16, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [7, 8, 9]},
{"id": None, "comp": "__system", "stream": "__tick", "task": -1, "tuple": [2]},
]
tups_json = "\nend\n".join([json.dumps(tup_dict) for tup_dict in self.tup_dicts] + [""])
self.tups = [
Tuple(tup_dict["id"], tup_dict["comp"], tup_dict["stream"], tup_dict["task"], tuple(tup_dict["tuple"]))
for tup_dict in self.tup_dicts
]
self.nontick_tups = [tup for tup in self.tups if tup.stream != "__tick"]
self.bolt = BatchingBolt(input_stream=BytesIO(tups_json.encode("utf-8")), output_stream=BytesIO())
self.bolt.initialize({}, {})
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [{'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]},
{'id': 15,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [4, 5, 6]},
{'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [1]},
{'id': 16,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [7, 8, 9]},
{'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [2]}]
tups_json = '\nend\n'.join([json.dumps(tup_dict) for tup_dict in
self.tup_dicts] + [''])
self.tups = [Tuple(tup_dict['id'], tup_dict['comp'], tup_dict['stream'],
tup_dict['task'], tup_dict['tuple']) for tup_dict in
self.tup_dicts]
self.nontick_tups = [tup for tup in self.tups if tup.stream != '__tick']
self.bolt = BatchingBolt(input_stream=BytesIO(tups_json.encode('utf-8')),
output_stream=BytesIO())
self.bolt.initialize({}, {})
class BatchingBoltTests(unittest.TestCase):
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [{
'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]
}, {
'id': 15,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [4, 5, 6]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [1]
}, {
'id': 16,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [7, 8, 9]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [2]
}]
tups_json = '\nend\n'.join(
[json.dumps(tup_dict) for tup_dict in self.tup_dicts] + [''])
self.tups = [
Tuple(tup_dict['id'],
tup_dict['comp'], tup_dict['stream'], tup_dict['task'],
tuple(tup_dict['tuple'])) for tup_dict in self.tup_dicts
]
self.nontick_tups = [
tup for tup in self.tups if tup.stream != '__tick'
]
self.bolt = BatchingBolt(input_stream=BytesIO(
tups_json.encode('utf-8')),
output_stream=BytesIO())
self.bolt.initialize({}, {})
@patch.object(BatchingBolt, 'process_batch', autospec=True)
def test_batching(self, process_batch_mock):
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_called_with(self.bolt, None,
self.nontick_tups)
@patch.object(BatchingBolt, 'process_batch', autospec=True)
def test_group_key(self, process_batch_mock):
# Change the group key to even/odd grouping
self.bolt.group_key = lambda t: sum(t.values) % 2
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_has_calls([
mock.call(self.bolt, 0,
[self.nontick_tups[0], self.nontick_tups[2]]),
mock.call(self.bolt, 1, [self.nontick_tups[1]])
],
any_order=True)
@patch.object(BatchingBolt, 'ack', autospec=True)
@patch.object(BatchingBolt, 'process_batch', new=lambda *args: None)
def test_auto_ack_on(self, ack_mock):
# Test auto-ack on (the default)
for __ in self.tups:
self.bolt._run()
ack_mock.assert_has_calls(
[mock.call(self.bolt, tup) for tup in self.tups], any_order=True)
self.assertEqual(ack_mock.call_count, 5)
@patch.object(BatchingBolt, 'ack', autospec=True)
@patch.object(BatchingBolt, 'process_batch', new=lambda *args: None)
def test_auto_ack_off(self, ack_mock):
# Test auto-ack off
self.bolt.auto_ack = False
for __ in self.tups:
self.bolt._run()
# Assert that this wasn't called, and print out what it was called with
# otherwise.
ack_mock.assert_has_calls([
mock.call(self.bolt, tup)
for tup in self.tups if self.bolt.is_tick(tup)
],
any_order=True)
self.assertEqual(ack_mock.call_count, 2)
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'raise_exception', new=lambda *a: None)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_on(self, fail_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail on (the default)
with self.assertRaises(SystemExit):
self.bolt.run()
# All waiting Tuples should have failed at this point
fail_mock.assert_has_calls([
mock.call(self.bolt, self.nontick_tups[0]),
mock.call(self.bolt, self.nontick_tups[1]),
mock.call(self.bolt, self.nontick_tups[2])
],
any_order=True)
self.assertEqual(fail_mock.call_count, 3)
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'raise_exception', new=lambda *a: None)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_off(self, fail_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail off
self.bolt.auto_fail = False
with self.assertRaises(SystemExit):
self.bolt.run()
# All waiting Tuples should have failed at this point
self.assertListEqual(fail_mock.call_args_list, [])
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'process_batch', autospec=True)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_partial_exit_on_exception_true(self, fail_mock,
process_batch_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Change the group key just be the sum of values, which makes 3 separate
# batches
self.bolt.group_key = lambda t: sum(t.values)
self.bolt.exit_on_exception = True
# Make sure we fail on the second batch
work = {'status': True} # to avoid scoping problems
def work_once(*args):
if work['status']:
work['status'] = False
else:
raise Exception('borkt')
process_batch_mock.side_effect = work_once
# Run the batches
with self.assertRaises(SystemExit):
self.bolt.run()
self.assertEqual(process_batch_mock.call_count, 2)
# Only some Tuples should have failed at this point. The key is that
# all un-acked Tuples should be failed, even for batches we haven't
# started processing yet.
self.assertEqual(fail_mock.call_count, 2)
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'process_batch', autospec=True)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_partial_exit_on_exception_false(self, fail_mock,
process_batch_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Change the group key just be the sum of values, which makes 3 separate
# batches
self.bolt.group_key = lambda t: sum(t.values)
self.bolt.exit_on_exception = False
# Make sure we fail on the second batch
work = {'status': True, 'raised': False} # to avoid scoping problems
def work_once(*args):
if work['status']:
work['status'] = False
else:
raise Exception('borkt')
process_batch_mock.side_effect = work_once
# Run the batches
with self.assertRaises(SystemExit) as raises_fixture:
self.bolt.run()
assert raises_fixture.exception.code == 2
self.assertEqual(process_batch_mock.call_count, 2)
# Only Tuples in the current batch should have failed at this point.
self.assertEqual(fail_mock.call_count, 1)
@patch.object(BatchingBolt, 'read_tuple', autospec=True)
@patch.object(BatchingBolt, 'send_message', autospec=True)
def test_heartbeat_response(self, send_message_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id='foo',
task=-1,
stream='__heartbeat',
values=(),
component='__system')
self.bolt._run()
send_message_mock.assert_called_with(self.bolt, {'command': 'sync'})
@patch.object(BatchingBolt, 'read_tuple', autospec=True)
@patch.object(BatchingBolt, 'process_tick', autospec=True)
def test_process_tick(self, process_tick_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id=None,
task=-1,
component='__system',
stream='__tick',
values=(50, ))
self.bolt._run()
process_tick_mock.assert_called_with(self.bolt,
read_tuple_mock.return_value)
class BatchingBoltTests(unittest.TestCase):
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [{'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]},
{'id': 15,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [4, 5, 6]},
{'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [1]},
{'id': 16,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [7, 8, 9]},
{'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [2]}]
tups_json = '\nend\n'.join([json.dumps(tup_dict) for tup_dict in
self.tup_dicts] + [''])
self.tups = [Tuple(tup_dict['id'], tup_dict['comp'], tup_dict['stream'],
tup_dict['task'], tup_dict['tuple']) for tup_dict in
self.tup_dicts]
self.nontick_tups = [tup for tup in self.tups if tup.stream != '__tick']
self.bolt = BatchingBolt(input_stream=BytesIO(tups_json.encode('utf-8')),
output_stream=BytesIO())
self.bolt.initialize({}, {})
@patch.object(BatchingBolt, 'process_batch', autospec=True)
def test_batching(self, process_batch_mock):
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_called_with(self.bolt, None,
self.nontick_tups)
@patch.object(BatchingBolt, 'process_batch', autospec=True)
def test_group_key(self, process_batch_mock):
# Change the group key to even/odd grouping
self.bolt.group_key = lambda t: sum(t.values) % 2
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_has_calls([mock.call(self.bolt, 0,
[self.nontick_tups[0],
self.nontick_tups[2]]),
mock.call(self.bolt, 1,
[self.nontick_tups[1]])],
any_order=True)
@patch.object(BatchingBolt, 'ack', autospec=True)
@patch.object(BatchingBolt, 'process_batch', new=lambda *args: None)
def test_auto_ack_on(self, ack_mock):
# Test auto-ack on (the default)
for __ in self.tups:
self.bolt._run()
ack_mock.assert_has_calls([mock.call(self.bolt, tup)
for tup in self.tups],
any_order=True)
self.assertEqual(ack_mock.call_count, 5)
@patch.object(BatchingBolt, 'ack', autospec=True)
@patch.object(BatchingBolt, 'process_batch', new=lambda *args: None)
def test_auto_ack_off(self, ack_mock):
# Test auto-ack off
self.bolt.auto_ack = False
for __ in self.tups:
self.bolt._run()
# Assert that this wasn't called, and print out what it was called with
# otherwise.
ack_mock.assert_has_calls([mock.call(self.bolt, tup)
for tup in self.tups
if self.bolt.is_tick(tup)],
any_order=True)
self.assertEqual(ack_mock.call_count, 2)
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'raise_exception', new=lambda *a: None)
@patch('sys.exit', autospec=True)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_on(self, fail_mock, exit_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail on (the default)
self.bolt.run()
# All waiting Tuples should have failed at this point
fail_mock.assert_has_calls([mock.call(self.bolt, self.nontick_tups[0]),
mock.call(self.bolt, self.nontick_tups[1]),
mock.call(self.bolt, self.nontick_tups[2])],
any_order=True)
self.assertEqual(fail_mock.call_count, 3)
self.assertEqual(exit_mock.call_count, 1)
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch.object(BatchingBolt, 'raise_exception', new=lambda *a: None)
@patch('sys.exit', autospec=True)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_off(self, fail_mock, exit_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail off
self.bolt.auto_fail = False
self.bolt.run()
# All waiting Tuples should have failed at this point
self.assertEqual(exit_mock.call_count, 1)
self.assertListEqual(fail_mock.call_args_list, [])
@patch.object(BatchingBolt, 'read_handshake', new=lambda x: ({}, {}))
@patch('sys.exit', autospec=True)
@patch.object(BatchingBolt, 'process_batch', autospec=True)
@patch.object(BatchingBolt, 'fail', autospec=True)
def test_auto_fail_partial(self, fail_mock, process_batch_mock, exit_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Change the group key just be the sum of values, which makes 3 separate
# batches
self.bolt.group_key = lambda t: sum(t.values)
# Make sure we fail on the second batch
work = {'status': True} # to avoid scoping problems
def work_once(*args):
if work['status']:
work['status'] = False
else:
raise Exception('borkt')
process_batch_mock.side_effect = work_once
# Run the batches
self.bolt.run()
# Only some Tuples should have failed at this point. The key is that
# all un-acked Tuples should be failed, even for batches we haven't
# started processing yet.
self.assertEqual(fail_mock.call_count, 2)
self.assertEqual(exit_mock.call_count, 1)
@patch.object(BatchingBolt, 'read_tuple', autospec=True)
@patch.object(BatchingBolt, 'send_message', autospec=True)
def test_heartbeat_response(self, send_message_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id='foo', task=-1,
stream='__heartbeat', values=[],
component='__system')
self.bolt._run()
send_message_mock.assert_called_with(self.bolt, {'command': 'sync'})
@patch.object(BatchingBolt, 'read_tuple', autospec=True)
@patch.object(BatchingBolt, 'process_tick', autospec=True)
def test_process_tick(self, process_tick_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id=None, task=-1,
component='__system',
stream='__tick', values=[50])
self.bolt._run()
process_tick_mock.assert_called_with(self.bolt,
read_tuple_mock.return_value)
class BatchingBoltTests(unittest.TestCase):
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [
{"id": 14, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [1, 2, 3]},
{"id": 15, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [4, 5, 6]},
{"id": None, "comp": "__system", "stream": "__tick", "task": -1, "tuple": [1]},
{"id": 16, "comp": "some_spout", "stream": "default", "task": "some_bolt", "tuple": [7, 8, 9]},
{"id": None, "comp": "__system", "stream": "__tick", "task": -1, "tuple": [2]},
]
tups_json = "\nend\n".join([json.dumps(tup_dict) for tup_dict in self.tup_dicts] + [""])
self.tups = [
Tuple(tup_dict["id"], tup_dict["comp"], tup_dict["stream"], tup_dict["task"], tuple(tup_dict["tuple"]))
for tup_dict in self.tup_dicts
]
self.nontick_tups = [tup for tup in self.tups if tup.stream != "__tick"]
self.bolt = BatchingBolt(input_stream=BytesIO(tups_json.encode("utf-8")), output_stream=BytesIO())
self.bolt.initialize({}, {})
@patch.object(BatchingBolt, "process_batch", autospec=True)
def test_batching(self, process_batch_mock):
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_called_with(self.bolt, None, self.nontick_tups)
@patch.object(BatchingBolt, "process_batch", autospec=True)
def test_group_key(self, process_batch_mock):
# Change the group key to even/odd grouping
self.bolt.group_key = lambda t: sum(t.values) % 2
# Add a bunch of Tuples
for __ in self.tups:
self.bolt._run()
process_batch_mock.assert_has_calls(
[
mock.call(self.bolt, 0, [self.nontick_tups[0], self.nontick_tups[2]]),
mock.call(self.bolt, 1, [self.nontick_tups[1]]),
],
any_order=True,
)
@patch.object(BatchingBolt, "ack", autospec=True)
@patch.object(BatchingBolt, "process_batch", new=lambda *args: None)
def test_auto_ack_on(self, ack_mock):
# Test auto-ack on (the default)
for __ in self.tups:
self.bolt._run()
ack_mock.assert_has_calls([mock.call(self.bolt, tup) for tup in self.tups], any_order=True)
self.assertEqual(ack_mock.call_count, 5)
@patch.object(BatchingBolt, "ack", autospec=True)
@patch.object(BatchingBolt, "process_batch", new=lambda *args: None)
def test_auto_ack_off(self, ack_mock):
# Test auto-ack off
self.bolt.auto_ack = False
for __ in self.tups:
self.bolt._run()
# Assert that this wasn't called, and print out what it was called with
# otherwise.
ack_mock.assert_has_calls(
[mock.call(self.bolt, tup) for tup in self.tups if self.bolt.is_tick(tup)], any_order=True
)
self.assertEqual(ack_mock.call_count, 2)
@patch.object(BatchingBolt, "read_handshake", new=lambda x: ({}, {}))
@patch.object(BatchingBolt, "raise_exception", new=lambda *a: None)
@patch.object(BatchingBolt, "fail", autospec=True)
def test_auto_fail_on(self, fail_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail on (the default)
with self.assertRaises(SystemExit):
self.bolt.run()
# All waiting Tuples should have failed at this point
fail_mock.assert_has_calls(
[
mock.call(self.bolt, self.nontick_tups[0]),
mock.call(self.bolt, self.nontick_tups[1]),
mock.call(self.bolt, self.nontick_tups[2]),
],
any_order=True,
)
self.assertEqual(fail_mock.call_count, 3)
@patch.object(BatchingBolt, "read_handshake", new=lambda x: ({}, {}))
@patch.object(BatchingBolt, "raise_exception", new=lambda *a: None)
@patch.object(BatchingBolt, "fail", autospec=True)
def test_auto_fail_off(self, fail_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Test auto-fail off
self.bolt.auto_fail = False
with self.assertRaises(SystemExit):
self.bolt.run()
# All waiting Tuples should have failed at this point
self.assertListEqual(fail_mock.call_args_list, [])
@patch.object(BatchingBolt, "read_handshake", new=lambda x: ({}, {}))
@patch.object(BatchingBolt, "process_batch", autospec=True)
@patch.object(BatchingBolt, "fail", autospec=True)
def test_auto_fail_partial(self, fail_mock, process_batch_mock):
# Need to re-register signal handler with mocked version, because
# mock gets created after handler was originally registered.
self.setUp()
# Change the group key just be the sum of values, which makes 3 separate
# batches
self.bolt.group_key = lambda t: sum(t.values)
# Make sure we fail on the second batch
work = {"status": True} # to avoid scoping problems
def work_once(*args):
if work["status"]:
work["status"] = False
else:
raise Exception("borkt")
process_batch_mock.side_effect = work_once
# Run the batches
with self.assertRaises(SystemExit):
self.bolt.run()
# Only some Tuples should have failed at this point. The key is that
# all un-acked Tuples should be failed, even for batches we haven't
# started processing yet.
self.assertEqual(fail_mock.call_count, 2)
@patch.object(BatchingBolt, "read_tuple", autospec=True)
@patch.object(BatchingBolt, "send_message", autospec=True)
def test_heartbeat_response(self, send_message_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id="foo", task=-1, stream="__heartbeat", values=(), component="__system")
self.bolt._run()
send_message_mock.assert_called_with(self.bolt, {"command": "sync"})
@patch.object(BatchingBolt, "read_tuple", autospec=True)
@patch.object(BatchingBolt, "process_tick", autospec=True)
def test_process_tick(self, process_tick_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id=None, task=-1, component="__system", stream="__tick", values=(50,))
self.bolt._run()
process_tick_mock.assert_called_with(self.bolt, read_tuple_mock.return_value)
