def test_read_tuple_named_fields(self):
inputs = [('{ "id": "-6955786537413359385", "comp": "example-spout", '
'"stream": "default", "task": 9, "tuple": ["snow white and '
'the seven dwarfs", "field2", 3]}\n'), 'end\n']
Example_SpoutDefaultTuple = namedtuple(
'Example_SpoutDefaultTuple',
field_names=['sentence', 'word', 'number'])
self.bolt = Bolt(input_stream=BytesIO(''.join(inputs).encode('utf-8')),
output_stream=BytesIO())
self.bolt._setup_component(self.conf, self.context)
outputs = []
for msg in inputs[::2]:
output = json.loads(msg)
output['component'] = output['comp']
output['values'] = Example_SpoutDefaultTuple(*output['tuple'])
del output['comp']
del output['tuple']
outputs.append(Tuple(**output))
for output in outputs:
log.info('Checking Tuple for %r', output)
tup = self.bolt.read_tuple()
self.assertEqual(output.values.sentence, tup.values.sentence)
self.assertEqual(output.values.word, tup.values.word)
self.assertEqual(output.values.number, tup.values.number)
self.assertEqual(output, tup)
def test_read_tuple(self):
inputs = [ # Tuple with all values
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'),
'end\n',
# Tick Tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__tick", "tuple": [50]}\n'),
'end\n',
# Heartbeat Tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__heartbeat", "tuple": []}\n'),
'end\n',
]
outputs = []
for msg in inputs[::2]:
output = json.loads(msg)
output['component'] = output['comp']
output['values'] = tuple(output['tuple'])
del output['comp']
del output['tuple']
outputs.append(Tuple(**output))
self.bolt = Bolt(input_stream=BytesIO(''.join(inputs).encode('utf-8')),
output_stream=BytesIO())
for output in outputs:
log.info('Checking Tuple for %r', output)
tup = self.bolt.read_tuple()
self.assertEqual(output, tup)
def test_heartbeat_response(self, send_message_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id='foo',
task=-1,
stream='__heartbeat',
values=(),
component='__system')
self.bolt._run()
send_message_mock.assert_called_with(self.bolt, {'command': 'sync'})
def test_process_tick(self, process_tick_mock, read_tuple_mock):
# Make sure we send sync for heartbeats
read_tuple_mock.return_value = Tuple(id=None,
task=-1,
component='__system',
stream='__tick',
values=(50, ))
self.bolt._run()
process_tick_mock.assert_called_with(self.bolt,
read_tuple_mock.return_value)
def setUp(self):
self.tup_dict = {'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_spout',
'tuple': [1, 2, 3]}
self.tup = Tuple(self.tup_dict['id'], self.tup_dict['comp'],
self.tup_dict['stream'], self.tup_dict['task'],
self.tup_dict['tuple'],)
self.spout = Spout(input_stream=BytesIO(),
output_stream=BytesIO())
self.spout.initialize({}, {})
self.spout.logger = logging.getLogger(__name__)
def setUp(self):
self.ticks_between_batches = 1
self.tup_dicts = [{
'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]
}, {
'id': 15,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [4, 5, 6]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [1]
}, {
'id': 16,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [7, 8, 9]
}, {
'id': None,
'comp': '__system',
'stream': '__tick',
'task': -1,
'tuple': [2]
}]
tups_json = '\nend\n'.join(
[json.dumps(tup_dict) for tup_dict in self.tup_dicts] + [''])
self.tups = [
Tuple(tup_dict['id'],
tup_dict['comp'], tup_dict['stream'], tup_dict['task'],
tuple(tup_dict['tuple'])) for tup_dict in self.tup_dicts
]
self.nontick_tups = [
tup for tup in self.tups if tup.stream != '__tick'
]
self.bolt = BatchingBolt(input_stream=BytesIO(
tups_json.encode('utf-8')),
output_stream=BytesIO())
self.bolt.initialize({}, {})
def setUp(self):
self.conf = {
"topology.message.timeout.secs": 3,
"topology.tick.tuple.freq.secs": 1,
"topology.debug": True,
"topology.name": "foo"
}
self.context = {
"task->component": {
"1": "example-spout",
"2": "__acker",
"3": "example-bolt1",
"4": "example-bolt2"
},
"taskid": 3,
# Everything below this line is only available in Storm 0.11.0+
"componentid": "example-bolt1",
"stream->target->grouping": {
"default": {
"example-bolt2": {
"type": "SHUFFLE"
}
}
},
"streams": ["default"],
"stream->outputfields": {
"default": ["word"]
},
"source->stream->grouping": {
"example-spout": {
"default": {
"type": "FIELDS",
"fields": ["word"]
}
}
},
"source->stream->fields": {
"example-spout": {
"default": ["sentence", "word", "number"]
}
}
}
self.tup_dict = {
'id': 14,
'comp': 'some_spout',
'stream': 'default',
'task': 'some_bolt',
'tuple': [1, 2, 3]
}
tup_json = "{}\nend\n".format(json.dumps(
self.tup_dict)).encode('utf-8')
self.tup = Tuple(
self.tup_dict['id'],
self.tup_dict['comp'],
self.tup_dict['stream'],
self.tup_dict['task'],
tuple(self.tup_dict['tuple']),
)
self.bolt = Bolt(input_stream=BytesIO(tup_json),
output_stream=BytesIO())
self.bolt.initialize(self.conf, self.context)