def test_read_command(self):
# Check that we properly queue task IDs and return only commands
inputs = [ # Task IDs
'[12, 22, 24]\n',
'end\n',
# Incoming tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'),
'end\n',
# next command for spout
'{"command": "next"}\n',
'end\n'
]
outputs = [json.loads(msg) for msg in inputs[::2]]
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
# Skip first output, because it's a task ID, and won't be returned by
# read_command
for output in outputs[1:]:
log.info('Checking msg for %s', output)
msg = component.read_command()
self.assertEqual(output, msg)
self.assertEqual(component._pending_task_ids.pop(), outputs[0])
def test_read_tuple(self):
# This is only valid for bolts, so we only need to test with task IDs
# and tuples
inputs = [# Tuple with all values
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'), 'end\n',
# Tick tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__tick", "tuple": [50]}\n'), 'end\n',
# Heartbeat tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__heartbeat", "tuple": []}\n'), 'end\n',
]
outputs = []
for msg in inputs[::2]:
output = json.loads(msg)
output['component'] = output['comp']
output['values'] = output['tuple']
del output['comp']
del output['tuple']
outputs.append(Tuple(**output))
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
for output in outputs:
log.info('Checking tuple for %s', output)
tup = component.read_tuple()
self.assertEqual(output, tup)
def test_read_tuple(self):
# This is only valid for bolts, so we only need to test with task IDs
# and Tuples
inputs = [ # Tuple with all values
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'),
'end\n',
# Tick Tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__tick", "tuple": [50]}\n'),
'end\n',
# Heartbeat Tuple
('{ "id": null, "task": -1, "comp": "__system", "stream": '
'"__heartbeat", "tuple": []}\n'),
'end\n',
]
outputs = []
for msg in inputs[::2]:
output = json.loads(msg)
output['component'] = output['comp']
output['values'] = output['tuple']
del output['comp']
del output['tuple']
outputs.append(Tuple(**output))
component = Component(input_stream=BytesIO(
''.join(inputs).encode('utf-8')),
output_stream=BytesIO())
for output in outputs:
log.info('Checking Tuple for %s', output)
tup = component.read_tuple()
self.assertEqual(output, tup)
def test_read_message_unicode(self):
inputs = [ # Task IDs
'[12, 22, 24]\n',
'end\n',
# Incoming Tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white \uFFE6 the seven dwarfs"'
', "field2", 3]}\n'),
'end\n',
# next command for spout
'{"command": "next"}\n',
'end\n',
# empty message, which should trigger sys.exit (end ignored)
'',
''
]
outputs = [json.loads(msg) for msg in inputs[::2] if msg]
outputs.append('')
component = Component(input_stream=BytesIO(
''.join(inputs).encode('utf8')),
output_stream=BytesIO())
for output in outputs:
log.info('Checking msg for %s', output)
if output:
msg = component.read_message()
self.assertEqual(output, msg)
else:
with self.assertRaises(SystemExit):
component.read_message()
def test_read_task_ids(self):
# Check that we properly queue commands and return only task IDs
inputs = [ # Task IDs
'[4, 8, 15]\n',
'end\n',
# Incoming Tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'),
'end\n',
# next command for spout
'{"command": "next"}\n',
'end\n',
# Task IDs
'[16, 23, 42]\n',
'end\n'
]
outputs = [json.loads(msg) for msg in inputs[::2]]
component = Component(input_stream=BytesIO(
''.join(inputs).encode('utf-8')),
output_stream=BytesIO())
# Skip middle outputs, because they're commands and won't be returned by
# read_task_ids
for output in (outputs[0], outputs[-1]):
log.info('Checking msg for %s', output)
msg = component.read_task_ids()
self.assertEqual(output, msg)
for output in outputs[1:-1]:
self.assertEqual(component._pending_commands.popleft(), output)
def test_read_handshake(self):
handshake_dict = {
"conf": {
"topology.message.timeout.secs": 3,
"topology.tick.tuple.freq.secs": 1,
"topology.debug": True
},
"pidDir": ".",
"context": {
"task->component": {
"1": "example-spout",
"2": "__acker",
"3": "example-bolt1",
"4": "example-bolt2"
},
"taskid": 3,
# Everything below this line is only available in Storm 0.10.0+
"componentid": "example-bolt1",
"stream->target->grouping": {
"default": {
"example-bolt2": {
"type": "SHUFFLE"
}
}
},
"streams": ["default"],
"stream->outputfields": {"default": ["word"]},
"source->stream->grouping": {
"example-spout": {
"default": {
"type": "FIELDS",
"fields": ["word"]
}
}
},
"source->stream->fields": {
"example-spout": {
"default": ["word"]
}
}
}
}
pid_dir = handshake_dict['pidDir']
expected_conf = handshake_dict['conf']
expected_context = handshake_dict['context']
inputs = ["{}\n".format(json.dumps(handshake_dict)),
"end\n"]
component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')),
output_stream=BytesIO())
given_conf, given_context = component.read_handshake()
pid_path = os.path.join(pid_dir, str(component.pid))
self.assertTrue(os.path.exists(pid_path))
os.remove(pid_path)
self.assertEqual(given_conf, expected_conf)
self.assertEqual(given_context, expected_context)
self.assertEqual("{}\nend\n".format(json.dumps({"pid": component.pid})).encode('utf-8'),
component.output_stream.buffer.getvalue())
def test_read_handshake(self):
handshake_dict = {
"conf": {
"topology.message.timeout.secs": 3,
"topology.tick.tuple.freq.secs": 1,
"topology.debug": True
},
"pidDir": ".",
"context": {
"task->component": {
"1": "example-spout",
"2": "__acker",
"3": "example-bolt1",
"4": "example-bolt2"
},
"taskid": 3,
# Everything below this line is only available in Storm 0.11.0+
"componentid": "example-bolt",
"stream->target->grouping": {
"default": {
"example-bolt2": {
"type": "SHUFFLE"
}
}
},
"streams": ["default"],
"stream->outputfields": {
"default": ["word"]
},
"source->stream->grouping": {
"example-spout": {
"default": {
"type": "FIELDS",
"fields": ["word"]
}
}
}
}
}
pid_dir = handshake_dict['pidDir']
expected_conf = handshake_dict['conf']
expected_context = handshake_dict['context']
inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"]
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
given_conf, given_context = component.read_handshake()
pid_path = os.path.join(pid_dir, str(component.pid))
self.assertTrue(os.path.exists(pid_path))
os.remove(pid_path)
self.assertEqual(given_conf, expected_conf)
self.assertEqual(given_context, expected_context)
self.assertEqual(
"{}\nend\n".format(json.dumps({"pid":
component.pid})).encode('utf-8'),
component.output_stream.getvalue())
def test_log(self, send_message_mock):
component = Component(input_stream=BytesIO(), output_stream=BytesIO())
inputs = [("I am a robot monkey.", None, 2),
("I am a monkey who learned to talk.", 'warning', 3)]
for msg, level, storm_level in inputs:
component.output_stream.close()
component.output_stream = BytesIO()
component.log(msg, level=level)
send_message_mock.assert_called_with(component, {'command': 'log',
'msg': msg,
'level': storm_level})
def test_log(self, send_message_mock):
component = Component(input_stream=BytesIO(), output_stream=BytesIO())
inputs = [("I am a robot monkey.", None, 2),
("I am a monkey who learned to talk.", 'warning', 3)]
for msg, level, storm_level in inputs:
component.output_stream.close()
component.output_stream = BytesIO()
component.log(msg, level=level)
send_message_mock.assert_called_with(component, {
'command': 'log',
'msg': msg,
'level': storm_level
})
def test_read_split_message(self):
# Make sure we can read something that's broken up into many "lines"
inputs = [
'{ "id": "-6955786537413359385", ', '"comp": "1", "stream": "1"\n',
'\n', ', "task": 9, "tuple": ["snow white and the seven dwarfs", ',
'"field2", 3]}\n', 'end\n'
]
output = json.loads(''.join(inputs[:-1]))
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
msg = component.read_message()
self.assertEqual(output, msg)
def test_read_split_message(self):
# Make sure we can read something that's broken up into many "lines"
inputs = ['{ "id": "-6955786537413359385", ',
'"comp": "1", "stream": "1"\n',
'\n',
', "task": 9, "tuple": ["snow white and the seven dwarfs", ',
'"field2", 3]}\n',
'end\n']
output = json.loads(''.join(inputs[:-1]))
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
msg = component.read_message()
self.assertEqual(output, msg)
def test_setup_component(self):
conf = {
"topology.message.timeout.secs": 3,
"topology.tick.tuple.freq.secs": 1,
"topology.debug": True,
"topology.name": "foo"
}
context = {
"task->component": {
"1": "example-spout",
"2": "__acker",
"3": "example-bolt1",
"4": "example-bolt2"
},
"taskid": 3,
# Everything below this line is only available in Storm 0.11.0+
"componentid": "example-bolt1",
"stream->target->grouping": {
"default": {
"example-bolt2": {
"type": "SHUFFLE"
}
}
},
"streams": ["default"],
"stream->outputfields": {
"default": ["word"]
},
"source->stream->grouping": {
"example-spout": {
"default": {
"type": "FIELDS",
"fields": ["word"]
}
}
},
"source->stream->fields": {
"example-spout": {
"default": ["word"]
}
}
}
component = Component(input_stream=BytesIO(), output_stream=BytesIO())
component._setup_component(conf, context)
self.assertEqual(component.topology_name, conf['topology.name'])
self.assertEqual(component.task_id, context['taskid'])
self.assertEqual(component.component_name,
context['task->component'][str(context['taskid'])])
self.assertEqual(component.storm_conf, conf)
self.assertEqual(component.context, context)
def test_send_message(self):
component = Component(input_stream=BytesIO(), output_stream=BytesIO())
inputs = [{"command": "emit", "id": 4, "stream": "", "task": 9,
"tuple": ["field1", 2, 3]},
{"command": "log", "msg": "I am a robot monkey."},
{"command": "next"},
{"command": "sync"}]
for cmd in inputs:
component.output_stream.close()
component.output_stream = BytesIO()
component.send_message(cmd)
self.assertEqual("{}\nend\n".format(json.dumps(cmd)).encode('utf-8'),
component.output_stream.getvalue())
# Check that we properly skip over invalid input
self.assertIsNone(component.send_message(['foo', 'bar']))
def test_setup_component(self):
conf = {"topology.message.timeout.secs": 3,
"topology.tick.tuple.freq.secs": 1,
"topology.debug": True,
"topology.name": "foo"}
context = {
"task->component": {
"1": "example-spout",
"2": "__acker",
"3": "example-bolt1",
"4": "example-bolt2"
},
"taskid": 3,
# Everything below this line is only available in Storm 0.11.0+
"componentid": "example-bolt1",
"stream->target->grouping": {
"default": {
"example-bolt2": {
"type": "SHUFFLE"
}
}
},
"streams": ["default"],
"stream->outputfields": {"default": ["word"]},
"source->stream->grouping": {
"example-spout": {
"default": {
"type": "FIELDS",
"fields": ["word"]
}
}
},
"source->stream->fields": {
"example-spout": {
"default": ["word"]
}
}
}
component = Component(input_stream=BytesIO(),
output_stream=BytesIO())
component._setup_component(conf, context)
self.assertEqual(component.topology_name, conf['topology.name'])
self.assertEqual(component.task_id, context['taskid'])
self.assertEqual(component.component_name,
context['task->component'][str(context['taskid'])])
self.assertEqual(component.storm_conf, conf)
self.assertEqual(component.context, context)
def test_send_message_unicode(self):
component = Component(input_stream=BytesIO(), output_stream=BytesIO())
inputs = [{
"command": "emit",
"id": 4,
"stream": "",
"task": 9,
"tuple": ["field\uFFE6", 2, 3]
}, {
"command": "log",
"msg": "I am a robot monkey."
}, {
"command": "next"
}, {
"command": "sync"
}]
for cmd in inputs:
component.output_stream.close()
component.output_stream = component._wrap_stream(BytesIO())
component.send_message(cmd)
self.assertEqual(
"{}\nend\n".format(json.dumps(cmd)).encode('utf-8'),
component.output_stream.buffer.getvalue())
# Check that we properly skip over invalid input
self.assertIsNone(component.send_message(['foo', 'bar']))
def test_read_command(self):
# Check that we properly queue task IDs and return only commands
inputs = [# Task IDs
'[12, 22, 24]\n', 'end\n',
# Incoming tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'), 'end\n',
# next command for spout
'{"command": "next"}\n', 'end\n']
outputs = [json.loads(msg) for msg in inputs[::2]]
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
# Skip first output, because it's a task ID, and won't be returned by
# read_command
for output in outputs[1:]:
log.info('Checking msg for %s', output)
msg = component.read_command()
self.assertEqual(output, msg)
self.assertEqual(component._pending_task_ids.pop(), outputs[0])
def test_read_message(self):
inputs = [# Task IDs
'[12, 22, 24]\n', 'end\n',
# Incoming tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'), 'end\n',
# next command for spout
'{"command": "next"}\n', 'end\n',
# empty message, which should trigger sys.exit (end ignored)
'', '']
outputs = [json.loads(msg) for msg in inputs[::2] if msg]
outputs.append('')
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
for output in outputs:
log.info('Checking msg for %s', output)
if output:
msg = component.read_message()
self.assertEqual(output, msg)
else:
with self.assertRaises(SystemExit):
component.read_message()
def test_read_task_ids(self):
# Check that we properly queue commands and return only task IDs
inputs = [# Task IDs
'[4, 8, 15]\n', 'end\n',
# Incoming tuple for bolt
('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"'
', "task": 9, "tuple": ["snow white and the seven dwarfs", '
'"field2", 3]}\n'), 'end\n',
# next command for spout
'{"command": "next"}\n', 'end\n',
# Task IDs
'[16, 23, 42]\n', 'end\n']
outputs = [json.loads(msg) for msg in inputs[::2]]
component = Component(input_stream=StringIO(''.join(inputs)),
output_stream=BytesIO())
# Skip middle outputs, because they're commands and won't be returned by
# read_task_ids
for output in (outputs[0], outputs[-1]):
log.info('Checking msg for %s', output)
msg = component.read_task_ids()
self.assertEqual(output, msg)
for output in outputs[1:-1]:
self.assertEqual(component._pending_commands.popleft(), output)