def test_read_task_ids(self): # Check that we properly queue commands and return only task IDs inputs = [ # Task IDs '[4, 8, 15]\n', 'end\n', # Incoming Tuple for bolt ('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"' ', "task": 9, "tuple": ["snow white and the seven dwarfs", ' '"field2", 3]}\n'), 'end\n', # next command for spout '{"command": "next"}\n', 'end\n', # Task IDs '[16, 23, 42]\n', 'end\n' ] outputs = [json.loads(msg) for msg in inputs[::2]] component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) # Skip middle outputs, because they're commands and won't be returned by # read_task_ids for output in (outputs[0], outputs[-1]): log.info('Checking msg for %r', output) msg = component.read_task_ids() self.assertEqual(output, msg) for output in outputs[1:-1]: self.assertEqual(component._pending_commands.popleft(), output)
def test_read_message_unicode(self): inputs = [ # Task IDs '[12, 22, 24]\n', 'end\n', # Incoming Tuple for bolt ('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"' ', "task": 9, "tuple": ["snow white \uFFE6 the seven dwarfs"' ', "field2", 3]}\n'), 'end\n', # next command for spout '{"command": "next"}\n', 'end\n', # empty message, which should trigger sys.exit (end ignored) '', '' ] outputs = [json.loads(msg) for msg in inputs[::2] if msg] outputs.append('') component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf8')), output_stream=BytesIO()) for output in outputs: log.info('Checking msg for %r', output) if output: msg = component.read_message() self.assertEqual(output, msg) else: with self.assertRaises(StormWentAwayError): component.read_message()
def test_send_message_unicode(self): component = Component(input_stream=BytesIO(), output_stream=BytesIO()) inputs = [{ "command": "emit", "id": 4, "stream": "", "task": 9, "tuple": ["field\uFFE6", 2, 3] }, { "command": "log", "msg": "I am a robot monkey." }, { "command": "next" }, { "command": "sync" }] for cmd in inputs: component.serializer.output_stream.close() component.serializer.output_stream = component.serializer._wrap_stream( BytesIO()) component.send_message(cmd) self.assertEqual( component.serializer.serialize_dict(cmd).encode('utf-8'), component.serializer.output_stream.buffer.getvalue()) # Check that we properly skip over invalid input self.assertIsNone(component.send_message(['foo', 'bar']))
def test_log(self, send_message_mock): component = Component(input_stream=BytesIO(), output_stream=BytesIO()) inputs = [("I am a robot monkey.", None, 2), ("I am a monkey who learned to talk.", 'warning', 3)] for msg, level, storm_level in inputs: component.serializer.output_stream.close() component.serializer.output_stream = component.serializer._wrap_stream(BytesIO()) component.log(msg, level=level) send_message_mock.assert_called_with(component, {'command': 'log', 'msg': msg, 'level': storm_level})
def test_exit_on_exception_true(self): handshake_dict = {"conf": self.conf, "pidDir": ".", "context": self.context} inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) with self.assertRaises(SystemExit) as raises_fixture: component.run() assert raises_fixture.exception.value == 1
def test_setup_component(self): conf = self.conf component = Component(input_stream=BytesIO(), output_stream=BytesIO()) component._setup_component(conf, self.context) self.assertEqual(component.topology_name, conf['topology.name']) self.assertEqual(component.task_id, self.context['taskid']) self.assertEqual(component.component_name, self.context['task->component'][str(self.context['taskid'])]) self.assertEqual(component.storm_conf, conf) self.assertEqual(component.context, self.context)
def test_setup_component(self): conf = self.conf component = Component(input_stream=BytesIO(), output_stream=BytesIO()) component._setup_component(conf, self.context) self.assertEqual(component.topology_name, conf['topology.name']) self.assertEqual(component.task_id, self.context['taskid']) self.assertEqual( component.component_name, self.context['task->component'][str(self.context['taskid'])]) self.assertEqual(component.storm_conf, conf) self.assertEqual(component.context, self.context)
def test_read_split_message(self): # Make sure we can read something that's broken up into many "lines" inputs = ['{ "id": "-6955786537413359385", ', '"comp": "1", "stream": "1"\n', '\n', ', "task": 9, "tuple": ["snow white and the seven dwarfs", ', '"field2", 3]}\n', 'end\n'] output = json.loads(''.join(inputs[:-1])) component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) msg = component.read_message() self.assertEqual(output, msg)
def test_exit_on_exception_true(self): handshake_dict = { "conf": self.conf, "pidDir": ".", "context": self.context } inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) component.exit_on_exception = True with self.assertRaises(SystemExit) as raises_fixture: component.run() assert raises_fixture.exception.code == 1
def test_read_split_message(self): # Make sure we can read something that's broken up into many "lines" inputs = [ '{ "id": "-6955786537413359385", ', '"comp": "1", "stream": "1"\n', '\n', ', "task": 9, "tuple": ["snow white and the seven dwarfs", ', '"field2", 3]}\n', 'end\n' ] output = json.loads(''.join(inputs[:-1])) component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) msg = component.read_message() self.assertEqual(output, msg)
def test_log(self, send_message_mock): component = Component(input_stream=BytesIO(), output_stream=BytesIO()) inputs = [("I am a robot monkey.", None, 2), ("I am a monkey who learned to talk.", 'warning', 3)] for msg, level, storm_level in inputs: component.serializer.output_stream.close() component.serializer.output_stream = component.serializer._wrap_stream( BytesIO()) component.log(msg, level=level) send_message_mock.assert_called_with(component, { 'command': 'log', 'msg': msg, 'level': storm_level })
def test_send_message_unicode(self): component = Component(input_stream=BytesIO(), output_stream=BytesIO()) inputs = [{"command": "emit", "id": 4, "stream": "", "task": 9, "tuple": ["field\uFFE6", 2, 3]}, {"command": "log", "msg": "I am a robot monkey."}, {"command": "next"}, {"command": "sync"}] for cmd in inputs: component.serializer.output_stream.close() component.serializer.output_stream = component.serializer._wrap_stream(BytesIO()) component.send_message(cmd) self.assertEqual(component.serializer.serialize_dict(cmd).encode('utf-8'), component.serializer.output_stream.buffer.getvalue()) # Check that we properly skip over invalid input self.assertIsNone(component.send_message(['foo', 'bar']))
def test_exit_on_exception_false(self, _run_mock): # Make sure _run raises an exception def raiser(self): # lambdas can't raise raise StormWentAwayError if _run_mock.called else NotImplementedError _run_mock.side_effect = raiser handshake_dict = {"conf": self.conf, "pidDir": ".", "context": self.context} inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) component.exit_on_exception = False with self.assertRaises(SystemExit) as raises_fixture: component.run() assert raises_fixture.exception.code == 2
def test_read_handshake(self): handshake_dict = {"conf": self.conf, "pidDir": ".", "context": self.context} pid_dir = handshake_dict['pidDir'] expected_conf = handshake_dict['conf'] expected_context = handshake_dict['context'] inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) given_conf, given_context = component.read_handshake() pid_path = os.path.join(pid_dir, str(component.pid)) self.assertTrue(os.path.exists(pid_path)) os.remove(pid_path) self.assertEqual(given_conf, expected_conf) self.assertEqual(given_context, expected_context) self.assertEqual(component.serializer.serialize_dict({"pid": component.pid}).encode('utf-8'), component.serializer.output_stream.buffer.getvalue())
def test_nested_exception(self, log_mock, _handle_run_exception_mock): # Make sure self._handle_run_exception raises an exception def raiser(self): # lambdas can't raise raise Exception('Oops') handshake_dict = {"conf": self.conf, "pidDir": ".", "context": self.context} inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) component.exit_on_exception = True _handle_run_exception_mock.side_effect = raiser with self.assertRaises(SystemExit) as raises_fixture: component.run() assert log_mock.error.call_count == 2 assert raises_fixture.exception.code == 1
def test_exit_on_exception_false(self, _run_mock): # Make sure _run raises an exception def raiser(self): # lambdas can't raise raise StormWentAwayError if _run_mock.called else NotImplementedError _run_mock.side_effect = raiser handshake_dict = { "conf": self.conf, "pidDir": ".", "context": self.context } inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) component.exit_on_exception = False with self.assertRaises(SystemExit) as raises_fixture: component.run() assert raises_fixture.exception.code == 2
def test_nested_exception(self, log_mock, _handle_run_exception_mock): # Make sure self._handle_run_exception raises an exception def raiser(self): # lambdas can't raise raise Exception('Oops') handshake_dict = { "conf": self.conf, "pidDir": ".", "context": self.context } inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) component.exit_on_exception = True _handle_run_exception_mock.side_effect = raiser with self.assertRaises(SystemExit) as raises_fixture: component.run() assert log_mock.error.call_count == 2 assert raises_fixture.exception.code == 1
def test_read_command(self): # Check that we properly queue task IDs and return only commands inputs = [# Task IDs '[12, 22, 24]\n', 'end\n', # Incoming Tuple for bolt ('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"' ', "task": 9, "tuple": ["snow white and the seven dwarfs", ' '"field2", 3]}\n'), 'end\n', # next command for spout '{"command": "next"}\n', 'end\n'] outputs = [json.loads(msg) for msg in inputs[::2]] component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) # Skip first output, because it's a task ID, and won't be returned by # read_command for output in outputs[1:]: log.info('Checking msg for %r', output) msg = component.read_command() self.assertEqual(output, msg) self.assertEqual(component._pending_task_ids.pop(), outputs[0])
def test_read_message(self): inputs = [# Task IDs '[12, 22, 24]\n', 'end\n', # Incoming Tuple for bolt ('{ "id": "-6955786537413359385", "comp": "1", "stream": "1"' ', "task": 9, "tuple": ["snow white and the seven dwarfs", ' '"field2", 3]}\n'), 'end\n', # next command for spout '{"command": "next"}\n', 'end\n', # empty message, which should trigger sys.exit (end ignored) '', ''] outputs = [json.loads(msg) for msg in inputs[::2] if msg] outputs.append('') component = Component(input_stream=BytesIO(''.join(inputs).encode('utf-8')), output_stream=BytesIO()) for output in outputs: log.info('Checking msg for %r', output) if output: msg = component.read_message() self.assertEqual(output, msg) else: with self.assertRaises(StormWentAwayError): component.read_message()
def test_read_handshake(self): handshake_dict = { "conf": self.conf, "pidDir": ".", "context": self.context } pid_dir = handshake_dict['pidDir'] expected_conf = handshake_dict['conf'] expected_context = handshake_dict['context'] inputs = ["{}\n".format(json.dumps(handshake_dict)), "end\n"] component = Component(input_stream=BytesIO( ''.join(inputs).encode('utf-8')), output_stream=BytesIO()) given_conf, given_context = component.read_handshake() pid_path = os.path.join(pid_dir, str(component.pid)) self.assertTrue(os.path.exists(pid_path)) os.remove(pid_path) self.assertEqual(given_conf, expected_conf) self.assertEqual(given_context, expected_context) self.assertEqual( component.serializer.serialize_dict({ "pid": component.pid }).encode('utf-8'), component.serializer.output_stream.buffer.getvalue())