def handle_assignment_msg(self, pplan):
"""Called when new NewInstanceAssignmentMessage arrives
Tells this instance to become either spout/bolt.
:param pplan: PhysicalPlan proto
"""
new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id,
self.topo_pex_file_abs_path)
if self.my_pplan_helper is not None and \
(self.my_pplan_helper.my_component_name != new_helper.my_component_name or
self.my_pplan_helper.my_task_id != new_helper.my_task_id):
raise RuntimeError(
"Our Assignment has changed. We will die to pick it.")
new_helper.set_topology_context(self.metrics_collector)
if self.my_pplan_helper is None:
Log.info("Received a new Physical Plan")
Log.info("Push the new pplan_helper to Heron Instance")
self._handle_assignment_msg(new_helper)
else:
Log.info(
"Received a new Physical Plan with the same assignment -- State Change"
)
Log.info("Old state: %s, new state: %s.",
self.my_pplan_helper.get_topology_state(),
new_helper.get_topology_state())
self._handle_state_change_msg(new_helper)
def handle_assignment_msg(self, pplan):
"""Called when new NewInstanceAssignmentMessage arrives
Tells this instance to become either spout/bolt.
:param pplan: PhysicalPlan proto
"""
new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id,
self.topo_pex_file_abs_path)
if self.my_pplan_helper is not None and \
(self.my_pplan_helper.my_component_name != new_helper.my_component_name or
self.my_pplan_helper.my_task_id != new_helper.my_task_id):
raise RuntimeError("Our Assignment has changed. We will die to pick it.")
new_helper.set_topology_context(self.metrics_collector)
if self.my_pplan_helper is None:
Log.info("Received a new Physical Plan")
Log.info("Push the new pplan_helper to Heron Instance")
self._handle_assignment_msg(new_helper)
else:
Log.info("Received a new Physical Plan with the same assignment -- State Change")
Log.info("Old state: %s, new state: %s.",
self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state())
self._handle_state_change_msg(new_helper)
def test_get_dict_from_config(self):
# testing creating auto-typed dict from protobuf Config message
# try with config <str -> str>
test_config_1 = {"key1": "string_value",
"key2": "12",
"key3": "1.50",
"key4": "false"}
expected_1 = {"key1": "string_value",
"key2": 12,
"key3": 1.50,
"key4": False}
proto_config = mock_protobuf.get_mock_config(config_dict=test_config_1)
ret = PhysicalPlanHelper._get_dict_from_config(proto_config)
self.assertEqual(ret, expected_1)
# try with config <str -> any object>
test_config_2 = {"key1": "string",
"key2": True,
"key3": 1.20,
"key4": (1, 2, 3, "hello"),
"key5": {"key5_1": 1, "key5_2": "value"},
"key6": [1, 2, 3, 4, 5.0]}
proto_config = mock_protobuf.get_mock_config(config_dict=test_config_2)
ret = PhysicalPlanHelper._get_dict_from_config(proto_config)
self.assertEqual(ret, test_config_2)
def test_sample_success(self):
# Instance 1 should be Spout 1
pplan, instances = mock_generator.get_a_sample_pplan()
instance_1 = instances[0]
pplan_helper = PhysicalPlanHelper(pplan, instance_1["instance_id"], "topology.pex.path")
self.assertIsNotNone(pplan_helper.my_instance)
self.assertTrue(pplan_helper.is_spout)
self.assertIsInstance(pplan_helper.get_my_spout(), topology_pb2.Spout)
self.assertIsNone(pplan_helper.get_my_bolt())
self.assertEqual(instance_1["task_id"], pplan_helper.my_task_id)
self.assertEqual(instance_1["comp_name"], pplan_helper.my_component_name)
def test_number_autotype(self):
# testing _is_number() and _get_number()
number_strings = ["1", "20", "30", "123410000000000", "12.40", "3.1", "3.000"]
for test_str in number_strings:
ret = PhysicalPlanHelper._is_number(test_str)
self.assertTrue(ret)
ret = PhysicalPlanHelper._get_number(test_str)
self.assertIsInstance(ret, (int, float))
non_number_strings = ["1a", "1,000", "string", "1.51hello"]
for test_str in non_number_strings:
ret = PhysicalPlanHelper._is_number(test_str)
self.assertFalse(ret)
def test_sample_success(self):
# Instance 1 should be Spout 1
pplan, instances = mock_generator.get_a_sample_pplan()
instance_1 = instances[0]
pplan_helper = PhysicalPlanHelper(pplan, instance_1["instance_id"],
"topology.pex.path")
self.assertIsNotNone(pplan_helper.my_instance)
self.assertTrue(pplan_helper.is_spout)
self.assertIsInstance(pplan_helper.get_my_spout(), topology_pb2.Spout)
self.assertIsNone(pplan_helper.get_my_bolt())
self.assertEqual(instance_1["task_id"], pplan_helper.my_task_id)
self.assertEqual(instance_1["comp_name"],
pplan_helper.my_component_name)
def test_number_autotype(self):
# testing _is_number() and _get_number()
number_strings = [
"1", "20", "30", "123410000000000", "12.40", "3.1", "3.000"
]
for test_str in number_strings:
ret = PhysicalPlanHelper._is_number(test_str)
self.assertTrue(ret)
ret = PhysicalPlanHelper._get_number(test_str)
self.assertIsInstance(ret, (int, float))
non_number_strings = ["1a", "1,000", "string", "1.51hello"]
for test_str in non_number_strings:
ret = PhysicalPlanHelper._is_number(test_str)
self.assertFalse(ret)
def test_get_dict_from_config(self):
# testing creating auto-typed dict from protobuf Config message
# try with config <str -> str>
test_config_1 = {
"key1": "string_value",
"key2": "12",
"key3": "1.50",
"key4": "false"
}
expected_1 = {
"key1": "string_value",
"key2": 12,
"key3": 1.50,
"key4": False
}
proto_config = mock_protobuf.get_mock_config(config_dict=test_config_1)
ret = PhysicalPlanHelper._get_dict_from_config(proto_config)
self.assertEqual(ret, expected_1)
# try with config <str -> any object>
test_config_2 = {
"key1": "string",
"key2": True,
"key3": 1.20,
"key4": (1, 2, 3, "hello"),
"key5": {
"key5_1": 1,
"key5_2": "value"
},
"key6": [1, 2, 3, 4, 5.0]
}
proto_config = mock_protobuf.get_mock_config(config_dict=test_config_2)
ret = PhysicalPlanHelper._get_dict_from_config(proto_config)
self.assertEqual(ret, test_config_2)
def _prepare_sample_success(): pplan, instances = get_a_sample_pplan() pplan_helper = PhysicalPlanHelper(pplan, instances[0]["instance_id"], "topology.pex.path") out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) return pplan_helper, out_stream