def handle_assignment_msg(self, pplan): """Called when new NewInstanceAssignmentMessage arrives Tells this instance to become either spout/bolt. :param pplan: PhysicalPlan proto """ new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id, self.topo_pex_file_abs_path) if self.my_pplan_helper is not None and \ (self.my_pplan_helper.my_component_name != new_helper.my_component_name or self.my_pplan_helper.my_task_id != new_helper.my_task_id): raise RuntimeError( "Our Assignment has changed. We will die to pick it.") new_helper.set_topology_context(self.metrics_collector) if self.my_pplan_helper is None: Log.info("Received a new Physical Plan") Log.info("Push the new pplan_helper to Heron Instance") self._handle_assignment_msg(new_helper) else: Log.info( "Received a new Physical Plan with the same assignment -- State Change" ) Log.info("Old state: %s, new state: %s.", self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state()) self._handle_state_change_msg(new_helper)
def handle_assignment_msg(self, pplan): """Called when new NewInstanceAssignmentMessage arrives Tells this instance to become either spout/bolt. :param pplan: PhysicalPlan proto """ new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id, self.topo_pex_file_abs_path) if self.my_pplan_helper is not None and \ (self.my_pplan_helper.my_component_name != new_helper.my_component_name or self.my_pplan_helper.my_task_id != new_helper.my_task_id): raise RuntimeError("Our Assignment has changed. We will die to pick it.") new_helper.set_topology_context(self.metrics_collector) if self.my_pplan_helper is None: Log.info("Received a new Physical Plan") Log.info("Push the new pplan_helper to Heron Instance") self._handle_assignment_msg(new_helper) else: Log.info("Received a new Physical Plan with the same assignment -- State Change") Log.info("Old state: %s, new state: %s.", self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state()) self._handle_state_change_msg(new_helper)
def test_get_dict_from_config(self): # testing creating auto-typed dict from protobuf Config message # try with config <str -> str> test_config_1 = {"key1": "string_value", "key2": "12", "key3": "1.50", "key4": "false"} expected_1 = {"key1": "string_value", "key2": 12, "key3": 1.50, "key4": False} proto_config = mock_protobuf.get_mock_config(config_dict=test_config_1) ret = PhysicalPlanHelper._get_dict_from_config(proto_config) self.assertEqual(ret, expected_1) # try with config <str -> any object> test_config_2 = {"key1": "string", "key2": True, "key3": 1.20, "key4": (1, 2, 3, "hello"), "key5": {"key5_1": 1, "key5_2": "value"}, "key6": [1, 2, 3, 4, 5.0]} proto_config = mock_protobuf.get_mock_config(config_dict=test_config_2) ret = PhysicalPlanHelper._get_dict_from_config(proto_config) self.assertEqual(ret, test_config_2)
def test_sample_success(self): # Instance 1 should be Spout 1 pplan, instances = mock_generator.get_a_sample_pplan() instance_1 = instances[0] pplan_helper = PhysicalPlanHelper(pplan, instance_1["instance_id"], "topology.pex.path") self.assertIsNotNone(pplan_helper.my_instance) self.assertTrue(pplan_helper.is_spout) self.assertIsInstance(pplan_helper.get_my_spout(), topology_pb2.Spout) self.assertIsNone(pplan_helper.get_my_bolt()) self.assertEqual(instance_1["task_id"], pplan_helper.my_task_id) self.assertEqual(instance_1["comp_name"], pplan_helper.my_component_name)
def test_number_autotype(self): # testing _is_number() and _get_number() number_strings = ["1", "20", "30", "123410000000000", "12.40", "3.1", "3.000"] for test_str in number_strings: ret = PhysicalPlanHelper._is_number(test_str) self.assertTrue(ret) ret = PhysicalPlanHelper._get_number(test_str) self.assertIsInstance(ret, (int, float)) non_number_strings = ["1a", "1,000", "string", "1.51hello"] for test_str in non_number_strings: ret = PhysicalPlanHelper._is_number(test_str) self.assertFalse(ret)
def test_sample_success(self): # Instance 1 should be Spout 1 pplan, instances = mock_generator.get_a_sample_pplan() instance_1 = instances[0] pplan_helper = PhysicalPlanHelper(pplan, instance_1["instance_id"], "topology.pex.path") self.assertIsNotNone(pplan_helper.my_instance) self.assertTrue(pplan_helper.is_spout) self.assertIsInstance(pplan_helper.get_my_spout(), topology_pb2.Spout) self.assertIsNone(pplan_helper.get_my_bolt()) self.assertEqual(instance_1["task_id"], pplan_helper.my_task_id) self.assertEqual(instance_1["comp_name"], pplan_helper.my_component_name)
def test_number_autotype(self): # testing _is_number() and _get_number() number_strings = [ "1", "20", "30", "123410000000000", "12.40", "3.1", "3.000" ] for test_str in number_strings: ret = PhysicalPlanHelper._is_number(test_str) self.assertTrue(ret) ret = PhysicalPlanHelper._get_number(test_str) self.assertIsInstance(ret, (int, float)) non_number_strings = ["1a", "1,000", "string", "1.51hello"] for test_str in non_number_strings: ret = PhysicalPlanHelper._is_number(test_str) self.assertFalse(ret)
def test_get_dict_from_config(self): # testing creating auto-typed dict from protobuf Config message # try with config <str -> str> test_config_1 = { "key1": "string_value", "key2": "12", "key3": "1.50", "key4": "false" } expected_1 = { "key1": "string_value", "key2": 12, "key3": 1.50, "key4": False } proto_config = mock_protobuf.get_mock_config(config_dict=test_config_1) ret = PhysicalPlanHelper._get_dict_from_config(proto_config) self.assertEqual(ret, expected_1) # try with config <str -> any object> test_config_2 = { "key1": "string", "key2": True, "key3": 1.20, "key4": (1, 2, 3, "hello"), "key5": { "key5_1": 1, "key5_2": "value" }, "key6": [1, 2, 3, 4, 5.0] } proto_config = mock_protobuf.get_mock_config(config_dict=test_config_2) ret = PhysicalPlanHelper._get_dict_from_config(proto_config) self.assertEqual(ret, test_config_2)
def _prepare_sample_success(): pplan, instances = get_a_sample_pplan() pplan_helper = PhysicalPlanHelper(pplan, instances[0]["instance_id"], "topology.pex.path") out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) return pplan_helper, out_stream