class TopologyHelpersTest(unittest.TestCase): def setUp(self): self.mock_proto = MockProto() def test_get_component_parallelism(self): topology = self.mock_proto.create_mock_medium_topology(1, 2, 3, 4) cmap = topology_helpers.get_component_parallelism(topology) self.assertEqual(1, cmap["mock_spout1"]) self.assertEqual(2, cmap["mock_bolt1"]) self.assertEqual(3, cmap["mock_bolt2"]) self.assertEqual(4, cmap["mock_bolt3"]) def test_get_disk_per_container(self): # We would have 9 instances topology = self.mock_proto.create_mock_medium_topology(1, 2, 3, 4) # First try with 1 container, so the disk request should be: # 10 * GB + Padding_Disk (12GB) = 22GB default_disk = topology_helpers.get_disk_per_container(topology) self.assertEqual(22 * constants.GB, default_disk) # Then try with 4 container, so the disk request should be: # 10/4 = 2.5 -> 3 (round to ceiling) + 12 = 15GB self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(15 * constants.GB, topology_helpers.get_disk_per_container(topology)) # Then let's set the disk_per_container explicitly self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_DISK_REQUESTED, 950109) # The add_topology_config will convert config into string self.assertEqual(str(950109), topology_helpers.get_disk_per_container(topology)) def test_get_total_instances(self): topology = self.mock_proto.create_mock_medium_topology(3, 4, 5, 6) num_instances = topology_helpers.get_total_instances(topology) self.assertEqual(18, num_instances) def test_sane(self): # Make wrong topology names topology = self.mock_proto.create_mock_simple_topology() topology.name = "" self.assertFalse(topology_helpers.sane(topology)) topology = self.mock_proto.create_mock_simple_topology() topology.name = "test.with.a.dot" self.assertFalse(topology_helpers.sane(topology)) topology = self.mock_proto.create_mock_simple_topology() topology.name = "test/with/a/slash" self.assertFalse(topology_helpers.sane(topology)) # Add another spout with the same name topology = self.mock_proto.create_mock_simple_topology() topology.spouts.extend([self.mock_proto.create_mock_spout("mock_spout", [], 1)]) self.assertFalse(topology_helpers.sane(topology)) # Add another bolt with the same name topology = self.mock_proto.create_mock_simple_topology() topology.bolts.extend([self.mock_proto.create_mock_bolt("mock_bolt", [], [], 1)]) self.assertFalse(topology_helpers.sane(topology)) # If num containers are greater than num instances topology = self.mock_proto.create_mock_simple_topology(1, 1) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertFalse(topology_helpers.sane(topology)) # If rammap is partial with less componenets topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1") self.assertTrue(topology_helpers.sane(topology)) # If rammap is not well formatted topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1:2,mock_bolt:2:3") self.assertFalse(topology_helpers.sane(topology)) # If rammap has wrong component name topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "wrong_mock_spout:1,mock_bolt:2") self.assertFalse(topology_helpers.sane(topology)) # If everything is right topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1,mock_bolt:2") self.assertTrue(topology_helpers.sane(topology)) def test_num_cpus_per_container(self): topology = self.mock_proto.create_mock_simple_topology(2, 2) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(2, topology_helpers.get_cpus_per_container(topology)) topology = self.mock_proto.create_mock_simple_topology(2, 2) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_CONTAINER_CPU_REQUESTED, 42) self.assertEqual(42, topology_helpers.get_cpus_per_container(topology)) def test_get_user_rammap(self): topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.assertEqual({"mock_spout":2, "mock_bolt":3}, topology_helpers.get_user_rammap(topology)) def test_get_component_distribution(self): topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) component_distribution = topology_helpers.get_component_distribution(topology) expected_component_distribution = { 1: [ ("mock_bolt", "1", "0"), ("mock_bolt", "5", "4"), ("mock_spout", "9", "0") ], 2: [ ("mock_bolt", "2", "1"), ("mock_bolt", "6", "5"), ("mock_spout", "10", "1") ], 3: [ ("mock_bolt", "3", "2"), ("mock_bolt", "7", "6"), ("mock_spout", "11", "2") ], 4: [ ("mock_bolt", "4", "3"), ("mock_bolt", "8", "7"), ("mock_spout", "12", "3") ] } self.assertEqual(expected_component_distribution, component_distribution) def test_get_component_rammap(self): # Mock a few methods # This is not a good way since this shows the internals # of the method. These methods need to be changed. # For example, ram_per_contaner could be taken as an argument. original_ram_for_stmgr = constants.RAM_FOR_STMGR constants.RAM_FOR_STMGR = 2 # When rammap is specified, it should be used. topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.assertEqual( {"mock_spout":2, "mock_bolt":3}, topology_helpers.get_component_rammap(topology)) # When partial rammap is specified, rest of the components should get default topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2") expected_component_rammap = { "mock_spout": 2, "mock_bolt": constants.DEFAULT_RAM_FOR_INSTANCE } self.assertEqual(expected_component_rammap, topology_helpers.get_component_rammap(topology)) # When container ram is specified. topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_CONTAINER_RAM_REQUESTED, 8) expected_component_rammap = { "mock_spout": 2, "mock_bolt": 2 } component_rammap = topology_helpers.get_component_rammap(topology) self.assertEqual(expected_component_rammap, component_rammap) # When nothing is specified. topology = self.mock_proto.create_mock_simple_topology(4, 8) component_rammap = topology_helpers.get_component_rammap(topology) expected_component_rammap = { "mock_spout": constants.DEFAULT_RAM_FOR_INSTANCE, "mock_bolt": constants.DEFAULT_RAM_FOR_INSTANCE } self.assertEqual(expected_component_rammap, component_rammap) # Unmock the things that we mocked. constants.RAM_FOR_STMGR = original_ram_for_stmgr def test_get_ram_per_container(self): # Mock a few things original_ram_for_stmgr = constants.RAM_FOR_STMGR constants.RAM_FOR_STMGR = 2 original_default_ram_for_instance = constants.DEFAULT_RAM_FOR_INSTANCE constants.DEFAULT_RAM_FOR_INSTANCE = 1 # When rammap is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(10, topology_helpers.get_ram_per_container(topology)) # When partial rammap is specified, rest of the components should get default topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2") self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) expected_ram_per_container = 6 self.assertEqual(expected_ram_per_container, topology_helpers.get_ram_per_container(topology)) # If container ram is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) requested_ram = 15000 self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_RAM_REQUESTED, str(requested_ram)) # Difference should be less than the total instances self.assertLess(abs(topology_helpers.get_ram_per_container(topology) - requested_ram), 12) # When nothing is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) expected_ram_per_container = 5 self.assertEqual(expected_ram_per_container, topology_helpers.get_ram_per_container(topology)) # Unmock the things that we mocked. constants.RAM_FOR_STMGR = original_ram_for_stmgr constants.DEFAULT_RAM_FOR_INSTANCE = original_default_ram_for_instance
class TopologyHelpersTest(unittest.TestCase): def setUp(self): self.mock_proto = MockProto() def test_get_component_parallelism(self): topology = self.mock_proto.create_mock_medium_topology(1, 2, 3, 4) cmap = topology_helpers.get_component_parallelism(topology) self.assertEqual(1, cmap["mock_spout1"]) self.assertEqual(2, cmap["mock_bolt1"]) self.assertEqual(3, cmap["mock_bolt2"]) self.assertEqual(4, cmap["mock_bolt3"]) def test_get_disk_per_container(self): # We would have 9 instances topology = self.mock_proto.create_mock_medium_topology(1, 2, 3, 4) # First try with 1 container, so the disk request should be: # 10 * GB + Padding_Disk (12GB) = 22GB default_disk = topology_helpers.get_disk_per_container(topology) self.assertEqual(22 * constants.GB, default_disk) # Then try with 4 container, so the disk request should be: # 10/4 = 2.5 -> 3 (round to ceiling) + 12 = 15GB self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(15 * constants.GB, topology_helpers.get_disk_per_container(topology)) # Then let's set the disk_per_container explicitly self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_DISK_REQUESTED, 950109) # The add_topology_config will convert config into string self.assertEqual(str(950109), topology_helpers.get_disk_per_container(topology)) def test_get_total_instances(self): topology = self.mock_proto.create_mock_medium_topology(3, 4, 5, 6) num_instances = topology_helpers.get_total_instances(topology) self.assertEqual(18, num_instances) def test_sane(self): # Make wrong topology names topology = self.mock_proto.create_mock_simple_topology() topology.name = "" self.assertFalse(topology_helpers.sane(topology)) topology = self.mock_proto.create_mock_simple_topology() topology.name = "test.with.a.dot" self.assertFalse(topology_helpers.sane(topology)) topology = self.mock_proto.create_mock_simple_topology() topology.name = "test/with/a/slash" self.assertFalse(topology_helpers.sane(topology)) # Add another spout with the same name topology = self.mock_proto.create_mock_simple_topology() topology.spouts.extend( [self.mock_proto.create_mock_spout("mock_spout", [], 1)]) self.assertFalse(topology_helpers.sane(topology)) # Add another bolt with the same name topology = self.mock_proto.create_mock_simple_topology() topology.bolts.extend( [self.mock_proto.create_mock_bolt("mock_bolt", [], [], 1)]) self.assertFalse(topology_helpers.sane(topology)) # If num containers are greater than num instances topology = self.mock_proto.create_mock_simple_topology(1, 1) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertFalse(topology_helpers.sane(topology)) # If rammap is partial with less componenets topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1") self.assertTrue(topology_helpers.sane(topology)) # If rammap is not well formatted topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1:2,mock_bolt:2:3") self.assertFalse(topology_helpers.sane(topology)) # If rammap has wrong component name topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "wrong_mock_spout:1,mock_bolt:2") self.assertFalse(topology_helpers.sane(topology)) # If everything is right topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:1,mock_bolt:2") self.assertTrue(topology_helpers.sane(topology)) def test_num_cpus_per_container(self): topology = self.mock_proto.create_mock_simple_topology(2, 2) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(2, topology_helpers.get_cpus_per_container(topology)) topology = self.mock_proto.create_mock_simple_topology(2, 2) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_CPU_REQUESTED, 42) self.assertEqual(42, topology_helpers.get_cpus_per_container(topology)) def test_get_user_rammap(self): topology = self.mock_proto.create_mock_simple_topology() self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.assertEqual({ "mock_spout": 2, "mock_bolt": 3 }, topology_helpers.get_user_rammap(topology)) def test_get_component_distribution(self): topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) component_distribution = topology_helpers.get_component_distribution( topology) expected_component_distribution = { 1: [("mock_bolt", "1", "0"), ("mock_bolt", "5", "4"), ("mock_spout", "9", "0")], 2: [("mock_bolt", "2", "1"), ("mock_bolt", "6", "5"), ("mock_spout", "10", "1")], 3: [("mock_bolt", "3", "2"), ("mock_bolt", "7", "6"), ("mock_spout", "11", "2")], 4: [("mock_bolt", "4", "3"), ("mock_bolt", "8", "7"), ("mock_spout", "12", "3")] } self.assertEqual(expected_component_distribution, component_distribution) def test_get_component_rammap(self): # Mock a few methods # This is not a good way since this shows the internals # of the method. These methods need to be changed. # For example, ram_per_contaner could be taken as an argument. original_ram_for_stmgr = constants.RAM_FOR_STMGR constants.RAM_FOR_STMGR = 2 # When rammap is specified, it should be used. topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.assertEqual({ "mock_spout": 2, "mock_bolt": 3 }, topology_helpers.get_component_rammap(topology)) # When partial rammap is specified, rest of the components should get default topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2") expected_component_rammap = { "mock_spout": 2, "mock_bolt": constants.DEFAULT_RAM_FOR_INSTANCE } self.assertEqual(expected_component_rammap, topology_helpers.get_component_rammap(topology)) # When container ram is specified. topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_RAM_REQUESTED, 8) expected_component_rammap = {"mock_spout": 2, "mock_bolt": 2} component_rammap = topology_helpers.get_component_rammap(topology) self.assertEqual(expected_component_rammap, component_rammap) # When nothing is specified. topology = self.mock_proto.create_mock_simple_topology(4, 8) component_rammap = topology_helpers.get_component_rammap(topology) expected_component_rammap = { "mock_spout": constants.DEFAULT_RAM_FOR_INSTANCE, "mock_bolt": constants.DEFAULT_RAM_FOR_INSTANCE } self.assertEqual(expected_component_rammap, component_rammap) # Unmock the things that we mocked. constants.RAM_FOR_STMGR = original_ram_for_stmgr def test_get_ram_per_container(self): # Mock a few things original_ram_for_stmgr = constants.RAM_FOR_STMGR constants.RAM_FOR_STMGR = 2 original_default_ram_for_instance = constants.DEFAULT_RAM_FOR_INSTANCE constants.DEFAULT_RAM_FOR_INSTANCE = 1 # When rammap is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2,mock_bolt:3") self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) self.assertEqual(10, topology_helpers.get_ram_per_container(topology)) # When partial rammap is specified, rest of the components should get default topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_COMPONENT_RAMMAP, "mock_spout:2") self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) expected_ram_per_container = 6 self.assertEqual(expected_ram_per_container, topology_helpers.get_ram_per_container(topology)) # If container ram is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) requested_ram = 15000 self.mock_proto.add_topology_config( topology, constants.TOPOLOGY_CONTAINER_RAM_REQUESTED, str(requested_ram)) # Difference should be less than the total instances self.assertLess( abs( topology_helpers.get_ram_per_container(topology) - requested_ram), 12) # When nothing is specified topology = self.mock_proto.create_mock_simple_topology(4, 8) self.mock_proto.add_topology_config(topology, constants.TOPOLOGY_STMGRS, 4) expected_ram_per_container = 5 self.assertEqual(expected_ram_per_container, topology_helpers.get_ram_per_container(topology)) # Unmock the things that we mocked. constants.RAM_FOR_STMGR = original_ram_for_stmgr constants.DEFAULT_RAM_FOR_INSTANCE = original_default_ram_for_instance