def topologies(tracker): def create_mock_execution_state(cluster, role, environ): estate = protoEState.ExecutionState() estate.cluster = cluster estate.role = role estate.environ = environ return estate topology1 = Topology('top_name1', 'mock_name1') topology1.execution_state = create_mock_execution_state('cluster1', 'mark', 'env1') topology2 = Topology('top_name2', 'mock_name1') topology2.execution_state = create_mock_execution_state('cluster1', 'bob', 'env1') topology3 = Topology('top_name3', 'mock_name1') topology3.execution_state = create_mock_execution_state('cluster1', 'tom', 'env2') topology4 = Topology('top_name4', 'mock_name2') topology4.execution_state = create_mock_execution_state('cluster2', 'x', 'env1') topology5 = Topology('top_name5', 'mock_name2') topology5.execution_state = create_mock_execution_state('cluster2', 'x', 'env2') tracker.topologies = [ topology1, topology2, topology3, topology4, topology5] return tracker.topologies[:]
def fill_tracker_topologies(self): def create_mock_execution_state(role): estate = protoEState.ExecutionState() estate.role = role return estate self.topology1 = Topology('top_name1', 'mock_name1') self.topology1.cluster = 'cluster1' self.topology1.environ = 'env1' self.topology1.execution_state = create_mock_execution_state('mark') self.topology2 = Topology('top_name2', 'mock_name1') self.topology2.cluster = 'cluster1' self.topology2.environ = 'env1' self.topology2.execution_state = create_mock_execution_state('bob') self.topology3 = Topology('top_name3', 'mock_name1') self.topology3.cluster = 'cluster1' self.topology3.environ = 'env2' self.topology3.execution_state = create_mock_execution_state('tom') self.topology4 = Topology('top_name4', 'mock_name2') self.topology4.cluster = 'cluster2' self.topology4.environ = 'env1' self.topology5 = Topology('top_name5', 'mock_name2') self.topology5.cluster = 'cluster2' self.topology5.environ = 'env2' self.tracker.topologies = [ self.topology1, self.topology2, self.topology3, self.topology4, self.topology5]
def get_topologies_for_state_location_side_effect(name): if name == 'mock_name1': return [Topology('top_name1', 'mock_name1'), Topology('top_name2', 'mock_name1')] if name == 'mock_name2': return [Topology('top_name3', 'mock_name2'), Topology('top_name4', 'mock_name2')] return []
def addNewTopology(self, state_manager, topologyName): """ Adds a topology in the local cache, and sets a watch on any changes on the topology. """ topology = Topology(topologyName, state_manager.name) Log.info("Adding new topology: %s, state_manager: %s", topologyName, state_manager.name) self.topologies.append(topology) # Register a watch on topology and change # the topologyInfo on any new change. topology.register_watch(self.setTopologyInfo) def on_topology_pplan(data): """watch physical plan""" Log.info("Watch triggered for topology pplan: " + topologyName) topology.set_physical_plan(data) if not data: Log.debug("No data to be set") def on_topology_packing_plan(data): """watch packing plan""" Log.info("Watch triggered for topology packing plan: " + topologyName) topology.set_packing_plan(data) if not data: Log.debug("No data to be set") def on_topology_execution_state(data): """watch execution state""" Log.info("Watch triggered for topology execution state: " + topologyName) topology.set_execution_state(data) if not data: Log.debug("No data to be set") def on_topology_tmaster(data): """set tmaster""" Log.info("Watch triggered for topology tmaster: " + topologyName) topology.set_tmaster(data) if not data: Log.debug("No data to be set") def on_topology_scheduler_location(data): """set scheduler location""" Log.info("Watch triggered for topology scheduler location: " + topologyName) topology.set_scheduler_location(data) if not data: Log.debug("No data to be set") # Set watches on the pplan, execution_state, tmaster and scheduler_location. state_manager.get_pplan(topologyName, on_topology_pplan) state_manager.get_packing_plan(topologyName, on_topology_packing_plan) state_manager.get_execution_state(topologyName, on_topology_execution_state) state_manager.get_tmaster(topologyName, on_topology_tmaster) state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)
def test_extract_packing_plan(self): # Create topology pb_pplan = MockProto().create_mock_simple_packing_plan() topology = Topology('topology_name', 'ExclamationTopology') topology.set_packing_plan(pb_pplan) # Extract packing plan packing_plan = self.tracker.extract_packing_plan(topology) self.assertEqual(packing_plan['id'], 'ExclamationTopology') self.assertEqual(packing_plan['container_plans'][0]['id'], 1) self.assertEqual( packing_plan['container_plans'][0]['required_resources'], { 'disk': 2048, 'ram': 1024, 'cpu': 1.0 }) self.assertEqual( packing_plan['container_plans'][0]['instances'][0], { 'component_index': 1, 'component_name': 'word', 'instance_resources': { 'cpu': 1.0, 'disk': 2048, 'ram': 1024 }, 'task_id': 1 })
def test_extract_packing_plan(self): # Create topology pb_pplan = MockProto().create_mock_simple_packing_plan() topology = Topology('topology_name', 'ExclamationTopology') topology.set_packing_plan(pb_pplan) # Extract packing plan packing_plan = self.tracker.extract_packing_plan(topology) self.assertEqual(packing_plan['id'], 'ExclamationTopology') self.assertEqual(packing_plan['container_plans'][0]['id'], 1) self.assertEqual( packing_plan['container_plans'][0]['required_resources'], { 'disk': 2048L, 'ram': 1024L, 'cpu': 1.0 })
def test_extract_physical_plan(self): # Create topology pb_pplan = MockProto().create_mock_simple_physical_plan() topology = Topology('topology_name', 'state_manager') topology.set_physical_plan(pb_pplan) # Extract physical plan pplan = self.tracker.extract_physical_plan(topology) # Mock topology doesn't have topology config and instances self.assertEqual(pplan['config'], {}) self.assertEqual(pplan['bolts'], {'mock_bolt': []}) self.assertEqual(pplan['spouts'], {'mock_spout': []}) self.assertEqual(pplan['components']['mock_bolt']['config'], {'topology.component.parallelism': '1'}) self.assertEqual(pplan['components']['mock_spout']['config'], {'topology.component.parallelism': '1'}) self.assertEqual(pplan['instances'], {}) self.assertEqual(pplan['stmgrs'], {})
def add_new_topology(self, state_manager, topology_name: str) -> None: """ Adds a topology in the local cache, and sets a watch on any changes on the topology. """ topology = Topology(topology_name, state_manager.name, self.config) Log.info("Adding new topology: %s, state_manager: %s", topology_name, state_manager.name) # populate the cache before making it addressable in the topologies to # avoid races due to concurrent execution self.topologies.append(topology) # Set watches on the pplan, execution_state, tmanager and scheduler_location. state_manager.get_pplan(topology_name, topology.set_physical_plan) state_manager.get_packing_plan(topology_name, topology.set_packing_plan) state_manager.get_execution_state(topology_name, topology.set_execution_state) state_manager.get_tmanager(topology_name, topology.set_tmanager) state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
def add_new_topology(self, state_manager, topology_name: str) -> None: """ Adds a topology in the local cache, and sets a watch on any changes on the topology. """ topology = Topology(topology_name, state_manager.name) Log.info("Adding new topology: %s, state_manager: %s", topology_name, state_manager.name) self.topologies.append(topology) # Register a watch on topology and change # the topology_info on any new change. topology.register_watch(self.set_topology_info) # Set watches on the pplan, execution_state, tmanager and scheduler_location. state_manager.get_pplan(topology_name, topology.set_physical_plan) state_manager.get_packing_plan(topology_name, topology.set_packing_plan) state_manager.get_execution_state(topology_name, topology.set_execution_state) state_manager.get_tmanager(topology_name, topology.set_tmanager) state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
def setUp(self): self.state_manager_name = "test_state_manager_name" self.topology = Topology(MockProto.topology_name, self.state_manager_name)
def topology(tracker): return Topology(MockProto.topology_name, "test_state_manager_name", tracker.config)