def addNewTopology(self, state_manager, topologyName):
"""
Adds a topology in the local cache, and sets a watch
on any changes on the topology.
"""
topology = Topology(topologyName, state_manager.name)
Log.info("Adding new topology: %s, state_manager: %s", topologyName,
state_manager.name)
self.topologies.append(topology)
# Register a watch on topology and change
# the topologyInfo on any new change.
topology.register_watch(self.setTopologyInfo)
def on_topology_pplan(data):
"""watch physical plan"""
Log.info("Watch triggered for topology pplan: " + topologyName)
topology.set_physical_plan(data)
if not data:
Log.debug("No data to be set")
def on_topology_packing_plan(data):
"""watch packing plan"""
Log.info("Watch triggered for topology packing plan: " +
topologyName)
topology.set_packing_plan(data)
if not data:
Log.debug("No data to be set")
def on_topology_execution_state(data):
"""watch execution state"""
Log.info("Watch triggered for topology execution state: " +
topologyName)
topology.set_execution_state(data)
if not data:
Log.debug("No data to be set")
def on_topology_tmaster(data):
"""set tmaster"""
Log.info("Watch triggered for topology tmaster: " + topologyName)
topology.set_tmaster(data)
if not data:
Log.debug("No data to be set")
def on_topology_scheduler_location(data):
"""set scheduler location"""
Log.info("Watch triggered for topology scheduler location: " +
topologyName)
topology.set_scheduler_location(data)
if not data:
Log.debug("No data to be set")
# Set watches on the pplan, execution_state, tmaster and scheduler_location.
state_manager.get_pplan(topologyName, on_topology_pplan)
state_manager.get_packing_plan(topologyName, on_topology_packing_plan)
state_manager.get_execution_state(topologyName,
on_topology_execution_state)
state_manager.get_tmaster(topologyName, on_topology_tmaster)
state_manager.get_scheduler_location(topologyName,
on_topology_scheduler_location)
def fill_tracker_topologies(self):
def create_mock_execution_state(role):
estate = protoEState.ExecutionState()
estate.role = role
return estate
self.topology1 = Topology('top_name1', 'mock_name1')
self.topology1.cluster = 'cluster1'
self.topology1.environ = 'env1'
self.topology1.execution_state = create_mock_execution_state('mark')
self.topology2 = Topology('top_name2', 'mock_name1')
self.topology2.cluster = 'cluster1'
self.topology2.environ = 'env1'
self.topology2.execution_state = create_mock_execution_state('bob')
self.topology3 = Topology('top_name3', 'mock_name1')
self.topology3.cluster = 'cluster1'
self.topology3.environ = 'env2'
self.topology3.execution_state = create_mock_execution_state('tom')
self.topology4 = Topology('top_name4', 'mock_name2')
self.topology4.cluster = 'cluster2'
self.topology4.environ = 'env1'
self.topology5 = Topology('top_name5', 'mock_name2')
self.topology5.cluster = 'cluster2'
self.topology5.environ = 'env2'
self.tracker.topologies = [
self.topology1,
self.topology2,
self.topology3,
self.topology4,
self.topology5]
def test_extract_packing_plan(self):
# Create topology
pb_pplan = MockProto().create_mock_simple_packing_plan()
topology = Topology('topology_name', 'ExclamationTopology')
topology.set_packing_plan(pb_pplan)
# Extract packing plan
packing_plan = self.tracker.extract_packing_plan(topology)
self.assertEqual(packing_plan['id'], 'ExclamationTopology')
self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
self.assertEqual(
packing_plan['container_plans'][0]['required_resources'], {
'disk': 2048,
'ram': 1024,
'cpu': 1.0
})
self.assertEqual(
packing_plan['container_plans'][0]['instances'][0], {
'component_index': 1,
'component_name': 'word',
'instance_resources': {
'cpu': 1.0,
'disk': 2048,
'ram': 1024
},
'task_id': 1
})
def get_topologies_for_state_location_side_effect(name):
if name == 'mock_name1':
return [Topology('top_name1', 'mock_name1'),
Topology('top_name2', 'mock_name1')]
if name == 'mock_name2':
return [Topology('top_name3', 'mock_name2'),
Topology('top_name4', 'mock_name2')]
return []
def addNewTopology(self, state_manager, topologyName):
"""
Adds a topology in the local cache, and sets a watch
on any changes on the topology.
"""
topology = Topology(topologyName, state_manager.name)
Log.info("Adding new topology: %s, state_manager: %s",
topologyName, state_manager.name)
self.topologies.append(topology)
# Register a watch on topology and change
# the topologyInfo on any new change.
topology.register_watch(self.setTopologyInfo)
def on_topology_pplan(data):
"""watch physical plan"""
Log.info("Watch triggered for topology pplan: " + topologyName)
topology.set_physical_plan(data)
if not data:
Log.debug("No data to be set")
def on_topology_packing_plan(data):
"""watch packing plan"""
Log.info("Watch triggered for topology packing plan: " + topologyName)
topology.set_packing_plan(data)
if not data:
Log.debug("No data to be set")
def on_topology_execution_state(data):
"""watch execution state"""
Log.info("Watch triggered for topology execution state: " + topologyName)
topology.set_execution_state(data)
if not data:
Log.debug("No data to be set")
def on_topology_tmaster(data):
"""set tmaster"""
Log.info("Watch triggered for topology tmaster: " + topologyName)
topology.set_tmaster(data)
if not data:
Log.debug("No data to be set")
def on_topology_scheduler_location(data):
"""set scheduler location"""
Log.info("Watch triggered for topology scheduler location: " + topologyName)
topology.set_scheduler_location(data)
if not data:
Log.debug("No data to be set")
# Set watches on the pplan, execution_state, tmaster and scheduler_location.
state_manager.get_pplan(topologyName, on_topology_pplan)
state_manager.get_packing_plan(topologyName, on_topology_packing_plan)
state_manager.get_execution_state(topologyName, on_topology_execution_state)
state_manager.get_tmaster(topologyName, on_topology_tmaster)
state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)
def test_extract_packing_plan(self):
# Create topology
pb_pplan = MockProto().create_mock_simple_packing_plan()
topology = Topology('topology_name', 'ExclamationTopology')
topology.set_packing_plan(pb_pplan)
# Extract packing plan
packing_plan = self.tracker.extract_packing_plan(topology)
self.assertEqual(packing_plan['id'], 'ExclamationTopology')
self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
self.assertEqual(
packing_plan['container_plans'][0]['required_resources'], {
'disk': 2048L,
'ram': 1024L,
'cpu': 1.0
})
def test_extract_physical_plan(self):
# Create topology
pb_pplan = MockProto().create_mock_simple_physical_plan()
topology = Topology('topology_name', 'state_manager')
topology.set_physical_plan(pb_pplan)
# Extract physical plan
pplan = self.tracker.extract_physical_plan(topology)
# Mock topology doesn't have topology config and instances
self.assertEqual(pplan['config'], {})
self.assertEqual(pplan['bolts'], {'mock_bolt': []})
self.assertEqual(pplan['spouts'], {'mock_spout': []})
self.assertEqual(pplan['components']['mock_bolt']['config'],
{'topology.component.parallelism': '1'})
self.assertEqual(pplan['components']['mock_spout']['config'],
{'topology.component.parallelism': '1'})
self.assertEqual(pplan['instances'], {})
self.assertEqual(pplan['stmgrs'], {})
def test_extract_physical_plan(self):
# Create topology
pb_pplan = MockProto().create_mock_simple_physical_plan()
topology = Topology('topology_name', 'state_manager')
topology.set_physical_plan(pb_pplan)
# Extract physical plan
pplan = self.tracker.extract_physical_plan(topology)
# Mock topology doesn't have topology config and instances
self.assertEqual(pplan['config'], {})
self.assertEqual(pplan['bolts'], {'mock_bolt': []})
self.assertEqual(pplan['spouts'], {'mock_spout': []})
self.assertEqual(pplan['components']['mock_bolt']['config'],
{'topology.component.parallelism': '1'})
self.assertEqual(pplan['components']['mock_spout']['config'],
{'topology.component.parallelism': '1'})
self.assertEqual(pplan['instances'], {})
self.assertEqual(pplan['stmgrs'], {})
def add_new_topology(self, state_manager, topology_name: str) -> None:
"""
Adds a topology in the local cache, and sets a watch
on any changes on the topology.
"""
topology = Topology(topology_name, state_manager.name)
Log.info("Adding new topology: %s, state_manager: %s",
topology_name, state_manager.name)
self.topologies.append(topology)
# Register a watch on topology and change
# the topology_info on any new change.
topology.register_watch(self.set_topology_info)
# Set watches on the pplan, execution_state, tmanager and scheduler_location.
state_manager.get_pplan(topology_name, topology.set_physical_plan)
state_manager.get_packing_plan(topology_name, topology.set_packing_plan)
state_manager.get_execution_state(topology_name, topology.set_execution_state)
state_manager.get_tmanager(topology_name, topology.set_tmanager)
state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
def add_new_topology(self, state_manager, topology_name: str) -> None:
"""
Adds a topology in the local cache, and sets a watch
on any changes on the topology.
"""
topology = Topology(topology_name, state_manager.name, self.config)
Log.info("Adding new topology: %s, state_manager: %s",
topology_name, state_manager.name)
# populate the cache before making it addressable in the topologies to
# avoid races due to concurrent execution
self.topologies.append(topology)
# Set watches on the pplan, execution_state, tmanager and scheduler_location.
state_manager.get_pplan(topology_name, topology.set_physical_plan)
state_manager.get_packing_plan(topology_name, topology.set_packing_plan)
state_manager.get_execution_state(topology_name, topology.set_execution_state)
state_manager.get_tmanager(topology_name, topology.set_tmanager)
state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
def setUp(self):
self.state_manager_name = "test_state_manager_name"
self.topology = Topology(MockProto.topology_name,
self.state_manager_name)
class TopologyTest(unittest.TestCase):
def setUp(self):
self.state_manager_name = "test_state_manager_name"
self.topology = Topology(MockProto.topology_name,
self.state_manager_name)
def test_set_physical_plan(self):
# Set it to None
self.topology.set_physical_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.physical_plan)
physical_plan = MockProto().create_mock_simple_physical_plan()
self.topology.set_physical_plan(physical_plan)
self.assertEqual(MockProto.topology_id, self.topology.id)
self.assertEqual(physical_plan, self.topology.physical_plan)
def test_set_packing_plan(self):
# Set it to None
self.topology.set_packing_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.packing_plan)
packing_plan = MockProto().create_mock_simple_packing_plan()
self.topology.set_packing_plan(packing_plan)
self.assertEqual(packing_plan, self.topology.packing_plan)
# testing with a packing plan with scheduled resources
self.topology.set_packing_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.packing_plan)
packing_plan = MockProto().create_mock_simple_packing_plan2()
self.topology.set_packing_plan(packing_plan)
self.assertEqual(packing_plan, self.topology.packing_plan)
def test_set_execution_state(self):
# Set it to None
self.topology.set_execution_state(None)
self.assertIsNone(self.topology.execution_state)
self.assertIsNone(self.topology.cluster)
self.assertIsNone(self.topology.environ)
estate = MockProto().create_mock_execution_state()
self.topology.set_execution_state(estate)
self.assertEqual(estate, self.topology.execution_state)
self.assertEqual(MockProto.cluster, self.topology.cluster)
self.assertEqual(MockProto.environ, self.topology.environ)
def test_set_tmanager(self):
# Set it to None
self.topology.set_tmanager(None)
self.assertIsNone(self.topology.tmanager)
tmanager = MockProto().create_mock_tmanager()
self.topology.set_tmanager(tmanager)
self.assertEqual(tmanager, self.topology.tmanager)
def test_spouts(self):
# When pplan is not set
self.assertEqual(0, len(self.topology.spouts()))
# Set pplan now
pplan = MockProto().create_mock_simple_physical_plan()
self.topology.set_physical_plan(pplan)
spouts = self.topology.spouts()
self.assertEqual(1, len(spouts))
self.assertEqual("mock_spout", spouts[0].comp.name)
self.assertEqual(["mock_spout"], self.topology.spout_names())
def test_bolts(self):
# When pplan is not set
self.assertEqual(0, len(self.topology.bolts()))
# Set pplan
pplan = MockProto().create_mock_medium_physical_plan()
self.topology.set_physical_plan(pplan)
bolts = self.topology.bolts()
self.assertEqual(3, len(bolts))
self.assertEqual(["mock_bolt1", "mock_bolt2", "mock_bolt3"],
self.topology.bolt_names())
def test_num_instances(self):
# When pplan is not set
self.assertEqual(0, self.topology.num_instances())
pplan = MockProto().create_mock_medium_physical_plan(1, 2, 3, 4)
self.topology.set_physical_plan(pplan)
self.assertEqual(10, self.topology.num_instances())
def test_trigger_watches(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument, unused-variable
def callback(something):
scope["is_called"] = True
uid = self.topology.register_watch(callback)
self.assertTrue(scope["is_called"])
scope["is_called"] = False
self.assertFalse(scope["is_called"])
print(scope)
self.topology.set_physical_plan(None)
print(scope)
self.assertTrue(scope["is_called"])
print(scope)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_execution_state(None)
self.assertTrue(scope["is_called"])
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_tmanager(None)
self.assertTrue(scope["is_called"])
def test_unregister_watch(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument
def callback(something):
scope["is_called"] = True
uid = self.topology.register_watch(callback)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertTrue(scope["is_called"])
self.topology.unregister_watch(uid)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertFalse(scope["is_called"])
def test_bad_watch(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument, unused-variable
def callback(something):
scope["is_called"] = True
raise Exception("Test Bad Trigger Exception")
uid = self.topology.register_watch(callback)
# is called the first time because of registeration
self.assertTrue(scope["is_called"])
# But should no longer be called
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertFalse(scope["is_called"])
def setUp(self):
self.state_manager_name = "test_state_manager_name"
self.topology = Topology(MockProto.topology_name,
self.state_manager_name)
class TopologyTest(unittest.TestCase):
def setUp(self):
self.state_manager_name = "test_state_manager_name"
self.topology = Topology(MockProto.topology_name,
self.state_manager_name)
def test_set_physical_plan(self):
# Set it to None
self.topology.set_physical_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.physical_plan)
physical_plan = MockProto().create_mock_simple_physical_plan()
self.topology.set_physical_plan(physical_plan)
self.assertEqual(MockProto.topology_id, self.topology.id)
self.assertEqual(physical_plan, self.topology.physical_plan)
def test_set_packing_plan(self):
# Set it to None
self.topology.set_packing_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.packing_plan)
packing_plan = MockProto().create_mock_simple_packing_plan()
self.topology.set_packing_plan(packing_plan)
self.assertEqual(packing_plan, self.topology.packing_plan)
# testing with a packing plan with scheduled resources
self.topology.set_packing_plan(None)
self.assertIsNone(self.topology.id)
self.assertIsNone(self.topology.packing_plan)
packing_plan = MockProto().create_mock_simple_packing_plan2()
self.topology.set_packing_plan(packing_plan)
self.assertEqual(packing_plan, self.topology.packing_plan)
def test_set_execution_state(self):
# Set it to None
self.topology.set_execution_state(None)
self.assertIsNone(self.topology.execution_state)
self.assertIsNone(self.topology.cluster)
self.assertIsNone(self.topology.environ)
estate = MockProto().create_mock_execution_state()
self.topology.set_execution_state(estate)
self.assertEqual(estate, self.topology.execution_state)
self.assertEqual(MockProto.cluster, self.topology.cluster)
self.assertEqual(MockProto.environ, self.topology.environ)
def test_set_tmaster(self):
# Set it to None
self.topology.set_tmaster(None)
self.assertIsNone(self.topology.tmaster)
tmaster = MockProto().create_mock_tmaster()
self.topology.set_tmaster(tmaster)
self.assertEqual(tmaster, self.topology.tmaster)
def test_spouts(self):
# When pplan is not set
self.assertEqual(0, len(self.topology.spouts()))
# Set pplan now
pplan = MockProto().create_mock_simple_physical_plan()
self.topology.set_physical_plan(pplan)
spouts = self.topology.spouts()
self.assertEqual(1, len(spouts))
self.assertEqual("mock_spout", spouts[0].comp.name)
self.assertEqual(["mock_spout"], self.topology.spout_names())
def test_bolts(self):
# When pplan is not set
self.assertEqual(0, len(self.topology.bolts()))
# Set pplan
pplan = MockProto().create_mock_medium_physical_plan()
self.topology.set_physical_plan(pplan)
bolts = self.topology.bolts()
self.assertEqual(3, len(bolts))
self.assertEqual(["mock_bolt1", "mock_bolt2", "mock_bolt3"],
self.topology.bolt_names())
def test_num_instances(self):
# When pplan is not set
self.assertEqual(0, self.topology.num_instances())
pplan = MockProto().create_mock_medium_physical_plan(1, 2, 3, 4)
self.topology.set_physical_plan(pplan)
self.assertEqual(10, self.topology.num_instances())
def test_trigger_watches(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument, unused-variable
def callback(something):
scope["is_called"] = True
uid = self.topology.register_watch(callback)
self.assertTrue(scope["is_called"])
scope["is_called"] = False
self.assertFalse(scope["is_called"])
print(scope)
self.topology.set_physical_plan(None)
print(scope)
self.assertTrue(scope["is_called"])
print(scope)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_execution_state(None)
self.assertTrue(scope["is_called"])
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_tmaster(None)
self.assertTrue(scope["is_called"])
def test_unregister_watch(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument
def callback(something):
scope["is_called"] = True
uid = self.topology.register_watch(callback)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertTrue(scope["is_called"])
self.topology.unregister_watch(uid)
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertFalse(scope["is_called"])
def test_bad_watch(self):
# Workaround
scope = {
"is_called": False
}
# pylint: disable=unused-argument, unused-variable
def callback(something):
scope["is_called"] = True
raise Exception("Test Bad Trigger Exception")
uid = self.topology.register_watch(callback)
# is called the first time because of registeration
self.assertTrue(scope["is_called"])
# But should no longer be called
scope["is_called"] = False
self.assertFalse(scope["is_called"])
self.topology.set_physical_plan(None)
self.assertFalse(scope["is_called"])
def topology(tracker):
return Topology(MockProto.topology_name,
"test_state_manager_name",
tracker.config)
def topologies(tracker):
def create_mock_execution_state(cluster, role, environ):
estate = protoEState.ExecutionState()
estate.cluster = cluster
estate.role = role
estate.environ = environ
return estate
topology1 = Topology('top_name1', 'mock_name1')
topology1.execution_state = create_mock_execution_state('cluster1', 'mark', 'env1')
topology2 = Topology('top_name2', 'mock_name1')
topology2.execution_state = create_mock_execution_state('cluster1', 'bob', 'env1')
topology3 = Topology('top_name3', 'mock_name1')
topology3.execution_state = create_mock_execution_state('cluster1', 'tom', 'env2')
topology4 = Topology('top_name4', 'mock_name2')
topology4.execution_state = create_mock_execution_state('cluster2', 'x', 'env1')
topology5 = Topology('top_name5', 'mock_name2')
topology5.execution_state = create_mock_execution_state('cluster2', 'x', 'env2')
tracker.topologies = [
topology1,
topology2,
topology3,
topology4,
topology5]
return tracker.topologies[:]
