Ejemplo n.º 1
0
    def addNewTopology(self, state_manager, topologyName):
        """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
        topology = Topology(topologyName, state_manager.name)
        Log.info("Adding new topology: %s, state_manager: %s", topologyName,
                 state_manager.name)
        self.topologies.append(topology)

        # Register a watch on topology and change
        # the topologyInfo on any new change.
        topology.register_watch(self.setTopologyInfo)

        def on_topology_pplan(data):
            """watch physical plan"""
            Log.info("Watch triggered for topology pplan: " + topologyName)
            topology.set_physical_plan(data)
            if not data:
                Log.debug("No data to be set")

        def on_topology_packing_plan(data):
            """watch packing plan"""
            Log.info("Watch triggered for topology packing plan: " +
                     topologyName)
            topology.set_packing_plan(data)
            if not data:
                Log.debug("No data to be set")

        def on_topology_execution_state(data):
            """watch execution state"""
            Log.info("Watch triggered for topology execution state: " +
                     topologyName)
            topology.set_execution_state(data)
            if not data:
                Log.debug("No data to be set")

        def on_topology_tmaster(data):
            """set tmaster"""
            Log.info("Watch triggered for topology tmaster: " + topologyName)
            topology.set_tmaster(data)
            if not data:
                Log.debug("No data to be set")

        def on_topology_scheduler_location(data):
            """set scheduler location"""
            Log.info("Watch triggered for topology scheduler location: " +
                     topologyName)
            topology.set_scheduler_location(data)
            if not data:
                Log.debug("No data to be set")

        # Set watches on the pplan, execution_state, tmaster and scheduler_location.
        state_manager.get_pplan(topologyName, on_topology_pplan)
        state_manager.get_packing_plan(topologyName, on_topology_packing_plan)
        state_manager.get_execution_state(topologyName,
                                          on_topology_execution_state)
        state_manager.get_tmaster(topologyName, on_topology_tmaster)
        state_manager.get_scheduler_location(topologyName,
                                             on_topology_scheduler_location)
Ejemplo n.º 2
0
  def fill_tracker_topologies(self):

    def create_mock_execution_state(role):
      estate = protoEState.ExecutionState()
      estate.role = role
      return estate

    self.topology1 = Topology('top_name1', 'mock_name1')
    self.topology1.cluster = 'cluster1'
    self.topology1.environ = 'env1'
    self.topology1.execution_state = create_mock_execution_state('mark')

    self.topology2 = Topology('top_name2', 'mock_name1')
    self.topology2.cluster = 'cluster1'
    self.topology2.environ = 'env1'
    self.topology2.execution_state = create_mock_execution_state('bob')

    self.topology3 = Topology('top_name3', 'mock_name1')
    self.topology3.cluster = 'cluster1'
    self.topology3.environ = 'env2'
    self.topology3.execution_state = create_mock_execution_state('tom')

    self.topology4 = Topology('top_name4', 'mock_name2')
    self.topology4.cluster = 'cluster2'
    self.topology4.environ = 'env1'

    self.topology5 = Topology('top_name5', 'mock_name2')
    self.topology5.cluster = 'cluster2'
    self.topology5.environ = 'env2'
    self.tracker.topologies = [
        self.topology1,
        self.topology2,
        self.topology3,
        self.topology4,
        self.topology5]
Ejemplo n.º 3
0
 def test_extract_packing_plan(self):
     # Create topology
     pb_pplan = MockProto().create_mock_simple_packing_plan()
     topology = Topology('topology_name', 'ExclamationTopology')
     topology.set_packing_plan(pb_pplan)
     # Extract packing plan
     packing_plan = self.tracker.extract_packing_plan(topology)
     self.assertEqual(packing_plan['id'], 'ExclamationTopology')
     self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
     self.assertEqual(
         packing_plan['container_plans'][0]['required_resources'], {
             'disk': 2048,
             'ram': 1024,
             'cpu': 1.0
         })
     self.assertEqual(
         packing_plan['container_plans'][0]['instances'][0], {
             'component_index': 1,
             'component_name': 'word',
             'instance_resources': {
                 'cpu': 1.0,
                 'disk': 2048,
                 'ram': 1024
             },
             'task_id': 1
         })
Ejemplo n.º 4
0
 def get_topologies_for_state_location_side_effect(name):
   if name == 'mock_name1':
     return [Topology('top_name1', 'mock_name1'),
             Topology('top_name2', 'mock_name1')]
   if name == 'mock_name2':
     return [Topology('top_name3', 'mock_name2'),
             Topology('top_name4', 'mock_name2')]
   return []
Ejemplo n.º 5
0
  def addNewTopology(self, state_manager, topologyName):
    """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
    topology = Topology(topologyName, state_manager.name)
    Log.info("Adding new topology: %s, state_manager: %s",
             topologyName, state_manager.name)
    self.topologies.append(topology)

    # Register a watch on topology and change
    # the topologyInfo on any new change.
    topology.register_watch(self.setTopologyInfo)

    def on_topology_pplan(data):
      """watch physical plan"""
      Log.info("Watch triggered for topology pplan: " + topologyName)
      topology.set_physical_plan(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_packing_plan(data):
      """watch packing plan"""
      Log.info("Watch triggered for topology packing plan: " + topologyName)
      topology.set_packing_plan(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_execution_state(data):
      """watch execution state"""
      Log.info("Watch triggered for topology execution state: " + topologyName)
      topology.set_execution_state(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_tmaster(data):
      """set tmaster"""
      Log.info("Watch triggered for topology tmaster: " + topologyName)
      topology.set_tmaster(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_scheduler_location(data):
      """set scheduler location"""
      Log.info("Watch triggered for topology scheduler location: " + topologyName)
      topology.set_scheduler_location(data)
      if not data:
        Log.debug("No data to be set")

    # Set watches on the pplan, execution_state, tmaster and scheduler_location.
    state_manager.get_pplan(topologyName, on_topology_pplan)
    state_manager.get_packing_plan(topologyName, on_topology_packing_plan)
    state_manager.get_execution_state(topologyName, on_topology_execution_state)
    state_manager.get_tmaster(topologyName, on_topology_tmaster)
    state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)
Ejemplo n.º 6
0
 def test_extract_packing_plan(self):
     # Create topology
     pb_pplan = MockProto().create_mock_simple_packing_plan()
     topology = Topology('topology_name', 'ExclamationTopology')
     topology.set_packing_plan(pb_pplan)
     # Extract packing plan
     packing_plan = self.tracker.extract_packing_plan(topology)
     self.assertEqual(packing_plan['id'], 'ExclamationTopology')
     self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
     self.assertEqual(
         packing_plan['container_plans'][0]['required_resources'], {
             'disk': 2048L,
             'ram': 1024L,
             'cpu': 1.0
         })
Ejemplo n.º 7
0
 def test_extract_physical_plan(self):
   # Create topology
   pb_pplan = MockProto().create_mock_simple_physical_plan()
   topology = Topology('topology_name', 'state_manager')
   topology.set_physical_plan(pb_pplan)
   # Extract physical plan
   pplan = self.tracker.extract_physical_plan(topology)
   # Mock topology doesn't have topology config and instances
   self.assertEqual(pplan['config'], {})
   self.assertEqual(pplan['bolts'], {'mock_bolt': []})
   self.assertEqual(pplan['spouts'], {'mock_spout': []})
   self.assertEqual(pplan['components']['mock_bolt']['config'],
                    {'topology.component.parallelism': '1'})
   self.assertEqual(pplan['components']['mock_spout']['config'],
                    {'topology.component.parallelism': '1'})
   self.assertEqual(pplan['instances'], {})
   self.assertEqual(pplan['stmgrs'], {})
Ejemplo n.º 8
0
 def test_extract_physical_plan(self):
   # Create topology
   pb_pplan = MockProto().create_mock_simple_physical_plan()
   topology = Topology('topology_name', 'state_manager')
   topology.set_physical_plan(pb_pplan)
   # Extract physical plan
   pplan = self.tracker.extract_physical_plan(topology)
   # Mock topology doesn't have topology config and instances
   self.assertEqual(pplan['config'], {})
   self.assertEqual(pplan['bolts'], {'mock_bolt': []})
   self.assertEqual(pplan['spouts'], {'mock_spout': []})
   self.assertEqual(pplan['components']['mock_bolt']['config'],
                    {'topology.component.parallelism': '1'})
   self.assertEqual(pplan['components']['mock_spout']['config'],
                    {'topology.component.parallelism': '1'})
   self.assertEqual(pplan['instances'], {})
   self.assertEqual(pplan['stmgrs'], {})
Ejemplo n.º 9
0
  def add_new_topology(self, state_manager, topology_name: str) -> None:
    """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
    topology = Topology(topology_name, state_manager.name)
    Log.info("Adding new topology: %s, state_manager: %s",
             topology_name, state_manager.name)
    self.topologies.append(topology)

    # Register a watch on topology and change
    # the topology_info on any new change.
    topology.register_watch(self.set_topology_info)

    # Set watches on the pplan, execution_state, tmanager and scheduler_location.
    state_manager.get_pplan(topology_name, topology.set_physical_plan)
    state_manager.get_packing_plan(topology_name, topology.set_packing_plan)
    state_manager.get_execution_state(topology_name, topology.set_execution_state)
    state_manager.get_tmanager(topology_name, topology.set_tmanager)
    state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
Ejemplo n.º 10
0
  def add_new_topology(self, state_manager, topology_name: str) -> None:
    """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
    topology = Topology(topology_name, state_manager.name, self.config)
    Log.info("Adding new topology: %s, state_manager: %s",
             topology_name, state_manager.name)
    # populate the cache before making it addressable in the topologies to
    # avoid races due to concurrent execution
    self.topologies.append(topology)

    # Set watches on the pplan, execution_state, tmanager and scheduler_location.
    state_manager.get_pplan(topology_name, topology.set_physical_plan)
    state_manager.get_packing_plan(topology_name, topology.set_packing_plan)
    state_manager.get_execution_state(topology_name, topology.set_execution_state)
    state_manager.get_tmanager(topology_name, topology.set_tmanager)
    state_manager.get_scheduler_location(topology_name, topology.set_scheduler_location)
Ejemplo n.º 11
0
 def setUp(self):
   self.state_manager_name = "test_state_manager_name"
   self.topology = Topology(MockProto.topology_name,
                            self.state_manager_name)
Ejemplo n.º 12
0
class TopologyTest(unittest.TestCase):
  def setUp(self):
    self.state_manager_name = "test_state_manager_name"
    self.topology = Topology(MockProto.topology_name,
                             self.state_manager_name)

  def test_set_physical_plan(self):
    # Set it to None
    self.topology.set_physical_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.physical_plan)

    physical_plan = MockProto().create_mock_simple_physical_plan()
    self.topology.set_physical_plan(physical_plan)
    self.assertEqual(MockProto.topology_id, self.topology.id)
    self.assertEqual(physical_plan, self.topology.physical_plan)

  def test_set_packing_plan(self):
    # Set it to None
    self.topology.set_packing_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.packing_plan)

    packing_plan = MockProto().create_mock_simple_packing_plan()
    self.topology.set_packing_plan(packing_plan)
    self.assertEqual(packing_plan, self.topology.packing_plan)

    # testing with a packing plan with scheduled resources
    self.topology.set_packing_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.packing_plan)

    packing_plan = MockProto().create_mock_simple_packing_plan2()
    self.topology.set_packing_plan(packing_plan)
    self.assertEqual(packing_plan, self.topology.packing_plan)

  def test_set_execution_state(self):
    # Set it to None
    self.topology.set_execution_state(None)
    self.assertIsNone(self.topology.execution_state)
    self.assertIsNone(self.topology.cluster)
    self.assertIsNone(self.topology.environ)

    estate = MockProto().create_mock_execution_state()
    self.topology.set_execution_state(estate)
    self.assertEqual(estate, self.topology.execution_state)
    self.assertEqual(MockProto.cluster, self.topology.cluster)
    self.assertEqual(MockProto.environ, self.topology.environ)

  def test_set_tmanager(self):
    # Set it to None
    self.topology.set_tmanager(None)
    self.assertIsNone(self.topology.tmanager)

    tmanager = MockProto().create_mock_tmanager()
    self.topology.set_tmanager(tmanager)
    self.assertEqual(tmanager, self.topology.tmanager)

  def test_spouts(self):
    # When pplan is not set
    self.assertEqual(0, len(self.topology.spouts()))

    # Set pplan now
    pplan = MockProto().create_mock_simple_physical_plan()
    self.topology.set_physical_plan(pplan)

    spouts = self.topology.spouts()
    self.assertEqual(1, len(spouts))
    self.assertEqual("mock_spout", spouts[0].comp.name)
    self.assertEqual(["mock_spout"], self.topology.spout_names())

  def test_bolts(self):
    # When pplan is not set
    self.assertEqual(0, len(self.topology.bolts()))

    # Set pplan
    pplan = MockProto().create_mock_medium_physical_plan()
    self.topology.set_physical_plan(pplan)

    bolts = self.topology.bolts()
    self.assertEqual(3, len(bolts))
    self.assertEqual(["mock_bolt1", "mock_bolt2", "mock_bolt3"],
                     self.topology.bolt_names())

  def test_num_instances(self):
    # When pplan is not set
    self.assertEqual(0, self.topology.num_instances())

    pplan = MockProto().create_mock_medium_physical_plan(1, 2, 3, 4)
    self.topology.set_physical_plan(pplan)

    self.assertEqual(10, self.topology.num_instances())

  def test_trigger_watches(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument, unused-variable
    def callback(something):
      scope["is_called"] = True
    uid = self.topology.register_watch(callback)
    self.assertTrue(scope["is_called"])

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    print(scope)
    self.topology.set_physical_plan(None)
    print(scope)
    self.assertTrue(scope["is_called"])
    print(scope)

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_execution_state(None)
    self.assertTrue(scope["is_called"])

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_tmanager(None)
    self.assertTrue(scope["is_called"])

  def test_unregister_watch(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument
    def callback(something):
      scope["is_called"] = True
    uid = self.topology.register_watch(callback)
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertTrue(scope["is_called"])

    self.topology.unregister_watch(uid)
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertFalse(scope["is_called"])

  def test_bad_watch(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument, unused-variable
    def callback(something):
      scope["is_called"] = True
      raise Exception("Test Bad Trigger Exception")

    uid = self.topology.register_watch(callback)
    # is called the first time because of registeration
    self.assertTrue(scope["is_called"])

    # But should no longer be called
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertFalse(scope["is_called"])
Ejemplo n.º 13
0
 def setUp(self):
   self.state_manager_name = "test_state_manager_name"
   self.topology = Topology(MockProto.topology_name,
                            self.state_manager_name)
Ejemplo n.º 14
0
class TopologyTest(unittest.TestCase):
  def setUp(self):
    self.state_manager_name = "test_state_manager_name"
    self.topology = Topology(MockProto.topology_name,
                             self.state_manager_name)

  def test_set_physical_plan(self):
    # Set it to None
    self.topology.set_physical_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.physical_plan)

    physical_plan = MockProto().create_mock_simple_physical_plan()
    self.topology.set_physical_plan(physical_plan)
    self.assertEqual(MockProto.topology_id, self.topology.id)
    self.assertEqual(physical_plan, self.topology.physical_plan)

  def test_set_packing_plan(self):
    # Set it to None
    self.topology.set_packing_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.packing_plan)

    packing_plan = MockProto().create_mock_simple_packing_plan()
    self.topology.set_packing_plan(packing_plan)
    self.assertEqual(packing_plan, self.topology.packing_plan)

    # testing with a packing plan with scheduled resources
    self.topology.set_packing_plan(None)
    self.assertIsNone(self.topology.id)
    self.assertIsNone(self.topology.packing_plan)

    packing_plan = MockProto().create_mock_simple_packing_plan2()
    self.topology.set_packing_plan(packing_plan)
    self.assertEqual(packing_plan, self.topology.packing_plan)

  def test_set_execution_state(self):
    # Set it to None
    self.topology.set_execution_state(None)
    self.assertIsNone(self.topology.execution_state)
    self.assertIsNone(self.topology.cluster)
    self.assertIsNone(self.topology.environ)

    estate = MockProto().create_mock_execution_state()
    self.topology.set_execution_state(estate)
    self.assertEqual(estate, self.topology.execution_state)
    self.assertEqual(MockProto.cluster, self.topology.cluster)
    self.assertEqual(MockProto.environ, self.topology.environ)

  def test_set_tmaster(self):
    # Set it to None
    self.topology.set_tmaster(None)
    self.assertIsNone(self.topology.tmaster)

    tmaster = MockProto().create_mock_tmaster()
    self.topology.set_tmaster(tmaster)
    self.assertEqual(tmaster, self.topology.tmaster)

  def test_spouts(self):
    # When pplan is not set
    self.assertEqual(0, len(self.topology.spouts()))

    # Set pplan now
    pplan = MockProto().create_mock_simple_physical_plan()
    self.topology.set_physical_plan(pplan)

    spouts = self.topology.spouts()
    self.assertEqual(1, len(spouts))
    self.assertEqual("mock_spout", spouts[0].comp.name)
    self.assertEqual(["mock_spout"], self.topology.spout_names())

  def test_bolts(self):
    # When pplan is not set
    self.assertEqual(0, len(self.topology.bolts()))

    # Set pplan
    pplan = MockProto().create_mock_medium_physical_plan()
    self.topology.set_physical_plan(pplan)

    bolts = self.topology.bolts()
    self.assertEqual(3, len(bolts))
    self.assertEqual(["mock_bolt1", "mock_bolt2", "mock_bolt3"],
                     self.topology.bolt_names())

  def test_num_instances(self):
    # When pplan is not set
    self.assertEqual(0, self.topology.num_instances())

    pplan = MockProto().create_mock_medium_physical_plan(1, 2, 3, 4)
    self.topology.set_physical_plan(pplan)

    self.assertEqual(10, self.topology.num_instances())

  def test_trigger_watches(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument, unused-variable
    def callback(something):
      scope["is_called"] = True
    uid = self.topology.register_watch(callback)
    self.assertTrue(scope["is_called"])

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    print(scope)
    self.topology.set_physical_plan(None)
    print(scope)
    self.assertTrue(scope["is_called"])
    print(scope)

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_execution_state(None)
    self.assertTrue(scope["is_called"])

    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_tmaster(None)
    self.assertTrue(scope["is_called"])

  def test_unregister_watch(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument
    def callback(something):
      scope["is_called"] = True
    uid = self.topology.register_watch(callback)
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertTrue(scope["is_called"])

    self.topology.unregister_watch(uid)
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertFalse(scope["is_called"])

  def test_bad_watch(self):
    # Workaround
    scope = {
        "is_called": False
    }
    # pylint: disable=unused-argument, unused-variable
    def callback(something):
      scope["is_called"] = True
      raise Exception("Test Bad Trigger Exception")

    uid = self.topology.register_watch(callback)
    # is called the first time because of registeration
    self.assertTrue(scope["is_called"])

    # But should no longer be called
    scope["is_called"] = False
    self.assertFalse(scope["is_called"])
    self.topology.set_physical_plan(None)
    self.assertFalse(scope["is_called"])
Ejemplo n.º 15
0
def topology(tracker):
  return Topology(MockProto.topology_name,
                  "test_state_manager_name",
                  tracker.config)
Ejemplo n.º 16
0
def topologies(tracker):

  def create_mock_execution_state(cluster, role, environ):
    estate = protoEState.ExecutionState()
    estate.cluster = cluster
    estate.role = role
    estate.environ = environ
    return estate

  topology1 = Topology('top_name1', 'mock_name1')
  topology1.execution_state = create_mock_execution_state('cluster1', 'mark', 'env1')

  topology2 = Topology('top_name2', 'mock_name1')
  topology2.execution_state = create_mock_execution_state('cluster1', 'bob', 'env1')

  topology3 = Topology('top_name3', 'mock_name1')
  topology3.execution_state = create_mock_execution_state('cluster1', 'tom', 'env2')

  topology4 = Topology('top_name4', 'mock_name2')
  topology4.execution_state = create_mock_execution_state('cluster2', 'x', 'env1')

  topology5 = Topology('top_name5', 'mock_name2')
  topology5.execution_state = create_mock_execution_state('cluster2', 'x', 'env2')
  tracker.topologies = [
      topology1,
      topology2,
      topology3,
      topology4,
      topology5]
  return tracker.topologies[:]