Exemplo n.º 1
0
    def setUp(self):
        self.store = self.setup_store()
        self.resource_client = MockResourceClient()
        self.epum_client = MockEPUMClient()
        self.registry = EngineRegistry.from_config(self.engine_conf, default='engine1')
        self.notifier = MockNotifier()
        self.service_name = "some_pd"
        self.definition_id = "pd_definition"
        self.definition = get_definition()
        self.base_domain_config = get_domain_config()
        self.run_type = "fake_run_type"
        self.restart_throttling_config = {}

        self.core = ProcessDispatcherCore(self.store, self.registry,
            self.resource_client, self.notifier)

        self.epum_client.add_domain_definition(self.definition_id, self.definition)
        self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
            self.registry, self.epum_client, self.notifier, self.service_name,
            self.definition_id, self.base_domain_config, self.run_type,
            self.restart_throttling_config)

        self.mmthread = None
Exemplo n.º 2
0
class PDMatchmakerTests(unittest.TestCase, StoreTestMixin):

    engine_conf = {
        'engine1': {
            'maximum_vms': 100,
            'slots': 1
        },
        'engine2': {
            'maximum_vms': 100,
            'slots': 2
        },
        'engine3': {
            'maximum_vms': 100,
            'slots': 2,
            'replicas': 2
        },
        'engine4': {
            'maximum_vms': 100,
            'slots': 1,
            'spare_slots': 1
        }

    }

    def setUp(self):
        self.store = self.setup_store()
        self.resource_client = MockResourceClient()
        self.epum_client = MockEPUMClient()
        self.registry = EngineRegistry.from_config(self.engine_conf, default='engine1')
        self.notifier = MockNotifier()
        self.service_name = "some_pd"
        self.definition_id = "pd_definition"
        self.definition = get_definition()
        self.base_domain_config = get_domain_config()
        self.run_type = "fake_run_type"
        self.restart_throttling_config = {}

        self.core = ProcessDispatcherCore(self.store, self.registry,
            self.resource_client, self.notifier)

        self.epum_client.add_domain_definition(self.definition_id, self.definition)
        self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
            self.registry, self.epum_client, self.notifier, self.service_name,
            self.definition_id, self.base_domain_config, self.run_type,
            self.restart_throttling_config)

        self.mmthread = None

    def tearDown(self):
        if self.mmthread:
            self.mm.cancel()
            self.mmthread.join()
            self.mmthread = None
        self.teardown_store()

    def setup_store(self):
        return ProcessDispatcherStore()

    def teardown_store(self):
        return

    def _run_in_thread(self):
        self.mmthread = tevent.spawn(self.mm.inaugurate)
        time.sleep(0.05)

    def test_run_cancel(self):
        self._run_in_thread()

        self.mm.cancel()
        self.mmthread.join()
        self.mmthread = None

    def test_match_writeconflict(self):
        self.mm.initialize()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        # now update the resource record so the matchmake() attempt to write will conflict
        r1.assigned = ["hats"]
        self.store.update_resource(r1)

        # this should bail out without resetting the needs_matchmaking flag
        # or registering any need
        self.assertTrue(self.mm.needs_matchmaking)
        self.mm.matchmake()
        self.assertFalse(self.epum_client.reconfigures)
        self.assertTrue(self.mm.needs_matchmaking)

        r1copy = self.store.get_resource(r1.resource_id)
        self.assertRecordVersions(r1, r1copy)

    def test_match_notfound(self):
        self.mm.initialize()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        # now update the resource record so the matchmake() attempt to write will conflict
        self.store.remove_resource("r1")

        # this should bail out without resetting the needs_matchmaking flag
        # or registering any need
        self.assertTrue(self.mm.needs_matchmaking)
        self.mm.matchmake()
        self.assertFalse(self.epum_client.reconfigures)
        self.assertTrue(self.mm.needs_matchmaking)

    def test_match_process_terminated(self):
        self.mm.initialize()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
            ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        # now update the process record to be TERMINATED so that
        # MM should bail out of matching this process

        p1.state = ProcessState.TERMINATED
        self.store.update_process(p1)
        self.store.remove_queued_process(*p1key)

        self.mm.matchmake()

        p1 = self.store.get_process(None, "p1")
        self.assertEqual(p1.state, ProcessState.TERMINATED)

    def test_match1(self):
        self._run_in_thread()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)

        self.store.enqueue_process(*p1key)

        self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned)
        time.sleep(0.05)
        self.resource_client.check_process_launched(p1, r1.resource_id)
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.assigned == r1.resource_id and
                                    p.state == ProcessState.PENDING)

    def test_match_double_queued_process(self):

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)
        r2 = ResourceRecord.new("r2", "n1", 1, properties=props)
        self.store.add_resource(r2)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)

        p2 = ProcessRecord.new(None, "p2", get_process_definition(),
                               ProcessState.REQUESTED)
        p2key = p2.get_key()
        self.store.add_process(p2)

        # enqueue p1 repeatedly. make sure that doesn't bomb anything
        self.store.enqueue_process(*p1key)
        self.store.enqueue_process(*p1key)
        self.store.enqueue_process(*p2key)
        self.store.enqueue_process(*p1key)

        self._run_in_thread()
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.state == ProcessState.PENDING)
        self.wait_process(p2.owner, p2.upid,
                          lambda p: p.state == ProcessState.PENDING)

    def test_node_exclusive_bug(self):
        """test_node_exclusive_bug

        If two processes with the same node exclusive attribute where scheduled
        in the same matchmaking cycle, they could be scheduled to the same
        resource, due to a caching issue. This test tests the fix.
        """
        self.mm.initialize()

        n1 = NodeRecord.new("n1", "d1")
        self.store.add_node(n1)

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 2, properties=props)
        self.store.add_resource(r1)

        n2 = NodeRecord.new("n2", "d1")
        self.store.add_node(n2)

        props = {"engine": "engine1"}
        r2 = ResourceRecord.new("r2", "n2", 2, properties=props)
        self.store.add_resource(r2)

        xattr_1 = "port5000"
        constraints = {}
        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_1)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        p2 = ProcessRecord.new(None, "p2", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_1)
        p2key = p2.get_key()
        self.store.add_process(p2)
        self.store.enqueue_process(*p2key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.mm.matchmake()

        # Ensure these processes are pending and scheduled to different nodes

        p1 = self.store.get_process(None, "p1")
        p2 = self.store.get_process(None, "p2")
        self.assertNotEqual(p1.assigned, p2.assigned)

    def test_node_exclusive(self):
        self._run_in_thread()

        n1 = NodeRecord.new("n1", "d1")
        self.store.add_node(n1)

        props = {"engine": "engine1"}
        n1_r1 = ResourceRecord.new("n1_r1", "n1", 2, properties=props)
        self.store.add_resource(n1_r1)

        n1_r2 = ResourceRecord.new("n1_r2", "n1", 2, properties=props)
        self.store.add_resource(n1_r2)

        xattr_1 = "port5000"
        constraints = {}
        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_1)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # The first process should be assigned, since nothing else needs this
        # attr
        # TODO: it's possible that this could be assigned to n1_r2, but hopefully not
        self.wait_resource(n1_r1.resource_id, lambda r: list(p1key) in r.assigned)
        time.sleep(0.05)
        self.resource_client.check_process_launched(p1, n1_r1.resource_id)
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.assigned == n1_r1.resource_id and
                                    p.state == ProcessState.PENDING)

        p2 = ProcessRecord.new(None, "p2", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_1)
        p2key = p2.get_key()
        self.store.add_process(p2)
        self.store.enqueue_process(*p2key)

        # The second process should wait, since first process wants this attr
        # as well
        self.wait_process(p2.owner, p2.upid,
                          lambda p: p.state == ProcessState.WAITING)

        # If we start another node, we should see that second process be
        # scheduled
        n2 = NodeRecord.new("n2", "d1")
        self.store.add_node(n2)

        props = {"engine": "engine1"}
        n2_r1 = ResourceRecord.new("n2_r1", "n2", 2, properties=props)
        self.store.add_resource(n2_r1)

        props = {"engine": "engine1"}
        n2_r2 = ResourceRecord.new("n2_r2", "n2", 2, properties=props)
        self.store.add_resource(n2_r2)

        # The second process should now be assigned
        self.wait_resource(n2_r1.resource_id, lambda r: list(p2key) in r.assigned)
        time.sleep(0.05)
        self.resource_client.check_process_launched(p2, n2_r1.resource_id)
        self.wait_process(p2.owner, p2.upid,
                          lambda p: p.assigned == n2_r1.resource_id and
                                    p.state == ProcessState.PENDING)

        # Now we submit another process with a different exclusive attribute
        # It should be assigned right away
        xattr_2 = "port5001"
        constraints = {}
        p3 = ProcessRecord.new(None, "p3", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_2)
        p3key = p3.get_key()
        self.store.add_process(p3)
        self.store.enqueue_process(*p3key)

        p3_resource = None
        for resource in [n1_r1, n1_r2, n2_r1, n2_r2]:
            try:
                self.wait_resource(resource.resource_id, lambda r: list(p3key) in r.assigned,
                    timeout=0.5)
            except Exception:
                continue
            time.sleep(0.05)
            self.resource_client.check_process_launched(p3, resource.resource_id)
            self.wait_process(p3.owner, p3.upid,
                              lambda p: p.assigned == resource.resource_id and
                                        p.state == ProcessState.PENDING)
            p3_resource = resource

        self.assertIsNotNone(p3_resource)

        # Now submit a fourth process, which should be scheduled to a different
        # node from p3
        p4 = ProcessRecord.new(None, "p4", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints,
                               node_exclusive=xattr_2)
        p4key = p4.get_key()
        self.store.add_process(p4)
        self.store.enqueue_process(*p4key)

        p4_resource = None
        for resource in [n1_r1, n1_r2, n2_r1, n2_r2]:
            try:
                self.wait_resource(resource.resource_id, lambda r: list(p4key) in r.assigned,
                    timeout=0.5)
            except Exception:
                continue
            time.sleep(0.05)
            self.resource_client.check_process_launched(p4, resource.resource_id)
            self.wait_process(p4.owner, p4.upid,
                              lambda p: p.assigned == resource.resource_id and
                                        p.state == ProcessState.PENDING)
            p4_resource = resource

        self.assertIsNotNone(p4_resource)

        self.assertNotEqual(p3_resource.node_id, p4_resource.node_id)

    def test_node_filo(self):
        """test_node_filo

        We prioritize shutting down the newest VMs as a workaround for OOI
        Testing strategy
        """
        self.mm.initialize()

        n1 = NodeRecord.new("n1", "d1")
        self.store.add_node(n1)

        props = {"engine": "engine4"}
        r1 = ResourceRecord.new("r1", "n1", 2, properties=props)
        self.store.add_resource(r1)

        n2 = NodeRecord.new("n2", "d1")
        self.store.add_node(n2)

        props = {"engine": "engine4"}
        r2 = ResourceRecord.new("r2", "n2", 2, properties=props)
        self.store.add_resource(r2)

        constraints = {"engine": "engine4"}
        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.mm.register_needs()
        self.epum_client.clear()

        self.mm.queued_processes = []

        self.mm.register_needs()
        conf = self.epum_client.reconfigures['pd_domain_engine4'][0]
        retired_nodes = conf['engine_conf']['retirable_nodes']
        assert len(retired_nodes) == 1

        # This should be the second node we started
        assert retired_nodes[0] == "n2"

    def test_match_copy_hostname(self):
        self._run_in_thread()

        props = {"engine": "engine1", "hostname": "vm123"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
            ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)

        self.store.enqueue_process(*p1key)
        self.wait_process(p1.owner, p1.upid,
            lambda p: p.assigned == r1.resource_id and
                      p.state == ProcessState.PENDING)
        p1 = self.store.get_process(None, "p1")
        self.assertEqual(p1.hostname, "vm123")

    def test_waiting(self):
        self._run_in_thread()

        # not-immediate process enqueued while there are no resources

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)

        self.store.enqueue_process(*p1key)
        self.wait_process(None, "p1", lambda p: p.state == ProcessState.WAITING)

        # now give it a resource. it should be scheduled
        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned)
        time.sleep(0.05)
        self.resource_client.check_process_launched(p1, r1.resource_id)

    def test_process_terminated(self):
        self._run_in_thread()

        event = threading.Event()

        # we set up a resource and a matching process that should be assigned
        # to it. we will simulate marking the process TERMINATED out-of-band
        # and ensure that is recognized before the dispatch.

        # when the matchmaker attempts to update the process, sneak in an update
        # first so the matchmaker request conflicts
        original_update_process = self.store.update_process

        def patched_update_process(process):
            original = self.store.get_process(process.owner, process.upid)
            original.state = ProcessState.TERMINATED
            original_update_process(original)

            try:
                original_update_process(process)
            finally:
                event.set()

        self.store.update_process = patched_update_process

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
            ProcessState.REQUESTED)
        self.store.add_process(p1)
        self.store.enqueue_process(*p1.key)

        # now give it a resource. it should be matched but in the meantime
        # the process will be terminated
        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        # wait for MM to hit our update conflict, kill it, and check that it
        # appropriately backed out the allocation
        assert event.wait(5)
        self.mm.cancel()
        self.mmthread.join()
        self.mmthread = None

        resource = self.store.get_resource("r1")
        self.assertEqual(len(resource.assigned), 0)

        self.assertEqual(self.resource_client.launch_count, 0)

    def test_process_already_assigned(self):

        # this is a recovery situation, probably. The process is assigned
        # to a resource already at the start of the matchmaker run, but
        # the process record hasn't been updated to reflect that. The
        # situation should be detected and handled by matchmaker.

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
            ProcessState.REQUESTED)
        self.store.add_process(p1)
        self.store.enqueue_process(*p1.key)

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        r1.assigned.append(p1.key)
        self.store.add_resource(r1)

        self._run_in_thread()

        self.wait_process(None, "p1", lambda p: p.state == ProcessState.PENDING)

        r1 = self.store.get_resource("r1")
        self.assertEqual(len(r1.assigned), 1)
        self.assertTrue(r1.is_assigned(p1.owner, p1.upid, p1.round))
        self.assertEqual(r1.available_slots, 0)

    def test_wait_resource(self):
        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)
        self.wait_resource("r1", lambda r: r.resource_id == "r1")

        def makeitso():
            r1.slot_count = 2
            self.store.update_resource(r1)

        tevent.spawn(makeitso)

        self.wait_resource("r1", lambda r: r.slot_count == 2)

    def test_disabled_resource(self):
        self._run_in_thread()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        r1.state = ExecutionResourceState.DISABLED

        self.store.add_resource(r1)
        self.wait_resource("r1", lambda r: r.resource_id == "r1")

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.key
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # the resource matches but it is disabled, process should
        # remain in the queue
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.state == ProcessState.WAITING)

    def test_queueing_order(self):
        self._run_in_thread()

        procnames = []
        # queue 10 processes
        for i in range(10):
            proc = ProcessRecord.new(None, "proc" + str(i), get_process_definition(),
                                     ProcessState.REQUESTED)
            prockey = proc.key
            self.store.add_process(proc)
            self.store.enqueue_process(*prockey)
            self.epum_client.clear()

            self.wait_process(proc.owner, proc.upid,
                              lambda p: p.state == ProcessState.WAITING)
            procnames.append(proc.upid)

            # potentially retry a few times to account for race between process
            # state updates and need reconfigures
            for i in range(5):
                try:
                    self.assert_one_reconfigure(preserve_n=i + 1, retirees=[])
                    break
                except AssertionError:
                    time.sleep(0.01)

            self.epum_client.clear()

        # now add 10 resources each with 1 slot. processes should start in order
        for i in range(10):
            props = {"engine": "engine1"}
            res = ResourceRecord.new("res" + str(i), "node" + str(i), 1,
                    properties=props)
            self.store.add_resource(res)

            self.wait_process(None, procnames[i],
                              lambda p: p.state >= ProcessState.PENDING and
                                        p.assigned == res.resource_id)

        # finally doublecheck that launch requests happened in order too
        for i in range(5):
            try:
                self.assertEqual(self.resource_client.launch_count, 10)
                for i, launch in enumerate(self.resource_client.launches):
                    self.assertEqual(launch[0], "res" + str(i))
                    self.assertEqual(launch[1], "proc" + str(i))

                break
            except AssertionError:
                time.sleep(0.01)

    def assert_one_reconfigure(self, domain_id=None, preserve_n=None, retirees=None):
        if domain_id is not None:
            reconfigures = self.epum_client.reconfigures[domain_id]
        else:
            reconfigures = self.epum_client.reconfigures.values()
            self.assertTrue(reconfigures)
            reconfigures = reconfigures[0]
        self.assertEqual(len(reconfigures), 1)
        reconfigure = reconfigures[0]
        engine_conf = reconfigure['engine_conf']
        if preserve_n is not None:
            self.assertEqual(engine_conf['preserve_n'], preserve_n)
        if retirees is not None:
            retirables = engine_conf.get('retirable_nodes', [])
            self.assertEqual(set(retirables), set(retirees))

    def enqueue_n_processes(self, n_processes, engine_id):
        pkeys = []
        for pid in range(n_processes):
            upid = uuid.uuid4().hex
            p = ProcessRecord.new(None, upid, get_process_definition(),
                ProcessState.REQUESTED, constraints={'engine': engine_id})
            pkey = p.get_key()
            pkeys.append(pkey)
            self.store.add_process(p)
            self.store.enqueue_process(*pkey)
            self.mm.queued_processes.append(pkey)
        return pkeys

    def create_n_pending_processes(self, n_processes, engine_id):
        pkeys = []
        for pid in range(n_processes):
            upid = uuid.uuid4().hex
            p = ProcessRecord.new(None, upid, get_process_definition(),
                ProcessState.UNSCHEDULED_PENDING, constraints={'engine': engine_id})
            self.store.add_process(p)
            pkey = p.get_key()
            pkeys.append(pkey)
        return pkeys

    def enqueue_pending_processes(self):
        """enqueue_pending_processes

        Normally, this would be done by the doctor in a full system
        """
        pkeys = []
        for pid in self.store.get_process_ids():
            owner, upid = pid
            process = self.store.get_process(owner, upid)
            if process.state == ProcessState.UNSCHEDULED_PENDING:
                pkey = process.get_key()
                process.state = ProcessState.REQUESTED
                self.store.update_process(process)
                self.store.enqueue_process(*pkey)
                self.mm.queued_processes.append(pkey)
                pkeys.append(pkey)
        return pkeys

    def create_engine_resources(self, engine_id, node_count=1, assignments=None):
        engine_spec = self.registry.get_engine_by_id(engine_id)
        assert len(assignments) <= engine_spec.slots * engine_spec.replicas * node_count

        records = []
        for i in range(node_count):
            node_id = uuid.uuid4().hex
            props = {"engine": engine_id}
            for i in range(engine_spec.replicas):
                res = ResourceRecord.new(uuid.uuid4().hex, node_id,
                    engine_spec.slots, properties=props)
                records.append(res)
                res.metadata['version'] = 0
                self.mm.resources[res.resource_id] = res

                # use fake process ids in the assigned list, til it matters
                if len(assignments) <= engine_spec.slots:
                    res.assigned = list(assignments)
                    assignments = []
                else:
                    res.assigned = assignments[:engine_spec.slots]
                    assignments[:] = assignments[engine_spec.slots:]

                print "added resource: %s" % res
        return records

    def test_engine_config(self):
        self.mm.cancel()
        maximum_vms = 10
        base_iaas_allocation = 'm1.small'
        engine_iaas_allocation = 'm1.medium'
        self.base_domain_config = {
            'engine_conf': {
                'iaas_allocation': base_iaas_allocation
            }
        }

        engine_conf = {
            'engine1': {
                'slots': 1,
                'iaas_allocation': engine_iaas_allocation,
                'maximum_vms': maximum_vms
            },
            'engine2': {
                'slots': 2,
                'iaas_allocation': engine_iaas_allocation,
                'maximum_vms': maximum_vms
            },
        }

        self.registry = EngineRegistry.from_config(engine_conf, default='engine1')
        self.mm = PDMatchmaker(self.core, self.store, self.resource_client,
            self.registry, self.epum_client, self.notifier, self.service_name,
            self.definition_id, self.base_domain_config, self.run_type,
            self.restart_throttling_config)

        self.mm.initialize()
        self.assertEqual(len(self.epum_client.domains), len(engine_conf.keys()))

        engine1_domain_conf = self.epum_client.domains['pd_domain_engine1']['engine_conf']
        engine2_domain_conf = self.epum_client.domains['pd_domain_engine2']['engine_conf']

        self.assertEqual(engine1_domain_conf['iaas_allocation'], engine_iaas_allocation)
        self.assertEqual(engine2_domain_conf['iaas_allocation'], engine_iaas_allocation)

        self.assertEqual(engine1_domain_conf['maximum_vms'], maximum_vms)
        self.assertEqual(engine2_domain_conf['maximum_vms'], maximum_vms)

    def test_needs(self):
        self.mm.initialize()

        self.assertFalse(self.epum_client.reconfigures)
        self.assertEqual(len(self.epum_client.domains), len(self.engine_conf.keys()))

        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            self.assertEqual(self.epum_client.domain_subs[domain_id],
                [(self.service_name, "node_state")])

        self.mm.register_needs()
        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            engine = self.engine_conf[engine_id]
            if engine.get('spare_slots', 0) == 0:
                self.assert_one_reconfigure(domain_id, 0, [])
        self.epum_client.clear()

        engine1_domain_id = domain_id_from_engine("engine1")
        engine2_domain_id = domain_id_from_engine("engine2")
        engine3_domain_id = domain_id_from_engine("engine3")
        engine4_domain_id = domain_id_from_engine("engine4")

        # engine1 has 1 slot and 1 replica per node, expect a VM per process
        engine1_procs = self.enqueue_n_processes(10, "engine1")

        # engine2 has 2 slots and 1 replica per node, expect a VM per 2 processes
        engine2_procs = self.enqueue_n_processes(10, "engine2")

        # engine3 has 2 slots and 2 replicas per node, expect a VM per 4 processes
        engine3_procs = self.enqueue_n_processes(10, "engine3")

        # engine4 has 2 slots and 1 replica per node, and a
        # minimum of 1 free slot, expect a VM per process + 1
        engine4_procs = self.enqueue_n_processes(10, "engine4")

        self.mm.register_needs()
        self.assert_one_reconfigure(engine1_domain_id, 10, [])
        self.assert_one_reconfigure(engine2_domain_id, 5, [])
        self.assert_one_reconfigure(engine3_domain_id, 3, [])
        self.assert_one_reconfigure(engine4_domain_id, 11, [])
        self.epum_client.clear()

        # now add some resources with assigned processes
        # and removed queued processes. need shouldn't change.
        engine1_resources = self.create_engine_resources("engine1",
            node_count=10, assignments=engine1_procs)
        self.assertEqual(len(engine1_resources), 10)
        engine2_resources = self.create_engine_resources("engine2",
            node_count=5, assignments=engine2_procs)
        self.assertEqual(len(engine2_resources), 5)
        engine3_resources = self.create_engine_resources("engine3",
            node_count=3, assignments=engine3_procs)
        self.assertEqual(len(engine3_resources), 6)
        engine4_resources = self.create_engine_resources("engine4",
            node_count=11, assignments=engine4_procs)
        self.assertEqual(len(engine4_resources), 11)
        self.mm.queued_processes = []

        self.mm.register_needs()
        self.assertFalse(self.epum_client.reconfigures)

        # now try scale down

        # empty 2 resources from engine1. 2 nodes should be terminated.
        engine1_retirees = set()
        for resource in engine1_resources[:2]:
            engine1_retirees.add(resource.node_id)
            resource.assigned = []

        # empty 2 resources from engine2. 2 nodes should be terminated
        engine2_retirees = set()
        for resource in engine2_resources[:2]:
            engine2_retirees.add(resource.node_id)
            resource.assigned = []

        # empty 3 resources from engine3.  1 node should be terminated
        for resource in engine3_resources[:3]:
            resource.assigned = []
        engine3_retirees = set([engine3_resources[0].node_id])

        # empty 2 resources from engine4.  2 nodes should be terminated
        engine4_retirees = set()
        for resource in engine4_resources:
            if len(resource.assigned) > 0:
                engine4_retirees.add(resource.node_id)
                resource.assigned = []

            if len(engine4_retirees) >= 2:
                break

        self.mm.register_needs()
        self.assert_one_reconfigure(engine1_domain_id, 8,
            engine1_retirees)
        self.assert_one_reconfigure(engine2_domain_id, 3,
            engine2_retirees)
        self.assert_one_reconfigure(engine3_domain_id, 2,
            engine3_retirees)
        # Note that we cannot check which nodes have retired, since the spare
        # one may be terminated
        self.assert_one_reconfigure(engine4_domain_id, 9)
        self.epum_client.clear()

    def test_needs_maximum_vms(self):
        self.mm.initialize()

        self.assertFalse(self.epum_client.reconfigures)
        self.assertEqual(len(self.epum_client.domains), len(self.engine_conf.keys()))

        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            self.assertEqual(self.epum_client.domain_subs[domain_id],
                [(self.service_name, "node_state")])

        self.mm.register_needs()
        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            engine = self.engine_conf[engine_id]
            if engine.get('spare_slots', 0) == 0:
                self.assert_one_reconfigure(domain_id, 0, [])
        self.epum_client.clear()

        engine1_domain_id = domain_id_from_engine("engine1")
        engine2_domain_id = domain_id_from_engine("engine2")

        # engine1 has 1 slot and 1 replica per node, expect a VM per process
        self.enqueue_n_processes(10, "engine1")

        # engine2 has 2 slots and 1 replica per node, expect a VM per 2 processes
        self.enqueue_n_processes(10, "engine2")

        self.mm.register_needs()
        self.assert_one_reconfigure(engine1_domain_id, 10, [])
        self.assert_one_reconfigure(engine2_domain_id, 5, [])
        self.epum_client.clear()

        # engine1 has 1 slot and 1 replica per node, expect a VM per process, normally
        self.enqueue_n_processes(1000, "engine1")

        # engine2 has 2 slots and 1 replica per node, expect a VM per 2 processes, normally
        self.enqueue_n_processes(1000, "engine2")

        # But, we set a maximum of 100 VMs, so even though we have thousands of processes
        # queued, we only start 100 VMs total
        self.mm.register_needs()
        self.assert_one_reconfigure(engine1_domain_id, 100, [])
        self.assert_one_reconfigure(engine2_domain_id, 100, [])
        self.epum_client.clear()

    def test_needs_unscheduled_pending(self):

        # engine1 has 1 slot, expect a VM per process
        engine1_pending_procs = self.create_n_pending_processes(10, "engine1")

        # engine2 has 2 slots, expect a VM per 2 processes
        engine2_pending_procs = self.create_n_pending_processes(10, "engine2")

        # Normally this is done by the doctor, but we do it manually here,
        # since there is no doctor in this test env
        self.store.set_initialized()
        self.store.set_pd_state(ProcessDispatcherState.SYSTEM_BOOTING)

        self.mm.initialize()

        self.assertFalse(self.epum_client.reconfigures)
        self.assertEqual(len(self.epum_client.domains), len(self.engine_conf.keys()))

        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            self.assertEqual(self.epum_client.domain_subs[domain_id],
                [(self.service_name, "node_state")])

        engine1_domain_id = domain_id_from_engine("engine1")
        engine2_domain_id = domain_id_from_engine("engine2")

        self.mm.register_needs()

        # we should see VMs even though we have no queued procs
        self.assert_one_reconfigure(engine1_domain_id, 10, [])
        self.assert_one_reconfigure(engine2_domain_id, 5, [])

        self.epum_client.clear()

        # engine1 has 1 slot, expect a VM per process
        engine1_queued_procs = self.enqueue_n_processes(10, "engine1")

        # engine2 has 2 slots, expect a VM per 2 processes
        engine2_queued_procs = self.enqueue_n_processes(10, "engine2")

        self.mm.register_needs()

        # we should see for the queued and pending procs
        self.assert_one_reconfigure(engine1_domain_id, 20, [])
        self.assert_one_reconfigure(engine2_domain_id, 10, [])
        self.epum_client.clear()

        # When we enqueue the procs and mark the PD OK
        self.enqueue_pending_processes()
        self.mm.register_needs()

        # The matchmaker won't have checked for the updated pending procs
        self.assertEqual(len(self.mm.unscheduled_pending_processes), 20)

        # But there should be no change to requested VMs, since we should
        # deduplicate processes
        self.assertFalse(self.epum_client.reconfigures)

        self.store.set_pd_state(ProcessDispatcherState.OK)

        self.mm.register_needs()

        # The matchmaker should have no pending processes
        self.assertEqual(len(self.mm.unscheduled_pending_processes), 0)

        # There should be no change to requested VMs
        self.assertFalse(self.epum_client.reconfigures)

        self.epum_client.clear()

        # now add some resources with assigned processes
        # and removed queued processes. need shouldn't change.
        engine1_procs = engine1_queued_procs + engine1_pending_procs
        engine2_procs = engine2_queued_procs + engine2_pending_procs
        engine1_resources = self.create_engine_resources("engine1",
            node_count=20, assignments=engine1_procs)
        self.assertEqual(len(engine1_resources), 20)
        engine2_resources = self.create_engine_resources("engine2",
            node_count=10, assignments=engine2_procs)
        self.assertEqual(len(engine2_resources), 10)
        self.mm.queued_processes = []

        self.mm.register_needs()
        self.assertFalse(self.epum_client.reconfigures)

        # empty resources from engine1. all nodes should be terminated.
        engine1_retirees = set()
        for resource in engine1_resources:
            engine1_retirees.add(resource.node_id)
            resource.assigned = []

        # empty resources from engine2. all nodes should be terminated
        engine2_retirees = set()
        for resource in engine2_resources:
            engine2_retirees.add(resource.node_id)
            resource.assigned = []

        self.mm.register_needs()

        # we should see for the queued and pending procs
        self.assert_one_reconfigure(engine1_domain_id, 0, engine1_retirees)
        self.assert_one_reconfigure(engine2_domain_id, 0, engine2_retirees)
        self.epum_client.clear()

    def test_needs_duplicate_process(self):

        # ensure processes represented in queue and in a resource are not
        # counted twice. This situation arises in between process and resource
        # record updates.
        self.mm.initialize()

        self.assertFalse(self.epum_client.reconfigures)
        self.assertEqual(len(self.epum_client.domains), len(self.engine_conf.keys()))

        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            self.assertEqual(self.epum_client.domain_subs[domain_id],
                [(self.service_name, "node_state")])

        self.mm.register_needs()
        for engine_id in self.engine_conf:
            domain_id = domain_id_from_engine(engine_id)
            engine = self.engine_conf[engine_id]
            if engine.get('spare_slots', 0) == 0:
                self.assert_one_reconfigure(domain_id, 0, [])
        self.epum_client.clear()

        engine1_domain_id = domain_id_from_engine("engine1")

        engine1_procs = self.enqueue_n_processes(10, "engine1")

        one_process_key = engine1_procs[0]
        owner, upid, rround = one_process_key

        self.mm.register_needs()
        self.assert_one_reconfigure(engine1_domain_id, 10, [])
        self.epum_client.clear()

        # now add some resources with assigned processes
        # and removed queued processes. need shouldn't change.
        engine1_resources = self.create_engine_resources("engine1",
            node_count=10, assignments=engine1_procs)
        self.assertEqual(len(engine1_resources), 10)
        self.mm.queued_processes = []

        self.mm.register_needs()
        self.assertFalse(self.epum_client.reconfigures)

        # now pretend one process fails and is requeued
        # the requue can happen before the resource update so we
        # simulate this to ensure that the process isn't counted twice

        self.mm.queued_processes = [(owner, upid, rround + 1)]
        self.mm.register_needs()
        self.assertFalse(self.epum_client.reconfigures)

    @attr('INT')
    def test_engine_types(self):
        self._run_in_thread()

        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        constraints = {"engine": "engine2"}
        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED, constraints=constraints)
        p1key = p1.get_key()
        self.store.add_process(p1)

        self.store.enqueue_process(*p1key)

        # We don't have a resource that can run this yet
        timed_out = False
        try:
            self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned,
                    timeout=2)
        except Exception:
            timed_out = True
        assert timed_out

        props = {"engine": "engine2"}
        r2 = ResourceRecord.new("r2", "n2", 1, properties=props)
        self.store.add_resource(r2)

        self.wait_resource(r2.resource_id, lambda r: list(p1key) in r.assigned)

        time.sleep(0.05)
        self.resource_client.check_process_launched(p1, r2.resource_id)
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.assigned == r2.resource_id and
                                    p.state == ProcessState.PENDING)

    @attr('INT')
    def test_default_engine_types(self):
        self._run_in_thread()

        props = {"engine": self.registry.default}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)

        self.store.enqueue_process(*p1key)

        self.wait_resource(r1.resource_id, lambda r: list(p1key) in r.assigned)

        time.sleep(0.05)
        self.resource_client.check_process_launched(p1, r1.resource_id)
        self.wait_process(p1.owner, p1.upid,
                          lambda p: p.assigned == r1.resource_id and
                                    p.state == ProcessState.PENDING)

    @attr('INT')
    def test_stale_procs(self):
        """test that the matchmaker doesn't try to schedule stale procs

        A stale proc is one that the matchmaker has attempted to scale before
        while the state have the resources hasn't changed.
        """
        if not os.environ.get('INT'):
            raise SkipTest("Skip slow integration test")

        self.mm.initialize()

        p1 = ProcessRecord.new(None, "p1", get_process_definition(),
                               ProcessState.REQUESTED)
        p1key = p1.get_key()
        self.store.add_process(p1)
        self.store.enqueue_process(*p1key)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.assertTrue(self.mm.needs_matchmaking)
        self.mm.matchmake()
        self.assertFalse(self.mm.needs_matchmaking)
        self.assertTrue(len(self.mm.stale_processes) > 0)

        self.mm._get_queued_processes()
        self.mm._get_resource_set()
        self.assertFalse(self.mm.needs_matchmaking)
        self.assertTrue(len(self.mm.stale_processes) > 0)

        p2 = ProcessRecord.new(None, "p2", get_process_definition(),
                               ProcessState.REQUESTED)
        p2key = p2.get_key()
        self.store.add_process(p2)
        self.store.enqueue_process(*p2key)

        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.assertTrue(self.mm.needs_matchmaking)
        self.assertTrue(len(self.mm.stale_processes) > 0)
        self.assertTrue(len(self.mm.queued_processes) > len(self.mm.stale_processes))

        self.mm.matchmake()

        self.assertFalse(self.mm.needs_matchmaking)
        self.assertTrue(len(self.mm.queued_processes) == len(self.mm.stale_processes))

        # Add a resource, and ensure that stale procs get dumped
        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        self.mm._get_queued_processes()
        self.mm._get_resources()
        self.mm._get_resource_set()

        self.assertTrue(len(self.mm.stale_processes) == 0)

    @attr('INT')
    def test_stale_optimization(self):
        # DL: not sure this test is really relevant anymore. It often fails
        # against zookeeper because the ratio isn't as good.
        raise SkipTest("Skip manual optimization test")

        from time import clock

        self.mm.initialize()

        n_start_proc = 10000

        for i in range(0, n_start_proc):

            p = ProcessRecord.new(None, "p%s" % i, get_process_definition(),
                                   ProcessState.REQUESTED)
            pkey = p.get_key()
            self.store.add_process(p)
            self.store.enqueue_process(*pkey)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.assertTrue(self.mm.needs_matchmaking)
        unoptimized_start = clock()
        self.mm.matchmake()
        unoptimized_end = clock()
        unoptimized_time = unoptimized_end - unoptimized_start
        self.assertFalse(self.mm.needs_matchmaking)
        self.assertTrue(len(self.mm.stale_processes) > 0)

        p = ProcessRecord.new(None, "px", get_process_definition(),
                               ProcessState.REQUESTED)
        pkey = p.get_key()
        self.store.add_process(p)
        self.store.enqueue_process(*pkey)

        # sneak into MM and force it to update this info from the store
        self.mm._get_queued_processes()
        self.mm._get_resource_set()

        self.assertTrue(self.mm.needs_matchmaking)
        optimized_start = clock()
        self.mm.matchmake()
        optimized_end = clock()
        optimized_time = optimized_end - optimized_start

        if optimized_time > 0:
            ratio = unoptimized_time / optimized_time
            print "Unoptimised Time: %s Optimised Time: %s ratio: %s" % (
                unoptimized_time, optimized_time, ratio)
            self.assertTrue(ratio >= 100,
                    "Our optimized matchmake didn't have a 100 fold improvement")
        else:
            print "optimized_time was zero. hmm"

        # Add a resource, and ensure that matchmake time is unoptimized
        props = {"engine": "engine1"}
        r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
        self.store.add_resource(r1)

        self.mm._get_queued_processes()
        self.mm._get_resources()
        self.mm._get_resource_set()

        self.assertTrue(self.mm.needs_matchmaking)
        addresource_start = clock()
        self.mm.matchmake()
        addresource_end = clock()
        addresource_time = addresource_end - addresource_start

        optimized_addresource_ratio = unoptimized_time / addresource_time
        print "Add resource ratio: %s" % optimized_addresource_ratio
        msg = "After adding a resource, matchmaking should be of the same order"
        self.assertTrue(optimized_addresource_ratio < 10, msg)