示例#1
0
class BaseProvisionerStoreTests(unittest.TestCase):
    def setUp(self):
        self.store = ProvisionerStore()

    def test_get_unknown_launch(self):
        launch = self.store.get_launch("not-a-real-launch")
        self.assertIsNone(launch)

    def test_get_unknown_node(self):
        node = self.store.get_node("not-a-real-node")
        self.assertIsNone(node)

    def test_put_get_launches(self):

        launch_id_1 = new_id()
        l1 = {'launch_id': launch_id_1, 'state': states.REQUESTED}
        l1_dupe = l1.copy()
        self.store.add_launch(l1)

        # adding it again should error
        try:
            self.store.add_launch(l1_dupe)
        except WriteConflictError:
            pass
        else:
            self.fail("expected WriteConflictError")

        l1_read = self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, l1_read['launch_id'])
        self.assertEqual(states.REQUESTED, l1_read['state'])

        # now make two changes, one from the original and one from what we read
        l2 = l1.copy()
        l2['state'] = states.PENDING
        self.store.update_launch(l2)
        l2_read = self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, l2_read['launch_id'])
        self.assertEqual(states.PENDING, l2_read['state'])

        # this one should hit a write conflict
        l1_read['state'] = states.PENDING
        try:
            self.store.update_launch(l1_read)
        except WriteConflictError:
            pass
        else:
            self.fail("expected WriteConflictError")

        # store another launch altogether
        launch_id_2 = new_id()
        l3 = {'launch_id': launch_id_2, 'state': states.REQUESTED}
        self.store.add_launch(l3)

        latest = self.store.get_launch(launch_id_2)
        self.assertEqual(launch_id_2, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        all = self.store.get_launches()
        self.assertEqual(2, len(all))
        for l in all:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2))

        # try some range queries
        requested = self.store.get_launches(state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        requested = self.store.get_launches(
            min_state=states.REQUESTED,
            max_state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        at_least_requested = self.store.get_launches(
            min_state=states.REQUESTED)
        self.assertEqual(2, len(at_least_requested))
        for l in at_least_requested:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2))

        at_least_pending = self.store.get_launches(
            min_state=states.PENDING)
        self.assertEqual(1, len(at_least_pending))
        self.assertEqual(at_least_pending[0]['launch_id'], launch_id_1)

        at_most_pending = self.store.get_launches(
            max_state=states.PENDING)
        self.assertEqual(2, len(at_most_pending))
        for l in at_most_pending:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2))

    def test_put_get_nodes(self):

        node_id_1 = new_id()
        n1 = {'node_id': node_id_1, 'state': states.REQUESTED}
        n1_dupe = n1.copy()
        self.store.add_node(n1)

        # adding it again should error
        try:
            self.store.add_node(n1_dupe)
        except WriteConflictError:
            pass
        else:
            self.fail("expected WriteConflictError")

        n1_read = self.store.get_node(node_id_1)
        self.assertEqual(node_id_1, n1_read['node_id'])
        self.assertEqual(states.REQUESTED, n1_read['state'])

        # now make two changes, one from the original and one from what we read
        n2 = n1.copy()
        n2['state'] = states.PENDING
        self.store.update_node(n2)
        n2_read = self.store.get_node(node_id_1)
        self.assertEqual(node_id_1, n2_read['node_id'])
        self.assertEqual(states.PENDING, n2_read['state'])

        # this one should hit a write conflict
        n1_read['state'] = states.PENDING
        try:
            self.store.update_node(n1_read)
        except WriteConflictError:
            pass
        else:
            self.fail("expected WriteConflictError")

        # store another node altogether
        node_id_2 = new_id()
        n3 = {'node_id': node_id_2, 'state': states.REQUESTED}
        self.store.add_node(n3)

        latest = self.store.get_node(node_id_2)
        self.assertEqual(node_id_2, latest['node_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        all = self.store.get_nodes()
        self.assertEqual(2, len(all))
        for n in all:
            self.assertTrue(n['node_id'] in (node_id_1, node_id_2))

        # try some range queries
        requested = self.store.get_nodes(state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(node_id_2, requested[0]['node_id'])

        requested = self.store.get_nodes(
            min_state=states.REQUESTED,
            max_state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(node_id_2, requested[0]['node_id'])

        at_least_requested = self.store.get_nodes(
            min_state=states.REQUESTED)
        self.assertEqual(2, len(at_least_requested))
        for n in at_least_requested:
            self.assertTrue(n['node_id'] in (node_id_1, node_id_2))

        at_least_pending = self.store.get_nodes(
            min_state=states.PENDING)
        self.assertEqual(1, len(at_least_pending))
        self.assertEqual(at_least_pending[0]['node_id'], node_id_1)

        at_most_pending = self.store.get_nodes(
            max_state=states.PENDING)
        self.assertEqual(2, len(at_most_pending))
        for n in at_most_pending:
            self.assertTrue(n['node_id'] in (node_id_1, node_id_2))
示例#2
0
文件: test_core.py 项目: timf/epu
class ProvisionerCoreTests(unittest.TestCase):
    """Testing the provisioner core functionality
    """
    def setUp(self):
        self.notifier = FakeProvisionerNotifier()
        self.store = ProvisionerStore()
        self.ctx = FakeContextClient()
        self.dtrs = FakeDTRS()

        self.site1_driver = FakeNodeDriver()
        self.site2_driver = FakeNodeDriver()

        drivers = {'site1': self.site1_driver, 'site2': self.site2_driver}
        self.core = ProvisionerCore(store=self.store,
                                    notifier=self.notifier,
                                    dtrs=self.dtrs,
                                    context=self.ctx,
                                    site_drivers=drivers)

    @defer.inlineCallbacks
    def test_prepare_dtrs_error(self):
        self.dtrs.error = DeployableTypeLookupError()

        nodes = {
            "i1": dict(ids=[_new_id()], site="chicago", allocation="small")
        }
        request = dict(launch_id=_new_id(),
                       deployable_type="foo",
                       subscribers=('blah', ),
                       nodes=nodes)
        yield self.core.prepare_provision(request)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_prepare_broker_error(self):
        self.ctx.create_error = BrokerError("fake ctx create failed")
        self.dtrs.result = {
            'document': "<fake>document</fake>",
            "nodes": {
                "i1": {}
            }
        }
        nodes = {"i1": dict(ids=[_new_id()], site="site1", allocation="small")}
        request = dict(launch_id=_new_id(),
                       deployable_type="foo",
                       subscribers=('blah', ),
                       nodes=nodes)
        yield self.core.prepare_provision(request)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_prepare_execute(self):
        yield self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.PENDING))

    @defer.inlineCallbacks
    def test_prepare_execute_iaas_fail(self):
        self.site1_driver.create_node_error = InvalidCredsError()
        yield self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def _prepare_execute(self):
        self.dtrs.result = {
            'document': _get_one_node_cluster_doc("node1", "image1"),
            "nodes": {
                "node1": {}
            }
        }
        request_node = dict(ids=[_new_id()], site="site1", allocation="small")
        request_nodes = {"node1": request_node}
        request = dict(launch_id=_new_id(),
                       deployable_type="foo",
                       subscribers=('blah', ),
                       nodes=request_nodes)

        launch, nodes = yield self.core.prepare_provision(request)

        self.assertEqual(len(nodes), 1)
        node = nodes[0]
        self.assertEqual(node['node_id'], request_node['ids'][0])
        self.assertEqual(launch['launch_id'], request['launch_id'])

        self.assertTrue(self.ctx.last_create)
        self.assertEqual(launch['context'], self.ctx.last_create)
        for key in ('uri', 'secret', 'context_id', 'broker_uri'):
            self.assertIn(key, launch['context'])
        self.assertTrue(self.notifier.assure_state(states.REQUESTED))

        yield self.core.execute_provision(launch, nodes)

    @defer.inlineCallbacks
    def test_execute_bad_doc(self):
        ctx = yield self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': "<this><isnt><a><real><doc>",
            'deployable_type': "dt",
            'context': ctx,
            'subscribers': [],
            'state': states.PENDING,
            'node_ids': ['node1']
        }
        nodes = [{
            'node_id': 'node1',
            'launch_id': "thelaunchid",
            'state': states.REQUESTED
        }]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

        # TODO this should be a better error coming from nimboss
        #self.assertEqual(self.notifier.nodes['node1']['state_desc'], "CONTEXT_DOC_INVALID")

    @defer.inlineCallbacks
    def test_execute_bad_doc_nodes(self):
        ctx = yield self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': _get_one_node_cluster_doc("node1", "image1"),
            'deployable_type': "dt",
            'context': ctx,
            'subscribers': [],
            'state': states.PENDING,
            'node_ids': ['node1']
        }
        nodes = [{
            'node_id': 'node1',
            'launch_id': "thelaunchid",
            'state': states.REQUESTED,
            'ctx_name': "adifferentname"
        }]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_execute_bad_doc_node_count(self):
        ctx = yield self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': _get_one_node_cluster_doc("node1", "image1"),
            'deployable_type': "dt",
            'context': ctx,
            'subscribers': [],
            'state': states.PENDING,
            'node_ids': ['node1']
        }

        # two nodes where doc expects 1
        nodes = [{
            'node_id': 'node1',
            'launch_id': "thelaunchid",
            'state': states.REQUESTED,
            'ctx_name': "node1"
        }, {
            'node_id': 'node1',
            'launch_id': "thelaunchid",
            'state': states.REQUESTED,
            'ctx_name': "node1"
        }]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_query_missing_node_within_window(self):
        launch_id = _new_id()
        node_id = _new_id()
        ts = time.time() - 30.0
        launch = {
            'launch_id': launch_id,
            'node_ids': [node_id],
            'state': states.PENDING,
            'subscribers': 'fake-subscribers'
        }
        node = {
            'launch_id': launch_id,
            'node_id': node_id,
            'state': states.PENDING,
            'pending_timestamp': ts
        }
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)

        yield self.core.query_one_site('fake-site', [node],
                                       driver=FakeEmptyNodeQueryDriver())
        self.assertEqual(len(self.notifier.nodes), 0)

    @defer.inlineCallbacks
    def test_query_missing_node_past_window(self):
        launch_id = _new_id()
        node_id = _new_id()

        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id,
            'node_ids': [node_id],
            'state': states.PENDING,
            'subscribers': 'fake-subscribers'
        }
        node = {
            'launch_id': launch_id,
            'node_id': node_id,
            'state': states.PENDING,
            'pending_timestamp': ts
        }
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)

        yield self.core.query_one_site('fake-site', [node],
                                       driver=FakeEmptyNodeQueryDriver())
        self.assertEqual(len(self.notifier.nodes), 1)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_query(self):
        launch_id = _new_id()
        node_id = _new_id()

        iaas_node = self.site1_driver.create_node()[0]
        self.site1_driver.set_node_running(iaas_node.id)

        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id,
            'node_ids': [node_id],
            'state': states.PENDING,
            'subscribers': 'fake-subscribers'
        }
        node = {
            'launch_id': launch_id,
            'node_id': node_id,
            'state': states.PENDING,
            'pending_timestamp': ts,
            'iaas_id': iaas_node.id,
            'site': 'site1'
        }

        req_node = {
            'launch_id': launch_id,
            'node_id': _new_id(),
            'state': states.REQUESTED
        }
        nodes = [node, req_node]
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)
        yield self.store.put_node(req_node)

        yield self.core.query_one_site('site1', nodes)

        node = yield self.store.get_node(node_id)
        self.assertEqual(node['public_ip'], iaas_node.public_ip)
        self.assertEqual(node['private_ip'], iaas_node.private_ip)
        self.assertEqual(node['state'], states.STARTED)

        # query again should detect no changes
        yield self.core.query_one_site('site1', nodes)

        # now destroy
        yield self.core.terminate_nodes([node_id])
        node = yield self.store.get_node(node_id)
        yield self.core.query_one_site('site1', [node])

        node = yield self.store.get_node(node_id)
        self.assertEqual(node['public_ip'], iaas_node.public_ip)
        self.assertEqual(node['private_ip'], iaas_node.private_ip)
        self.assertEqual(node['state'], states.TERMINATED)

    @defer.inlineCallbacks
    def test_query_ctx(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [
            make_node(launch_id, states.STARTED) for i in range(node_count)
        ]
        launch_record = make_launch(launch_id, states.PENDING, node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        #first query with no ctx nodes. zero records should be updated
        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_record_count(0))

        # all but 1 node have reported ok
        self.ctx.nodes = [
            _one_fake_ctx_node_ok(node_records[i]['public_ip'], _new_id(),
                                  _new_id()) for i in range(node_count - 1)
        ]

        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertEqual(len(self.notifier.nodes), node_count - 1)

        #last node reports ok
        self.ctx.nodes.append(
            _one_fake_ctx_node_ok(node_records[-1]['public_ip'], _new_id(),
                                  _new_id()))

        self.ctx.complete = True
        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertTrue(self.notifier.assure_record_count(1))

    @defer.inlineCallbacks
    def test_query_ctx_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [
            make_node(launch_id, states.STARTED) for i in range(node_count)
        ]
        launch_record = make_launch(launch_id, states.PENDING, node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        # all but 1 node have reported ok
        self.ctx.nodes = [
            _one_fake_ctx_node_ok(node_records[i]['public_ip'], _new_id(),
                                  _new_id()) for i in range(node_count - 1)
        ]
        self.ctx.nodes.append(
            _one_fake_ctx_node_error(node_records[-1]['public_ip'], _new_id(),
                                     _new_id()))

        ok_ids = [node_records[i]['node_id'] for i in range(node_count - 1)]
        error_ids = [node_records[-1]['node_id']]

        self.ctx.complete = True
        self.ctx.error = True

        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING, ok_ids))
        self.assertTrue(
            self.notifier.assure_state(states.RUNNING_FAILED, error_ids))

    @defer.inlineCallbacks
    def test_query_ctx_nodes_not_started(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.PENDING) for i in range(3)]
        node_records.append(make_node(launch_id, states.STARTED))
        launch_record = make_launch(launch_id, states.PENDING, node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.query_contexts()

        # ensure that no context was actually queried. See the note in
        # _query_one_context for the reason why this is important.
        self.assertEqual(len(self.ctx.queried_uris), 0)

    @defer.inlineCallbacks
    def test_query_ctx_permanent_broker_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [
            make_node(launch_id, states.STARTED) for i in range(node_count)
        ]
        node_ids = [node['node_id'] for node in node_records]
        launch_record = make_launch(launch_id, states.PENDING, node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.query_error = ContextNotFoundError()
        yield self.core.query_contexts()

        self.assertTrue(
            self.notifier.assure_state(states.RUNNING_FAILED, node_ids))
        launch = yield self.store.get_launch(launch_id)
        self.assertEqual(launch['state'], states.FAILED)

    def test_update_node_ip_info(self):
        node = dict(public_ip=None)
        iaas_node = Mock(public_ip=None, private_ip=None)
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], None)
        self.assertEqual(node['private_ip'], None)

        iaas_node = Mock(public_ip=["pub1"], private_ip=["priv1"])
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")

        iaas_node = Mock(public_ip=[], private_ip=[])
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")

    def test_update_nodes_from_ctx(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED) for i in range(5)]
        ctx_nodes = [
            _one_fake_ctx_node_ok(node['public_ip'], _new_id(), _new_id())
            for node in nodes
        ]

        self.assertEquals(len(nodes),
                          len(update_nodes_from_context(nodes, ctx_nodes)))

    def test_update_nodes_from_ctx_with_hostname(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED) for i in range(5)]
        #libcloud puts the hostname in the public_ip field
        ctx_nodes = [
            _one_fake_ctx_node_ok(ip=_new_id(),
                                  hostname=node['public_ip'],
                                  pubkey=_new_id()) for node in nodes
        ]

        self.assertEquals(len(nodes),
                          len(update_nodes_from_context(nodes, ctx_nodes)))

    @defer.inlineCallbacks
    def test_query_broker_exception(self):
        for i in range(2):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING,
                                        node_records)

            yield self.store.put_launch(launch_record)
            yield self.store.put_nodes(node_records)

        # no guaranteed order here so grabbing first launch from store
        # and making that one return a BrokerError during context query.
        # THe goal is to ensure that one error doesn't prevent querying
        # for other contexts.

        launches = yield self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]
        error_launch_ctx = error_launch['context']['uri']
        ok_node_id = launches[1]['node_ids'][0]
        ok_node = yield self.store.get_node(ok_node_id)

        self.ctx.uri_query_error[error_launch_ctx] = BrokerError("bad broker")
        self.ctx.nodes = [
            _one_fake_ctx_node_ok(ok_node['public_ip'], _new_id(), _new_id())
        ]
        self.ctx.complete = True
        yield self.core.query_contexts()

        launches = yield self.store.get_launches()
        for launch in launches:
            self.assertIn(launch['context']['uri'], self.ctx.queried_uris)

            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED
            else:
                self.assertEqual(launch['state'], states.RUNNING)
                expected_node_state = states.RUNNING

            node = yield self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)

    @defer.inlineCallbacks
    def test_query_ctx_without_valid_nodes(self):

        # if there are no nodes < TERMINATING, no broker query should happen
        for i in range(3):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING,
                                        node_records)

            yield self.store.put_launch(launch_record)
            yield self.store.put_nodes(node_records)

        launches = yield self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]

        # mark first launch's node as TERMINATING, should prevent
        # context query and result in launch being marked FAILED
        error_launch_node = yield self.store.get_node(
            error_launch['node_ids'][0])
        error_launch_node['state'] = states.TERMINATING
        yield self.store.put_node(error_launch_node)

        yield self.core.query_contexts()
        self.assertNotIn(error_launch['context']['uri'], self.ctx.queried_uris)

        launches = yield self.store.get_launches()
        for launch in launches:
            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.FAILED)
                expected_node_state = states.TERMINATING
            else:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED

            node = yield self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)

    @defer.inlineCallbacks
    def test_query_unexpected_exception(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)]
        launch_record = make_launch(launch_id, states.PENDING, node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)
        self.ctx.query_error = ValueError("bad programmer")

        # digging into internals a bit: patching one of the methods query()
        # calls to raise an exception. This will let us ensure exceptions do
        # not bubble up
        def raiser(self):
            raise KeyError("notreallyaproblem")

        self.patch(self.core, 'query_nodes', raiser)

        yield self.core.query()  # ensure that exception doesn't bubble up

    @defer.inlineCallbacks
    def test_dump_state(self):
        node_ids = []
        node_records = []
        for i in range(3):
            launch_id = _new_id()
            nodes = [make_node(launch_id, states.PENDING)]
            node_ids.append(nodes[0]['node_id'])
            node_records.extend(nodes)
            launch = make_launch(launch_id, states.PENDING, nodes)
            yield self.store.put_launch(launch)
            yield self.store.put_nodes(nodes)

        yield self.core.dump_state(node_ids[:2])

        # should have gotten notifications about the 2 nodes
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[0]], 1)
        self.assertEqual(node_records[0], self.notifier.nodes[node_ids[0]])
        self.assertEqual(node_records[1], self.notifier.nodes[node_ids[1]])
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[1]], 1)
        self.assertNotIn(node_ids[2], self.notifier.nodes)

    @defer.inlineCallbacks
    def test_mark_nodes_terminating(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.RUNNING) for i in range(3)]
        launch_record = make_launch(launch_id, states.PENDING, node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        first_two_node_ids = [
            node_records[0]['node_id'], node_records[1]['node_id']
        ]
        yield self.core.mark_nodes_terminating(first_two_node_ids)

        self.assertTrue(
            self.notifier.assure_state(states.TERMINATING,
                                       nodes=first_two_node_ids))
        self.assertNotIn(node_records[2]['node_id'], self.notifier.nodes)

        for node_id in first_two_node_ids:
            terminating_node = yield self.store.get_node(node_id)
            self.assertEqual(terminating_node['state'], states.TERMINATING)
示例#3
0
文件: test_core.py 项目: timf/epu
class ProvisionerCoreRecoveryTests(unittest.TestCase):
    def setUp(self):
        self.notifier = FakeProvisionerNotifier()
        self.store = ProvisionerStore()
        self.ctx = FakeContextClient()
        self.driver = FakeNodeDriver()
        self.dtrs = FakeDTRS()
        drivers = {'fake': self.driver}
        self.core = ProvisionerCore(store=self.store,
                                    notifier=self.notifier,
                                    dtrs=self.dtrs,
                                    site_drivers=drivers,
                                    context=self.ctx)

    @defer.inlineCallbacks
    def test_recover_launch_incomplete(self):
        """Ensures that launches in REQUESTED state are completed
        """
        launch_id = _new_id()
        doc = "<cluster><workspace><name>node</name><image>fake</image>"+\
              "<quantity>3</quantity>"+\
              "</workspace><workspace><name>running</name><image>fake"+\
              "</image><quantity>1</quantity></workspace></cluster>"
        context = {
            'broker_uri': _new_id(),
            'context_id': _new_id(),
            'secret': _new_id(),
            'uri': _new_id()
        }

        requested_node_ids = [_new_id(), _new_id()]

        node_records = [
            make_node(launch_id,
                      states.RUNNING,
                      site='fake',
                      ctx_name='running'),
            make_node(launch_id,
                      states.REQUESTED,
                      site='fake',
                      node_id=requested_node_ids[0],
                      ctx_name='node'),
            make_node(launch_id,
                      states.REQUESTED,
                      site='fake',
                      node_id=requested_node_ids[1],
                      ctx_name='node'),
            make_node(launch_id, states.RUNNING, ctx_name='node')
        ]
        launch_record = make_launch(launch_id,
                                    states.REQUESTED,
                                    node_records,
                                    document=doc,
                                    context=context)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        # 2 nodes are in REQUESTED state, so those should be launched
        yield self.core.recover()

        # because we rely on IaaS idempotency, we get full Node responses
        # for all nodes in the group. What really would cause this scenario
        # is successfully launching the full group but failing before records
        # could be written for the two REQUESTED nodes.
        self.assertEqual(3, len(self.driver.created))
        iaas_ids = set(node.id for node in self.driver.created)
        self.assertEqual(3, len(iaas_ids))

        for node_id in requested_node_ids:
            node = yield self.store.get_node(node_id)
            self.assertEqual(states.PENDING, node['state'])
            self.assertTrue(node['iaas_id'] in iaas_ids)

        launch = yield self.store.get_launch(launch_id)
        self.assertEqual(states.PENDING, launch['state'])

    @defer.inlineCallbacks
    def test_recovery_nodes_terminating(self):
        launch_id = _new_id()

        terminating_iaas_id = _new_id()

        node_records = [
            make_node(launch_id,
                      states.TERMINATING,
                      iaas_id=terminating_iaas_id,
                      site='fake'),
            make_node(launch_id, states.TERMINATED),
            make_node(launch_id, states.RUNNING)
        ]

        launch_record = make_launch(launch_id, states.RUNNING, node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.recover()

        self.assertEqual(1, len(self.driver.destroyed))
        self.assertEqual(self.driver.destroyed[0].id, terminating_iaas_id)

        terminated = yield self.store.get_nodes(state=states.TERMINATED)
        self.assertEqual(2, len(terminated))

    @defer.inlineCallbacks
    def test_recovery_launch_terminating(self):
        launch_id = _new_id()

        terminating_iaas_ids = [_new_id(), _new_id()]

        node_records = [
            make_node(launch_id,
                      states.TERMINATING,
                      iaas_id=terminating_iaas_ids[0],
                      site='fake'),
            make_node(launch_id, states.TERMINATED),
            make_node(launch_id,
                      states.RUNNING,
                      iaas_id=terminating_iaas_ids[1],
                      site='fake')
        ]

        launch_record = make_launch(launch_id, states.TERMINATING,
                                    node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.recover()

        self.assertEqual(2, len(self.driver.destroyed))
        self.assertTrue(self.driver.destroyed[0].id in terminating_iaas_ids)
        self.assertTrue(self.driver.destroyed[1].id in terminating_iaas_ids)

        terminated = yield self.store.get_nodes(state=states.TERMINATED)
        self.assertEqual(3, len(terminated))

        launch_record = yield self.store.get_launch(launch_id)
        self.assertEqual(launch_record['state'], states.TERMINATED)

    @defer.inlineCallbacks
    def test_terminate_all(self):
        running_launch_id = _new_id()
        running_launch, running_nodes = make_launch_and_nodes(
            running_launch_id, 3, states.RUNNING)
        yield self.store.put_launch(running_launch)
        yield self.store.put_nodes(running_nodes)

        pending_launch_id = _new_id()
        pending_launch, pending_nodes = make_launch_and_nodes(
            pending_launch_id, 3, states.PENDING)
        yield self.store.put_launch(pending_launch)
        yield self.store.put_nodes(pending_nodes)

        terminated_launch_id = _new_id()
        terminated_launch, terminated_nodes = make_launch_and_nodes(
            terminated_launch_id, 3, states.TERMINATED)
        yield self.store.put_launch(terminated_launch)
        yield self.store.put_nodes(terminated_nodes)

        yield self.core.terminate_all()

        self.assertEqual(6, len(self.driver.destroyed))

        all_launches = yield self.store.get_launches()
        self.assertEqual(3, len(all_launches))
        self.assertTrue(
            all(l['state'] == states.TERMINATED for l in all_launches))

        all_nodes = yield self.store.get_nodes()
        self.assertEqual(9, len(all_nodes))
        self.assertTrue(all(n['state'] == states.TERMINATED
                            for n in all_nodes))

        state = yield self.core.check_terminate_all()
        self.assertTrue(state)
示例#4
0
class ProvisionerCoreTests(unittest.TestCase):
    """Testing the provisioner core functionality
    """
    def setUp(self):
        self.notifier = FakeProvisionerNotifier()
        self.store = ProvisionerStore()
        self.ctx = FakeContextClient()
        self.dtrs = FakeDTRS()

        self.dtrs.sites["site1"] = self.dtrs.sites["site2"] = {
            "type": "fake"
        }

        self.dtrs.credentials[("asterix", "site1")] = self.dtrs.credentials[("asterix", "site2")] = {
            "access_key": "mykey",
            "secret_key": "mysecret"
        }

        self.site1_driver = FakeNodeDriver()
        self.site2_driver = FakeNodeDriver()
        self.site1_driver.initialize()
        self.site2_driver.initialize()

        self.core = ProvisionerCore(store=self.store, notifier=self.notifier,
                                    dtrs=self.dtrs, context=self.ctx)

    def test_terminate_all(self):
        caller = 'asterix'
        running_launch_id = _new_id()
        running_launch, running_nodes = make_launch_and_nodes(
            running_launch_id, 3, states.RUNNING, caller=caller)
        self.store.add_launch(running_launch)
        for node in running_nodes:
            self.store.add_node(node)

        pending_launch_id = _new_id()
        pending_launch, pending_nodes = make_launch_and_nodes(
            pending_launch_id, 3, states.PENDING, caller=caller)
        self.store.add_launch(pending_launch)
        for node in pending_nodes:
            self.store.add_node(node)

        terminated_launch_id = _new_id()
        terminated_launch, terminated_nodes = make_launch_and_nodes(
            terminated_launch_id, 3, states.TERMINATED, caller=caller)
        self.store.add_launch(terminated_launch)
        for node in terminated_nodes:
            self.store.add_node(node)

        self.core.terminate_all()

        all_nodes = self.store.get_nodes()
        self.assertEqual(9, len(all_nodes))
        self.assertTrue(all(n['state'] == states.TERMINATING or
                        n['state'] == states.TERMINATED for n in all_nodes))

    def test_prepare_dtrs_error(self):
        self.dtrs.error = DeployableTypeLookupError()

        self.core.prepare_provision(
            launch_id=_new_id(), deployable_type="foo",
            instance_ids=[_new_id()], site="chicago")
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_prepare_broker_error(self):
        self.ctx.create_error = BrokerError("fake ctx create failed")
        self.dtrs.result = {'document': "<fake>document</fake>",
                            "node": {}}
        self.core.prepare_provision(
            launch_id=_new_id(), deployable_type="foo",
            instance_ids=[_new_id()], site="chicago")
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_prepare_execute(self):
        self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.PENDING))

    def test_prepare_execute_iaas_fail(self):
        with patch('epu.provisioner.test.util.FakeNodeDriver.create_node') as mock_method:
            mock_method.return_value = InvalidCredsError()
            self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_prepare_execute_no_ctx(self):
        self.core.context = None

        # just in case
        self.ctx.create_error = NotImplementedError()
        self.ctx.query_error = NotImplementedError()

        self._prepare_execute(context_enabled=False)
        self.assertTrue(self.notifier.assure_state(states.PENDING))

    def test_prepare_execute_existing_launch(self):
        self.core.context = None
        launch_id = _new_id()
        instance_id = _new_id()

        self._prepare_execute(
            launch_id=launch_id, instance_ids=[instance_id],
            context_enabled=False)
        self._prepare_execute(
            launch_id=launch_id, instance_ids=[instance_id],
            context_enabled=False, assure_state=False)

        self.assertTrue(self.notifier.assure_state(states.PENDING))

    def _prepare_execute(self, launch_id=None, instance_ids=None,
                         context_enabled=True, assure_state=True):
        self.dtrs.result = {'document': _get_one_node_cluster_doc("node1", "image1"),
                            "node": {}}

        caller = "asterix"
        if not launch_id:
            launch_id = _new_id()
        if not instance_ids:
            instance_ids = [_new_id()]
        launch, nodes = self.core.prepare_provision(
            launch_id=launch_id,
            deployable_type="foo", instance_ids=instance_ids,
            site="site1", caller=caller)

        self.assertEqual(len(nodes), 1)
        node = nodes[0]
        self.assertEqual(node['node_id'], instance_ids[0])
        self.assertEqual(launch['launch_id'], launch_id)
        self.assertEqual(launch['node_ids'], instance_ids)

        if context_enabled:
            self.assertTrue(self.ctx.last_create)
            self.assertEqual(launch['context'], self.ctx.last_create)
            for key in ('uri', 'secret', 'context_id', 'broker_uri'):
                self.assertIn(key, launch['context'])
        else:
            self.assertEqual(launch['context'], None)

        if assure_state:
            self.assertTrue(self.notifier.assure_state(states.REQUESTED))

        self.core.execute_provision(launch, nodes, caller)

    def test_execute_bad_doc(self):
        caller = "asterix"
        ctx = self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': "<this><isnt><a><real><doc>",
            'deployable_type': "dt",
            'context': ctx,
            'state': states.PENDING,
            'node_ids': ['node1']}
        nodes = [{'node_id': 'node1', 'launch_id': "thelaunchid",
                 'state': states.REQUESTED, 'creator': caller}]

        self.store.add_launch(launch_record)
        self.store.add_node(nodes[0])

        self.core.execute_provision(launch_record, nodes, caller)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

        # TODO this should be a better error coming from nimboss
        # self.assertEqual(self.notifier.nodes['node1']['state_desc'], "CONTEXT_DOC_INVALID")

    def test_execute_bad_doc_nodes(self):
        caller = 'asterix'
        ctx = self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': _get_one_node_cluster_doc("node1", "image1"),
            'deployable_type': "dt",
            'context': ctx,
            'state': states.PENDING,
            'node_ids': ['node1']}
        nodes = [{'node_id': 'node1', 'launch_id': "thelaunchid",
                  'state': states.REQUESTED, 'ctx_name': "adifferentname",
                  'creator': caller}]

        self.store.add_launch(launch_record)
        self.store.add_node(nodes[0])

        self.core.execute_provision(launch_record, nodes, caller)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_execute_bad_doc_node_count(self):
        caller = "asterix"
        ctx = self.ctx.create()
        launch_record = {
            'launch_id': "thelaunchid",
            'document': _get_one_node_cluster_doc("node1", "image1"),
            'deployable_type': "dt",
            'context': ctx,
            'state': states.PENDING,
            'node_ids': ['node1']}

        # two nodes where doc expects 1
        nodes = [{'node_id': 'node1', 'launch_id': "thelaunchid",
                 'state': states.REQUESTED, 'ctx_name': "node1", 'creator': caller},
                 {'node_id': 'node2', 'launch_id': "thelaunchid",
                     'state': states.REQUESTED, 'ctx_name': "node1",
                     'creator': caller}]

        self.store.add_launch(launch_record)
        for node in nodes:
            self.store.add_node(node)

        self.core.execute_provision(launch_record, nodes, caller)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_query_missing_node_within_window(self):
        launch_id = _new_id()
        node_id = _new_id()
        caller = 'asterix'
        ts = time.time() - 30.0
        launch = {'launch_id': launch_id, 'node_ids': [node_id],
                  'state': states.PENDING,
                  'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.PENDING,
                'pending_timestamp': ts,
                'creator': caller}
        self.store.add_launch(launch)
        self.store.add_node(node)

        with patch.object(FakeNodeDriver, 'list_nodes', return_value=[]):
            self.core.query_one_site('site1', [node], caller=caller)
        self.assertEqual(len(self.notifier.nodes), 0)

    def test_query_missing_started_node_within_window(self):
        launch_id = _new_id()
        node_id = _new_id()
        caller = 'asterix'
        ts = time.time() - 30.0
        launch = {'launch_id': launch_id, 'node_ids': [node_id],
                  'state': states.PENDING,
                  'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.STARTED,
                'pending_timestamp': ts,
                'creator': caller}
        self.store.add_launch(launch)
        self.store.add_node(node)

        with patch.object(FakeNodeDriver, 'list_nodes', return_value=[]):
            self.core.query_one_site('site1', [node], caller=caller)
        self.assertEqual(len(self.notifier.nodes), 1)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_query_missing_node_past_window(self):
        launch_id = _new_id()
        node_id = _new_id()

        caller = 'asterix'
        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.PENDING,
            'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.PENDING,
                'pending_timestamp': ts,
                'creator': caller}
        self.store.add_launch(launch)
        self.store.add_node(node)

        with patch.object(FakeNodeDriver, 'list_nodes', return_value=[]):
            self.core.query_one_site('site1', [node], caller=caller)
        self.assertEqual(len(self.notifier.nodes), 1)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    def test_query_missing_node_terminating(self):
        launch_id = _new_id()
        node_id = _new_id()

        caller = 'asterix'
        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.RUNNING,
            'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.TERMINATING,
                'creator': caller}
        self.store.add_launch(launch)
        self.store.add_node(node)

        with patch.object(FakeNodeDriver, 'list_nodes', return_value=[]):
            self.core.query_one_site('site1', [node], caller=caller)
        self.assertEqual(len(self.notifier.nodes), 1)
        self.assertTrue(self.notifier.assure_state(states.TERMINATED))

    def test_query(self):
        caller = "asterix"
        launch_id = _new_id()
        node_id = _new_id()

        iaas_node = self.site1_driver.create_node()[0]
        self.site1_driver.set_node_running(iaas_node.id)

        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.PENDING,
            'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.PENDING,
                'pending_timestamp': ts,
                'iaas_id': iaas_node.id,
                'creator': caller,
                'site': 'site1'}

        req_node = {'launch_id': launch_id,
                    'node_id': _new_id(),
                    'state': states.REQUESTED}
        nodes = [node, req_node]
        self.store.add_launch(launch)
        self.store.add_node(node)
        self.store.add_node(req_node)

        self.core.query_one_site('site1', nodes, caller=caller)

        node = self.store.get_node(node_id)
        self.assertEqual(node.get('public_ip'), iaas_node.public_ip)
        self.assertEqual(node.get('private_ip'), iaas_node.private_ip)
        self.assertEqual(node.get('state'), states.STARTED)

        # query again should detect no changes
        self.core.query_one_site('site1', nodes, caller=caller)

        # now destroy
        self.core.terminate_nodes([node_id], remove_terminating=False)
        node = self.store.get_node(node_id)
        self.core.query_one_site('site1', [node], caller=caller)

        node = self.store.get_node(node_id)
        self.assertEqual(node['public_ip'], iaas_node.public_ip)
        self.assertEqual(node['private_ip'], iaas_node.private_ip)
        self.assertEqual(node['state'], states.TERMINATED)

    def test_terminate_requested_node(self):
        caller = "asterix"
        launch_id = _new_id()
        node_id = _new_id()

        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.PENDING,
            'creator': caller}
        req_node = {
            'launch_id': launch_id,
            'node_id': node_id,
            'state': states.REQUESTED,
            'site': 'site1'}
        self.store.add_launch(launch)
        self.store.add_node(req_node)

        # destroy
        self.core.terminate_nodes([node_id], remove_terminating=False)
        node = self.store.get_node(node_id)
        self.assertEqual(node['state'], states.TERMINATED)

    def test_query_no_contextualization(self):

        self.core.context = None

        launch_id = _new_id()
        node_id = _new_id()

        caller = 'asterix'

        iaas_node = self.site1_driver.create_node()[0]
        self.site1_driver.set_node_running(iaas_node.id)

        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.PENDING,
            'creator': caller}
        node = {'launch_id': launch_id,
                'node_id': node_id,
                'state': states.PENDING,
                'pending_timestamp': ts,
                'iaas_id': iaas_node.id,
                'site': 'site1',
                'creator': caller}

        req_node = {'launch_id': launch_id,
                    'node_id': _new_id(),
                    'state': states.REQUESTED}
        nodes = [node, req_node]
        self.store.add_launch(launch)
        self.store.add_node(node)
        self.store.add_node(req_node)

        self.core.query_one_site('site1', nodes, caller=caller)

        node = self.store.get_node(node_id)
        self.assertEqual(node.get('public_ip'), iaas_node.public_ip)
        self.assertEqual(node.get('private_ip'), iaas_node.private_ip)

        # since contextualization is disabled we should jump straight
        # to RUNNING
        self.assertEqual(node.get('state'), states.RUNNING)

    @raises(timeout)
    def test_query_iaas_timeout(self):
        launch_id = _new_id()
        node_id = _new_id()

        iaas_node = self.site1_driver.create_node()[0]
        self.site1_driver.set_node_running(iaas_node.id)

        caller = 'asterix'
        ts = time.time() - 120.0
        launch = {
            'launch_id': launch_id, 'node_ids': [node_id],
            'state': states.PENDING,
            'creator': caller}
        node = {'name': 'hello',
                'launch_id': launch_id,
                'node_id': node_id,
                'state': states.PENDING,
                'pending_timestamp': ts,
                'iaas_id': iaas_node.id,
                'site': 'site1',
                'creator': caller}

        req_node = {'launch_id': launch_id,
                    'node_id': _new_id(),
                    'state': states.REQUESTED}
        nodes = [node, req_node]
        self.store.add_launch(launch)
        self.store.add_node(node)
        self.store.add_node(req_node)

        def x():
            raise timeout("Took too long to query iaas")
        self.core._IAAS_DEFAULT_TIMEOUT = 0.5

        with patch.object(FakeNodeDriver, 'list_nodes', side_effect=x):
            self.core.query_one_site('site1', nodes, caller=caller)

    def test_launch_one_iaas_full(self):
        def x(**kwargs):
            raise Exception("InstanceLimitExceeded: too many vms :(")

        with patch.object(FakeNodeDriver, 'create_node', side_effect=x):
            self.core._IAAS_DEFAULT_TIMEOUT = 0.5

            node_id = _new_id()
            launch_id = _new_id()

            self._prepare_execute(launch_id=launch_id, instance_ids=[node_id])

            self.assertTrue(self.notifier.assure_state(states.FAILED))
            self.assertIn('IAAS_FULL', self.notifier.nodes[node_id]['state_desc'])
            launch = self.store.get_launch(launch_id)
            self.assertEqual(launch['state'], states.FAILED)

    def test_launch_one_iaas_timeout(self):
        def x(**kwargs):
            raise timeout("Launch took too long")

        with patch.object(FakeNodeDriver, 'create_node', side_effect=x):
            self.core._IAAS_DEFAULT_TIMEOUT = 0.5

            node_id = _new_id()
            launch_id = _new_id()

            self._prepare_execute(launch_id=launch_id, instance_ids=[node_id])

            self.assertTrue(self.notifier.assure_state(states.FAILED))
            self.assertEqual(self.notifier.nodes[node_id]['state_desc'], 'IAAS_TIMEOUT')
            launch = self.store.get_launch(launch_id)
            self.assertEqual(launch['state'], states.FAILED)

    def test_query_ctx(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                        for i in range(node_count)]
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records)

        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        # first query with no ctx nodes. zero records should be updated
        self.core.query_contexts()
        self.assertTrue(self.notifier.assure_record_count(0))

        # all but 1 node have reported ok
        self.ctx.nodes = [_one_fake_ctx_node_ok(node_records[i]['public_ip'],
                          _new_id(), _new_id()) for i in range(node_count - 1)]

        self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertEqual(len(self.notifier.nodes), node_count - 1)

        # last node reports ok
        self.ctx.nodes.append(_one_fake_ctx_node_ok(node_records[-1]['public_ip'],
                              _new_id(), _new_id()))

        self.ctx.complete = True
        self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertTrue(self.notifier.assure_record_count(1))

    def test_query_ctx_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                        for i in range(node_count)]
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records)

        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        # all but 1 node have reported ok
        self.ctx.nodes = [_one_fake_ctx_node_ok(node_records[i]['public_ip'],
                          _new_id(), _new_id()) for i in range(node_count - 1)]
        self.ctx.nodes.append(_one_fake_ctx_node_error(node_records[-1]['public_ip'],
                              _new_id(), _new_id()))

        ok_ids = [node_records[i]['node_id'] for i in range(node_count - 1)]
        error_ids = [node_records[-1]['node_id']]

        self.ctx.complete = True
        self.ctx.error = True

        self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING, ok_ids))
        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED, error_ids))

    def test_query_ctx_nodes_not_pending(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.REQUESTED)
                        for i in range(3)]
        node_records.append(make_node(launch_id, states.STARTED))
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records)
        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.core.query_contexts()

        # ensure that no context was actually queried. See the note in
        # _query_one_context for the reason why this is important.
        self.assertEqual(len(self.ctx.queried_uris), 0)

    def test_query_ctx_nodes_pending_but_actually_running(self):
        """
        When doing large runs, a few EC2 instances get their status changed to
        "running" a long time after having requested them (up to 15 minutes,
        compared to about 30 seconds normally).
        It appears that these instances have been booted successfully for a
        while, because they are reachable through SSH and the context broker
        has OK'ed them.
        Test that we detect these "pending but actually running" instances
        early.
        """
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.PENDING)
                        for i in range(3)]
        node_records.append(make_node(launch_id, states.STARTED))
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records)
        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.ctx.nodes = [_one_fake_ctx_node_ok(node['public_ip'], _new_id(),
                          _new_id()) for node in node_records]
        self.ctx.complete = True

        self.core.query_contexts()

        launch = self.store.get_launch(launch_id)
        self.assertEqual(launch['state'], states.RUNNING)

        for node_id in launch['node_ids']:
            node = self.store.get_node(node_id)
            self.assertEqual(states.RUNNING, node['state'])

    def test_query_ctx_permanent_broker_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                        for i in range(node_count)]
        node_ids = [node['node_id'] for node in node_records]
        launch_record = make_launch(launch_id, states.PENDING, node_records)
        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.ctx.query_error = ContextNotFoundError()
        self.core.query_contexts()

        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED, node_ids))
        launch = self.store.get_launch(launch_id)
        self.assertEqual(launch['state'], states.FAILED)

    def test_query_ctx_with_one_node_timeout(self):
        launch_id = _new_id()
        node_record = make_node(launch_id, states.STARTED)
        launch_record = make_launch(launch_id, states.PENDING, [node_record])

        ts = time.time()
        node_record['running_timestamp'] = ts - INSTANCE_READY_TIMEOUT - 10

        self.store.add_launch(launch_record)
        self.store.add_node(node_record)

        self.ctx.expected_count = 1
        self.ctx.complete = False
        self.ctx.error = False

        self.ctx.nodes = []
        self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED))
        self.assertTrue(self.notifier.assure_record_count(1))

    def test_query_ctx_with_several_nodes_timeout(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                        for i in range(node_count)]
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records)
        node_ids = map(lambda node: node['node_id'], node_records)

        ts = time.time()
        for i in range(node_count - 1):
            node_records[i]['running_timestamp'] = ts - INSTANCE_READY_TIMEOUT + 10
        node_records[-1]['running_timestamp'] = ts - INSTANCE_READY_TIMEOUT - 10

        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        # all but 1 node have reported ok
        self.ctx.nodes = [_one_fake_ctx_node_ok(node_records[i]['public_ip'],
                          _new_id(), _new_id()) for i in range(node_count - 1)]

        self.core.query_contexts()

        self.assertTrue(self.notifier.assure_state(states.RUNNING, node_ids[:node_count - 1]))
        self.assertEqual(len(self.notifier.nodes), node_count)
        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED, node_ids[node_count - 1:]))
        self.assertTrue(self.notifier.assure_record_count(1, node_ids[node_count - 1:]))

    def test_query_ctx_with_no_timeout(self):
        caller = "asterix"
        launch_id = _new_id()
        node_record = make_node(launch_id, states.STARTED)
        launch_record = make_launch(launch_id, states.PENDING, [node_record],
                                    caller=caller)

        ts = time.time()
        node_record['running_timestamp'] = ts - INSTANCE_READY_TIMEOUT - 10

        self.store.add_launch(launch_record)
        self.store.add_node(node_record)

        self.ctx.expected_count = 1
        self.ctx.complete = False
        self.ctx.error = False

        self.ctx.nodes = [_one_fake_ctx_node_not_done(node_record['public_ip'],
                          _new_id(), _new_id())]
        self.core.query_contexts()

        self.assertTrue(self.notifier.assure_record_count(0))

    def test_update_node_ip_info(self):
        node = dict(public_ip=None)
        iaas_node = Mock(public_ip=None, private_ip=None, extra={'dns_name': None})
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], None)
        self.assertEqual(node['private_ip'], None)

        iaas_node = Mock(public_ip=["pub1"], private_ip=["priv1"], extra={'dns_name': 'host'})
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")
        self.assertEqual(node['hostname'], "host")

        iaas_node = Mock(public_ip=[], private_ip=[], extra={'dns_name': []})
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")
        self.assertEqual(node['hostname'], "host")

    def test_update_nodes_from_ctx(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED)
                 for i in range(5)]
        ctx_nodes = [_one_fake_ctx_node_ok(node['public_ip'], _new_id(),
                     _new_id()) for node in nodes]

        self.assertEquals(
            len(nodes),
            len(update_nodes_from_context(match_nodes_from_context(nodes, ctx_nodes))))

    def test_update_nodes_from_ctx_with_hostname(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED)
                 for i in range(5)]
        # libcloud puts the hostname in the public_ip field
        ctx_nodes = [_one_fake_ctx_node_ok(ip=_new_id(), hostname=node['public_ip'],
                     pubkey=_new_id()) for node in nodes]

        self.assertEquals(
            len(nodes),
            len(update_nodes_from_context(match_nodes_from_context(nodes, ctx_nodes))))

    def test_query_broker_exception(self):
        caller = "asterix"
        for i in range(2):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING, node_records, caller=caller)

            self.store.add_launch(launch_record)
            for node in node_records:
                self.store.add_node(node)

        # no guaranteed order here so grabbing first launch from store
        # and making that one return a BrokerError during context query.
        # THe goal is to ensure that one error doesn't prevent querying
        # for other contexts.

        launches = self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]
        error_launch_ctx = error_launch['context']['uri']
        ok_node_id = launches[1]['node_ids'][0]
        ok_node = self.store.get_node(ok_node_id)

        self.ctx.uri_query_error[error_launch_ctx] = BrokerError("bad broker")
        self.ctx.nodes = [_one_fake_ctx_node_ok(ok_node['public_ip'],
                          _new_id(), _new_id())]
        self.ctx.complete = True
        self.core.query_contexts()

        launches = self.store.get_launches()
        for launch in launches:
            self.assertIn(launch['context']['uri'], self.ctx.queried_uris)

            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED
            else:
                self.assertEqual(launch['state'], states.RUNNING)
                expected_node_state = states.RUNNING

            node = self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)

    def test_query_ctx_without_valid_nodes(self):

        caller = "asterix"
        # if there are no nodes < TERMINATING, no broker query should happen
        for i in range(3):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING,
                                        node_records, caller=caller)

            self.store.add_launch(launch_record)
            for node in node_records:
                self.store.add_node(node)

        launches = self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]

        # mark first launch's node as TERMINATING, should prevent
        # context query and result in launch being marked FAILED
        error_launch_node = self.store.get_node(error_launch['node_ids'][0])
        error_launch_node['state'] = states.TERMINATING
        self.store.update_node(error_launch_node)

        self.core.query_contexts()
        self.assertNotIn(error_launch['context']['uri'], self.ctx.queried_uris)

        launches = self.store.get_launches()
        for launch in launches:
            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.FAILED)
                expected_node_state = states.TERMINATING
            else:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED

            node = self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)

    def test_dump_state(self):
        caller = "asterix"
        node_ids = []
        node_records = []
        for i in range(3):
            launch_id = _new_id()
            nodes = [make_node(launch_id, states.PENDING)]
            node_ids.append(nodes[0]['node_id'])
            node_records.extend(nodes)
            launch = make_launch(launch_id, states.PENDING,
                                 nodes, caller=caller)
            self.store.add_launch(launch)
            for node in nodes:
                self.store.add_node(node)

        self.core.dump_state(node_ids[:2])

        # should have gotten notifications about the 2 nodes
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[0]], 1)
        self.assertEqual(node_records[0], self.notifier.nodes[node_ids[0]])
        self.assertEqual(node_records[1], self.notifier.nodes[node_ids[1]])
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[1]], 1)
        self.assertNotIn(node_ids[2], self.notifier.nodes)

    def test_mark_nodes_terminating(self):
        caller = "asterix"
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.RUNNING)
                        for i in range(3)]
        launch_record = make_launch(launch_id, states.PENDING,
                                    node_records, caller=caller)

        self.store.add_launch(launch_record)
        for node in node_records:
            self.store.add_node(node)

        first_two_node_ids = [node_records[0]['node_id'],
                              node_records[1]['node_id']]
        self.core.mark_nodes_terminating(first_two_node_ids)

        self.assertTrue(self.notifier.assure_state(states.TERMINATING,
                                                   nodes=first_two_node_ids))
        self.assertNotIn(node_records[2]['node_id'], self.notifier.nodes)

        for node_id in first_two_node_ids:
            terminating_node = self.store.get_node(node_id)
            self.assertEqual(terminating_node['state'], states.TERMINATING)

    def test_describe(self):
        caller = "asterix"
        node_ids = []
        for _ in range(3):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.RUNNING)]
            node_ids.append(node_records[0]['node_id'])
            launch_record = make_launch(
                launch_id, states.PENDING,
                node_records, caller=caller)
            self.store.add_launch(launch_record)
            for node in node_records:
                self.store.add_node(node)

        all_nodes = self.core.describe_nodes()
        all_node_ids = [n['node_id'] for n in all_nodes]
        self.assertEqual(set(all_node_ids), set(node_ids))
        self.assertFalse(any(VERSION_KEY in n for n in all_nodes))

        all_nodes = self.core.describe_nodes(node_ids)
        all_node_ids = [m['node_id'] for m in all_nodes]
        self.assertEqual(set(all_node_ids), set(node_ids))

        subset_nodes = self.core.describe_nodes(node_ids[1:])
        subset_node_ids = [o['node_id'] for o in subset_nodes]
        self.assertEqual(set(subset_node_ids), set(node_ids[1:]))

        one_node = self.core.describe_nodes([node_ids[0]])
        self.assertEqual(len(one_node), 1)
        self.assertEqual(one_node[0]['node_id'], node_ids[0])
        self.assertEqual(one_node[0]['state'], states.RUNNING)

        self.assertNotIn(VERSION_KEY, one_node[0])

        try:
            self.core.describe_nodes([node_ids[0], "not-a-real-node"])
        except KeyError:
            pass
        else:
            self.fail("Expected exception for bad node_id")

    def test_maybe_update_node(self):

        node = dict(launch_id="somelaunch", node_id="anode",
                    state=states.REQUESTED)
        self.store.add_node(node)

        node2 = self.store.get_node("anode")

        node['state'] = states.PENDING
        self.store.update_node(node)

        # this should succeed even though we are basing off of an older copy
        node2['state'] = states.RUNNING
        node3, updated = self.core.maybe_update_node(node2)
        self.assertTrue(updated)
        self.assertEqual(node3['state'], states.RUNNING)

        node4 = self.store.get_node("anode")
        self.assertEqual(node4['state'], states.RUNNING)

    def test_out_of_order_launch_and_terminate(self):

        # test case where a node terminate request arrives before
        # the launch request.
        self.core.context = None
        launch_id = _new_id()
        instance_id = _new_id()

        self.core.mark_nodes_terminating([instance_id])
        self.assertTrue(self.notifier.assure_state(states.TERMINATED,
                                                   nodes=[instance_id]))
        self._prepare_execute(
            launch_id=launch_id, instance_ids=[instance_id],
            context_enabled=False, assure_state=False)
        self.assertTrue(self.notifier.assure_state(states.TERMINATED,
                                                   nodes=[instance_id]))
        # make sure nothing was launched
        self.assertFalse(self.site1_driver.list_nodes())
示例#5
0
文件: test_core.py 项目: timf/epu
class ProvisionerCoreRecoveryTests(unittest.TestCase):

    def setUp(self):
        self.notifier = FakeProvisionerNotifier()
        self.store = ProvisionerStore()
        self.ctx = FakeContextClient()
        self.driver = FakeNodeDriver()
        self.dtrs = FakeDTRS()
        drivers = {'fake' : self.driver}
        self.core = ProvisionerCore(store=self.store, notifier=self.notifier,
                                    dtrs=self.dtrs, site_drivers=drivers,
                                    context=self.ctx)

    @defer.inlineCallbacks
    def test_recover_launch_incomplete(self):
        """Ensures that launches in REQUESTED state are completed
        """
        launch_id = _new_id()
        doc = "<cluster><workspace><name>node</name><image>fake</image>"+\
              "<quantity>3</quantity>"+\
              "</workspace><workspace><name>running</name><image>fake"+\
              "</image><quantity>1</quantity></workspace></cluster>"
        context = {'broker_uri' : _new_id(), 'context_id' : _new_id(),
                  'secret' : _new_id(), 'uri' : _new_id()}

        requested_node_ids = [_new_id(), _new_id()]

        node_records = [make_node(launch_id, states.RUNNING,
                                              site='fake',
                                              ctx_name='running'),
                        make_node(launch_id, states.REQUESTED,
                                              site='fake',
                                              node_id=requested_node_ids[0],
                                              ctx_name='node'),
                        make_node(launch_id, states.REQUESTED,
                                              site='fake',
                                              node_id=requested_node_ids[1],
                                              ctx_name='node'),
                        make_node(launch_id, states.RUNNING,
                                              ctx_name='node')]
        launch_record = make_launch(launch_id, states.REQUESTED,
                                                node_records, document=doc,
                                                context=context)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        # 2 nodes are in REQUESTED state, so those should be launched
        yield self.core.recover()

        # because we rely on IaaS idempotency, we get full Node responses
        # for all nodes in the group. What really would cause this scenario
        # is successfully launching the full group but failing before records
        # could be written for the two REQUESTED nodes.
        self.assertEqual(3, len(self.driver.created))
        iaas_ids = set(node.id for node in self.driver.created)
        self.assertEqual(3, len(iaas_ids))

        for node_id in requested_node_ids:
            node = yield self.store.get_node(node_id)
            self.assertEqual(states.PENDING, node['state'])
            self.assertTrue(node['iaas_id'] in iaas_ids)

        launch = yield self.store.get_launch(launch_id)
        self.assertEqual(states.PENDING, launch['state'])

    @defer.inlineCallbacks
    def test_recovery_nodes_terminating(self):
        launch_id = _new_id()

        terminating_iaas_id = _new_id()

        node_records = [make_node(launch_id, states.TERMINATING,
                                              iaas_id=terminating_iaas_id,
                                              site='fake'),
                        make_node(launch_id, states.TERMINATED),
                        make_node(launch_id, states.RUNNING)]

        launch_record = make_launch(launch_id, states.RUNNING,
                                                node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.recover()

        self.assertEqual(1, len(self.driver.destroyed))
        self.assertEqual(self.driver.destroyed[0].id, terminating_iaas_id)

        terminated = yield self.store.get_nodes(state=states.TERMINATED)
        self.assertEqual(2, len(terminated))

    @defer.inlineCallbacks
    def test_recovery_launch_terminating(self):
        launch_id = _new_id()

        terminating_iaas_ids = [_new_id(), _new_id()]

        node_records = [make_node(launch_id, states.TERMINATING,
                                              iaas_id=terminating_iaas_ids[0],
                                              site='fake'),
                        make_node(launch_id, states.TERMINATED),
                        make_node(launch_id, states.RUNNING,
                                              iaas_id=terminating_iaas_ids[1],
                                              site='fake')]

        launch_record = make_launch(launch_id, states.TERMINATING,
                                                node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.recover()

        self.assertEqual(2, len(self.driver.destroyed))
        self.assertTrue(self.driver.destroyed[0].id in terminating_iaas_ids)
        self.assertTrue(self.driver.destroyed[1].id in terminating_iaas_ids)

        terminated = yield self.store.get_nodes(state=states.TERMINATED)
        self.assertEqual(3, len(terminated))

        launch_record = yield self.store.get_launch(launch_id)
        self.assertEqual(launch_record['state'], states.TERMINATED)

    @defer.inlineCallbacks
    def test_terminate_all(self):
        running_launch_id = _new_id()
        running_launch, running_nodes = make_launch_and_nodes(
                running_launch_id, 3, states.RUNNING)
        yield self.store.put_launch(running_launch)
        yield self.store.put_nodes(running_nodes)

        pending_launch_id = _new_id()
        pending_launch, pending_nodes = make_launch_and_nodes(
                pending_launch_id, 3, states.PENDING)
        yield self.store.put_launch(pending_launch)
        yield self.store.put_nodes(pending_nodes)

        terminated_launch_id = _new_id()
        terminated_launch, terminated_nodes = make_launch_and_nodes(
                terminated_launch_id, 3, states.TERMINATED)
        yield self.store.put_launch(terminated_launch)
        yield self.store.put_nodes(terminated_nodes)

        yield self.core.terminate_all()

        self.assertEqual(6, len(self.driver.destroyed))

        all_launches = yield self.store.get_launches()
        self.assertEqual(3, len(all_launches))
        self.assertTrue(all(l['state'] == states.TERMINATED
                           for l in all_launches))

        all_nodes = yield self.store.get_nodes()
        self.assertEqual(9, len(all_nodes))
        self.assertTrue(all(n['state'] == states.TERMINATED
                           for n in all_nodes))

        state = yield self.core.check_terminate_all()
        self.assertTrue(state)
示例#6
0
文件: test_core.py 项目: timf/epu
class ProvisionerCoreTests(unittest.TestCase):
    """Testing the provisioner core functionality
    """
    def setUp(self):
        self.notifier = FakeProvisionerNotifier()
        self.store = ProvisionerStore()
        self.ctx = FakeContextClient()
        self.dtrs = FakeDTRS()

        self.site1_driver = FakeNodeDriver()
        self.site2_driver = FakeNodeDriver()

        drivers = {'site1' : self.site1_driver, 'site2' : self.site2_driver}
        self.core = ProvisionerCore(store=self.store, notifier=self.notifier,
                                    dtrs=self.dtrs, context=self.ctx,
                                    site_drivers=drivers)

    @defer.inlineCallbacks
    def test_prepare_dtrs_error(self):
        self.dtrs.error = DeployableTypeLookupError()

        nodes = {"i1" : dict(ids=[_new_id()], site="chicago", allocation="small")}
        request = dict(launch_id=_new_id(), deployable_type="foo",
                       subscribers=('blah',), nodes=nodes)
        yield self.core.prepare_provision(request)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_prepare_broker_error(self):
        self.ctx.create_error = BrokerError("fake ctx create failed")
        self.dtrs.result = {'document' : "<fake>document</fake>",
                            "nodes" : {"i1" : {}}}
        nodes = {"i1" : dict(ids=[_new_id()], site="site1", allocation="small")}
        request = dict(launch_id=_new_id(), deployable_type="foo",
                       subscribers=('blah',), nodes=nodes)
        yield self.core.prepare_provision(request)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_prepare_execute(self):
        yield self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.PENDING))

    @defer.inlineCallbacks
    def test_prepare_execute_iaas_fail(self):
        self.site1_driver.create_node_error = InvalidCredsError()
        yield self._prepare_execute()
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def _prepare_execute(self):
        self.dtrs.result = {'document' : _get_one_node_cluster_doc("node1", "image1"),
                            "nodes" : {"node1" : {}}}
        request_node = dict(ids=[_new_id()], site="site1", allocation="small")
        request_nodes = {"node1" : request_node}
        request = dict(launch_id=_new_id(), deployable_type="foo",
                       subscribers=('blah',), nodes=request_nodes)

        launch, nodes = yield self.core.prepare_provision(request)

        self.assertEqual(len(nodes), 1)
        node = nodes[0]
        self.assertEqual(node['node_id'], request_node['ids'][0])
        self.assertEqual(launch['launch_id'], request['launch_id'])

        self.assertTrue(self.ctx.last_create)
        self.assertEqual(launch['context'], self.ctx.last_create)
        for key in ('uri', 'secret', 'context_id', 'broker_uri'):
            self.assertIn(key, launch['context'])
        self.assertTrue(self.notifier.assure_state(states.REQUESTED))

        yield self.core.execute_provision(launch, nodes)

    @defer.inlineCallbacks
    def test_execute_bad_doc(self):
        ctx = yield self.ctx.create()
        launch_record = {
                'launch_id' : "thelaunchid",
                'document' : "<this><isnt><a><real><doc>",
                'deployable_type' : "dt",
                'context' : ctx,
                'subscribers' : [],
                'state' : states.PENDING,
                'node_ids' : ['node1']}
        nodes = [{'node_id' : 'node1', 'launch_id' : "thelaunchid",
                  'state' : states.REQUESTED}]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

        # TODO this should be a better error coming from nimboss
        #self.assertEqual(self.notifier.nodes['node1']['state_desc'], "CONTEXT_DOC_INVALID")

    @defer.inlineCallbacks
    def test_execute_bad_doc_nodes(self):
        ctx = yield self.ctx.create()
        launch_record = {
                'launch_id' : "thelaunchid",
                'document' : _get_one_node_cluster_doc("node1", "image1"),
                'deployable_type' : "dt",
                'context' : ctx,
                'subscribers' : [],
                'state' : states.PENDING,
                'node_ids' : ['node1']}
        nodes = [{'node_id' : 'node1', 'launch_id' : "thelaunchid",
                  'state' : states.REQUESTED, 'ctx_name' : "adifferentname"}]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_execute_bad_doc_node_count(self):
        ctx = yield self.ctx.create()
        launch_record = {
                'launch_id' : "thelaunchid",
                'document' : _get_one_node_cluster_doc("node1", "image1"),
                'deployable_type' : "dt",
                'context' : ctx,
                'subscribers' : [],
                'state' : states.PENDING,
                'node_ids' : ['node1']}

        # two nodes where doc expects 1
        nodes = [{'node_id' : 'node1', 'launch_id' : "thelaunchid",
                  'state' : states.REQUESTED, 'ctx_name' : "node1"},
                 {'node_id' : 'node1', 'launch_id' : "thelaunchid",
                  'state' : states.REQUESTED, 'ctx_name' : "node1"}]

        yield self.core.execute_provision(launch_record, nodes)
        self.assertTrue(self.notifier.assure_state(states.FAILED))


    @defer.inlineCallbacks
    def test_query_missing_node_within_window(self):
        launch_id = _new_id()
        node_id = _new_id()
        ts = time.time() - 30.0
        launch = {'launch_id' : launch_id, 'node_ids' : [node_id],
                'state' : states.PENDING,
                'subscribers' : 'fake-subscribers'}
        node = {'launch_id' : launch_id,
                'node_id' : node_id,
                'state' : states.PENDING,
                'pending_timestamp' : ts}
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)

        yield self.core.query_one_site('fake-site', [node],
                driver=FakeEmptyNodeQueryDriver())
        self.assertEqual(len(self.notifier.nodes), 0)
    
    @defer.inlineCallbacks
    def test_query_missing_node_past_window(self):
        launch_id = _new_id()
        node_id = _new_id()

        ts = time.time() - 120.0
        launch = {
                'launch_id' : launch_id, 'node_ids' : [node_id],
                'state' : states.PENDING,
                'subscribers' : 'fake-subscribers'}
        node = {'launch_id' : launch_id,
                'node_id' : node_id,
                'state' : states.PENDING,
                'pending_timestamp' : ts}
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)

        yield self.core.query_one_site('fake-site', [node],
                driver=FakeEmptyNodeQueryDriver())
        self.assertEqual(len(self.notifier.nodes), 1)
        self.assertTrue(self.notifier.assure_state(states.FAILED))

    @defer.inlineCallbacks
    def test_query(self):
        launch_id = _new_id()
        node_id = _new_id()

        iaas_node = self.site1_driver.create_node()[0]
        self.site1_driver.set_node_running(iaas_node.id)

        ts = time.time() - 120.0
        launch = {
                'launch_id' : launch_id, 'node_ids' : [node_id],
                'state' : states.PENDING,
                'subscribers' : 'fake-subscribers'}
        node = {'launch_id' : launch_id,
                'node_id' : node_id,
                'state' : states.PENDING,
                'pending_timestamp' : ts,
                'iaas_id' : iaas_node.id,
                'site':'site1'}

        req_node = {'launch_id' : launch_id,
                'node_id' : _new_id(),
                'state' : states.REQUESTED}
        nodes = [node, req_node]
        yield self.store.put_launch(launch)
        yield self.store.put_node(node)
        yield self.store.put_node(req_node)

        yield self.core.query_one_site('site1', nodes)

        node = yield self.store.get_node(node_id)
        self.assertEqual(node['public_ip'], iaas_node.public_ip)
        self.assertEqual(node['private_ip'], iaas_node.private_ip)
        self.assertEqual(node['state'], states.STARTED)

        # query again should detect no changes
        yield self.core.query_one_site('site1', nodes)

        # now destroy
        yield self.core.terminate_nodes([node_id])
        node = yield self.store.get_node(node_id)
        yield self.core.query_one_site('site1', [node])

        node = yield self.store.get_node(node_id)
        self.assertEqual(node['public_ip'], iaas_node.public_ip)
        self.assertEqual(node['private_ip'], iaas_node.private_ip)
        self.assertEqual(node['state'], states.TERMINATED)


    @defer.inlineCallbacks
    def test_query_ctx(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                for i in range(node_count)]
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        #first query with no ctx nodes. zero records should be updated
        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_record_count(0))
        
        # all but 1 node have reported ok
        self.ctx.nodes = [_one_fake_ctx_node_ok(node_records[i]['public_ip'], 
            _new_id(),  _new_id()) for i in range(node_count-1)]

        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertEqual(len(self.notifier.nodes), node_count-1)

        #last node reports ok
        self.ctx.nodes.append(_one_fake_ctx_node_ok(node_records[-1]['public_ip'],
            _new_id(), _new_id()))

        self.ctx.complete = True
        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING))
        self.assertTrue(self.notifier.assure_record_count(1))
    
    @defer.inlineCallbacks
    def test_query_ctx_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                for i in range(node_count)]
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.expected_count = len(node_records)
        self.ctx.complete = False
        self.ctx.error = False

        # all but 1 node have reported ok
        self.ctx.nodes = [_one_fake_ctx_node_ok(node_records[i]['public_ip'], 
            _new_id(),  _new_id()) for i in range(node_count-1)]
        self.ctx.nodes.append(_one_fake_ctx_node_error(node_records[-1]['public_ip'],
            _new_id(), _new_id()))

        ok_ids = [node_records[i]['node_id'] for i in range(node_count-1)]
        error_ids = [node_records[-1]['node_id']]

        self.ctx.complete = True
        self.ctx.error = True

        yield self.core.query_contexts()
        self.assertTrue(self.notifier.assure_state(states.RUNNING, ok_ids))
        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED, error_ids))

    @defer.inlineCallbacks
    def test_query_ctx_nodes_not_started(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.PENDING)
                for i in range(3)]
        node_records.append(make_node(launch_id, states.STARTED))
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        yield self.core.query_contexts()

        # ensure that no context was actually queried. See the note in
        # _query_one_context for the reason why this is important.
        self.assertEqual(len(self.ctx.queried_uris), 0)

    @defer.inlineCallbacks
    def test_query_ctx_permanent_broker_error(self):
        node_count = 3
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)
                for i in range(node_count)]
        node_ids = [node['node_id'] for node in node_records]
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        self.ctx.query_error = ContextNotFoundError()
        yield self.core.query_contexts()

        self.assertTrue(self.notifier.assure_state(states.RUNNING_FAILED, node_ids))
        launch = yield self.store.get_launch(launch_id)
        self.assertEqual(launch['state'], states.FAILED)

    def test_update_node_ip_info(self):
        node = dict(public_ip=None)
        iaas_node = Mock(public_ip=None, private_ip=None)
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], None)
        self.assertEqual(node['private_ip'], None)

        iaas_node = Mock(public_ip=["pub1"], private_ip=["priv1"])
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")

        iaas_node = Mock(public_ip=[], private_ip=[])
        update_node_ip_info(node, iaas_node)
        self.assertEqual(node['public_ip'], "pub1")
        self.assertEqual(node['private_ip'], "priv1")

    def test_update_nodes_from_ctx(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED)
                for i in range(5)]
        ctx_nodes = [_one_fake_ctx_node_ok(node['public_ip'], _new_id(), 
            _new_id()) for node in nodes]

        self.assertEquals(len(nodes), len(update_nodes_from_context(nodes, ctx_nodes)))
        
    def test_update_nodes_from_ctx_with_hostname(self):
        launch_id = _new_id()
        nodes = [make_node(launch_id, states.STARTED)
                for i in range(5)]
        #libcloud puts the hostname in the public_ip field
        ctx_nodes = [_one_fake_ctx_node_ok(ip=_new_id(), hostname=node['public_ip'],
            pubkey=_new_id()) for node in nodes]

        self.assertEquals(len(nodes), len(update_nodes_from_context(nodes, ctx_nodes)))

    @defer.inlineCallbacks
    def test_query_broker_exception(self):
        for i in range(2):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING,
                                                    node_records)

            yield self.store.put_launch(launch_record)
            yield self.store.put_nodes(node_records)

        # no guaranteed order here so grabbing first launch from store
        # and making that one return a BrokerError during context query.
        # THe goal is to ensure that one error doesn't prevent querying
        # for other contexts.

        launches = yield self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]
        error_launch_ctx = error_launch['context']['uri']
        ok_node_id = launches[1]['node_ids'][0]
        ok_node = yield self.store.get_node(ok_node_id)

        self.ctx.uri_query_error[error_launch_ctx] = BrokerError("bad broker")
        self.ctx.nodes = [_one_fake_ctx_node_ok(ok_node['public_ip'],
            _new_id(), _new_id())]
        self.ctx.complete = True
        yield self.core.query_contexts()

        launches = yield self.store.get_launches()
        for launch in launches:
            self.assertIn(launch['context']['uri'], self.ctx.queried_uris)

            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED
            else:
                self.assertEqual(launch['state'], states.RUNNING)
                expected_node_state = states.RUNNING

            node = yield self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)

    @defer.inlineCallbacks
    def test_query_ctx_without_valid_nodes(self):

        # if there are no nodes < TERMINATING, no broker query should happen
        for i in range(3):
            launch_id = _new_id()
            node_records = [make_node(launch_id, states.STARTED)]
            launch_record = make_launch(launch_id, states.PENDING,
                                                    node_records)

            yield self.store.put_launch(launch_record)
            yield self.store.put_nodes(node_records)

        launches = yield self.store.get_launches(state=states.PENDING)
        error_launch = launches[0]

        # mark first launch's node as TERMINATING, should prevent
        # context query and result in launch being marked FAILED
        error_launch_node = yield self.store.get_node(error_launch['node_ids'][0])
        error_launch_node['state'] = states.TERMINATING
        yield self.store.put_node(error_launch_node)

        yield self.core.query_contexts()
        self.assertNotIn(error_launch['context']['uri'], self.ctx.queried_uris)

        launches = yield self.store.get_launches()
        for launch in launches:
            if launch['launch_id'] == error_launch['launch_id']:
                self.assertEqual(launch['state'], states.FAILED)
                expected_node_state = states.TERMINATING
            else:
                self.assertEqual(launch['state'], states.PENDING)
                expected_node_state = states.STARTED

            node = yield self.store.get_node(launch['node_ids'][0])
            self.assertEqual(node['state'], expected_node_state)


    @defer.inlineCallbacks
    def test_query_unexpected_exception(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.STARTED)]
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)
        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)
        self.ctx.query_error = ValueError("bad programmer")


        # digging into internals a bit: patching one of the methods query()
        # calls to raise an exception. This will let us ensure exceptions do
        # not bubble up
        def raiser(self):
            raise KeyError("notreallyaproblem")
        self.patch(self.core, 'query_nodes', raiser)

        yield self.core.query() # ensure that exception doesn't bubble up

    @defer.inlineCallbacks
    def test_dump_state(self):
        node_ids = []
        node_records = []
        for i in range(3):
            launch_id = _new_id()
            nodes = [make_node(launch_id, states.PENDING)]
            node_ids.append(nodes[0]['node_id'])
            node_records.extend(nodes)
            launch = make_launch(launch_id, states.PENDING,
                                                    nodes)
            yield self.store.put_launch(launch)
            yield self.store.put_nodes(nodes)

        yield self.core.dump_state(node_ids[:2])

        # should have gotten notifications about the 2 nodes
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[0]], 1)
        self.assertEqual(node_records[0], self.notifier.nodes[node_ids[0]])
        self.assertEqual(node_records[1], self.notifier.nodes[node_ids[1]])
        self.assertEqual(self.notifier.nodes_rec_count[node_ids[1]], 1)
        self.assertNotIn(node_ids[2], self.notifier.nodes)

    @defer.inlineCallbacks
    def test_mark_nodes_terminating(self):
        launch_id = _new_id()
        node_records = [make_node(launch_id, states.RUNNING)
                        for i in range(3)]
        launch_record = make_launch(launch_id, states.PENDING,
                                                node_records)

        yield self.store.put_launch(launch_record)
        yield self.store.put_nodes(node_records)

        first_two_node_ids = [node_records[0]['node_id'],
                              node_records[1]['node_id']]
        yield self.core.mark_nodes_terminating(first_two_node_ids)

        self.assertTrue(self.notifier.assure_state(states.TERMINATING,
                                                   nodes=first_two_node_ids))
        self.assertNotIn(node_records[2]['node_id'], self.notifier.nodes)

        for node_id in first_two_node_ids:
            terminating_node = yield self.store.get_node(node_id)
            self.assertEqual(terminating_node['state'], states.TERMINATING)
示例#7
0
文件: test_store.py 项目: timf/epu
class BaseProvisionerStoreTests(unittest.TestCase):
    def setUp(self):
        self.store = ProvisionerStore()
    def tearDown(self):
        self.store = None

    @defer.inlineCallbacks
    def test_put_get_launches(self):

        launch_id_1 = new_id()
        l1 = {'launch_id' : launch_id_1, 'state' : states.REQUESTED}
        yield self.store.put_launch(l1)

        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        l2 = l1.copy()
        l2['state'] = states.PENDING
        yield self.store.put_launch(l2)
        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.PENDING, latest['state'])

        # store another launch altogether
        launch_id_2 = new_id()
        l3 = {'launch_id' : launch_id_2, 'state' : states.REQUESTED}
        yield self.store.put_launch(l3)

        latest = yield self.store.get_launch(launch_id_2)
        self.assertEqual(launch_id_2, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        # put the first launch record again, should not overwrite l2
        # because state is lower
        yield self.store.put_launch(l2)
        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.PENDING, latest['state'])

        latest = yield self.store.get_launch(launch_id_2)
        self.assertEqual(launch_id_2, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        # add a third launch with request, pending, and running records
        launch_id_3 = new_id()
        l4 = {'launch_id' : launch_id_3, 'state' : states.REQUESTED}
        yield self.store.put_launch(l4)
        l5 = {'launch_id' : launch_id_3, 'state' : states.PENDING}
        yield self.store.put_launch(l5)
        l6 = {'launch_id' : launch_id_3, 'state' : states.RUNNING}
        yield self.store.put_launch(l6)

        all = yield self.store.get_launches()
        self.assertEqual(3, len(all))
        for l in all:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2,
                                               launch_id_3))

        # try some range queries
        requested = yield self.store.get_launches(state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        requested = yield self.store.get_launches(
                min_state=states.REQUESTED,
                max_state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        at_least_requested = yield self.store.get_launches(
                min_state=states.REQUESTED)
        self.assertEqual(3, len(at_least_requested))
        for l in at_least_requested:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2,
                                               launch_id_3))

        at_least_pending = yield self.store.get_launches(
                min_state=states.PENDING)
        self.assertEqual(2, len(at_least_pending))
        for l in at_least_pending:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_3))

        at_most_pending = yield self.store.get_launches(
            max_state=states.PENDING)
        self.assertEqual(2, len(at_most_pending))
        for l in at_most_pending:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2))

    @defer.inlineCallbacks
    def put_node(self, node_id, *states):
        for state in states:
            record = {'node_id' : node_id, 'state':state}
            yield self.store.put_node(record)

    @defer.inlineCallbacks
    def put_many_nodes(self, count, *states):
        node_ids = set(str(uuid.uuid4()) for i in range(count))
        for node_id in node_ids:
            yield self.put_node(node_id, *states)
        defer.returnValue(node_ids)

    def assertNodesInSet(self, nodes, *sets):
        node_ids = set(node["node_id"] for node in nodes)
        self.assertEqual(len(nodes), len(node_ids))

        for node_id in node_ids:
            found = False
            for aset in sets:
                if node_id in aset:
                    found = True
                    break
            if not found:
                self.fail("node %s not in any set" % node_id)
示例#8
0
文件: test_store.py 项目: timf/epu
class BaseProvisionerStoreTests(unittest.TestCase):
    def setUp(self):
        self.store = ProvisionerStore()

    def tearDown(self):
        self.store = None

    @defer.inlineCallbacks
    def test_put_get_launches(self):

        launch_id_1 = new_id()
        l1 = {'launch_id': launch_id_1, 'state': states.REQUESTED}
        yield self.store.put_launch(l1)

        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        l2 = l1.copy()
        l2['state'] = states.PENDING
        yield self.store.put_launch(l2)
        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.PENDING, latest['state'])

        # store another launch altogether
        launch_id_2 = new_id()
        l3 = {'launch_id': launch_id_2, 'state': states.REQUESTED}
        yield self.store.put_launch(l3)

        latest = yield self.store.get_launch(launch_id_2)
        self.assertEqual(launch_id_2, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        # put the first launch record again, should not overwrite l2
        # because state is lower
        yield self.store.put_launch(l2)
        latest = yield self.store.get_launch(launch_id_1)
        self.assertEqual(launch_id_1, latest['launch_id'])
        self.assertEqual(states.PENDING, latest['state'])

        latest = yield self.store.get_launch(launch_id_2)
        self.assertEqual(launch_id_2, latest['launch_id'])
        self.assertEqual(states.REQUESTED, latest['state'])

        # add a third launch with request, pending, and running records
        launch_id_3 = new_id()
        l4 = {'launch_id': launch_id_3, 'state': states.REQUESTED}
        yield self.store.put_launch(l4)
        l5 = {'launch_id': launch_id_3, 'state': states.PENDING}
        yield self.store.put_launch(l5)
        l6 = {'launch_id': launch_id_3, 'state': states.RUNNING}
        yield self.store.put_launch(l6)

        all = yield self.store.get_launches()
        self.assertEqual(3, len(all))
        for l in all:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2,
                                               launch_id_3))

        # try some range queries
        requested = yield self.store.get_launches(state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        requested = yield self.store.get_launches(min_state=states.REQUESTED,
                                                  max_state=states.REQUESTED)
        self.assertEqual(1, len(requested))
        self.assertEqual(launch_id_2, requested[0]['launch_id'])

        at_least_requested = yield self.store.get_launches(
            min_state=states.REQUESTED)
        self.assertEqual(3, len(at_least_requested))
        for l in at_least_requested:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2,
                                               launch_id_3))

        at_least_pending = yield self.store.get_launches(
            min_state=states.PENDING)
        self.assertEqual(2, len(at_least_pending))
        for l in at_least_pending:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_3))

        at_most_pending = yield self.store.get_launches(
            max_state=states.PENDING)
        self.assertEqual(2, len(at_most_pending))
        for l in at_most_pending:
            self.assertTrue(l['launch_id'] in (launch_id_1, launch_id_2))

    @defer.inlineCallbacks
    def put_node(self, node_id, *states):
        for state in states:
            record = {'node_id': node_id, 'state': state}
            yield self.store.put_node(record)

    @defer.inlineCallbacks
    def put_many_nodes(self, count, *states):
        node_ids = set(str(uuid.uuid4()) for i in range(count))
        for node_id in node_ids:
            yield self.put_node(node_id, *states)
        defer.returnValue(node_ids)

    def assertNodesInSet(self, nodes, *sets):
        node_ids = set(node["node_id"] for node in nodes)
        self.assertEqual(len(nodes), len(node_ids))

        for node_id in node_ids:
            found = False
            for aset in sets:
                if node_id in aset:
                    found = True
                    break
            if not found:
                self.fail("node %s not in any set" % node_id)