Example #1
0
 def render(self):
     # TODO: demo specific should be changed afterwards
     if self.provider.name == "maas":
         DEMO_BUNDLE = os.path.join(
             Config.share_path(), "data-analytics-with-sql-like.yaml")
         DEMO_METADATA = os.path.join(
             Config.share_path(),
             "data-analytics-with-sql-like-metadata.yaml")
         bundleplacer_cfg = Config('bundle-placer',
                                   {'bundle_filename': DEMO_BUNDLE,
                                    'metadata_filename': DEMO_METADATA})
         placement_controller = PlacementController(
             config=bundleplacer_cfg,
             maas_state=FakeMaasState())
         mainview = PlacerView(placement_controller,
                               bundleplacer_cfg)
         self.common['ui'].set_header(
             title="Bundle Editor: {}".format(
                 self.common['config']['summary']),
             excerpt="Choose where your services should be "
             "placed in your available infrastructure"
         )
         self.common['ui'].set_subheader("Machine Placement")
         self.common['ui'].set_body(mainview)
         mainview.update()
Example #2
0
def main():
    opts = parse_options(sys.argv[1:])

    config = Config('bundle-placer', opts.__dict__)
    config.save()

    setup_logger(cfg_path=config.cfg_path)
    log = logging.getLogger('bundleplacer')
    log.debug(opts.__dict__)

    log.info("Editing file: {}".format(opts.bundle_filename))

    if opts.maas_ip and opts.maas_cred:
        creds = dict(api_host=opts.maas_ip,
                     api_key=opts.maas_cred)
        maas, maas_state = connect_to_maas(creds)
    else:
        maas_state = FakeMaasState()

    placement_controller = PlacementController(config=config,
                                               maas_state=maas_state)

    mainview = PlacerView(placement_controller, config)
    ui = PlacerUI(mainview)

    def unhandled_input(key):
        if key in ['q', 'Q']:
            raise urwid.ExitMainLoop()
    EventLoop.build_loop(ui, STYLES, unhandled_input=unhandled_input)
    mainview.loop = EventLoop.loop
    mainview.update()
    EventLoop.run()
Example #3
0
def setup_metadata_controller():
    """ Pulls in a local bundle or via charmstore api and sets up our
    controller. You can also further customize the bundle by providing a local
    bundle-custom.yaml that will be deep merged over whatever bundle is
    referenced. """
    spell_dir = Path(app.config['spell-dir'])
    bundle_filename = spell_dir / 'bundle.yaml'
    bundle_custom_filename = spell_dir / 'bundle-custom.yaml'

    if bundle_filename.exists():
        # Load bundle data early so we can merge any additional charm options
        bundle_data = yaml.load(bundle_filename.read_text())
    else:
        bundle_name = app.config['metadata'].get('bundle-name', None)
        if bundle_name is None:
            raise Exception(
                "Could not determine a bundle to download, please make sure "
                "the spell contains a 'bundle-name' field.")
        bundle_channel = app.argv.channel

        app.log.debug("Pulling bundle for {} from channel: {}".format(
            bundle_name, bundle_channel))
        bundle_data = charm.get_bundle(bundle_name, bundle_channel)

    if bundle_custom_filename.exists():
        bundle_custom = yaml.load(slurp(bundle_custom_filename))
        bundle_data = merge_dicts(bundle_data, bundle_custom)

    for name in app.selected_addons:
        addon = app.addons[name]
        bundle_data = merge_dicts(bundle_data, addon.bundle)

    bundle = Bundle(bundle_data=bundle_data)
    app.metadata_controller = MetadataController(bundle, Config('bundle-cfg'))
Example #4
0
def setup_metadata_controller():
    bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
    if not os.path.isfile(bundle_filename):
        if 'bundle-location' not in app.config['metadata']:
            raise Exception(
                "Could not determine bundle location: no local bundle "
                "was found and bundle-location not set in spell metadata.")
        bundle_filename = charm.get_bundle(
            app.config['metadata']['bundle-location'], True)

    # Load bundle data early so we can merge any additional charm options
    with open(bundle_filename) as f:
        bundle_data = yaml.load(f)

    bundle_custom_filename = os.path.join(app.config['spell-dir'],
                                          'bundle-custom.yaml')
    if os.path.isfile(bundle_custom_filename):
        with open(bundle_custom_filename) as f:
            bundle_custom = yaml.load(f)
        bundle_data = merge_dicts(bundle_data, bundle_custom)

    bundle = Bundle(bundle_data=bundle_data)
    bundleplacer_cfg = Config('bundle-placer', {
        'bundle_filename': bundle_filename,
        'bundle_key': None,
    })

    app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
    def setUp(self):
        self.mock_maas_state = MagicMock()
        with NamedTemporaryFile(mode='w+', encoding='utf-8') as tempf:
            utils.spew(tempf.name, yaml.dump(dict()))
            self.conf = Config(tempf.name, {}, save_backups=False)

        temp_bundle_f = NamedTemporaryFile(mode='r')
        self.conf.setopt('bundle_filename', temp_bundle_f.name)
        self.service_1 = create_service("nova-compute", {
            "num_units": 1,
            "charm": "cs:trusty/nova-compute-100"}, {}, [])
        self.service_2 = create_service("keystone", {
            "num_units": 1,
            "charm": "cs:trusty/keystone-100"}, {}, [])

        self.bundle_patcher = patch("bundleplacer.controller.Bundle")
        self.mock_bundle = self.bundle_patcher.start()
        self.mock_bundle_i = self.mock_bundle.return_value
        pm = PropertyMock(return_value=[self.service_1, self.service_2])
        type(self.mock_bundle_i).services = pm

        self.pc = PlacementController(self.mock_maas_state,
                                      self.conf)
        self.mock_machine = MagicMock(name='machine1')
        pmid = PropertyMock(return_value='fake-instance-id-1')
        type(self.mock_machine).instance_id = pmid

        self.mock_machine_2 = MagicMock(name='machine2')
        pmid2 = PropertyMock(return_value='fake-instance-id-2')
        type(self.mock_machine_2).instance_id = pmid2

        self.mock_machines = [self.mock_machine, self.mock_machine_2]

        self.mock_maas_state.machines.return_value = self.mock_machines
Example #6
0
def setup_metadata_controller():
    bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
    bundle = Bundle(filename=bundle_filename)
    bundleplacer_cfg = Config('bundle-placer', {
        'bundle_filename': bundle_filename,
        'bundle_key': None,
    })

    app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
Example #7
0
 def render(self):
     # TODO: demo specific should be changed afterwards
     if self.provider.name == "maas":
         DEMO_BUNDLE = os.path.join(Config.share_path(),
                                    "data-analytics-with-sql-like.yaml")
         DEMO_METADATA = os.path.join(
             Config.share_path(),
             "data-analytics-with-sql-like-metadata.yaml")
         bundleplacer_cfg = Config('bundle-placer', {
             'bundle_filename': DEMO_BUNDLE,
             'metadata_filename': DEMO_METADATA
         })
         placement_controller = PlacementController(
             config=bundleplacer_cfg, maas_state=FakeMaasState())
         mainview = PlacerView(placement_controller, bundleplacer_cfg)
         self.common['ui'].set_header(
             title="Bundle Editor: {}".format(
                 self.common['config']['summary']),
             excerpt="Choose where your services should be "
             "placed in your available infrastructure")
         self.common['ui'].set_subheader("Machine Placement")
         self.common['ui'].set_body(mainview)
         mainview.update()
Example #8
0
def setup_metadata_controller():
    bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
    if not os.path.isfile(bundle_filename):
        if 'bundle-location' not in app.config['metadata']:
            raise Exception(
                "Could not determine bundle location: no local bundle "
                "was found and bundle-location not set in spell metadata.")
        bundle_filename = charm.get_bundle(
            app.config['metadata']['bundle-location'], True)

    bundle = Bundle(filename=bundle_filename)
    bundleplacer_cfg = Config('bundle-placer', {
        'bundle_filename': bundle_filename,
        'bundle_key': None,
    })

    app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
Example #9
0
def main():
    if os.getenv("BUNDLE_EDITOR_TESTING"):
        test_args = True
    else:
        test_args = False

    opts = parse_options(sys.argv[1:], test_args)

    config = Config('bundle-placer', opts.__dict__)
    config.save()

    setup_logger(cfg_path=config.cfg_path)
    log = logging.getLogger('bundleplacer')
    log.debug(opts.__dict__)

    log.info("Editing file: {}".format(opts.bundle_filename))

    if opts.maas_ip and opts.maas_cred:
        creds = dict(api_host=opts.maas_ip,
                     api_key=opts.maas_cred)
        maas, maas_state = connect_to_maas(creds)
    elif 'fake_maas' in opts and opts.fake_maas:
        maas = None
        maas_state = FakeMaasState()
    else:
        maas = None
        maas_state = None

    try:
        placement_controller = PlacementController(config=config,
                                                   maas_state=maas_state)
    except Exception as e:
        print("Error: " + e.args[0])
        return

    def cb():
        if maas:
            maas.tag_name(maas.nodes)

        bw = BundleWriter(placement_controller)
        if opts.out_filename:
            outfn = opts.out_filename
        else:
            outfn = opts.bundle_filename
            if os.path.exists(outfn):
                shutil.copy2(outfn, outfn + '~')
        bw.write_bundle(outfn)
        async.shutdown()
        raise urwid.ExitMainLoop()

    has_maas = (maas_state is not None)
    mainview = PlacerView(placement_controller, config, cb, has_maas=has_maas)
    ui = PlacerUI(mainview)

    def unhandled_input(key):
        if key in ['q', 'Q']:
            async.shutdown()
            raise urwid.ExitMainLoop()
    EventLoop.build_loop(ui, STYLES, unhandled_input=unhandled_input)
    mainview.loop = EventLoop.loop
    mainview.update()
    EventLoop.run()
class PlacementControllerTestCase(unittest.TestCase):

    def setUp(self):
        self.mock_maas_state = MagicMock()
        with NamedTemporaryFile(mode='w+', encoding='utf-8') as tempf:
            utils.spew(tempf.name, yaml.dump(dict()))
            self.conf = Config(tempf.name, {}, save_backups=False)

        temp_bundle_f = NamedTemporaryFile(mode='r')
        self.conf.setopt('bundle_filename', temp_bundle_f.name)
        self.service_1 = create_service("nova-compute", {
            "num_units": 1,
            "charm": "cs:trusty/nova-compute-100"}, {}, [])
        self.service_2 = create_service("keystone", {
            "num_units": 1,
            "charm": "cs:trusty/keystone-100"}, {}, [])

        self.bundle_patcher = patch("bundleplacer.controller.Bundle")
        self.mock_bundle = self.bundle_patcher.start()
        self.mock_bundle_i = self.mock_bundle.return_value
        pm = PropertyMock(return_value=[self.service_1, self.service_2])
        type(self.mock_bundle_i).services = pm

        self.pc = PlacementController(self.mock_maas_state,
                                      self.conf)
        self.mock_machine = MagicMock(name='machine1')
        pmid = PropertyMock(return_value='fake-instance-id-1')
        type(self.mock_machine).instance_id = pmid

        self.mock_machine_2 = MagicMock(name='machine2')
        pmid2 = PropertyMock(return_value='fake-instance-id-2')
        type(self.mock_machine_2).instance_id = pmid2

        self.mock_machines = [self.mock_machine, self.mock_machine_2]

        self.mock_maas_state.machines.return_value = self.mock_machines

    def tearDown(self):
        self.bundle_patcher.stop()

    def test_get_assignments_atype(self):
        self.assertEqual(0,
                         len(self.pc.get_assignments(self.service_1)))
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        md = self.pc.get_assignments(self.service_1)
        self.assertEqual(1, len(md))
        self.assertEqual(2, len(md[AssignmentType.LXC]))

    def _do_test_simple_assign_type(self, assignment_type):
        self.pc.assign(self.mock_machine, self.service_1, assignment_type)
        print("assignments is {}".format(self.pc.assignments))
        machines = self.pc.get_assignments(self.service_1)
        print('machines for charm is {}'.format(machines))
        self.assertEqual(machines,
                         {assignment_type: [self.mock_machine]})

        ma = self.pc.assignments_for_machine(self.mock_machine)

        self.assertEqual(ma[assignment_type], [self.service_1])

    def test_simple_assign_bare(self):
        self._do_test_simple_assign_type(AssignmentType.BareMetal)

    def test_simple_assign_lxc(self):
        self._do_test_simple_assign_type(AssignmentType.LXC)

    def test_simple_assign_kvm(self):
        self._do_test_simple_assign_type(AssignmentType.KVM)

    def test_assign_multi(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.assertEqual(self.pc.get_assignments(self.service_1),
                         {AssignmentType.LXC: [self.mock_machine]})

        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
        self.assertEqual(self.pc.get_assignments(self.service_1),
                         {AssignmentType.LXC: [self.mock_machine],
                          AssignmentType.KVM: [self.mock_machine]})

        ma = self.pc.assignments_for_machine(self.mock_machine)
        self.assertEqual(ma[AssignmentType.LXC], [self.service_1])
        self.assertEqual(ma[AssignmentType.KVM], [self.service_1])

    def test_remove_assignment_multi(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine_2, self.service_1,
                       AssignmentType.LXC)

        mfc = self.pc.get_assignments(self.service_1)

        mfc_lxc = set(mfc[AssignmentType.LXC])
        self.assertEqual(mfc_lxc, set(self.mock_machines))

        self.pc.clear_assignments(self.mock_machine)
        self.assertEqual(self.pc.get_assignments(self.service_1),
                         {AssignmentType.LXC: [self.mock_machine_2]})

    def test_remove_one_assignment_sametype(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)

        self.pc.remove_one_assignment(self.mock_machine, self.service_1)
        md = self.pc.assignments[self.mock_machine.instance_id]
        lxcs = md[AssignmentType.LXC]
        self.assertEqual(lxcs, [self.service_1])

        self.pc.remove_one_assignment(self.mock_machine, self.service_1)
        md = self.pc.assignments[self.mock_machine.instance_id]
        lxcs = md[AssignmentType.LXC]
        self.assertEqual(lxcs, [])

    def test_remove_one_assignment_othertype(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)

        self.pc.remove_one_assignment(self.mock_machine, self.service_1)
        md = self.pc.assignments[self.mock_machine.instance_id]
        lxcs = md[AssignmentType.LXC]
        kvms = md[AssignmentType.KVM]
        self.assertEqual(1, len(lxcs) + len(kvms))

        self.pc.remove_one_assignment(self.mock_machine, self.service_1)
        md = self.pc.assignments[self.mock_machine.instance_id]
        lxcs = md[AssignmentType.LXC]
        kvms = md[AssignmentType.KVM]
        self.assertEqual(0, len(lxcs) + len(kvms))

    def test_clear_all(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine_2,
                       self.service_1, AssignmentType.KVM)
        self.pc.clear_all_assignments()
        # check that it's empty:
        self.assertEqual(self.pc.assignments, {})
        # and that it's still a defaultdict(lambda: defaultdict(list))
        mid = self.mock_machine.machine_id
        lxcs = self.pc.assignments[mid][AssignmentType.LXC]
        self.assertEqual(lxcs, [])

    def test_unassigned_starts_full(self):
        self.assertEqual(len(self.pc.unassigned_undeployed_services()),
                         len(self.pc.services()))

    def test_assigned_services_starts_empty(self):
        self.assertEqual(0, len(self.pc.assigned_services))

    def test_reset_unassigned_undeployed_none(self):
        """Assign all charms, ensure that unassigned is empty"""
        for cc in self.pc.services():
            self.pc.assign(self.mock_machine, cc, AssignmentType.LXC)

        self.pc.reset_assigned_deployed()

        self.assertEqual(0, len(self.pc.unassigned_undeployed_services()))

    def test_reset_unassigned_undeployed_two(self):
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
        self.pc.assign(self.mock_machine_2, self.service_2, AssignmentType.KVM)
        self.pc.reset_assigned_deployed()
        self.assertEqual(len(self.pc.services()) - 2,
                         len(self.pc.unassigned_undeployed_services()))

    def test_reset_excepting_compute(self):
        for cc in self.pc.services():
            if cc.charm_name == 'nova-compute':
                continue
            self.pc.assign(self.mock_machine, cc, AssignmentType.LXC)

        self.pc.reset_assigned_deployed()
        self.assertEqual(len(self.pc.unassigned_undeployed_services()), 1)

    def test_unassigned_undeployed(self):
        all_charms = set(self.pc.services())
        self.pc.assign(self.mock_machine, self.service_2, AssignmentType.KVM)
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
        self.pc.mark_deployed(self.mock_machine, self.service_2,
                              AssignmentType.KVM)

        self.assertTrue(self.service_2 not in
                        self.pc.unassigned_undeployed_services())
        self.assertTrue(self.service_1 not in
                        self.pc.unassigned_undeployed_services())
        self.assertTrue(self.pc.is_deployed(self.service_2))
        self.assertTrue(self.pc.is_assigned(self.service_1))

        self.assertEqual(len(all_charms) - 2,
                         len(self.pc.unassigned_undeployed_services()))

        n_k_as = self.pc.assignment_machine_count_for_service(self.service_2)
        self.assertEqual(n_k_as, 0)
        n_k_dl = self.pc.deployment_machine_count_for_service(self.service_2)
        self.assertEqual(n_k_dl, 1)
        n_nc_as = self.pc.assignment_machine_count_for_service(self.service_1)
        self.assertEqual(n_nc_as, 1)
        n_nc_dl = self.pc.deployment_machine_count_for_service(self.service_1)
        self.assertEqual(n_nc_dl, 0)

    def test_deployed_charms_starts_empty(self):
        "Initially there are no deployed charms"
        self.assertEqual(0, len(self.pc.deployed_services))

    def test_mark_deployed_unsets_assignment(self):
        "Setting a placement to deployed removes it from assignment dict"
        self.pc.assign(self.mock_machine, self.service_2, AssignmentType.KVM)
        self.assertEqual([self.service_2], self.pc.assigned_services)
        self.pc.mark_deployed(self.mock_machine, self.service_2,
                              AssignmentType.KVM)
        self.assertEqual([self.service_2], self.pc.deployed_services)
        self.assertEqual([], self.pc.assigned_services)

    def test_set_deployed_unsets_assignment_only_once(self):
        "Setting a placement to deployed removes it from assignment dict"
        self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
        self.pc.assign(self.mock_machine_2, self.service_1,
                       AssignmentType.KVM)
        self.assertEqual([self.service_1], self.pc.assigned_services)
        ad = self.pc.get_assignments(self.service_1)
        dd = self.pc.get_deployments(self.service_1)
        from pprint import pformat
        print("Assignments is {}".format(pformat(ad)))
        print("Deployments is {}".format(pformat(dd)))
        self.assertEqual(set([self.mock_machine, self.mock_machine_2]),
                         set(ad[AssignmentType.KVM]))
        self.assertEqual(len(dd.items()), 0)

        self.pc.mark_deployed(self.mock_machine, self.service_1,
                              AssignmentType.KVM)
        self.assertEqual([self.service_1], self.pc.deployed_services)
        self.assertEqual([self.service_1], self.pc.assigned_services)
        ad = self.pc.get_assignments(self.service_1)
        dd = self.pc.get_deployments(self.service_1)
        self.assertEqual([self.mock_machine_2], ad[AssignmentType.KVM])
        self.assertEqual([self.mock_machine], dd[AssignmentType.KVM])

    def test_is_assigned_to_is_deployed_to(self):
        self.assertFalse(self.pc.is_assigned_to(self.service_2,
                                                self.mock_machine))
        self.assertFalse(self.pc.is_deployed_to(self.service_2,
                                                self.mock_machine))
        self.pc.assign(self.mock_machine, self.service_2, AssignmentType.LXC)
        self.assertFalse(self.pc.is_deployed_to(self.service_2,
                                                self.mock_machine))
        self.assertTrue(self.pc.is_assigned_to(self.service_2,
                                               self.mock_machine))
        self.pc.mark_deployed(self.mock_machine, self.service_2,
                              AssignmentType.LXC)
        self.assertTrue(self.pc.is_deployed_to(self.service_2,
                                               self.mock_machine))
        self.assertFalse(self.pc.is_assigned_to(self.service_2,
                                                self.mock_machine))

    def test_double_clear_ok(self):
        """clearing assignments for a machine that isn't assigned (anymore) is
        OK and should do nothing
        """
        self.pc.assign(self.mock_machine, self.service_2, AssignmentType.LXC)
        self.pc.clear_assignments(self.mock_machine)
        self.pc.clear_assignments(self.mock_machine)
        self.pc.clear_assignments(self.mock_machine_2)
 def setUp(self):
     self.conf = Config('test-temp', GOOD_CONFIG, save_backups=False)
class TestGoodConfig(unittest.TestCase):

    def setUp(self):
        self.conf = Config('test-temp', GOOD_CONFIG, save_backups=False)

    def test_save_maas_creds(self):
        """ Save maas credentials """
        self.conf.setopt('maascreds', dict(api_host='127.0.0.1',
                                           api_key='1234567'))
        self.conf.save()
        self.assertEqual(
            '127.0.0.1', self.conf.getopt('maascreds')['api_host'])

    def test_clear_empty_args(self):
        """ Empty cli options are not populated
        in generated config
        """
        cfg_file = path.join(DATA_DIR, 'good_config.yaml')
        cfg = utils.populate_config(parse_opts(['--config', cfg_file]))
        self.assertEqual(True, 'http-proxy' not in cfg)

    def test_config_file_persists(self):
        """ CLI options override options in config file
        """
        cfg_file = path.join(DATA_DIR, 'good_config.yaml')
        cfg = utils.populate_config(
            parse_opts(['--config', cfg_file,
                        '--headless']))
        self.assertEqual(True, cfg['headless'])

    def test_config_file_persists_new_cli_opts(self):
        """ Generated config object appends new options
        passed via cli
        """
        cfg_file = path.join(DATA_DIR, 'good_config.yaml')
        cfg = utils.populate_config(
            parse_opts(['--config', cfg_file,
                        '--install-only',
                        '--killcloud-noprompt']))
        self.assertEqual(True, cfg['install_only'])
        self.assertEqual(True, cfg['killcloud_noprompt'])

    def test_config_overrides_from_cli(self):
        """ Config object item is not overridden by unset cli option
        """
        cfg_file = path.join(DATA_DIR, 'good_config.yaml')
        cfg = utils.populate_config(
            parse_opts(['--http-proxy',
                        'http://localhost:2222',
                        '--killcloud-noprompt',
                        '--config', cfg_file]))
        self.assertEqual(cfg['https_proxy'], GOOD_CONFIG['https_proxy'])

    def test_default_opts_not_override_config(self):
        """ Verify that default cli opts that are False
        do not override their config_option whose option
        is True.
        """
        cfg_file = path.join(DATA_DIR, 'good_config.yaml')
        cfg_opts_raw = parse_opts(['--config', cfg_file])
        cfg = utils.populate_config(cfg_opts_raw)
        self.assertEqual(True, cfg['headless'])

    def test_default_opts_no_config(self):
        """ Verify that default cli opts are sanitized
        and that no options set to False or None exist
        in the config object
        """
        cfg = utils.sanitize_cli_opts(parse_opts([]))
        self.assertEqual(True, 'headless' not in cfg)