def render(self):
# TODO: demo specific should be changed afterwards
if self.provider.name == "maas":
DEMO_BUNDLE = os.path.join(
Config.share_path(), "data-analytics-with-sql-like.yaml")
DEMO_METADATA = os.path.join(
Config.share_path(),
"data-analytics-with-sql-like-metadata.yaml")
bundleplacer_cfg = Config('bundle-placer',
{'bundle_filename': DEMO_BUNDLE,
'metadata_filename': DEMO_METADATA})
placement_controller = PlacementController(
config=bundleplacer_cfg,
maas_state=FakeMaasState())
mainview = PlacerView(placement_controller,
bundleplacer_cfg)
self.common['ui'].set_header(
title="Bundle Editor: {}".format(
self.common['config']['summary']),
excerpt="Choose where your services should be "
"placed in your available infrastructure"
)
self.common['ui'].set_subheader("Machine Placement")
self.common['ui'].set_body(mainview)
mainview.update()
def main():
opts = parse_options(sys.argv[1:])
config = Config('bundle-placer', opts.__dict__)
config.save()
setup_logger(cfg_path=config.cfg_path)
log = logging.getLogger('bundleplacer')
log.debug(opts.__dict__)
log.info("Editing file: {}".format(opts.bundle_filename))
if opts.maas_ip and opts.maas_cred:
creds = dict(api_host=opts.maas_ip,
api_key=opts.maas_cred)
maas, maas_state = connect_to_maas(creds)
else:
maas_state = FakeMaasState()
placement_controller = PlacementController(config=config,
maas_state=maas_state)
mainview = PlacerView(placement_controller, config)
ui = PlacerUI(mainview)
def unhandled_input(key):
if key in ['q', 'Q']:
raise urwid.ExitMainLoop()
EventLoop.build_loop(ui, STYLES, unhandled_input=unhandled_input)
mainview.loop = EventLoop.loop
mainview.update()
EventLoop.run()
def setup_metadata_controller():
""" Pulls in a local bundle or via charmstore api and sets up our
controller. You can also further customize the bundle by providing a local
bundle-custom.yaml that will be deep merged over whatever bundle is
referenced. """
spell_dir = Path(app.config['spell-dir'])
bundle_filename = spell_dir / 'bundle.yaml'
bundle_custom_filename = spell_dir / 'bundle-custom.yaml'
if bundle_filename.exists():
# Load bundle data early so we can merge any additional charm options
bundle_data = yaml.load(bundle_filename.read_text())
else:
bundle_name = app.config['metadata'].get('bundle-name', None)
if bundle_name is None:
raise Exception(
"Could not determine a bundle to download, please make sure "
"the spell contains a 'bundle-name' field.")
bundle_channel = app.argv.channel
app.log.debug("Pulling bundle for {} from channel: {}".format(
bundle_name, bundle_channel))
bundle_data = charm.get_bundle(bundle_name, bundle_channel)
if bundle_custom_filename.exists():
bundle_custom = yaml.load(slurp(bundle_custom_filename))
bundle_data = merge_dicts(bundle_data, bundle_custom)
for name in app.selected_addons:
addon = app.addons[name]
bundle_data = merge_dicts(bundle_data, addon.bundle)
bundle = Bundle(bundle_data=bundle_data)
app.metadata_controller = MetadataController(bundle, Config('bundle-cfg'))
def setup_metadata_controller():
bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
if not os.path.isfile(bundle_filename):
if 'bundle-location' not in app.config['metadata']:
raise Exception(
"Could not determine bundle location: no local bundle "
"was found and bundle-location not set in spell metadata.")
bundle_filename = charm.get_bundle(
app.config['metadata']['bundle-location'], True)
# Load bundle data early so we can merge any additional charm options
with open(bundle_filename) as f:
bundle_data = yaml.load(f)
bundle_custom_filename = os.path.join(app.config['spell-dir'],
'bundle-custom.yaml')
if os.path.isfile(bundle_custom_filename):
with open(bundle_custom_filename) as f:
bundle_custom = yaml.load(f)
bundle_data = merge_dicts(bundle_data, bundle_custom)
bundle = Bundle(bundle_data=bundle_data)
bundleplacer_cfg = Config('bundle-placer', {
'bundle_filename': bundle_filename,
'bundle_key': None,
})
app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
def setUp(self):
self.mock_maas_state = MagicMock()
with NamedTemporaryFile(mode='w+', encoding='utf-8') as tempf:
utils.spew(tempf.name, yaml.dump(dict()))
self.conf = Config(tempf.name, {}, save_backups=False)
temp_bundle_f = NamedTemporaryFile(mode='r')
self.conf.setopt('bundle_filename', temp_bundle_f.name)
self.service_1 = create_service("nova-compute", {
"num_units": 1,
"charm": "cs:trusty/nova-compute-100"}, {}, [])
self.service_2 = create_service("keystone", {
"num_units": 1,
"charm": "cs:trusty/keystone-100"}, {}, [])
self.bundle_patcher = patch("bundleplacer.controller.Bundle")
self.mock_bundle = self.bundle_patcher.start()
self.mock_bundle_i = self.mock_bundle.return_value
pm = PropertyMock(return_value=[self.service_1, self.service_2])
type(self.mock_bundle_i).services = pm
self.pc = PlacementController(self.mock_maas_state,
self.conf)
self.mock_machine = MagicMock(name='machine1')
pmid = PropertyMock(return_value='fake-instance-id-1')
type(self.mock_machine).instance_id = pmid
self.mock_machine_2 = MagicMock(name='machine2')
pmid2 = PropertyMock(return_value='fake-instance-id-2')
type(self.mock_machine_2).instance_id = pmid2
self.mock_machines = [self.mock_machine, self.mock_machine_2]
self.mock_maas_state.machines.return_value = self.mock_machines
def setup_metadata_controller():
bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
bundle = Bundle(filename=bundle_filename)
bundleplacer_cfg = Config('bundle-placer', {
'bundle_filename': bundle_filename,
'bundle_key': None,
})
app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
def render(self):
# TODO: demo specific should be changed afterwards
if self.provider.name == "maas":
DEMO_BUNDLE = os.path.join(Config.share_path(),
"data-analytics-with-sql-like.yaml")
DEMO_METADATA = os.path.join(
Config.share_path(),
"data-analytics-with-sql-like-metadata.yaml")
bundleplacer_cfg = Config('bundle-placer', {
'bundle_filename': DEMO_BUNDLE,
'metadata_filename': DEMO_METADATA
})
placement_controller = PlacementController(
config=bundleplacer_cfg, maas_state=FakeMaasState())
mainview = PlacerView(placement_controller, bundleplacer_cfg)
self.common['ui'].set_header(
title="Bundle Editor: {}".format(
self.common['config']['summary']),
excerpt="Choose where your services should be "
"placed in your available infrastructure")
self.common['ui'].set_subheader("Machine Placement")
self.common['ui'].set_body(mainview)
mainview.update()
def setup_metadata_controller():
bundle_filename = os.path.join(app.config['spell-dir'], 'bundle.yaml')
if not os.path.isfile(bundle_filename):
if 'bundle-location' not in app.config['metadata']:
raise Exception(
"Could not determine bundle location: no local bundle "
"was found and bundle-location not set in spell metadata.")
bundle_filename = charm.get_bundle(
app.config['metadata']['bundle-location'], True)
bundle = Bundle(filename=bundle_filename)
bundleplacer_cfg = Config('bundle-placer', {
'bundle_filename': bundle_filename,
'bundle_key': None,
})
app.metadata_controller = MetadataController(bundle, bundleplacer_cfg)
def main():
if os.getenv("BUNDLE_EDITOR_TESTING"):
test_args = True
else:
test_args = False
opts = parse_options(sys.argv[1:], test_args)
config = Config('bundle-placer', opts.__dict__)
config.save()
setup_logger(cfg_path=config.cfg_path)
log = logging.getLogger('bundleplacer')
log.debug(opts.__dict__)
log.info("Editing file: {}".format(opts.bundle_filename))
if opts.maas_ip and opts.maas_cred:
creds = dict(api_host=opts.maas_ip,
api_key=opts.maas_cred)
maas, maas_state = connect_to_maas(creds)
elif 'fake_maas' in opts and opts.fake_maas:
maas = None
maas_state = FakeMaasState()
else:
maas = None
maas_state = None
try:
placement_controller = PlacementController(config=config,
maas_state=maas_state)
except Exception as e:
print("Error: " + e.args[0])
return
def cb():
if maas:
maas.tag_name(maas.nodes)
bw = BundleWriter(placement_controller)
if opts.out_filename:
outfn = opts.out_filename
else:
outfn = opts.bundle_filename
if os.path.exists(outfn):
shutil.copy2(outfn, outfn + '~')
bw.write_bundle(outfn)
async.shutdown()
raise urwid.ExitMainLoop()
has_maas = (maas_state is not None)
mainview = PlacerView(placement_controller, config, cb, has_maas=has_maas)
ui = PlacerUI(mainview)
def unhandled_input(key):
if key in ['q', 'Q']:
async.shutdown()
raise urwid.ExitMainLoop()
EventLoop.build_loop(ui, STYLES, unhandled_input=unhandled_input)
mainview.loop = EventLoop.loop
mainview.update()
EventLoop.run()
class PlacementControllerTestCase(unittest.TestCase):
def setUp(self):
self.mock_maas_state = MagicMock()
with NamedTemporaryFile(mode='w+', encoding='utf-8') as tempf:
utils.spew(tempf.name, yaml.dump(dict()))
self.conf = Config(tempf.name, {}, save_backups=False)
temp_bundle_f = NamedTemporaryFile(mode='r')
self.conf.setopt('bundle_filename', temp_bundle_f.name)
self.service_1 = create_service("nova-compute", {
"num_units": 1,
"charm": "cs:trusty/nova-compute-100"}, {}, [])
self.service_2 = create_service("keystone", {
"num_units": 1,
"charm": "cs:trusty/keystone-100"}, {}, [])
self.bundle_patcher = patch("bundleplacer.controller.Bundle")
self.mock_bundle = self.bundle_patcher.start()
self.mock_bundle_i = self.mock_bundle.return_value
pm = PropertyMock(return_value=[self.service_1, self.service_2])
type(self.mock_bundle_i).services = pm
self.pc = PlacementController(self.mock_maas_state,
self.conf)
self.mock_machine = MagicMock(name='machine1')
pmid = PropertyMock(return_value='fake-instance-id-1')
type(self.mock_machine).instance_id = pmid
self.mock_machine_2 = MagicMock(name='machine2')
pmid2 = PropertyMock(return_value='fake-instance-id-2')
type(self.mock_machine_2).instance_id = pmid2
self.mock_machines = [self.mock_machine, self.mock_machine_2]
self.mock_maas_state.machines.return_value = self.mock_machines
def tearDown(self):
self.bundle_patcher.stop()
def test_get_assignments_atype(self):
self.assertEqual(0,
len(self.pc.get_assignments(self.service_1)))
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
md = self.pc.get_assignments(self.service_1)
self.assertEqual(1, len(md))
self.assertEqual(2, len(md[AssignmentType.LXC]))
def _do_test_simple_assign_type(self, assignment_type):
self.pc.assign(self.mock_machine, self.service_1, assignment_type)
print("assignments is {}".format(self.pc.assignments))
machines = self.pc.get_assignments(self.service_1)
print('machines for charm is {}'.format(machines))
self.assertEqual(machines,
{assignment_type: [self.mock_machine]})
ma = self.pc.assignments_for_machine(self.mock_machine)
self.assertEqual(ma[assignment_type], [self.service_1])
def test_simple_assign_bare(self):
self._do_test_simple_assign_type(AssignmentType.BareMetal)
def test_simple_assign_lxc(self):
self._do_test_simple_assign_type(AssignmentType.LXC)
def test_simple_assign_kvm(self):
self._do_test_simple_assign_type(AssignmentType.KVM)
def test_assign_multi(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.assertEqual(self.pc.get_assignments(self.service_1),
{AssignmentType.LXC: [self.mock_machine]})
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
self.assertEqual(self.pc.get_assignments(self.service_1),
{AssignmentType.LXC: [self.mock_machine],
AssignmentType.KVM: [self.mock_machine]})
ma = self.pc.assignments_for_machine(self.mock_machine)
self.assertEqual(ma[AssignmentType.LXC], [self.service_1])
self.assertEqual(ma[AssignmentType.KVM], [self.service_1])
def test_remove_assignment_multi(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine_2, self.service_1,
AssignmentType.LXC)
mfc = self.pc.get_assignments(self.service_1)
mfc_lxc = set(mfc[AssignmentType.LXC])
self.assertEqual(mfc_lxc, set(self.mock_machines))
self.pc.clear_assignments(self.mock_machine)
self.assertEqual(self.pc.get_assignments(self.service_1),
{AssignmentType.LXC: [self.mock_machine_2]})
def test_remove_one_assignment_sametype(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.remove_one_assignment(self.mock_machine, self.service_1)
md = self.pc.assignments[self.mock_machine.instance_id]
lxcs = md[AssignmentType.LXC]
self.assertEqual(lxcs, [self.service_1])
self.pc.remove_one_assignment(self.mock_machine, self.service_1)
md = self.pc.assignments[self.mock_machine.instance_id]
lxcs = md[AssignmentType.LXC]
self.assertEqual(lxcs, [])
def test_remove_one_assignment_othertype(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
self.pc.remove_one_assignment(self.mock_machine, self.service_1)
md = self.pc.assignments[self.mock_machine.instance_id]
lxcs = md[AssignmentType.LXC]
kvms = md[AssignmentType.KVM]
self.assertEqual(1, len(lxcs) + len(kvms))
self.pc.remove_one_assignment(self.mock_machine, self.service_1)
md = self.pc.assignments[self.mock_machine.instance_id]
lxcs = md[AssignmentType.LXC]
kvms = md[AssignmentType.KVM]
self.assertEqual(0, len(lxcs) + len(kvms))
def test_clear_all(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine_2,
self.service_1, AssignmentType.KVM)
self.pc.clear_all_assignments()
# check that it's empty:
self.assertEqual(self.pc.assignments, {})
# and that it's still a defaultdict(lambda: defaultdict(list))
mid = self.mock_machine.machine_id
lxcs = self.pc.assignments[mid][AssignmentType.LXC]
self.assertEqual(lxcs, [])
def test_unassigned_starts_full(self):
self.assertEqual(len(self.pc.unassigned_undeployed_services()),
len(self.pc.services()))
def test_assigned_services_starts_empty(self):
self.assertEqual(0, len(self.pc.assigned_services))
def test_reset_unassigned_undeployed_none(self):
"""Assign all charms, ensure that unassigned is empty"""
for cc in self.pc.services():
self.pc.assign(self.mock_machine, cc, AssignmentType.LXC)
self.pc.reset_assigned_deployed()
self.assertEqual(0, len(self.pc.unassigned_undeployed_services()))
def test_reset_unassigned_undeployed_two(self):
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.LXC)
self.pc.assign(self.mock_machine_2, self.service_2, AssignmentType.KVM)
self.pc.reset_assigned_deployed()
self.assertEqual(len(self.pc.services()) - 2,
len(self.pc.unassigned_undeployed_services()))
def test_reset_excepting_compute(self):
for cc in self.pc.services():
if cc.charm_name == 'nova-compute':
continue
self.pc.assign(self.mock_machine, cc, AssignmentType.LXC)
self.pc.reset_assigned_deployed()
self.assertEqual(len(self.pc.unassigned_undeployed_services()), 1)
def test_unassigned_undeployed(self):
all_charms = set(self.pc.services())
self.pc.assign(self.mock_machine, self.service_2, AssignmentType.KVM)
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
self.pc.mark_deployed(self.mock_machine, self.service_2,
AssignmentType.KVM)
self.assertTrue(self.service_2 not in
self.pc.unassigned_undeployed_services())
self.assertTrue(self.service_1 not in
self.pc.unassigned_undeployed_services())
self.assertTrue(self.pc.is_deployed(self.service_2))
self.assertTrue(self.pc.is_assigned(self.service_1))
self.assertEqual(len(all_charms) - 2,
len(self.pc.unassigned_undeployed_services()))
n_k_as = self.pc.assignment_machine_count_for_service(self.service_2)
self.assertEqual(n_k_as, 0)
n_k_dl = self.pc.deployment_machine_count_for_service(self.service_2)
self.assertEqual(n_k_dl, 1)
n_nc_as = self.pc.assignment_machine_count_for_service(self.service_1)
self.assertEqual(n_nc_as, 1)
n_nc_dl = self.pc.deployment_machine_count_for_service(self.service_1)
self.assertEqual(n_nc_dl, 0)
def test_deployed_charms_starts_empty(self):
"Initially there are no deployed charms"
self.assertEqual(0, len(self.pc.deployed_services))
def test_mark_deployed_unsets_assignment(self):
"Setting a placement to deployed removes it from assignment dict"
self.pc.assign(self.mock_machine, self.service_2, AssignmentType.KVM)
self.assertEqual([self.service_2], self.pc.assigned_services)
self.pc.mark_deployed(self.mock_machine, self.service_2,
AssignmentType.KVM)
self.assertEqual([self.service_2], self.pc.deployed_services)
self.assertEqual([], self.pc.assigned_services)
def test_set_deployed_unsets_assignment_only_once(self):
"Setting a placement to deployed removes it from assignment dict"
self.pc.assign(self.mock_machine, self.service_1, AssignmentType.KVM)
self.pc.assign(self.mock_machine_2, self.service_1,
AssignmentType.KVM)
self.assertEqual([self.service_1], self.pc.assigned_services)
ad = self.pc.get_assignments(self.service_1)
dd = self.pc.get_deployments(self.service_1)
from pprint import pformat
print("Assignments is {}".format(pformat(ad)))
print("Deployments is {}".format(pformat(dd)))
self.assertEqual(set([self.mock_machine, self.mock_machine_2]),
set(ad[AssignmentType.KVM]))
self.assertEqual(len(dd.items()), 0)
self.pc.mark_deployed(self.mock_machine, self.service_1,
AssignmentType.KVM)
self.assertEqual([self.service_1], self.pc.deployed_services)
self.assertEqual([self.service_1], self.pc.assigned_services)
ad = self.pc.get_assignments(self.service_1)
dd = self.pc.get_deployments(self.service_1)
self.assertEqual([self.mock_machine_2], ad[AssignmentType.KVM])
self.assertEqual([self.mock_machine], dd[AssignmentType.KVM])
def test_is_assigned_to_is_deployed_to(self):
self.assertFalse(self.pc.is_assigned_to(self.service_2,
self.mock_machine))
self.assertFalse(self.pc.is_deployed_to(self.service_2,
self.mock_machine))
self.pc.assign(self.mock_machine, self.service_2, AssignmentType.LXC)
self.assertFalse(self.pc.is_deployed_to(self.service_2,
self.mock_machine))
self.assertTrue(self.pc.is_assigned_to(self.service_2,
self.mock_machine))
self.pc.mark_deployed(self.mock_machine, self.service_2,
AssignmentType.LXC)
self.assertTrue(self.pc.is_deployed_to(self.service_2,
self.mock_machine))
self.assertFalse(self.pc.is_assigned_to(self.service_2,
self.mock_machine))
def test_double_clear_ok(self):
"""clearing assignments for a machine that isn't assigned (anymore) is
OK and should do nothing
"""
self.pc.assign(self.mock_machine, self.service_2, AssignmentType.LXC)
self.pc.clear_assignments(self.mock_machine)
self.pc.clear_assignments(self.mock_machine)
self.pc.clear_assignments(self.mock_machine_2)
def setUp(self):
self.conf = Config('test-temp', GOOD_CONFIG, save_backups=False)
class TestGoodConfig(unittest.TestCase):
def setUp(self):
self.conf = Config('test-temp', GOOD_CONFIG, save_backups=False)
def test_save_maas_creds(self):
""" Save maas credentials """
self.conf.setopt('maascreds', dict(api_host='127.0.0.1',
api_key='1234567'))
self.conf.save()
self.assertEqual(
'127.0.0.1', self.conf.getopt('maascreds')['api_host'])
def test_clear_empty_args(self):
""" Empty cli options are not populated
in generated config
"""
cfg_file = path.join(DATA_DIR, 'good_config.yaml')
cfg = utils.populate_config(parse_opts(['--config', cfg_file]))
self.assertEqual(True, 'http-proxy' not in cfg)
def test_config_file_persists(self):
""" CLI options override options in config file
"""
cfg_file = path.join(DATA_DIR, 'good_config.yaml')
cfg = utils.populate_config(
parse_opts(['--config', cfg_file,
'--headless']))
self.assertEqual(True, cfg['headless'])
def test_config_file_persists_new_cli_opts(self):
""" Generated config object appends new options
passed via cli
"""
cfg_file = path.join(DATA_DIR, 'good_config.yaml')
cfg = utils.populate_config(
parse_opts(['--config', cfg_file,
'--install-only',
'--killcloud-noprompt']))
self.assertEqual(True, cfg['install_only'])
self.assertEqual(True, cfg['killcloud_noprompt'])
def test_config_overrides_from_cli(self):
""" Config object item is not overridden by unset cli option
"""
cfg_file = path.join(DATA_DIR, 'good_config.yaml')
cfg = utils.populate_config(
parse_opts(['--http-proxy',
'http://localhost:2222',
'--killcloud-noprompt',
'--config', cfg_file]))
self.assertEqual(cfg['https_proxy'], GOOD_CONFIG['https_proxy'])
def test_default_opts_not_override_config(self):
""" Verify that default cli opts that are False
do not override their config_option whose option
is True.
"""
cfg_file = path.join(DATA_DIR, 'good_config.yaml')
cfg_opts_raw = parse_opts(['--config', cfg_file])
cfg = utils.populate_config(cfg_opts_raw)
self.assertEqual(True, cfg['headless'])
def test_default_opts_no_config(self):
""" Verify that default cli opts are sanitized
and that no options set to False or None exist
in the config object
"""
cfg = utils.sanitize_cli_opts(parse_opts([]))
self.assertEqual(True, 'headless' not in cfg)