def test_load__more_complex_ref(self):
"""Test loading references links via dictionaries and lists."""
self.sample = """
stable: 9
ref_to_s1: "$(sample:stable)"
ref_to_s2: "$(sample2:stable)"
ref_to_s3: "$(sample3:stable)"
sample:
stable: "$(sample:stable)"
ref_to_s1: "$(sample:ref_to_s1)"
ref_to_s2: "$(sample:ref_to_s2)"
list:
- "$(sample:stable)"
- "$(sample2:stable)"
- "$(sample3:stable)"
- "$(sample:ref_to_s1)"
- "$(sample:ref_to_s2)"
- "$(sample:ref_to_s3)"
- "$(sample:sample)"
dict:
stable: "$(sample:stable)"
sample: "$(sample:sample)"
list: "$(sample:list)"
"""
self.sample2 = """stable: 10"""
self.sample3 = """stable: 11"""
self._write_samples()
processed = self.loader.load('sample')
self.assertEqual(processed['stable'], 9)
self.assertEqual(processed['ref_to_s1'], 9)
self.assertEqual(processed['ref_to_s2'], 10)
self.assertEqual(processed['ref_to_s3'], 11)
self.assertEqual(
processed['sample'],
utils.OrderedDict([('ref_to_s1', 9), ('ref_to_s2', 10),
('stable', 9)]))
self.assertEqual(processed['list'],
[9, 10, 11, 9, 10, 11, processed['sample']])
self.assertEqual(
processed['dict'],
utils.OrderedDict([
('list', processed['list']),
('sample', processed['sample']),
('stable', 9),
]))
def test_load__dict_reference(self):
self.sample = """
sample2:
opt: "$(sample2:opt)"
"""
self.sample2 = """opt: 10"""
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict([('sample2',
utils.OrderedDict([('opt', 10)]))])
self.assertEqual(processed, should_be)
def test_load__nested_dict(self):
self.sample = """
dict:
dict1:
default: default_value
integer: 11
dict2:
default: default_value
string: "string sample"
"""
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({
'dict': {
'dict1': {
'default': 'default_value',
'integer': 11
},
'dict2': {
'default': 'default_value',
'string': 'string sample'
}
}
})
self.assertEqual(processed, should_be)
def env_exports(self):
to_set = utils.OrderedDict()
to_set['OS_COMPUTE_API_VERSION'] = self.get_option('nova_version')
n_params = nhelper.get_shared_params(**self.options)
for (endpoint, details) in n_params['endpoints'].items():
to_set[("NOVA_%s_URI" % (endpoint.upper()))] = details['uri']
return to_set
def test_load__bool(self):
self.sample = "bool: true"
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'bool': True})
self.assertEqual(processed, should_be)
def test_load__integer(self):
self.sample = "integer: 11"
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'integer': 11})
self.assertEqual(processed, should_be)
def test_load__string(self):
self.sample = 'string: "string sample"'
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'string': "string sample"})
self.assertEqual(processed, should_be)
def test_load__default(self):
self.sample = "default: default_value"
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'default': 'default_value'})
self.assertEqual(processed, should_be)
def test_load__empty2(self):
self.sample = "empty: "
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'empty': None})
self.assertEqual(processed, should_be)
def _get_dir_opts(self, component):
component_dir = sh.joinpths(self._root_dir, component)
trace_dir = sh.joinpths(component_dir, 'traces')
app_dir = sh.joinpths(component_dir, 'app')
return utils.OrderedDict([('app_dir', app_dir),
('component_dir', component_dir),
('root_dir', self._root_dir),
('trace_dir', trace_dir)])
def _process_dict(self, value):
"""Process dictionary values."""
processed = utils.OrderedDict()
for opt, val in sorted(value.items()):
res = self._process(val)
processed[opt] = res
return processed
def test_load__dict(self):
self.sample = """
dict:
integer: 11
default: default_value
string: "string sample"
"""
self._write_samples()
# Note: dictionaries are always sorted by options names.
processed = self.loader.load('sample')
should_be = utils.OrderedDict([
('dict',
utils.OrderedDict([('default', 'default_value'), ('integer', 11),
('string', 'string sample')]))
])
self.assertEqual(processed, should_be)
def test_load__simple_reference(self):
self.sample = 'opt: $(sample2:opt)'
self.sample2 = 'opt: 10'
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'opt': 10})
self.assertEqual(processed, should_be)
def test_load__list(self):
self.sample = """
list:
- first
- second
- 100
"""
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict({'list': ['first', 'second', 100]})
self.assertEqual(processed, should_be)
def test_load__self_reference(self):
self.sample = """
opt1: "$(sample:opt2)"
opt2: "$(sample:opt3)"
opt3: 10
"""
self._write_samples()
processed = self.loader.load('sample')
should_be = utils.OrderedDict([('opt1', 10), ('opt2', 10),
('opt3', 10)])
self.assertEqual(processed, should_be)
def load(self, conf):
"""Load config `conf` from same yaml file with and resolve all
references.
"""
self._precache()
self._cache(conf)
# NOTE(imelnikov): some confs may be partially processed, so
# we have to ensure all the options got loaded.
for opt in self._cached[conf].iterkeys():
self._load_option(conf, opt)
# TODO(imelnikov: can we really restore original order here?
self._processed[conf] = utils.OrderedDict(
sorted(self._processed.get(conf, {}).iteritems()))
return self._processed[conf]
def test_load(self):
self.general = """
unique-general: True
redefined-in-general: 1
redefined-in-component: 1
"""
self.component = """
unique-component: True
redefined-in-component: 2
redefined-in-specific: 0
"""
self._write_samples()
merged = self.loader.load(self.distro, 'component', self.persona)
should_be = utils.OrderedDict([
('app_dir', os.path.join(self.temp_dir, 'component', 'app')),
('component_dir', os.path.join(self.temp_dir, 'component')),
('root_dir', os.path.join(self.temp_dir)),
('trace_dir', os.path.join(self.temp_dir, 'component', 'traces')),
('unique-distro', True),
('redefined-in-general', 1),
('redefined-in-component', 2),
('redefined-in-specific', 1),
('unique-general', True),
('unique-specific', True),
('unique-component', True),
])
self.assertEqual(merged, should_be)
# yet once loading with changed values.
self.persona.component_options['component'][
'redefined-in-specific'] = 2
merged = self.loader.load(self.distro, 'component', self.persona)
self.assertEqual(merged['redefined-in-specific'], 2)
def _run_many_phase(self, functors, group, instances, phase_name,
*inv_phase_names):
"""Run a given 'functor' across all of the components, passing *all* instances to run."""
# This phase recorder will be used to check if a given component
# and action has ran in the past, if so that components action
# will not be ran again. It will also be used to mark that a given
# component has completed a phase (if that phase runs).
if not phase_name:
phase_recorder = phase.NullPhaseRecorder()
else:
phase_recorder = phase.PhaseRecorder(
self._get_phase_filename(phase_name))
# These phase recorders will be used to undo other actions activities
# ie, when an install completes you want the uninstall phase to be
# removed from that actions phase file (and so on). This list will be
# used to accomplish that.
neg_phase_recs = []
if inv_phase_names:
for n in inv_phase_names:
if not n:
neg_phase_recs.append(phase.NullPhaseRecorder())
else:
neg_phase_recs.append(
phase.PhaseRecorder(self._get_phase_filename(n)))
def change_activate(instance, on_off):
# Activate/deactivate a component instance and there siblings (if any)
#
# This is used when you say are looking at components
# that have been activated before your component has been.
#
# Typically this is useful for checking if a previous component
# has a shared dependency with your component and if so then there
# is no need to reinstall said dependency...
instance.activated = on_off
for (_name, sibling_instance) in instance.siblings.items():
sibling_instance.activated = on_off
def run_inverse_recorders(c_name):
for n in neg_phase_recs:
n.unmark(c_name)
# Reset all activations
for c, instance in six.iteritems(instances):
change_activate(instance, False)
# Run all components which have not been ran previously (due to phase tracking)
instances_started = utils.OrderedDict()
for c, instance in six.iteritems(instances):
if c in SPECIAL_GROUPS:
c = "%s_%s" % (c, group)
if c in phase_recorder:
LOG.debug(
"Skipping phase named %r for component %r since it already happened.",
phase_name, c)
else:
try:
with phase_recorder.mark(c):
if functors.start:
functors.start(instance)
instances_started[c] = instance
except excp.NoTraceException:
pass
if functors.run:
results = functors.run(list(six.itervalues(instances_started)))
else:
results = [None] * len(instances_started)
instances_ran = instances_started
for i, (c, instance) in enumerate(six.iteritems(instances_ran)):
result = results[i]
try:
with phase_recorder.mark(c):
if functors.end:
functors.end(instance, result)
except excp.NoTraceException:
pass
for c, instance in six.iteritems(instances_ran):
change_activate(instance, True)
run_inverse_recorders(c)
def _construct_instances(self, persona):
"""Create component objects for each component in the persona."""
# Keeps track of all sibling instances across all components + actions
# so that each instance or sibling instance will be connected to the
# right set of siblings....
sibling_instances = {}
components_created = set()
groups = []
for group in persona.matched_components:
instances = utils.OrderedDict()
for c in group:
if c in components_created:
raise RuntimeError("Can not duplicate component %s in a"
" later group %s" % (c, group.id))
d_component = self.distro.extract_component(
c,
self.lookup_name,
default_entry_point_creator=self._make_default_entry_points
)
LOG.debug("Constructing component %r (%s)", c,
d_component.entry_point)
d_subsystems = d_component.options.pop('subsystems', {})
sibling_params = {}
sibling_params['name'] = c
# First create its siblings with a 'minimal' set of options
# This is done, so that they will work in a minimal state, they do not
# get access to the persona options since those are action specific (or could be),
# if this is not useful, we can give them full access, unsure if its worse or better...
active_subsystems = self._merge_subsystems(
distro_subsystems=d_subsystems,
desired_subsystems=persona.wanted_subsystems.get(c, []))
sibling_params['subsystems'] = active_subsystems
sibling_params['siblings'] = {
} # This gets adjusted during construction
sibling_params['distro'] = self.distro
sibling_params['options'] = self.config_loader.load(
distro=d_component,
component=c,
origins_patch=self.cli_opts.get('origins_patch'))
LOG.debug("Constructing %r %s siblings...", c,
len(d_component.siblings))
my_siblings = self._construct_siblings(c, d_component.siblings,
sibling_params,
sibling_instances)
# Now inject the full options and create the target instance
# with the full set of options and not the restricted set that
# siblings get...
instance_params = dict(sibling_params)
instance_params['instances'] = instances
instance_params['options'] = self.config_loader.load(
distro=d_component,
component=c,
persona=persona,
origins_patch=self.cli_opts.get('origins_patch'))
instance_params['siblings'] = my_siblings
instance_params = utils.merge_dicts(instance_params,
self.cli_opts,
preserve=True)
instances[c] = importer.construct_entry_point(
d_component.entry_point, **instance_params)
if c not in SPECIAL_GROUPS:
components_created.add(c)
groups.append((group.id, instances))
return groups
def test_load__complex_reference(self):
self.sample = """
stable: 9
ref0: "$(sample:stable)"
ref1: "$(sample2:stable)"
ref2: "$(sample2:ref1)"
ref3: "$(sample2:ref2)"
ref4: "$(sample2:ref3)"
ref5: "$(sample3:ref1)"
sample:
stable: "$(sample:stable)"
ref0: "$(sample:ref0)"
ref1: "$(sample:ref1)"
sample2:
stable: "$(sample2:stable)"
ref3: "$(sample2:ref3)"
sample3:
stable: "$(sample3:stable)"
ref1: "$(sample3:ref1)"
list:
- "$(sample:sample2)"
- "$(sample:sample3)"
dict:
sample3: "$(sample:sample3)"
sample2: "$(sample:sample2)"
"""
self.sample2 = """
stable: 10
ref1: "$(sample:stable)"
ref2: "$(sample3:stable)"
ref3: "$(sample3:ref1)"
ref4: "$(sample2:stable)"
"""
self.sample3 = """
stable: 11
ref1: "$(sample:stable)"
"""
self._write_samples()
processed = self.loader.load('sample')
self.assertTrue(isinstance(processed, utils.OrderedDict))
#self.assertEqual(len(processed), 11)
self.assertEqual(processed['stable'], 9)
self.assertEqual(processed['ref0'], 9)
self.assertEqual(processed['ref1'], 10)
self.assertEqual(processed['ref2'], 9)
self.assertEqual(processed['ref3'], 11)
self.assertEqual(processed['ref4'], 9)
self.assertEqual(processed['ref5'], 9)
sample = utils.OrderedDict([
('ref0', 9),
('ref1', 10),
('stable', 9),
])
self.assertEqual(processed['sample'], sample)
sample2 = utils.OrderedDict([
('ref3', 9),
('stable', 10),
])
self.assertEqual(processed['sample2'], sample2)
sample3 = utils.OrderedDict([
('ref1', 9),
('stable', 11),
])
self.assertEqual(processed['sample3'], sample3)
self.assertEqual(processed['list'], [sample2, sample3])
self.assertEqual(
processed['dict'],
utils.OrderedDict([
('sample2', sample2),
('sample3', sample3),
]))
processed = self.loader.load('sample2')
self.assertEqual(processed, {
'stable': 10,
'ref1': 9,
'ref2': 11,
'ref3': 9,
'ref4': 10,
})
processed = self.loader.load('sample3')
self.assertEqual(len(processed), 2)
self.assertEqual(processed['stable'], 11)
self.assertEqual(processed['ref1'], 9)
