def test_load__more_complex_ref(self): """Test loading references links via dictionaries and lists.""" self.sample = """ stable: 9 ref_to_s1: "$(sample:stable)" ref_to_s2: "$(sample2:stable)" ref_to_s3: "$(sample3:stable)" sample: stable: "$(sample:stable)" ref_to_s1: "$(sample:ref_to_s1)" ref_to_s2: "$(sample:ref_to_s2)" list: - "$(sample:stable)" - "$(sample2:stable)" - "$(sample3:stable)" - "$(sample:ref_to_s1)" - "$(sample:ref_to_s2)" - "$(sample:ref_to_s3)" - "$(sample:sample)" dict: stable: "$(sample:stable)" sample: "$(sample:sample)" list: "$(sample:list)" """ self.sample2 = """stable: 10""" self.sample3 = """stable: 11""" self._write_samples() processed = self.loader.load('sample') self.assertEqual(processed['stable'], 9) self.assertEqual(processed['ref_to_s1'], 9) self.assertEqual(processed['ref_to_s2'], 10) self.assertEqual(processed['ref_to_s3'], 11) self.assertEqual( processed['sample'], utils.OrderedDict([('ref_to_s1', 9), ('ref_to_s2', 10), ('stable', 9)])) self.assertEqual(processed['list'], [9, 10, 11, 9, 10, 11, processed['sample']]) self.assertEqual( processed['dict'], utils.OrderedDict([ ('list', processed['list']), ('sample', processed['sample']), ('stable', 9), ]))
def test_load__dict_reference(self): self.sample = """ sample2: opt: "$(sample2:opt)" """ self.sample2 = """opt: 10""" self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict([('sample2', utils.OrderedDict([('opt', 10)]))]) self.assertEqual(processed, should_be)
def test_load__nested_dict(self): self.sample = """ dict: dict1: default: default_value integer: 11 dict2: default: default_value string: "string sample" """ self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({ 'dict': { 'dict1': { 'default': 'default_value', 'integer': 11 }, 'dict2': { 'default': 'default_value', 'string': 'string sample' } } }) self.assertEqual(processed, should_be)
def env_exports(self): to_set = utils.OrderedDict() to_set['OS_COMPUTE_API_VERSION'] = self.get_option('nova_version') n_params = nhelper.get_shared_params(**self.options) for (endpoint, details) in n_params['endpoints'].items(): to_set[("NOVA_%s_URI" % (endpoint.upper()))] = details['uri'] return to_set
def test_load__bool(self): self.sample = "bool: true" self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'bool': True}) self.assertEqual(processed, should_be)
def test_load__integer(self): self.sample = "integer: 11" self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'integer': 11}) self.assertEqual(processed, should_be)
def test_load__string(self): self.sample = 'string: "string sample"' self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'string': "string sample"}) self.assertEqual(processed, should_be)
def test_load__default(self): self.sample = "default: default_value" self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'default': 'default_value'}) self.assertEqual(processed, should_be)
def test_load__empty2(self): self.sample = "empty: " self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'empty': None}) self.assertEqual(processed, should_be)
def _get_dir_opts(self, component): component_dir = sh.joinpths(self._root_dir, component) trace_dir = sh.joinpths(component_dir, 'traces') app_dir = sh.joinpths(component_dir, 'app') return utils.OrderedDict([('app_dir', app_dir), ('component_dir', component_dir), ('root_dir', self._root_dir), ('trace_dir', trace_dir)])
def _process_dict(self, value): """Process dictionary values.""" processed = utils.OrderedDict() for opt, val in sorted(value.items()): res = self._process(val) processed[opt] = res return processed
def test_load__dict(self): self.sample = """ dict: integer: 11 default: default_value string: "string sample" """ self._write_samples() # Note: dictionaries are always sorted by options names. processed = self.loader.load('sample') should_be = utils.OrderedDict([ ('dict', utils.OrderedDict([('default', 'default_value'), ('integer', 11), ('string', 'string sample')])) ]) self.assertEqual(processed, should_be)
def test_load__simple_reference(self): self.sample = 'opt: $(sample2:opt)' self.sample2 = 'opt: 10' self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'opt': 10}) self.assertEqual(processed, should_be)
def test_load__list(self): self.sample = """ list: - first - second - 100 """ self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict({'list': ['first', 'second', 100]}) self.assertEqual(processed, should_be)
def test_load__self_reference(self): self.sample = """ opt1: "$(sample:opt2)" opt2: "$(sample:opt3)" opt3: 10 """ self._write_samples() processed = self.loader.load('sample') should_be = utils.OrderedDict([('opt1', 10), ('opt2', 10), ('opt3', 10)]) self.assertEqual(processed, should_be)
def load(self, conf): """Load config `conf` from same yaml file with and resolve all references. """ self._precache() self._cache(conf) # NOTE(imelnikov): some confs may be partially processed, so # we have to ensure all the options got loaded. for opt in self._cached[conf].iterkeys(): self._load_option(conf, opt) # TODO(imelnikov: can we really restore original order here? self._processed[conf] = utils.OrderedDict( sorted(self._processed.get(conf, {}).iteritems())) return self._processed[conf]
def test_load(self): self.general = """ unique-general: True redefined-in-general: 1 redefined-in-component: 1 """ self.component = """ unique-component: True redefined-in-component: 2 redefined-in-specific: 0 """ self._write_samples() merged = self.loader.load(self.distro, 'component', self.persona) should_be = utils.OrderedDict([ ('app_dir', os.path.join(self.temp_dir, 'component', 'app')), ('component_dir', os.path.join(self.temp_dir, 'component')), ('root_dir', os.path.join(self.temp_dir)), ('trace_dir', os.path.join(self.temp_dir, 'component', 'traces')), ('unique-distro', True), ('redefined-in-general', 1), ('redefined-in-component', 2), ('redefined-in-specific', 1), ('unique-general', True), ('unique-specific', True), ('unique-component', True), ]) self.assertEqual(merged, should_be) # yet once loading with changed values. self.persona.component_options['component'][ 'redefined-in-specific'] = 2 merged = self.loader.load(self.distro, 'component', self.persona) self.assertEqual(merged['redefined-in-specific'], 2)
def _run_many_phase(self, functors, group, instances, phase_name, *inv_phase_names): """Run a given 'functor' across all of the components, passing *all* instances to run.""" # This phase recorder will be used to check if a given component # and action has ran in the past, if so that components action # will not be ran again. It will also be used to mark that a given # component has completed a phase (if that phase runs). if not phase_name: phase_recorder = phase.NullPhaseRecorder() else: phase_recorder = phase.PhaseRecorder( self._get_phase_filename(phase_name)) # These phase recorders will be used to undo other actions activities # ie, when an install completes you want the uninstall phase to be # removed from that actions phase file (and so on). This list will be # used to accomplish that. neg_phase_recs = [] if inv_phase_names: for n in inv_phase_names: if not n: neg_phase_recs.append(phase.NullPhaseRecorder()) else: neg_phase_recs.append( phase.PhaseRecorder(self._get_phase_filename(n))) def change_activate(instance, on_off): # Activate/deactivate a component instance and there siblings (if any) # # This is used when you say are looking at components # that have been activated before your component has been. # # Typically this is useful for checking if a previous component # has a shared dependency with your component and if so then there # is no need to reinstall said dependency... instance.activated = on_off for (_name, sibling_instance) in instance.siblings.items(): sibling_instance.activated = on_off def run_inverse_recorders(c_name): for n in neg_phase_recs: n.unmark(c_name) # Reset all activations for c, instance in six.iteritems(instances): change_activate(instance, False) # Run all components which have not been ran previously (due to phase tracking) instances_started = utils.OrderedDict() for c, instance in six.iteritems(instances): if c in SPECIAL_GROUPS: c = "%s_%s" % (c, group) if c in phase_recorder: LOG.debug( "Skipping phase named %r for component %r since it already happened.", phase_name, c) else: try: with phase_recorder.mark(c): if functors.start: functors.start(instance) instances_started[c] = instance except excp.NoTraceException: pass if functors.run: results = functors.run(list(six.itervalues(instances_started))) else: results = [None] * len(instances_started) instances_ran = instances_started for i, (c, instance) in enumerate(six.iteritems(instances_ran)): result = results[i] try: with phase_recorder.mark(c): if functors.end: functors.end(instance, result) except excp.NoTraceException: pass for c, instance in six.iteritems(instances_ran): change_activate(instance, True) run_inverse_recorders(c)
def _construct_instances(self, persona): """Create component objects for each component in the persona.""" # Keeps track of all sibling instances across all components + actions # so that each instance or sibling instance will be connected to the # right set of siblings.... sibling_instances = {} components_created = set() groups = [] for group in persona.matched_components: instances = utils.OrderedDict() for c in group: if c in components_created: raise RuntimeError("Can not duplicate component %s in a" " later group %s" % (c, group.id)) d_component = self.distro.extract_component( c, self.lookup_name, default_entry_point_creator=self._make_default_entry_points ) LOG.debug("Constructing component %r (%s)", c, d_component.entry_point) d_subsystems = d_component.options.pop('subsystems', {}) sibling_params = {} sibling_params['name'] = c # First create its siblings with a 'minimal' set of options # This is done, so that they will work in a minimal state, they do not # get access to the persona options since those are action specific (or could be), # if this is not useful, we can give them full access, unsure if its worse or better... active_subsystems = self._merge_subsystems( distro_subsystems=d_subsystems, desired_subsystems=persona.wanted_subsystems.get(c, [])) sibling_params['subsystems'] = active_subsystems sibling_params['siblings'] = { } # This gets adjusted during construction sibling_params['distro'] = self.distro sibling_params['options'] = self.config_loader.load( distro=d_component, component=c, origins_patch=self.cli_opts.get('origins_patch')) LOG.debug("Constructing %r %s siblings...", c, len(d_component.siblings)) my_siblings = self._construct_siblings(c, d_component.siblings, sibling_params, sibling_instances) # Now inject the full options and create the target instance # with the full set of options and not the restricted set that # siblings get... instance_params = dict(sibling_params) instance_params['instances'] = instances instance_params['options'] = self.config_loader.load( distro=d_component, component=c, persona=persona, origins_patch=self.cli_opts.get('origins_patch')) instance_params['siblings'] = my_siblings instance_params = utils.merge_dicts(instance_params, self.cli_opts, preserve=True) instances[c] = importer.construct_entry_point( d_component.entry_point, **instance_params) if c not in SPECIAL_GROUPS: components_created.add(c) groups.append((group.id, instances)) return groups
def test_load__complex_reference(self): self.sample = """ stable: 9 ref0: "$(sample:stable)" ref1: "$(sample2:stable)" ref2: "$(sample2:ref1)" ref3: "$(sample2:ref2)" ref4: "$(sample2:ref3)" ref5: "$(sample3:ref1)" sample: stable: "$(sample:stable)" ref0: "$(sample:ref0)" ref1: "$(sample:ref1)" sample2: stable: "$(sample2:stable)" ref3: "$(sample2:ref3)" sample3: stable: "$(sample3:stable)" ref1: "$(sample3:ref1)" list: - "$(sample:sample2)" - "$(sample:sample3)" dict: sample3: "$(sample:sample3)" sample2: "$(sample:sample2)" """ self.sample2 = """ stable: 10 ref1: "$(sample:stable)" ref2: "$(sample3:stable)" ref3: "$(sample3:ref1)" ref4: "$(sample2:stable)" """ self.sample3 = """ stable: 11 ref1: "$(sample:stable)" """ self._write_samples() processed = self.loader.load('sample') self.assertTrue(isinstance(processed, utils.OrderedDict)) #self.assertEqual(len(processed), 11) self.assertEqual(processed['stable'], 9) self.assertEqual(processed['ref0'], 9) self.assertEqual(processed['ref1'], 10) self.assertEqual(processed['ref2'], 9) self.assertEqual(processed['ref3'], 11) self.assertEqual(processed['ref4'], 9) self.assertEqual(processed['ref5'], 9) sample = utils.OrderedDict([ ('ref0', 9), ('ref1', 10), ('stable', 9), ]) self.assertEqual(processed['sample'], sample) sample2 = utils.OrderedDict([ ('ref3', 9), ('stable', 10), ]) self.assertEqual(processed['sample2'], sample2) sample3 = utils.OrderedDict([ ('ref1', 9), ('stable', 11), ]) self.assertEqual(processed['sample3'], sample3) self.assertEqual(processed['list'], [sample2, sample3]) self.assertEqual( processed['dict'], utils.OrderedDict([ ('sample2', sample2), ('sample3', sample3), ])) processed = self.loader.load('sample2') self.assertEqual(processed, { 'stable': 10, 'ref1': 9, 'ref2': 11, 'ref3': 9, 'ref4': 10, }) processed = self.loader.load('sample3') self.assertEqual(len(processed), 2) self.assertEqual(processed['stable'], 11) self.assertEqual(processed['ref1'], 9)