def parse_container_name(container_name): """ Parse names optionally also encoding ICC links :param container_name: Stripped string of container name :return: tuple of container_name, [(child, alias), ...] or None :raise ValueError: On invalid/unparsable container_name """ # Don't assume already stripped or string-like container_name = str(container_name).strip() links = None parsed_name = None if len(container_name) < 4: if (container_name.find(",") > -1) or (container_name.find("/") > -1): raise ValueError("Linked container name '%s' invalid" % container_name) if container_name == "": raise ValueError("Container name is empty string") # Name may additionally contain CSV child/alias links if container_name.find(",") > -1: link_names = get_as_list(container_name) # Any item w/o a '/' is the real container name links = [] for link_name in link_names: if link_name.find("/") > -1: child, alias = get_as_list(link_name, sep="/") links.append((child, alias)) else: parsed_name = link_name else: parsed_name = container_name # Could be a ',' or '/' w/o expected content or short list if parsed_name is None or parsed_name == "" or links == []: raise ValueError("container_name(%s) unrecognized format" % container_name) return (parsed_name, links)
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...") # Private to this instance, outside of __init__ if (self.config.get('subsubtests') is None and len(self.subsubtest_names) == 0): raise DockerTestNAError("Missing|empty 'subsubtests' in config.") if len(self.subsubtest_names) == 0: sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) # Turn a short name into a fully-qualified name mkfull = lambda short: os.path.join(self.config_section, short) # Could be None or empty dictionary if self.bugzilla_config: bzexclude = set( config.get_as_list(self.bugzilla_config['excluded'])) else: bzexclude = set() # Also could be None or empty dictionary if self.control_config: subthings = set( config.get_as_list(self.control_config['subthings'])) includes = set( config.get_as_list(self.control_config['include'])) excludes = set( config.get_as_list(self.control_config['exclude'])) # covers all items from command-line, config, control, and BZ allem = subthings | excludes | includes if len(allem) < 1: return # Empty set implies ALL fullnames = set([mkfull(ssn) for ssn in self.subsubtest_names]) # Make sure every control-config referenced sub-subtest exists # in subsubtests option, unless it is excluded by bugzilla children = set() # filter out all names not a subsubtest of this subtest for testname in allem: if testname.startswith(self.config_section + '/'): children.add(testname) if len(children) < 1: return # No children specified # Any children NOT listed in fullnames are 'undefined' # unless they were excluded by (an old) bugzilla missing_section = children - fullnames - bzexclude if missing_section: msg = ("Sub-subtest(s) %s referenced in control include, " "exclude, and/or subthings but not defined in " 'subsubtests = %s' % (missing_section, self.subsubtest_names)) raise DockerTestError(msg)
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...") # Private to this instance, outside of __init__ if (self.config.get('subsubtests') is None and not self.subsubtest_names): raise DockerTestNAError("Missing|empty 'subsubtests' in config.") if not self.subsubtest_names: sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) # Turn a short name into a fully-qualified name mkfull = lambda short: os.path.join(self.config_section, short) # Could be None or empty dictionary if self.bugzilla_config: bzexclude = set( config.get_as_list(self.bugzilla_config['excluded'])) else: bzexclude = set() # Also could be None or empty dictionary if self.control_config: subthings = set( config.get_as_list(self.control_config['subthings'])) includes = set( config.get_as_list(self.control_config['include'])) excludes = set( config.get_as_list(self.control_config['exclude'])) # covers all items from command-line, config, control, and BZ allem = subthings | excludes | includes if len(allem) < 1: return # Empty set implies ALL fullnames = set([mkfull(ssn) for ssn in self.subsubtest_names]) # Make sure every control-config referenced sub-subtest exists # in subsubtests option, unless it is excluded by bugzilla children = set() # filter out all names not a subsubtest of this subtest for testname in allem: if testname.startswith(self.config_section + '/'): children.add(testname) if len(children) < 1: return # No children specified # Any children NOT listed in fullnames are 'undefined' # unless they were excluded missing_section = children - fullnames - bzexclude - excludes if missing_section: msg = ("Sub-subtest(s) %s referenced in control include, " "and/or subthings but not defined in " 'subsubtests = %s' % (missing_section, self.subsubtest_names)) raise DockerTestError(msg)
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...") # Private to this instance, outside of __init__ if (self.config.get('subsubtests') is None and len(self.subsubtest_names) == 0): raise DockerTestNAError("Missing|empty 'subsubtests' in config.") if len(self.subsubtest_names) == 0: sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) else: sst_names = self.subsubtest_names if self.control_config is not None: subthings = set( config.get_as_list(self.control_config['subthings'], omit_empty=True)) includes = set( config.get_as_list(self.control_config['include'], omit_empty=True)) exclude = set( config.get_as_list(self.control_config['exclude'], omit_empty=True)) allem = subthings | exclude | includes if len(allem) < 1: return # Empty set implies ALL # Make sure every control-config referenced sub-subtest exists # in subsubtests option fullnames = set([ os.path.join(self.config_section, ssn) for ssn in self.subsubtest_names ]) children = set() # filter out all names not a subsubtest of this subtest for testname in allem: if testname.startswith(self.config_section + '/'): children.add(testname) if len(children) < 1: return # No children specified # Any children NOT listed in fullnames are 'undefined' if children - fullnames: msg = ("Sub-subtest(s) %s referenced in control include, " "exclude, and/or subthings but not defined in " "subsubtests configuration option: %s" % (children - fullnames, self.subsubtest_names)) raise DockerTestError(msg)
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...") # Private to this instance, outside of __init__ if (self.config.get('subsubtests') is None and len(self.subsubtest_names) == 0): raise DockerTestNAError("Missing|empty 'subsubtests' in config.") if len(self.subsubtest_names) == 0: sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) else: sst_names = self.subsubtest_names if self.control_config is not None: subthings = set( config.get_as_list(self.control_config['subthings'], omit_empty=True)) includes = set( config.get_as_list(self.control_config['include'], omit_empty=True)) exclude = set( config.get_as_list(self.control_config['exclude'], omit_empty=True)) allem = subthings | exclude | includes if len(allem) < 1: return # Empty set implies ALL # Make sure every control-config referenced sub-subtest exists # in subsubtests option fullnames = set([os.path.join(self.config_section, ssn) for ssn in self.subsubtest_names]) children = set() # filter out all names not a subsubtest of this subtest for testname in allem: if testname.startswith(self.config_section + '/'): children.add(testname) if len(children) < 1: return # No children specified # Any children NOT listed in fullnames are 'undefined' if children - fullnames: msg = ("Sub-subtest(s) %s referenced in control include, " "exclude, and/or subthings but not defined in " "subsubtests configuration option: %s" % (children - fullnames, self.subsubtest_names)) raise DockerTestError(msg)
def clean_all(self, containers): """ Remove all containers not configured to preserve :param containers: Iterable sequence of container **names** """ if not hasattr(containers, "__iter__"): raise TypeError("clean_all() called with non-iterable.") if isinstance(containers, basestring): raise TypeError("clean_all() called with a string, " "instead of an interable of strings.") preserve_cnames = self.subtest.config.get('preserve_cnames') if preserve_cnames is not None and preserve_cnames.strip() != '': preserve_cnames = get_as_list(preserve_cnames) else: preserve_cnames = [] preserve_cnames_set = set(preserve_cnames) preserve_cnames_set.discard(None) preserve_cnames_set.discard('') self.verbose = False try: for name in containers: name = name.strip() if name in preserve_cnames_set: continue try: self.subtest.logdebug("Cleaning %s", name) self.docker_cmd("rm --force --volumes %s" % name, self.timeout) except error.CmdError: continue finally: self.verbose = DockerContainers.verbose
def clean_all(self, containers): """ Remove all containers not configured to preserve :param containers: Iterable sequence of container **names** """ if not hasattr(containers, "__iter__"): raise TypeError("clean_all() called with non-iterable.") if isinstance(containers, basestring): raise TypeError("clean_all() called with a string, " "instead of an interable of strings.") preserve_cnames = self.subtest.config.get('preserve_cnames') if preserve_cnames is not None and preserve_cnames.strip() != '': preserve_cnames = get_as_list(preserve_cnames) else: preserve_cnames = [] preserve_cnames = set(preserve_cnames) preserve_cnames.discard(None) preserve_cnames.discard('') self.verbose = False try: for name in containers: name = name.strip() if name in preserve_cnames: continue try: self.subtest.logdebug("Cleaning %s", name) self.docker_cmd("rm --force --volumes %s" % name, self.timeout) except error.CmdError: continue finally: self.verbose = DockerContainers.verbose
def clean_all(self, fqins): """ Remove all image fqins not configured to preserve :param fqins: Iterable sequence of image **fqins** """ if not hasattr(fqins, "__iter__"): raise TypeError("clean_all() called with non-iterable.") if isinstance(fqins, basestring): raise ValueError("clean_all() called with a string, " "instead of an interable of strings.") preserve_fqins = self.subtest.config.get('preserve_fqins') if preserve_fqins is not None and preserve_fqins.strip() != '': preserve_fqins = get_as_list(preserve_fqins) else: preserve_fqins = [] preserve_fqins.append(self.default_image) preserve_fqins = set(preserve_fqins) preserve_fqins.discard(None) preserve_fqins.discard('') self.verbose = False try: for name in fqins: if name in preserve_fqins: continue try: self.subtest.logdebug("Cleaning %s", name) self.docker_cmd("rmi --force %s" % name, self.timeout) except error.CmdError: continue finally: self.verbose = self.__class__.verbose
def subsub_control_enabled(self, name, control_config): """ Return True if name not excluded in control.ini """ subthings_csv = control_config.get('subthings', '') exclude_csv = control_config.get('exclude', '') include_csv = control_config.get('include', '') if subthings_csv != '': subthings = [ subthing.strip() for subthing in config.get_as_list(subthings_csv) ] else: return False # nothing is suppose to run?!?!?!?!? if exclude_csv != '': excludes = [ exclude.strip() for exclude in config.get_as_list(exclude_csv) ] if name in excludes: return False # else more checking reqired else: excludes = [] # exclude nothing if include_csv != '': includes = [ include.strip() for include in config.get_as_list(include_csv) ] # Can't use self.config['subsubtests'] b/c initialize() could # have modified/augmented it. specifics = self.subsubtests_in_list(self.subsubtest_names, includes) if specifics: return name in includes else: # All self.subsubtest_names included if none appear pass # everything included, name not excluded, specific sub-subtests? specifics = self.subsubtests_in_list(self.subsubtest_names, subthings) if specifics: return name in subthings else: # This code is running, assume all sub-subtest should run. return True
def make_config(cls, all_configs, parent_config, name): """ Form subsubtest configuration by inheriting parent subtest config """ # Many branches needed to keep this method compact # pylint: disable=R0912 subsubtest_config = all_configs.get(name, {}) # don't redefine the module _config = copy.deepcopy(parent_config) # a copy # global defaults mixed in, even if overridden in parent :( for key, val in subsubtest_config.items(): if key == 'subsubtests': continue # only applicable to parent if key == '__example__': # Compose from parent + subsub par_val = parent_config.get(key, '').strip() if par_val is not '': par_val = set(config.get_as_list(par_val)) else: par_val = set() sst_val = val.strip() if sst_val is not '': sst_val = set(config.get_as_list(sst_val)) else: sst_val = set() _config[key] = ", ".join(par_val | sst_val) elif key in all_configs['DEFAULTS']: def_val = all_configs['DEFAULTS'][key] par_val = parent_config[key] if val == def_val: if par_val != def_val: # Parent overrides default, subsubtest inherited # default _config[key] = par_val else: # Parent uses default, subsubtest did not override _config[key] = def_val else: _config[key] = val else: _config[key] = val return _config
def make_config(cls, all_configs, parent_config, name): """ Form subsubtest configuration by inheriting parent subtest config """ # Many branches needed to keep this method compact # pylint: disable=R0912 subsubtest_config = all_configs.get(name, {}) # don't redefine the module _config = copy.deepcopy(parent_config) # a copy # global defaults mixed in, even if overridden in parent :( for key, val in subsubtest_config.items(): if key == 'subsubtests': continue # only applicable to parent if key == '__example__': # Compose from parent + subsub par_val = parent_config.get(key, '').strip() if par_val: par_val = set(config.get_as_list(par_val)) else: par_val = set() sst_val = val.strip() if sst_val: sst_val = set(config.get_as_list(sst_val)) else: sst_val = set() _config[key] = ", ".join(par_val | sst_val) elif key in all_configs['DEFAULTS']: def_val = all_configs['DEFAULTS'][key] par_val = parent_config[key] if val == def_val: if par_val != def_val: # Parent overrides default, subsubtest inherited # default _config[key] = par_val else: # Parent uses default, subsubtest did not override _config[key] = def_val else: _config[key] = val else: _config[key] = val return _config
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() # Private to this instance, outside of __init__ if not self.config['subsubtests']: raise DockerTestNAError("No subsubtests enabled in configuration.") self.subsubtest_names = config.get_as_list(self.config['subsubtests'])
def __init__(self, image_name, command, ports=None, container_name=None): """ Create a new container representation based on parameter content. :param image_name: FQIN, fully qualified image name :param command: String of command container is/was running :param ports: String of comma-separated port mappings :param container_name: String representing name of container optionally prefixed by CSV <child>/<alias> format link strings """ self.image_name = image_name self.command = command if ports is None: self.ports = '' else: self.ports = ports container_name = str(container_name).strip() # Name may additionally contain CSV child/alias links if container_name.find(',') > 0: link_names = get_as_list(container_name) # Any item w/o a '/' is the real container name self.links = [] for link_name in link_names: if link_name.find('/') > 0: child, alias = get_as_list(link_name, sep='/') self.links.append((child, alias)) else: self.container_name = link_name if getattr(self, 'container_name') is None: raise ValueError("container_name(%s) unrecognized format" % container_name) else: self.container_name = container_name self.links = None #: These are typically all generated at runtime self.long_id = None self.created = None self.status = None self.size = None
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() # Private to this instance, outside of __init__ if self.config.get('subsubtests') is None: raise DockerTestNAError("Missing|empty 'subsubtests' in config.") sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) # Make sure every control-config referenced sub-subtest exists # in subsubtests option fullnames = [ os.path.join(self.config_section, ssn) for ssn in self.subsubtest_names ] if self.control_config is not None: includes = config.get_as_list(self.control_config['include']) exclude = config.get_as_list(self.control_config['exclude']) subthings = config.get_as_list(self.control_config['subthings']) # filter out all names not a subsubtest of this test for testlist in (includes, exclude, subthings): remove = [] for testname in testlist: if testname == self.config_section: remove.append(testname) if not testname.startswith(self.config_section): remove.append(testname) for item in remove: testlist.remove(item) # Verify all runtime subsubtests are defined for item in testlist: if item not in fullnames: msg = ("Sub-subtest %s referenced by control " "but not defined in subsubtests option" % item) raise DockerTestError(msg) self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...")
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() # Private to this instance, outside of __init__ if self.config.get('subsubtests') is None: raise DockerTestNAError("Missing|empty 'subsubtests' in config.") sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names)
def initialize(self): """ Perform initialization steps needed before loading subsubtests. Split up the ``subsubtests`` config. option by commas, into instance attribute ``subsubtest_names`` (list). """ super(SubSubtestCaller, self).initialize() # Private to this instance, outside of __init__ if self.config.get('subsubtests') is None: raise DockerTestNAError("Missing|empty 'subsubtests' in config.") sst_names = self.config['subsubtests'] self.subsubtest_names = config.get_as_list(sst_names) # Make sure every control-config referenced sub-subtest exists # in subsubtests option fullnames = [os.path.join(self.config_section, ssn) for ssn in self.subsubtest_names] if self.control_config is not None: includes = config.get_as_list(self.control_config['include']) exclude = config.get_as_list(self.control_config['exclude']) subthings = config.get_as_list(self.control_config['subthings']) # filter out all names not a subsubtest of this test for testlist in (includes, exclude, subthings): remove = [] for testname in testlist: if testname == self.config_section: remove.append(testname) if not testname.startswith(self.config_section): remove.append(testname) for item in remove: testlist.remove(item) # Verify all runtime subsubtests are defined for item in testlist: if item not in fullnames: msg = ("Sub-subtest %s referenced by control " "but not defined in subsubtests option" % item) raise DockerTestError(msg) self.step_log_msgs['run_once'] = "Running sub-subtests..." self.step_log_msgs['postprocess'] = ("Postprocess sub-subtest " "results...")
def subsub_control_enabled(self, name, control_config): """ Return True if name not excluded in control.ini """ subthings_csv = control_config.get('subthings', '') exclude_csv = control_config.get('exclude', '') include_csv = control_config.get('include', '') if subthings_csv != '': subthings = [subthing.strip() for subthing in config.get_as_list(subthings_csv)] else: return False # nothing is suppose to run?!?!?!?!? if exclude_csv != '': excludes = [exclude.strip() for exclude in config.get_as_list(exclude_csv)] if name in excludes: return False # else more checking reqired else: excludes = [] # exclude nothing if include_csv != '': includes = [include.strip() for include in config.get_as_list(include_csv)] # Can't use self.config['subsubtests'] b/c initialize() could # have modified/augmented it. specifics = self.subsubtests_in_list(self.subsubtest_names, includes) if specifics: return name in includes else: # All self.subsubtest_names included if none appear pass # everything included, name not excluded, specific sub-subtests? specifics = self.subsubtests_in_list(self.subsubtest_names, subthings) if specifics: return name in subthings else: # This code is running, assume all sub-subtest should run. return True
def parse_container_name(container_name): """ Parse names optionally also encoding ICC links :param container_name: Stripped string of container name :return: tuple of container_name, [(child, alias), ...] or None :raise ValueError: On invalid/unparsable container_name """ # Don't assume already stripped or string-like container_name = str(container_name).strip() links = None parsed_name = None if len(container_name) < 4: if ((container_name.find(',') > -1) or (container_name.find('/') > -1)): raise ValueError("Linked container name '%s' invalid" % container_name) if container_name == '': raise ValueError("Container name is empty string") # Name may additionally contain CSV child/alias links if container_name.find(',') > -1: link_names = get_as_list(container_name) # Any item w/o a '/' is the real container name links = [] for link_name in link_names: if link_name.find('/') > -1: child, alias = get_as_list(link_name, sep='/') links.append((child, alias)) else: parsed_name = link_name else: parsed_name = container_name # Could be a ',' or '/' w/o expected content or short list if parsed_name is None or parsed_name == '' or links == []: raise ValueError("container_name(%s) unrecognized format" % container_name) return (parsed_name, links)
def initialize(self): """ Called every time the test is run. """ self.log_step_msg('initialize') # Issue warnings for failed to customize suggested options not_customized = self.config.get('__example__', None) if not_customized: self.logdebug("WARNING: Recommended options not customized:") for nco in get_as_list(not_customized): self.logdebug("WARNING: %s" % nco) msg = "%s configuration:\n" % self.__class__.__name__ for key, value in self.config.items(): if key == '__example__' or key.startswith('envcheck'): continue msg += '\t\t%s = "%s"\n' % (key, value) self.logdebug(msg)