def _execute_python_test(self, testid, testspec): from cStringIO import StringIO import sys execinfo = self._details['exec_info'][testid] try: rescue_stdout = sys.stdout rescue_stderr = sys.stderr sys.stdout = capture_stdout = StringIO() sys.stderr = capture_stderr = StringIO() try: if 'code' in testspec: exec(testspec['code'], {}, {}) elif 'file' in testspec: execfile(testspec['file'], {}, {}) else: raise ValueError("no test code found") except Exception as e: execinfo['exception'] = dict(type=e.__class__.__name__, info=str(e)) if not 'shouldfail' in testspec or testspec['shouldfail'] == False: lgr.error("%s: %s" % (e.__class__.__name__, str(e))) self.assertThat(e, Annotate("exception occured while executing Python test code in test '%s': %s (%s)" % (testid, str(e), e.__class__.__name__), Equals(None))) return if 'shouldfail' in testspec and testspec['shouldfail'] == True: self.assertThat(e, Annotate("an expected failure did not occur in test '%s': %s (%s)" % (testid, str(e), e.__class__.__name__), Equals(None))) finally: execinfo['stdout'] = capture_stdout.getvalue() execinfo['stderr'] = capture_stderr.getvalue() sys.stdout = rescue_stdout sys.stderr = rescue_stderr
def test_origin_runpath_migrated_to_rpath(self): # We stage libc6 for this to work on non xenial snapcraft_yaml = fixture_setup.SnapcraftYaml(self.path, confinement='classic') snapcraft_yaml.update_part( 'test-part', { 'plugin': 'nil', 'build-packages': ['patchelf'], 'stage-packages': ['hello'], # Without --force-rpath this is essentially a RUNPATH 'override-build': ("patchelf --set-rpath '$ORIGIN/foobar' " "$SNAPCRAFT_PART_INSTALL/usr/bin/hello") }) self.useFixture(snapcraft_yaml) self.run_snapcraft('prime') bin_path = os.path.join(self.prime_dir, 'usr', 'bin', 'hello') self.assertThat(bin_path, FileExists()) # Verify there aren't any duplicate rpath entries readelf_d = subprocess.check_output(['readelf', '-d', bin_path]).decode().strip() lines_with_runpath = [ l for l in readelf_d.splitlines() if 'RUNPATH' in l ] lines_with_rpath = [l for l in readelf_d.splitlines() if 'RPATH' in l] self.assertThat(len(lines_with_runpath), Annotate('Expected no RUNPATH entries', Equals(0))) self.assertThat(len(lines_with_rpath), Annotate('Expected one RPATH entry', Equals(1)))
def test_origin_runpath_migrated_to_rpath(self): # We stage libc6 for this to work on non xenial snapcraft_yaml = fixture_setup.SnapcraftYaml(self.path, confinement="classic") snapcraft_yaml.update_part( "test-part", { "plugin": "nil", "build-packages": ["patchelf"], "stage-packages": ["hello"], # Without --force-rpath this is essentially a RUNPATH "override-build": ("patchelf --set-rpath '$ORIGIN/foobar' " "$SNAPCRAFT_PART_INSTALL/usr/bin/hello"), }, ) self.useFixture(snapcraft_yaml) self.run_snapcraft("prime") bin_path = os.path.join(self.prime_dir, "usr", "bin", "hello") self.assertThat(bin_path, FileExists()) # Verify there aren't any duplicate rpath entries readelf_d = (subprocess.check_output(["readelf", "-d", bin_path]).decode().strip()) lines_with_runpath = [ l for l in readelf_d.splitlines() if "RUNPATH" in l ] lines_with_rpath = [l for l in readelf_d.splitlines() if "RPATH" in l] self.assertThat(len(lines_with_runpath), Annotate("Expected no RUNPATH entries", Equals(0))) self.assertThat(len(lines_with_rpath), Annotate("Expected one RPATH entry", Equals(1)))
class TestAnnotate(TestCase, TestMatchersInterface): matches_matcher = Annotate("foo", Equals(1)) matches_matches = [1] matches_mismatches = [2] str_examples = [("Annotate('foo', Equals(1))", Annotate("foo", Equals(1)))] describe_examples = [("1 != 2: foo", 2, Annotate('foo', Equals(1)))] def test_if_message_no_message(self): # Annotate.if_message returns the given matcher if there is no # message. matcher = Equals(1) not_annotated = Annotate.if_message('', matcher) self.assertIs(matcher, not_annotated) def test_if_message_given_message(self): # Annotate.if_message returns an annotated version of the matcher if a # message is provided. matcher = Equals(1) expected = Annotate('foo', matcher) annotated = Annotate.if_message('foo', matcher) self.assertThat( annotated, MatchesStructure.fromExample(expected, 'annotation', 'matcher'))
class TestAnnotate(TestCase, TestMatchersInterface): matches_matcher = Annotate("foo", Equals(1)) matches_matches = [1] matches_mismatches = [2] str_examples = [("Annotate('foo', Equals(1))", Annotate("foo", Equals(1)))] describe_examples = [("1 != 2: foo", 2, Annotate('foo', Equals(1)))]
def test_if_message_given_message(self): # Annotate.if_message returns an annotated version of the matcher if a # message is provided. matcher = Equals(1) expected = Annotate('foo', matcher) annotated = Annotate.if_message('foo', matcher) self.assertThat( annotated, MatchesStructure.fromExample(expected, 'annotation', 'matcher'))
def __init__(self, path, type=None, size=None, mtime=None, mtime_skew=None, mode=None, linkname=None, uid=None, gid=None, uname=None, gname=None, content=None, content_matcher=None): """Create a TarfileHasFile Matcher. :param path: the path that must be present. :type path: str :param type: the type of TarInfo that must be at `path`, or None to not check. :param size: the size that the entry at `path` must have, or None to not check. :param mtime: the mtime that the entry at `path` must have, or None to not check. :param mtime_skew: the number of seconds that the file mtime can be different to the required. :param mode: the mode that the entry at `path` must have, or None to not check. :param linkname: the linkname that the entry at `path` must have, or None to not check. :param uid: the user id that the entry at `path` must have, or None to not check. :param gid: the group id that the entry at `path` must have, or None to not check. :param uname: the username that the entry at `path` must have, or None to not check. :param gname: the group name that the entry at `path` must have, or None to not check. :param content: the content that `path` must have when extracted, or None to not check. :param content_matcher: a matcher to match the content that `path` has when extracted, or None to not check. You can't specify both content_matcher and content. """ self.path = path self.type = type self.size = size self.mtime = mtime self.mtime_skew = mtime_skew self.mode = mode self.linkname = linkname self.uid = uid self.gid = gid self.uname = uname self.gname = gname if content is not None: if content_matcher is not None: raise ValueError( "doesn't make sense to specify content and " "content_matcher") content_matcher = Equals(content) if content_matcher is not None: self.content_matcher = Annotate( 'The content of path "%s" did not match' % path, content_matcher) else: self.content_matcher = None
def test_since_clause(self): contains_since_clause = Annotate(self.since_clause_missing_message, Contains(":since:")) since_clause_contains_version = Annotate( self.since_clause_version_not_recognised, MatchesRegex(".*^:since: *[1-9][.][0-9]+([.][0-9]+)?$", re.DOTALL | re.MULTILINE)) self.assertThat( getdoc(self.command), MatchesAll(contains_since_clause, since_clause_contains_version))
def test_warn_once_only(self): fake_logger = fixtures.FakeLogger(level=logging.INFO) self.useFixture(fake_logger) self.config_data["confinement"] = "devmode" self.config_data["passthrough"] = {"foo": "bar", "spam": "eggs"} self.config_data["apps"] = { "foo": {"command": "echo", "passthrough": {"foo": "bar", "spam": "eggs"}} } self.config_data["hooks"] = { "foo": {"plugs": ["network"], "passthrough": {"foo": "bar", "spam": "eggs"}} } self.generate_meta_yaml() output_lines = fake_logger.output.splitlines() self.assertThat( output_lines, Contains( "The 'passthrough' property is being used to propagate " "experimental properties to snap.yaml that have not been " "validated." ), ) self.assertThat( len(output_lines), Annotate( "There were duplicate lines logged.", Equals(len(set(output_lines))) ), )
def match(self, value): matchers = [] values = [] for attr, matcher in self.kws.iteritems(): matchers.append(Annotate(attr, matcher)) values.append(getattr(value, attr)) return EachOf(matchers).match(values)
def test_if_message_given_message(self): # Annotate.if_message returns an annotated version of the matcher if a # message is provided. matcher = Equals(1) expected = Annotate("foo", matcher) annotated = Annotate.if_message("foo", matcher) self.assertThat(annotated, MatchesStructure.fromExample(expected, "annotation", "matcher"))
def _execute_nipype_test(self, testid, testspec): # TODO merge/refactor this one with the plain python code method from cStringIO import StringIO import sys execinfo = self._details['exec_info'][testid] try: import nipype except ImportError: self.skipTest("Nipype not found, skipping test") # where is the workflow if 'file' in testspec: testwffilepath = testspec['file'] lgr.debug("using custom workflow file name '%s'" % testwffilepath) else: testwffilepath = 'workflow.py' lgr.debug("using default workflow file name '%s'" % testwffilepath) # execute the script and extract the workflow locals = dict() try: execfile(testwffilepath, dict(), locals) except Exception, e: lgr.error("%s: %s" % (e.__class__.__name__, str(e))) self.assertThat( e, Annotate( "test workflow setup failed: %s (%s)" % (e.__class__.__name__, str(e)), Equals(None)))
def match(self, child): """ Assert ``child`` is beneath the core path. """ return Annotate( "%s in not beneath %s" % (child, self._parent), Contains(self._parent)).match(child.parents())
def assertThat(self, matchee, matcher, message='', verbose=False): """Assert that matchee is matched by matcher. :param matchee: An object to match with matcher. :param matcher: An object meeting the testtools.Matcher protocol. :raises self.failureException: When matcher does not match thing. """ matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return existing_details = self.getDetails() for (name, content) in mismatch.get_details().items(): full_name = name suffix = 1 while full_name in existing_details: full_name = "%s-%d" % (name, suffix) suffix += 1 self.addDetail(full_name, content) if verbose: message = ( 'Match failed. Matchee: "%s"\nMatcher: %s\nDifference: %s\n' % (matchee, matcher, mismatch.describe())) else: message = mismatch.describe() self.fail(message)
def _matchHelper(self, matchee, matcher, message, verbose): matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return for (name, value) in mismatch.get_details().items(): self.addDetailUniqueName(name, value) return MismatchError(matchee, matcher, mismatch, verbose)
def assertMismatchWithDescriptionMatching(self, value, matcher, description_matcher): mismatch = matcher.match(value) if mismatch is None: self.fail("%s matched %s" % (matcher, value)) actual_description = mismatch.describe() self.assertThat( actual_description, Annotate("%s matching %s" % (matcher, value), description_matcher))
def match(self, mock): matcher = MatchesAll( IsCallableMock(), Annotate( "mock has been called", AfterPreprocessing(get_mock_calls, HasLength(0)), ), first_only=True, ) return matcher.match(mock)
def match(self, mock): matcher = MatchesAll( IsCallableMock(), Annotate( "calls do not match", AfterPreprocessing(get_mock_calls, Equals(self.calls)), ), first_only=True, ) return matcher.match(mock)
def assertEqual(self, expected, observed, message=''): """Assert that 'expected' is equal to 'observed'. :param expected: The expected value. :param observed: The observed value. :param message: An optional message to include in the error. """ matcher = Equals(expected) if message: matcher = Annotate(message, matcher) self.assertThat(observed, matcher)
def match(self, values): mismatches = [] length_mismatch = Annotate( "Length mismatch", Equals(len(self.matchers))).match(len(values)) if length_mismatch: mismatches.append(length_mismatch) for matcher, value in zip(self.matchers, values): mismatch = matcher.match(value) if mismatch: mismatches.append(mismatch) if mismatches: return MismatchesAll(mismatches)
def _execute_shell_test(self, testid, testspec): import subprocess cmd = testspec['command'] execinfo = self._details['exec_info'][testid] if isinstance(cmd, list): # convert into a cmd string to execute via shell # to get all envvar expansion ... cmd = subprocess.list2cmdline(cmd) # for the rest we need to execute stuff in the root of the testbed try: lgr.debug("attempting to execute command '%s'" % cmd) texec = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) texec.wait() # record the exit code execinfo['exitcode'] = texec.returncode # store test output for chan in ('stderr', 'stdout'): execinfo[chan] = getattr(texec, chan).read() #lgr.debug('%s: %s' % (chan, execinfo[chan])) if texec.returncode != 0 and 'shouldfail' in testspec \ and testspec['shouldfail'] == True: # failure is expected return self.assertThat( texec.returncode, Annotate("test shell command '%s' yielded non-zero exit code" % cmd, Equals(0))) except OSError as e: lgr.error("%s: %s" % (e.__class__.__name__, str(e))) if not 'shouldfail' in testspec or testspec['shouldfail'] == False: self.assertThat(e, Annotate("test command execution failed: %s (%s)" % (e.__class__.__name__, str(e)), Equals(None))) if 'shouldfail' in testspec and testspec['shouldfail'] == True: self.assertThat(e, Annotate("an expected failure did not occur in test '%s': %s (%s)" % (testid, str(e), e.__class__.__name__), Equals(None)))
def _check_output_presence(self, spec): outspec = spec.get('outputs', {}) unmatched_output = [] for ospec_id in outspec: ospec = outspec[ospec_id] ospectype = ospec['type'] if ospectype == 'file': self.assertThat( ospec['value'], Annotate('expected output file missing', FileExists())) elif ospectype == 'directory': self.assertThat( ospec['value'], Annotate('expected output directory missing', DirExists())) elif ospectype == 'string' and ospec_id.startswith('tests'): execinfo = self._details['exec_info'] sec, idx, field = ospec_id.split('::') for f, matcher in __spec_matchers__.iteritems(): if f in ospec: # allow for multiple target values (given a matcher) being # specified. For some matchers it might make no sense # (e.g. "endswith") targets = ospec[f] for target in (targets if isinstance(targets, list) else [targets]): target = unicode.replace(target, "<NEWLINE>", os.linesep) # TODO: This replacement may be should be done elsewhere # to have a general solution. It's now affecting string-type only. # Additionally, "<NEWLINE>" may appear in some output intentionally, # so let's find sth closer to be 'unique'. self.assertThat( execinfo[idx][field], Annotate( "unexpected output for '%s'" % ospec_id, matcher(target))) else: raise NotImplementedError( "dunno how to handle output type '%s' yet" % ospectype)
def _tx(): SearchEntry.insert(s, SearchClasses.EXACT, u'e', u'i', u'RESULT', u'type', value) for mutation in [casefold, spaced, punctuated]: self.assertThat( list( SearchEntry.search(s, SearchClasses.EXACT, u'e', u'i', mutation(value))), Annotate( 'Not found for {!r}({!r}) == {!r}'.format( mutation, value, mutation(value)), Equals([{ u'result': u'RESULT', u'type': u'type' }])))
def check_table_row_counts(test, _last=[None]): """Check that all tables have expected row counts. This considers only tables in the database's public schema, which, for MAAS, means only application tables. :param test: An instance of :class:`testtools.TestCase`. """ culprit, _last[0] = _last[0], test.id() observed = get_table_row_counts() expected = dict.fromkeys(observed, Equals(0)) expected.update(expected_table_counts) test.assertThat(observed, Annotate( "Table counts are unexpected; most likely culprit " "is %s." % culprit, MatchesDict(expected)))
def assertThat(self, matchee, matcher, message='', verbose=False): """Assert that matchee is matched by matcher. :param matchee: An object to match with matcher. :param matcher: An object meeting the testtools.Matcher protocol. :raises MismatchError: When matcher does not match thing. """ matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return existing_details = self.getDetails() for (name, content) in mismatch.get_details().items(): self.addDetailUniqueName(name, content) raise MismatchError(matchee, matcher, mismatch, verbose)
def _expectThat(self, matchee, matcher, message='', verbose=False): matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return existing_details = self.getDetails() for (name, content) in mismatch.get_details().items(): full_name = name suffix = 1 while full_name in existing_details: full_name = "%s-%d" % (name, suffix) suffix += 1 self.addDetail(full_name, content) raise MismatchError(matchee, matcher, mismatch, verbose)
def assert_that(matchee, matcher, message='', verbose=False): """Assert that matchee is matched by matcher. This should only be used when you need to use a function based matcher, assertThat in Testtools.Testcase is prefered and has more features :param matchee: An object to match with matcher. :param matcher: An object meeting the testtools.Matcher protocol. :raises MismatchError: When matcher does not match thing. """ matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return raise MismatchError(matchee, matcher, mismatch, verbose)
def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: error_msg = '%s not raised.' % self.exc_type.__name__ if self.msg: error_msg = error_msg + ' : ' + self.msg raise AssertionError(error_msg) if exc_type != self.exc_type: return False if self.value_re: matcher = MatchesException(self.exc_type, self.value_re) if self.msg: matcher = Annotate(self.msg, matcher) mismatch = matcher.match((exc_type, exc_value, traceback)) if mismatch: raise AssertionError(mismatch.describe()) return True
def assertThat(self, matchee, matcher, message="", verbose=False): """Assert that matchee is matched by matcher. :param matchee: An object to match with matcher. :param matcher: An object meeting the testtools.Matcher protocol. :raises MismatchError: When matcher does not match thing. """ matcher = Annotate.if_message(message, matcher) mismatch = matcher.match(matchee) if not mismatch: return existing_details = self.getDetails() for (name, content) in mismatch.get_details().items(): full_name = name suffix = 1 while full_name in existing_details: full_name = "%s-%d" % (name, suffix) suffix += 1 self.addDetail(full_name, content) raise MismatchError(matchee, matcher, mismatch, verbose)
def test_origin(self): # We stage libc6 for this to work on non xenial snapcraft_yaml = fixture_setup.SnapcraftYaml(self.path) snapcraft_yaml.update_part('test-part', { 'plugin': 'nil', 'stage-packages': ['libc6', 'python3'] }) self.useFixture(snapcraft_yaml) self.run_snapcraft('prime') bin_path = os.path.join(self.prime_dir, 'usr', 'bin', 'python3') self.assertThat(bin_path, FileExists()) # Verify there aren't any duplicate rpath entries rpath = subprocess.check_output([ self.patchelf_command, '--print-rpath', bin_path]).decode().strip() seen = set() for i in rpath.split(':'): self.assertThat( seen, Annotate('A duplicate rpath was found!', Not(Contains(i)))) seen.add(i)
def test_origin(self): # We stage libc6 for this to work on non xenial snapcraft_yaml = fixture_setup.SnapcraftYaml(self.path, base=None) snapcraft_yaml.update_part("test-part", { "plugin": "nil", "stage-packages": ["libc6", "python3"] }) self.useFixture(snapcraft_yaml) self.run_snapcraft("prime") bin_path = os.path.join(self.prime_dir, "usr", "bin", "python3") self.assertThat(bin_path, FileExists()) # Verify there aren't any duplicate rpath entries rpath = (subprocess.check_output( [self.patchelf_command, "--print-rpath", bin_path]).decode().strip()) seen = set() for i in rpath.split(":"): self.assertThat( seen, Annotate("A duplicate rpath was found!", Not(Contains(i)))) seen.add(i)
def test_if_message_no_message(self): # Annotate.if_message returns the given matcher if there is no # message. matcher = Equals(1) not_annotated = Annotate.if_message('', matcher) self.assertIs(matcher, not_annotated)
class TarfileHasFile(Matcher): """Check that a tarfile has an entry with certain values.""" def __init__(self, path, type=None, size=None, mtime=None, mtime_skew=None, mode=None, linkname=None, uid=None, gid=None, uname=None, gname=None, content=None, content_matcher=None): """Create a TarfileHasFile Matcher. :param path: the path that must be present. :type path: str :param type: the type of TarInfo that must be at `path`, or None to not check. :param size: the size that the entry at `path` must have, or None to not check. :param mtime: the mtime that the entry at `path` must have, or None to not check. :param mtime_skew: the number of seconds that the file mtime can be different to the required. :param mode: the mode that the entry at `path` must have, or None to not check. :param linkname: the linkname that the entry at `path` must have, or None to not check. :param uid: the user id that the entry at `path` must have, or None to not check. :param gid: the group id that the entry at `path` must have, or None to not check. :param uname: the username that the entry at `path` must have, or None to not check. :param gname: the group name that the entry at `path` must have, or None to not check. :param content: the content that `path` must have when extracted, or None to not check. :param content_matcher: a matcher to match the content that `path` has when extracted, or None to not check. You can't specify both content_matcher and content. """ self.path = path self.type = type self.size = size self.mtime = mtime self.mtime_skew = mtime_skew self.mode = mode self.linkname = linkname self.uid = uid self.gid = gid self.uname = uname self.gname = gname if content is not None: if content_matcher is not None: raise ValueError( "doesn't make sense to specify content and " "content_matcher") content_matcher = Equals(content) if content_matcher is not None: self.content_matcher = Annotate( 'The content of path "%s" did not match' % path, content_matcher) else: self.content_matcher = None def match(self, tarball): """Match a tarfile.TarFile against the expected values.""" if self.path not in tarball.getnames(): return TarfileMissingPathMismatch(tarball, self.path) info = tarball.getmember(self.path) for attr in ("type", "size", "mode", "linkname", "uid", "gid", "uname", "gname"): expected = getattr(self, attr, None) if expected is not None: actual = getattr(info, attr) if expected != actual: return TarfileWrongValueMismatch( attr, tarball, self.path, expected, actual) if self.mtime is not None: mtime_skew = self.mtime_skew or 0 if abs(self.mtime - info.mtime) > mtime_skew: return TarfileWrongValueMismatch( "mtime", tarball, self.path, self.mtime, info.mtime) if self.content_matcher is not None: if info.type == tarfile.DIRTYPE: contents = [] path_frags = self.path.split('/') for name in tarball.getnames(): name_frags = name.split('/') if (len(name_frags) == len(path_frags) + 1 and name_frags[:-1] == path_frags): contents.append(name_frags[-1]) content_mismatch = self.content_matcher.match(contents) if content_mismatch: return content_mismatch else: actual = tarball.extractfile(self.path).read() content_mismatch = self.content_matcher.match(actual) if content_mismatch: return content_mismatch return None def __str__(self): return 'tarfile has file "%s"' % (self.path, )
def test_assertThat_message_is_annotated(self): matchee = 'foo' matcher = Equals('bar') expected = Annotate('woo', matcher).match(matchee).describe() self.assertFails(expected, self.assert_that_callable, matchee, matcher, 'woo')
Annotate( "test shell command '%s' yielded non-zero exit code" % cmd, Equals(0))) except OSError, e: lgr.error("%s: %s" % (e.__class__.__name__, str(e))) if not 'shouldfail' in testspec or testspec['shouldfail'] == False: self.assertThat( e, Annotate( "test command execution failed: %s (%s)" % (e.__class__.__name__, str(e)), Equals(None))) if 'shouldfail' in testspec and testspec['shouldfail'] == True: self.assertThat( e, Annotate( "an expected failure did not occur in test '%s': %s (%s)" % (testid, str(e), e.__class__.__name__), Equals(None))) def _execute_nipype_test(self, testid, testspec): # TODO merge/refactor this one with the plain python code method from cStringIO import StringIO import sys execinfo = self._details['exec_info'][testid] try: import nipype except ImportError: self.skipTest("Nipype not found, skipping test") # where is the workflow if 'file' in testspec: testwffilepath = testspec['file'] lgr.debug("using custom workflow file name '%s'" % testwffilepath)