def main():
parser = argparse.ArgumentParser(
description="Critic testing framework: instance upgrade utility")
parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
parser.add_argument("--vm-hostname", help="VirtualBox instance hostname", required=True)
parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to upgrade", default="clean")
parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
parser.add_argument("--pause-before-upgrade", help="Pause before upgrading", action="store_true")
parser.add_argument("--pause-after-upgrade", help="Pause after upgrading", action="store_true")
arguments = parser.parse_args()
logger = testing.configureLogging(arguments)
logger.info("Critic Testing Framework: Instance Upgrade")
instance = testing.virtualbox.Instance(
identifier=arguments.vm_identifier,
snapshot=arguments.vm_snapshot,
hostname=arguments.vm_hostname,
ssh_port=arguments.vm_ssh_port)
with instance:
instance.start()
logger.debug("Upgrading guest OS ...")
update_output = instance.execute(
["sudo", "DEBIAN_FRONTEND=noninteractive",
"apt-get", "-q", "-y", "update"])
logger.debug("Output from 'apt-get -q -y update':\n" + update_output)
upgrade_output = instance.execute(
["sudo", "DEBIAN_FRONTEND=noninteractive",
"apt-get", "-q", "-y", "upgrade"])
logger.debug("Output from 'apt-get -q -y upgrade':\n" + upgrade_output)
if "The following packages will be upgraded:" in upgrade_output.splitlines():
logger.info("Upgraded guest OS")
logger.debug("Retaking snapshot ...")
instance.retake_snapshot(arguments.vm_snapshot)
logger.info("Snapshot '%s' upgraded!" % arguments.vm_snapshot)
else:
logger.info("No packages upgraded in guest OS")
def main():
parser = argparse.ArgumentParser(description="Critic testing framework")
parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
parser.add_argument("--debug-mails", help="Log every mail sent by the tested system", action="store_true")
parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
parser.add_argument("--coverage", help="Enable coverage measurement mode", action="store_true")
parser.add_argument("--commit", help="Commit (symbolic ref or SHA-1) to test [default=HEAD]", default="HEAD")
parser.add_argument("--upgrade-from", help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
parser.add_argument("--strict-fs-permissions", help="Set strict file-system permissions in guest OS", action="store_true")
parser.add_argument("--vbox-host", help="Host that's running VirtualBox [default=host]", default="host")
parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
parser.add_argument("--vm-hostname", help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to restore [default=clean]", default="clean")
parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
parser.add_argument("--vm-http-port", help="VirtualBox instance HTTP port [default=80]", type=int, default=80)
parser.add_argument("--git-daemon-port", help="Port to tell 'git daemon' to bind to", type=int)
parser.add_argument("--pause-before", help="Pause testing before specified test(s)", action="append")
parser.add_argument("--pause-after", help="Pause testing before specified test(s)", action="append")
parser.add_argument("--pause-on-failure", help="Pause testing after each failed test", action="store_true")
parser.add_argument("--pause-upgrade-loop", help="Support upgrading the tested system while paused", action="store_true")
parser.add_argument("--pause-upgrade-hook", help="Command to run (locally) before upgrading", action="append")
parser.add_argument("test", help="Specific tests to run [default=all]", nargs="*")
arguments = parser.parse_args()
class Counters:
def __init__(self):
self.tests_run = 0
self.tests_failed = 0
self.errors_logged = 0
self.warnings_logged = 0
counters = Counters()
class CountingLogger(object):
def __init__(self, real, counters):
self.real = real
self.counters = counters
def log(self, level, message):
if level == logging.ERROR:
self.counters.errors_logged += 1
elif level == logging.WARNING:
self.counters.warnings_logged += 1
for line in message.splitlines() or [""]:
self.real.log(level, line)
def debug(self, message):
self.log(logging.DEBUG, message)
def info(self, message):
self.log(logging.INFO, message)
def warning(self, message):
self.log(logging.WARNING, message)
def error(self, message):
self.log(logging.ERROR, message)
def exception(self, message):
self.log(logging.ERROR, message + "\n" + traceback.format_exc())
logger = testing.configureLogging(
arguments, wrap=lambda logger: CountingLogger(logger, counters))
logger.info("Critic Testing Framework")
import_errors = False
try:
import requests
except ImportError:
logger.error("Failed to import 'requests'!")
import_errors = True
try:
import BeautifulSoup
except ImportError:
logger.error("Failed to import 'BeautifulSoup'!")
import_errors = True
git_version = subprocess.check_output(["git", "--version"]).strip()
m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
if not m:
logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
else:
version_tuple = tuple(map(int, m.groups()))
if version_tuple >= (1, 8, 5):
logger.debug("Using Git version %s on host." % git_version)
else:
logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
import_errors = True
if import_errors:
logger.error("Required software missing; see testing/USAGE.md for details.")
return
# Note: we are not ignoring typical temporary editor files such as the
# ".#<name>" files created by Emacs when a buffer has unsaved changes. This
# is because unsaved changes in an editor is probably also something you
# don't want to test with.
locally_modified_paths = []
status_output = subprocess.check_output(
["git", "status", "--porcelain"])
for line in status_output.splitlines():
locally_modified_paths.extend(line[3:].split(" -> "))
tests_modified = []
input_modified = []
other_modified = []
for path in locally_modified_paths:
if path.startswith("testing/input/"):
input_modified.append(path)
elif path.startswith("testing/"):
tests_modified.append(path)
else:
other_modified.append(path)
if input_modified:
logger.error("Test input files locally modified:\n " + "\n ".join(input_modified))
if other_modified:
logger.error("Critic files locally modified:\n " + "\n ".join(other_modified))
if input_modified or other_modified:
logger.error("Please commit or stash local modifications before running tests.")
return
if tests_modified:
logger.warning("Running tests using locally modified files:\n " + "\n ".join(tests_modified))
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
try:
frontend = testing.frontend.Frontend(
hostname=arguments.vm_hostname or arguments.vm_identifier,
http_port=arguments.vm_http_port)
instance = testing.virtualbox.Instance(
vboxhost=arguments.vbox_host,
identifier=arguments.vm_identifier,
snapshot=arguments.vm_snapshot,
hostname=arguments.vm_hostname,
ssh_port=arguments.vm_ssh_port,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description),
frontend=frontend,
strict_fs_permissions=arguments.strict_fs_permissions,
coverage=arguments.coverage)
except testing.Error as error:
logger.error(error.message)
return
if arguments.test:
tests = set(test.strip("/") for test in arguments.test)
groups = {}
for test in sorted(tests):
components = test.split("/")
if test.endswith(".py"):
del components[-1]
else:
test += "/"
for index in range(len(components)):
groups.setdefault("/".join(components[:index + 1]), set()).add(test)
def directory_enabled(path):
if path in groups:
# Directory (or sub-directory of or file in it) was directly named.
return True
# Check if an ancestor directory was directly name:
directory = path.rpartition("/")[0]
while directory:
if directory in groups and directory + "/" in groups[directory]:
return True
directory = directory.rpartition("/")[0]
return False
def file_enabled(path):
if path in tests:
# File was directly named.
return True
directory, _, name = path.rpartition("/")
if directory in groups:
# Loop over all enabled items in the same directory as 'path':
for item in groups[directory]:
if item != directory:
local, slash, _ = item[len(directory + "/"):].partition("/")
if local > name and (slash or "/" not in directory):
# A file in a later sub-directory is enabled; this
# means 'path' is a dependency of the other file.
return "dependency"
# Check if the file's directory or an ancestor directory thereof was
# directly name:
while directory:
if directory in groups and directory + "/" in groups[directory]:
return True
directory = directory.rpartition("/")[0]
return False
else:
def directory_enabled(path):
return True
def file_enabled(path):
return True
def pause():
if arguments.pause_upgrade_loop:
print "Testing paused."
while True:
testing.pause("Press ENTER to upgrade (to HEAD), CTRL-c to stop: ")
for command in arguments.pause_upgrade_hook:
subprocess.check_call(command, shell=True)
repository.push("HEAD")
instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
instance.upgrade_commit = "FETCH_HEAD"
instance.upgrade()
else:
testing.pause("Testing paused. Press ENTER to continue: ")
pause_before = set(arguments.pause_before or [])
pause_after = set(arguments.pause_after or [])
def run_group(group_name):
try:
instance.start()
instance.mailbox = mailbox
def run_tests(directory):
has_failed = False
if directory in pause_before:
pause()
for name in sorted(os.listdir(os.path.join("testing/tests", directory))):
if not re.match("\d{3}-", name):
continue
path = os.path.join(directory, name)
if os.path.isdir(os.path.join("testing/tests", path)):
if not directory_enabled(path):
logger.debug("Skipping: %s/" % path)
elif has_failed:
logger.info("Skipping: %s/ (failed dependency)" % path)
else:
run_tests(path)
continue
elif not re.search("\\.py$", name):
continue
enabled = file_enabled(path)
if not enabled:
logger.debug("Skipping: %s" % path)
continue
if path in pause_before:
pause()
if enabled is True:
mode = ""
else:
mode = " (%s)" % enabled
logger.info("Running: %s%s" % (path, mode))
counters.tests_run += 1
try:
execfile(os.path.join("testing/tests", path),
{ "testing": testing,
"logger": logger,
"instance": instance,
"frontend": frontend,
"repository": repository,
"mailbox": mailbox })
except testing.TestFailure as failure:
counters.tests_failed += 1
if failure.message:
logger.error(failure.message)
while True:
mail = mailbox.pop(
accept=testing.mailbox.to_recipient("*****@*****.**"),
timeout=1)
if not mail:
break
logger.error("Administrator message: %s\n %s"
% (mail.header("Subject"),
"\n ".join(mail.lines)))
if arguments.pause_on_failure:
pause()
if "/" in directory:
has_failed = True
else:
return
else:
if path in pause_after:
pause()
if directory in pause_after:
pause()
run_tests(group_name)
except KeyboardInterrupt:
logger.error("Testing aborted.")
return False
except testing.Error as error:
if error.message:
logger.exception(error.message)
if arguments.pause_on_failure:
pause()
return False
except Exception:
logger.exception("Unexpected exception!")
if arguments.pause_on_failure:
pause()
return False
else:
return True
for group_name in sorted(os.listdir("testing/tests")):
if not re.match("\d{3}-", group_name):
continue
if not directory_enabled(group_name):
logger.debug("Skipping: %s/" % group_name)
continue
repository = testing.repository.Repository(
arguments.vbox_host,
arguments.git_daemon_port,
tested_commit,
arguments.vm_hostname)
mailbox = testing.mailbox.Mailbox({ "username": "******",
"password": "******" },
arguments.debug_mails)
with repository:
with mailbox:
if not repository.export():
return
with instance:
if run_group(group_name):
instance.finish()
mailbox.check_empty()
logger.info("""
Test summary
============
Tests run: %3d
Tests failed: %3d
Errors logged: %3d
Warnings logged: %3d
""" % (counters.tests_run,
counters.tests_failed,
counters.errors_logged,
counters.warnings_logged))
def main():
parser = argparse.ArgumentParser(
description="Critic testing framework: Quick install utility")
parser.add_argument("--debug",
help="Enable DEBUG level logging", action="store_true")
parser.add_argument("--quiet",
help="Disable INFO level logging", action="store_true")
parser.add_argument("--commit", default="HEAD",
help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
parser.add_argument("--upgrade-from",
help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
parser.add_argument("--vm-identifier", required=True,
help="VirtualBox instance name or UUID")
parser.add_argument("--vm-hostname",
help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
parser.add_argument("--vm-snapshot", default="clean",
help="VirtualBox snapshot (name or UUID) to upgrade [default=clean]")
parser.add_argument("--vm-ssh-port", type=int, default=22,
help="VirtualBox instance SSH port [default=22]")
parser.add_argument("--git-daemon-port", type=int,
help="Port to tell 'git daemon' to bind to")
parser.add_argument("--interactive", "-i", action="store_true",
help="Install interactively (without arguments)")
arguments = parser.parse_args()
logger = testing.configureLogging(arguments)
logger.info("Critic testing framework: Quick install")
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
instance = testing.virtualbox.Instance(
arguments,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description))
repository = testing.repository.Repository(
arguments.git_daemon_port,
install_commit,
arguments.vm_hostname)
mailbox = testing.mailbox.Mailbox()
with repository, mailbox, instance:
if not repository.export():
return
instance.mailbox = mailbox
instance.start()
if arguments.interactive:
print """
Note: To use the simple SMTP server built into the Critic testing framework,
enter "host" as the SMTP host and "%d" as the SMTP port.
Also note: The administrator user's password will be "testing" (password
input doesn't work over this channel.)""" % mailbox.port
instance.install(repository, quick=True,
interactive=arguments.interactive)
instance.upgrade(interactive=arguments.interactive)
testing.pause("Press ENTER to stop VM: ")
try:
while True:
mail = mailbox.pop()
logger.info("Mail to <%s>:\n%s" % (mail.recipient, mail))
except testing.mailbox.MissingMail:
pass
def main():
parser = argparse.ArgumentParser(
description="Critic testing framework: instance upgrade utility")
parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
parser.add_argument("--vm-hostname", help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to upgrade", default="clean")
parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
parser.add_argument("--pause-before-upgrade", help="Pause before upgrading", action="store_true")
parser.add_argument("--pause-after-upgrade", help="Pause after upgrading", action="store_true")
parser.add_argument("--no-upgrade", action="store_true", help="Do not upgrade installed packages")
parser.add_argument("--install", action="append", help="Install named package")
parser.add_argument("--custom", action="store_true",
help="Stop for custom maintenance, and always retake snapshot")
parser.add_argument("--reboot", action="store_true",
help="Reboot VM before retaking snapshot")
arguments = parser.parse_args()
logger = testing.configureLogging(arguments)
logger.info("Critic Testing Framework: Instance Upgrade")
instance = testing.virtualbox.Instance(
identifier=arguments.vm_identifier,
snapshot=arguments.vm_snapshot,
hostname=arguments.vm_hostname,
ssh_port=arguments.vm_ssh_port)
with instance:
instance.start()
if not arguments.no_upgrade:
logger.debug("Upgrading guest OS ...")
update_output = instance.execute(
["sudo", "DEBIAN_FRONTEND=noninteractive",
"apt-get", "-q", "-y", "update"])
logger.debug("Output from 'apt-get -q -y update':\n" + update_output)
upgrade_output = instance.execute(
["sudo", "DEBIAN_FRONTEND=noninteractive",
"apt-get", "-q", "-y", "upgrade"])
logger.debug("Output from 'apt-get -q -y upgrade':\n" + upgrade_output)
retake_snapshot = False
if "The following packages will be upgraded:" in upgrade_output.splitlines():
retake_snapshot = True
if arguments.install:
install_output = instance.execute(
["sudo", "DEBIAN_FRONTEND=noninteractive",
"apt-get", "-q", "-y", "install"] + arguments.install)
logger.debug("Output from 'apt-get -q -y install':\n" +
install_output)
retake_snapshot = True
if arguments.custom:
testing.pause()
retake_snapshot = True
if retake_snapshot:
if arguments.reboot:
instance.execute(["sudo", "reboot"])
logger.debug("Sleeping 10 seconds ...")
time.sleep(10)
instance.wait()
logger.debug("Sleeping 10 seconds ...")
time.sleep(10)
logger.info("Rebooted VM: %s" % arguments.vm_identifier)
logger.info("Upgraded guest OS")
logger.debug("Retaking snapshot ...")
instance.retake_snapshot(arguments.vm_snapshot)
logger.info("Snapshot '%s' upgraded!" % arguments.vm_snapshot)
else:
logger.info("No packages upgraded in guest OS")
def main():
parser = argparse.ArgumentParser(description="Critic testing framework")
parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
parser.add_argument("--commit", help="Commit (symbolic ref or SHA-1) to test [default=HEAD]", default="HEAD")
parser.add_argument("--upgrade-from", help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
parser.add_argument("--strict-fs-permissions", help="Set strict file-system permissions in guest OS", action="store_true")
parser.add_argument("--vbox-host", help="Host that's running VirtualBox [default=host]", default="host")
parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
parser.add_argument("--vm-hostname", help="VirtualBox instance hostname", required=True)
parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to restore", default="clean")
parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
parser.add_argument("--vm-http-port", help="VirtualBox instance HTTP port [default=80]", type=int, default=80)
parser.add_argument("--git-daemon-port", help="Port to tell 'git daemon' to bind to", type=int)
parser.add_argument("--pause-before", help="Pause testing before specified test(s)", action="append")
parser.add_argument("--pause-after", help="Pause testing before specified test(s)", action="append")
parser.add_argument("--pause-on-failure", help="Pause testing after each failed test", action="store_true")
parser.add_argument("test", help="Specific tests to run [default=all]", nargs="*")
arguments = parser.parse_args()
logger = testing.configureLogging(arguments)
logger.info("Critic Testing Framework")
import_errors = False
try:
import requests
except ImportError:
logger.error("Failed to import 'requests'!")
import_errors = True
try:
import BeautifulSoup
except ImportError:
logger.error("Failed to import 'BeautifulSoup'!")
import_errors = True
if import_errors:
logger.error("Required software missing; see testing/USAGE.md for details.")
return
locally_modified_paths = subprocess.check_output(
["git", "diff", "--name-only"])
tests_modified = []
input_modified = []
other_modified = []
for path in locally_modified_paths.splitlines():
if path.startswith("testing/input/"):
input_modified.append(path)
elif path.startswith("testing/"):
tests_modified.append(path)
else:
other_modified.append(path)
if input_modified:
logger.error("Test input files locally modified:\n " + "\n ".join(input_modified))
if other_modified:
logger.error("Critic files locally modified:\n " + "\n ".join(other_modified))
if input_modified or other_modified:
logger.error("Please commit or stash local modifications before running tests.")
return
if tests_modified:
logger.warning("Running tests using locally modified files:\n " + "\n ".join(tests_modified))
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
try:
frontend = testing.frontend.Frontend(
hostname=arguments.vm_hostname,
http_port=arguments.vm_http_port)
instance = testing.virtualbox.Instance(
vboxhost=arguments.vbox_host,
identifier=arguments.vm_identifier,
snapshot=arguments.vm_snapshot,
hostname=arguments.vm_hostname,
ssh_port=arguments.vm_ssh_port,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description),
frontend=frontend,
strict_fs_permissions=arguments.strict_fs_permissions)
except testing.Error as error:
logger.error(error.message)
return
if arguments.test:
tests = set(test.strip("/") for test in arguments.test)
groups = {}
for test in sorted(tests):
components = test.split("/")
if test.endswith(".py"):
del components[-1]
else:
test += "/"
for index in range(len(components)):
groups.setdefault("/".join(components[:index + 1]), set()).add(test)
def directory_enabled(path):
if path in groups:
# Directory (or sub-directory of or file in it) was directly named.
return True
# Check if an ancestor directory was directly name:
directory = path.rpartition("/")[0]
while directory:
if directory in groups and directory + "/" in groups[directory]:
return True
directory = directory.rpartition("/")[0]
return False
def file_enabled(path):
if path in tests:
# File was directly named.
return True
directory, _, name = path.rpartition("/")
if directory in groups:
# Loop over all enabled items in the same directory as 'path':
for item in groups[directory]:
if item != directory:
local, slash, _ = item[len(directory + "/"):].partition("/")
if local > name and (slash or "/" not in directory):
# A file in a later sub-directory is enabled; this
# means 'path' is a dependency of the other file.
return "dependency"
# Check if the file's directory or an ancestor directory thereof was
# directly name:
while directory:
if directory in groups and directory + "/" in groups[directory]:
return True
directory = directory.rpartition("/")[0]
return False
else:
def directory_enabled(path):
return True
def file_enabled(path):
return True
def pause():
print
try:
raw_input("Testing paused. Press ENTER to continue: ")
except KeyboardInterrupt:
print
print
raise
print
pause_before = set(arguments.pause_before or [])
pause_after = set(arguments.pause_after or [])
def run_group(group_name):
try:
instance.start()
instance.mailbox = mailbox
def run_tests(directory):
has_failed = False
for name in sorted(os.listdir(os.path.join("testing/tests", directory))):
if not re.match("\d{3}-", name):
continue
path = os.path.join(directory, name)
if os.path.isdir(os.path.join("testing/tests", path)):
if not directory_enabled(path):
logger.debug("Skipping: %s/" % path)
elif has_failed:
logger.info("Skipping: %s/ (failed dependency)" % path)
else:
run_tests(path)
continue
elif not re.search("\\.py$", name):
continue
enabled = file_enabled(path)
if not enabled:
logger.debug("Skipping: %s" % path)
continue
if path in pause_before:
pause()
if enabled is True:
mode = ""
else:
mode = " (%s)" % enabled
logger.info("Running: %s%s" % (path, mode))
try:
execfile(os.path.join("testing/tests", path),
{ "testing": testing,
"logger": logger,
"instance": instance,
"frontend": frontend,
"repository": repository,
"mailbox": mailbox })
except testing.TestFailure as failure:
if failure.message:
logger.error(failure.message)
while True:
mail = mailbox.pop(
accept=testing.mailbox.to_recipient("*****@*****.**"),
timeout=1)
if not mail:
break
logger.error("Administrator message: %s\n %s"
% (mail.header("Subject"),
"\n ".join(mail.lines)))
if arguments.pause_on_failure:
pause()
if "/" in directory:
has_failed = True
else:
return
else:
if path in pause_after:
pause()
run_tests(group_name)
except KeyboardInterrupt:
logger.error("Testing aborted.")
except testing.Error as error:
if error.message:
logger.exception(error.message)
if arguments.pause_on_failure:
pause()
except Exception:
logger.exception("Unexpected exception!")
if arguments.pause_on_failure:
pause()
for group_name in sorted(os.listdir("testing/tests")):
if not re.match("\d{3}-", group_name):
continue
if not directory_enabled(group_name):
logger.debug("Skipping: %s/" % group_name)
continue
repository = testing.repository.Repository(
arguments.vbox_host,
arguments.git_daemon_port,
tested_commit,
arguments.vm_hostname)
mailbox = testing.mailbox.Mailbox()
with repository:
with mailbox:
if not repository.export():
return
with instance:
run_group(group_name)
mailbox.check_empty()
def run():
global logger
parser = argparse.ArgumentParser(description="Critic testing framework")
parser.add_argument("--debug", action="store_true",
help="Enable DEBUG level logging")
parser.add_argument("--debug-mails", action="store_true",
help="Log every mail sent by the tested system")
parser.add_argument("--quiet", action="store_true",
help="Disable INFO level logging")
parser.add_argument("--coverage", action="store_true",
help="Enable coverage measurement mode")
parser.add_argument("--commit",
help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
parser.add_argument("--upgrade-from",
help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
parser.add_argument("--strict-fs-permissions", action="store_true",
help="Set strict file-system permissions in guest OS")
parser.add_argument("--test-extensions", action="store_true",
help="Test extensions")
parser.add_argument("--local", action="store_true",
help="Run local standalone tests only")
parser.add_argument("--vbox-host", default="host",
help="Host that's running VirtualBox [default=host]")
parser.add_argument("--vm-identifier",
help="VirtualBox instance name or UUID")
parser.add_argument("--vm-hostname",
help="VirtualBox instance hostname [default=VM_IDENTIFIER")
parser.add_argument("--vm-snapshot", default="clean",
help="VirtualBox snapshot (name or UUID) to restore [default=clean]")
parser.add_argument("--vm-ssh-port", type=int, default=22,
help="VirtualBox instance SSH port [default=22]")
parser.add_argument("--vm-http-port", type=int, default=80,
help="VirtualBox instance HTTP port [default=80]")
parser.add_argument("--git-daemon-port", type=int,
help="Port to tell 'git daemon' to bind to")
parser.add_argument("--cache-dir", default="testing/cache",
help="Directory where cache files are stored")
parser.add_argument("--pause-before", action="append",
help="Pause testing before specified test(s)")
parser.add_argument("--pause-after", action="append",
help="Pause testing before specified test(s)")
parser.add_argument("--pause-on-failure", action="store_true",
help="Pause testing after each failed test")
parser.add_argument("--pause-upgrade-loop", action="store_true",
help="Support upgrading the tested system while paused")
parser.add_argument("--pause-upgrade-hook", action="append",
help="Command to run (locally) before upgrading")
parser.add_argument("test", nargs="*",
help="Specific tests to run [default=all]")
arguments = parser.parse_args()
class CountingLogger(object):
def __init__(self, real, counters):
self.real = real
self.counters = counters
def log(self, level, message):
if level == logging.ERROR:
self.counters.errors_logged += 1
elif level == logging.WARNING:
self.counters.warnings_logged += 1
for line in message.splitlines() or [""]:
self.real.log(level, line)
def debug(self, message):
self.log(logging.DEBUG, message)
def info(self, message):
self.log(logging.INFO, message)
def warning(self, message):
self.log(logging.WARNING, message)
def error(self, message):
self.log(logging.ERROR, message)
def exception(self, message):
self.log(logging.ERROR, message + "\n" + traceback.format_exc())
logger = testing.configureLogging(
arguments, wrap=lambda logger: CountingLogger(logger, counters))
logger.info("""\
Critic Testing Framework
========================
""")
if not arguments.local and not arguments.vm_identifier:
logger.error("Must specify one of --local and --vm-identifier!")
return
if arguments.local:
incompatible_arguments = []
# This is not a complete list; just those that are most significantly
# incompatible or irrelevant with --local.
if arguments.commit:
incompatible_arguments.append("--commit")
if arguments.upgrade_from:
incompatible_arguments.append("--upgrade-from")
if arguments.coverage:
incompatible_arguments.append("--coverage")
if arguments.test_extensions:
incompatible_arguments.append("--strict-fs-permissions")
if arguments.test_extensions:
incompatible_arguments.append("--test-extensions")
if arguments.vm_identifier:
incompatible_arguments.append("--vm-identifier")
if incompatible_arguments:
logger.error("These arguments can't be combined with --local:\n " +
"\n ".join(incompatible_arguments))
return
import_errors = False
try:
import requests
except ImportError:
logger.error("Failed to import 'requests'!")
import_errors = True
try:
import BeautifulSoup
except ImportError:
logger.error("Failed to import 'BeautifulSoup'!")
import_errors = True
git_version = subprocess.check_output(["git", "--version"]).strip()
m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
if not m:
logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
else:
version_tuple = tuple(map(int, m.groups()))
if version_tuple >= (1, 8, 5):
logger.debug("Using Git version %s on host." % git_version)
else:
logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
import_errors = True
if import_errors:
logger.error("Required software missing; see testing/USAGE.md for details.")
return
if arguments.test_extensions:
# Check that the v8-jsshell submodule is checked out if extension
# testing was requested.
output = subprocess.check_output(["git", "submodule", "status",
"installation/externals/v8-jsshell"])
if output.startswith("-"):
logger.error("""\
The v8-jsshell submodule must be checked for extension testing. Please run
git submodule update --init installation/externals/v8-jsshell
first or run this script without --test-extensions.""")
return
if not arguments.local:
# Note: we are not ignoring typical temporary editor files such as the
# ".#<name>" files created by Emacs when a buffer has unsaved changes.
# This is because unsaved changes in an editor is probably also
# something you don't want to test with.
locally_modified_paths = []
status_output = subprocess.check_output(
["git", "status", "--porcelain"])
for line in status_output.splitlines():
locally_modified_paths.extend(line[3:].split(" -> "))
tests_modified = []
input_modified = []
other_modified = []
for path in locally_modified_paths:
if path.startswith("testing/input/"):
input_modified.append(path)
elif path.startswith("testing/"):
tests_modified.append(path)
else:
other_modified.append(path)
if input_modified:
logger.error("Test input files locally modified:\n " +
"\n ".join(input_modified))
if other_modified:
logger.error("Critic files locally modified:\n " +
"\n ".join(other_modified))
if input_modified or other_modified:
logger.error("Please commit or stash local modifications before "
"running tests.")
return
if tests_modified:
logger.warning("Running tests using locally modified files:\n " +
"\n ".join(tests_modified))
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit or "HEAD"]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
flags_on = set()
flags_off = set()
try:
if arguments.local:
frontend = None
instance = testing.local.Instance()
flags_on.add("local")
else:
frontend = testing.frontend.Frontend(
hostname=arguments.vm_hostname or arguments.vm_identifier,
http_port=arguments.vm_http_port)
instance = testing.virtualbox.Instance(
arguments,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description),
frontend=frontend)
except testing.Error as error:
logger.error(error.message)
return
tests, dependencies = testing.findtests.selectTests(
arguments.test, strict=False, flags_on=flags_on, flags_off=flags_off)
if not tests:
logger.error("No tests selected!")
return
def pause():
if arguments.pause_upgrade_loop:
print "Testing paused."
while True:
testing.pause("Press ENTER to upgrade (to HEAD), CTRL-c to stop: ")
for command in arguments.pause_upgrade_hook:
subprocess.check_call(command, shell=True)
repository.push("HEAD")
instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
instance.upgrade_commit = "FETCH_HEAD"
instance.upgrade()
else:
testing.pause("Testing paused. Press ENTER to continue: ")
pause_before = pause_after = set()
if arguments.pause_before:
pause_before = testing.findtests.filterPatterns(arguments.pause_before)
pause_before_tests, _ = testing.findtests.selectTests(pause_before,
strict=True)
pause_before_tests = set(pause_before_tests)
pause_before_groups = set(pause_before)
def maybe_pause_before(test):
def do_pause(what):
logger.info("Pausing before: %s" % what)
pause()
if test in pause_before_tests:
do_pause(test)
else:
for group in test.groups:
if group in pause_before_groups \
and test == all_groups[group][0]:
do_pause(group)
break
else:
def maybe_pause_before(test):
pass
if arguments.pause_after:
pause_after = testing.findtests.filterPatterns(arguments.pause_after)
pause_after_tests, _ = testing.findtests.selectTests(pause_after,
strict=True)
pause_after_tests = set(pause_after_tests)
pause_after_groups = set(pause_after)
def maybe_pause_after(test):
def do_pause(what):
logger.info("Pausing after: %s" % what)
pause()
if test in pause_after_tests:
do_pause(test)
else:
for group in test.groups:
if group in pause_after_groups \
and test == all_groups[group][-1]:
do_pause(group)
break
else:
def maybe_pause_after(test):
pass
root_groups = {}
all_groups = {}
for test in tests:
for group in test.groups:
all_groups.setdefault(group, []).append(test)
root_groups.setdefault(test.groups[0], []).append(test)
failed_tests = set()
def run_group(group_name, tests):
scope = { "testing": testing,
"logger": logger,
"instance": instance }
if not arguments.local:
scope.update({ "frontend": frontend,
"repository": repository,
"mailbox": mailbox })
try:
for test in tests:
if test.dependencies & failed_tests:
logger.info("Skipping %s (failed dependency)" % test)
continue
maybe_pause_before(test)
if test in dependencies:
logger.info("Running: %s (dependency)" % test)
else:
logger.info("Running: %s" % test)
counters.tests_run += 1
try:
errors_before = counters.errors_logged
execfile(os.path.join("testing/tests", test.filename),
scope.copy())
if errors_before < counters.errors_logged:
raise testing.TestFailure
except testing.TestFailure as failure:
counters.tests_failed += 1
failed_tests.add(test)
if failure.message:
logger.error(failure.message)
if mailbox:
while True:
try:
mail = mailbox.pop(
accept=testing.mailbox.ToRecipient(
"*****@*****.**"),
timeout=1)
except testing.mailbox.MissingMail:
break
else:
logger.error("System message: %s\n %s"
% (mail.header("Subject"),
"\n ".join(mail.lines)))
if arguments.pause_on_failure:
pause()
except testing.NotSupported as not_supported:
failed_tests.add(test)
logger.info("Test not supported: %s" % not_supported.message)
else:
maybe_pause_after(test)
except KeyboardInterrupt:
logger.error("Testing aborted.")
return False
except testing.Error as error:
if error.message:
logger.exception(error.message)
if arguments.pause_on_failure:
pause()
return False
except Exception:
logger.exception("Unexpected exception!")
if arguments.pause_on_failure:
pause()
return False
else:
return True
for group_name in sorted(root_groups.keys()):
if arguments.local:
repository = None
mailbox = None
run_group(group_name, all_groups[group_name])
else:
repository = testing.repository.Repository(
arguments.vbox_host,
arguments.git_daemon_port,
tested_commit,
arguments.vm_hostname)
mailbox = testing.mailbox.Mailbox({ "username": "******",
"password": "******" },
arguments.debug_mails)
with repository:
with mailbox:
if not repository.export():
return
with instance:
instance.mailbox = mailbox
if run_group(group_name, all_groups[group_name]):
instance.finish()
mailbox.check_empty()
def run():
global logger
parser = argparse.ArgumentParser(description="Critic testing framework")
parser.add_argument("--debug", action="store_true",
help="Enable DEBUG level logging")
parser.add_argument("--debug-mails", action="store_true",
help="Log every mail sent by the tested system")
parser.add_argument("--quiet", action="store_true",
help="Disable INFO level logging")
parser.add_argument("--quickstart", action="store_true",
help="Test against a quick-start instance")
parser.add_argument("--coverage", action="store_true",
help="Enable coverage measurement mode")
parser.add_argument("--commit",
help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
parser.add_argument("--upgrade-from",
help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
parser.add_argument("--strict-fs-permissions", action="store_true",
help="Set strict file-system permissions in guest OS")
parser.add_argument("--test-extensions", action="store_true",
help="Test extensions")
parser.add_argument("--local", action="store_true",
help="Run local standalone tests only")
parser.add_argument("--vbox-host", default="host",
help="Host that's running VirtualBox [default=host]")
parser.add_argument("--vm-identifier",
help="VirtualBox instance name or UUID")
parser.add_argument("--vm-hostname",
help="VirtualBox instance hostname [default=VM_IDENTIFIER")
parser.add_argument("--vm-snapshot", default="clean",
help="VirtualBox snapshot (name or UUID) to restore [default=clean]")
parser.add_argument("--vm-ssh-port", type=int, default=22,
help="VirtualBox instance SSH port [default=22]")
parser.add_argument("--vm-http-port", type=int, default=80,
help="VirtualBox instance HTTP port [default=80]")
parser.add_argument("--git-daemon-port", type=int,
help="Port to tell 'git daemon' to bind to")
parser.add_argument("--cache-dir", default="testing/cache",
help="Directory where cache files are stored")
parser.add_argument("--upgrade-after",
help="Upgrade after specified test")
parser.add_argument("--pause-before", action="append",
help="Pause testing before specified test(s)")
parser.add_argument("--pause-after", action="append",
help="Pause testing before specified test(s)")
parser.add_argument("--pause-on-failure", action="store_true",
help="Pause testing after each failed test")
parser.add_argument("--pause-upgrade-loop", action="store_true",
help="Support upgrading the tested system while paused")
parser.add_argument("--pause-upgrade-retry", action="store_true",
help=("Support upgrading the tested system while paused "
"after a failed test, and retrying the failed test"))
parser.add_argument("--pause-upgrade-hook", action="append",
help="Command to run (locally) before upgrading")
parser.add_argument("test", nargs="*",
help="Specific tests to run [default=all]")
arguments = parser.parse_args()
class CountingLogger(object):
def __init__(self, real, counters):
self.real = real
self.counters = counters
def log(self, level, message):
if level == logging.ERROR:
self.counters.errors_logged += 1
elif level == logging.WARNING:
self.counters.warnings_logged += 1
for line in message.splitlines() or [""]:
self.real.log(level, line)
def debug(self, message):
self.log(logging.DEBUG, message)
def info(self, message):
self.log(logging.INFO, message)
def warning(self, message):
self.log(logging.WARNING, message)
def error(self, message):
self.log(logging.ERROR, message)
def exception(self, message):
self.log(logging.ERROR, message + "\n" + traceback.format_exc())
logger = testing.configureLogging(
arguments, wrap=lambda logger: CountingLogger(logger, counters))
logger.info("""\
Critic Testing Framework
========================
""")
key_arguments = [arguments.local,
arguments.quickstart,
arguments.vm_identifier]
if len(filter(None, key_arguments)) != 1:
logger.error("Must specify exactly one of --local, --quickstart and "
"--vm-identifier!")
return
if arguments.local or arguments.quickstart:
incompatible_arguments = []
# This is not a complete list; just those that are most significantly
# incompatible or irrelevant with --local/--quickstart.
if arguments.commit:
incompatible_arguments.append("--commit")
if arguments.upgrade_from:
incompatible_arguments.append("--upgrade-from")
if arguments.coverage:
incompatible_arguments.append("--coverage")
if arguments.test_extensions:
incompatible_arguments.append("--strict-fs-permissions")
if arguments.test_extensions:
incompatible_arguments.append("--test-extensions")
if arguments.vm_identifier:
incompatible_arguments.append("--vm-identifier")
if incompatible_arguments:
logger.error("These arguments can't be combined with "
"--local/--quickstart:\n " +
"\n ".join(incompatible_arguments))
return
import_errors = False
try:
import requests
except ImportError:
logger.error("Failed to import 'requests'!")
import_errors = True
try:
import BeautifulSoup
except ImportError:
logger.error("Failed to import 'BeautifulSoup'!")
import_errors = True
git_version = subprocess.check_output(["git", "--version"]).strip()
m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
if not m:
logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
else:
version_tuple = tuple(map(int, m.groups()))
if version_tuple >= (1, 8, 5):
logger.debug("Using Git version %s on host." % git_version)
else:
logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
import_errors = True
if import_errors:
logger.error("Required software missing; see testing/USAGE.md for details.")
return
if arguments.test_extensions:
# Check that the v8-jsshell submodule is checked out if extension
# testing was requested.
output = subprocess.check_output(["git", "submodule", "status",
"installation/externals/v8-jsshell"])
if output.startswith("-"):
logger.error("""\
The v8-jsshell submodule must be checked for extension testing. Please run
git submodule update --init installation/externals/v8-jsshell
first or run this script without --test-extensions.""")
return
if arguments.vm_identifier:
# Note: we are not ignoring typical temporary editor files such as the
# ".#<name>" files created by Emacs when a buffer has unsaved changes.
# This is because unsaved changes in an editor is probably also
# something you don't want to test with.
locally_modified_paths = []
status_output = subprocess.check_output(
["git", "status", "--porcelain"])
for line in status_output.splitlines():
locally_modified_paths.extend(line[3:].split(" -> "))
tests_modified = []
input_modified = []
other_modified = []
for path in locally_modified_paths:
if path.startswith("testing/input/"):
input_modified.append(path)
elif path.startswith("testing/"):
tests_modified.append(path)
else:
other_modified.append(path)
if input_modified:
logger.error("Test input files locally modified:\n " +
"\n ".join(input_modified))
if other_modified:
logger.error("Critic files locally modified:\n " +
"\n ".join(other_modified))
if input_modified or other_modified:
logger.error("Please commit or stash local modifications before "
"running tests.")
return
if tests_modified:
logger.warning("Running tests using locally modified files:\n " +
"\n ".join(tests_modified))
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit or "HEAD"]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
flags_on = set()
flags_off = set()
try:
if arguments.local:
frontend = None
instance = testing.local.Instance()
else:
frontend = testing.frontend.Frontend(
hostname=arguments.vm_hostname or arguments.vm_identifier,
http_port=arguments.vm_http_port)
if arguments.quickstart:
instance = testing.quickstart.Instance(
frontend=frontend)
else:
instance = testing.virtualbox.Instance(
arguments,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description),
frontend=frontend)
except testing.Error as error:
logger.error(error.message)
return
if not arguments.test_extensions:
flags_off.add("extensions")
flags_on.update(instance.flags_on)
flags_off.update(instance.flags_off)
tests, dependencies = testing.findtests.selectTests(
arguments.test, strict=False, flags_on=flags_on, flags_off=flags_off)
if not tests:
logger.error("No tests selected!")
return
if arguments.upgrade_after:
upgrade_after = testing.findtests.filterPatterns([arguments.upgrade_after])
upgrade_after_tests, _ = testing.findtests.selectTests(upgrade_after,
strict=True)
upgrade_after_tests = set(upgrade_after_tests)
upgrade_after_groups = set(upgrade_after)
def maybe_upgrade_after(test):
def do_upgrade(what):
logger.info("Upgrading after: %s" % what)
instance.upgrade(is_after_test=True)
if test in upgrade_after_tests:
do_upgrade(test)
else:
for group in test.groups:
if group in upgrade_after_groups \
and test == all_groups[group][-1]:
do_upgrade(group)
break
else:
def maybe_upgrade_after(test):
pass
def pause(failed_test=None):
if arguments.pause_upgrade_loop \
or (failed_test and arguments.pause_upgrade_retry):
print "Testing paused."
while True:
if failed_test and arguments.pause_upgrade_retry:
testing.pause("Press ENTER to upgrade (to HEAD) and "
"retry %s, CTRL-c to stop: "
% os.path.basename(failed_test))
else:
testing.pause("Press ENTER to upgrade (to HEAD), "
"CTRL-c to stop: ")
for command in arguments.pause_upgrade_hook:
subprocess.check_call(command, shell=True)
if isinstance(instance, testing.virtualbox.Instance):
repository.push("HEAD")
instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
instance.upgrade_commit = "FETCH_HEAD"
instance.upgrade()
if failed_test and arguments.pause_upgrade_retry:
return "retry"
else:
testing.pause("Testing paused. Press ENTER to continue: ")
if arguments.pause_before:
pause_before = testing.findtests.filterPatterns(arguments.pause_before)
pause_before_tests, _ = testing.findtests.selectTests(pause_before,
strict=True)
pause_before_tests = set(pause_before_tests)
pause_before_groups = set(pause_before)
def maybe_pause_before(test):
def do_pause(what):
logger.info("Pausing before: %s" % what)
pause()
if test in pause_before_tests:
do_pause(test)
else:
for group in test.groups:
if group in pause_before_groups \
and test == all_groups[group][0]:
do_pause(group)
break
else:
def maybe_pause_before(test):
pass
if arguments.pause_after:
pause_after = testing.findtests.filterPatterns(arguments.pause_after)
pause_after_tests, _ = testing.findtests.selectTests(pause_after,
strict=True)
pause_after_tests = set(pause_after_tests)
pause_after_groups = set(pause_after)
def maybe_pause_after(test):
def do_pause(what):
logger.info("Pausing after: %s" % what)
pause()
if test in pause_after_tests:
do_pause(test)
else:
for group in test.groups:
if group in pause_after_groups \
and test == all_groups[group][-1]:
do_pause(group)
break
else:
def maybe_pause_after(test):
pass
root_groups = {}
all_groups = {}
for test in tests:
for group in test.groups:
all_groups.setdefault(group, []).append(test)
root_groups.setdefault(test.groups[0], []).append(test)
failed_tests = set()
def run_test(test, scope):
prefix = "testing/tests"
path = ""
for component in test.filename.split("/")[:-1]:
path = os.path.join(path, component)
init_filename = os.path.join(path, "__init__.py")
if os.path.isfile(os.path.join(prefix, init_filename)):
logger.debug("Including: %s" % init_filename)
execfile(os.path.join(prefix, init_filename), scope)
execfile(os.path.join(prefix, test.filename), scope)
def run_group(group_name, tests):
scope = { "testing": testing,
"logger": logger,
"instance": instance }
if not arguments.local:
scope.update({ "frontend": frontend,
"repository": repository,
"mailbox": mailbox })
try:
for test in tests:
if test.dependencies & failed_tests:
logger.info("Skipping %s (failed dependency)" % test)
continue
maybe_pause_before(test)
if test in dependencies:
logger.info("Running: %s (dependency)" % test)
else:
logger.info("Running: %s" % test)
counters.tests_run += 1
while True:
try:
errors_before = counters.errors_logged
run_test(test, scope.copy())
if mailbox:
mailbox.check_empty()
instance.check_service_logs()
if errors_before < counters.errors_logged:
raise testing.TestFailure
except testing.TestFailure as failure:
counters.tests_failed += 1
failed_tests.add(test)
if failure.message:
logger.error(failure.message)
if mailbox:
try:
while True:
mail = mailbox.pop(
accept=testing.mailbox.ToRecipient(
"*****@*****.**"))
logger.error("System message: %s\n %s"
% (mail.header("Subject"),
"\n ".join(mail.lines)))
except testing.mailbox.MissingMail:
pass
instance.check_service_logs()
if arguments.pause_on_failure \
or arguments.pause_upgrade_retry:
if pause(test.filename) == "retry":
# Re-run test due to --pause-upgrade-retry.
continue
except testing.NotSupported as not_supported:
failed_tests.add(test)
logger.info("Test not supported: %s"
% not_supported.message)
else:
maybe_upgrade_after(test)
maybe_pause_after(test)
break
except KeyboardInterrupt:
raise TestingAborted
except testing.Error as error:
if error.message:
logger.exception(error.message)
if arguments.pause_on_failure:
pause()
return False
except Exception:
logger.exception("Unexpected exception!")
if arguments.pause_on_failure:
pause()
return False
else:
return True
for group_name in sorted(root_groups.keys()):
if arguments.local:
repository = None
mailbox = None
run_group(group_name, all_groups[group_name])
else:
repository = testing.repository.Repository(
"localhost" if arguments.quickstart else arguments.vbox_host,
arguments.git_daemon_port,
tested_commit,
instance)
mailbox = testing.mailbox.Mailbox(instance,
{ "username": "******",
"password": "******" },
arguments.debug_mails)
with repository:
with mailbox:
if not repository.export():
return
with instance:
instance.mailbox = mailbox
testing.utils.instance = instance
testing.utils.frontend = frontend
if run_group(group_name, all_groups[group_name]):
instance.finish()
mailbox.instance = None
mailbox.check_empty()
def main():
parser = argparse.ArgumentParser(
description="Critic testing framework: Quick install utility")
parser.add_argument("--debug",
help="Enable DEBUG level logging",
action="store_true")
parser.add_argument("--quiet",
help="Disable INFO level logging",
action="store_true")
parser.add_argument(
"--commit",
default="HEAD",
help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
parser.add_argument(
"--upgrade-from",
help="Commit (symbolic ref or SHA-1) to install first and upgrade from"
)
parser.add_argument("--vm-identifier",
required=True,
help="VirtualBox instance name or UUID")
parser.add_argument(
"--vm-hostname",
help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
parser.add_argument(
"--vm-snapshot",
default="clean",
help="VirtualBox snapshot (name or UUID) to upgrade [default=clean]")
parser.add_argument("--vm-ssh-port",
type=int,
default=22,
help="VirtualBox instance SSH port [default=22]")
parser.add_argument("--git-daemon-port",
type=int,
help="Port to tell 'git daemon' to bind to")
parser.add_argument("--interactive",
"-i",
action="store_true",
help="Install interactively (without arguments)")
arguments = parser.parse_args()
logger = testing.configureLogging(arguments)
logger.info("Critic testing framework: Quick install")
tested_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.commit]).strip()
if arguments.upgrade_from:
install_commit = subprocess.check_output(
["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
upgrade_commit = tested_commit
else:
install_commit = tested_commit
upgrade_commit = None
install_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", install_commit]).strip()
if upgrade_commit:
upgrade_commit_description = subprocess.check_output(
["git", "log", "--oneline", "-1", upgrade_commit]).strip()
else:
upgrade_commit_description = None
instance = testing.virtualbox.Instance(
arguments,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description))
repository = testing.repository.Repository(arguments.git_daemon_port,
install_commit,
arguments.vm_hostname)
mailbox = testing.mailbox.Mailbox()
with repository, mailbox, instance:
if not repository.export():
return
instance.mailbox = mailbox
instance.start()
if arguments.interactive:
print """
Note: To use the simple SMTP server built into the Critic testing framework,
enter "host" as the SMTP host and "%d" as the SMTP port.
Also note: The administrator user's password will be "testing" (password
input doesn't work over this channel.)""" % mailbox.port
instance.install(repository,
quick=True,
interactive=arguments.interactive)
instance.upgrade(interactive=arguments.interactive)
testing.pause("Press ENTER to stop VM: ")
try:
while True:
mail = mailbox.pop()
logger.info("Mail to <%s>:\n%s" % (mail.recipient, mail))
except testing.mailbox.MissingMail:
pass