示例#1
0
文件: upgrade.py 项目: ryfow/critic
def main():
    parser = argparse.ArgumentParser(
        description="Critic testing framework: instance upgrade utility")
    parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
    parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
    parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
    parser.add_argument("--vm-hostname", help="VirtualBox instance hostname", required=True)
    parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to upgrade", default="clean")
    parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
    parser.add_argument("--pause-before-upgrade", help="Pause before upgrading", action="store_true")
    parser.add_argument("--pause-after-upgrade", help="Pause after upgrading", action="store_true")

    arguments = parser.parse_args()

    logger = testing.configureLogging(arguments)
    logger.info("Critic Testing Framework: Instance Upgrade")

    instance = testing.virtualbox.Instance(
        identifier=arguments.vm_identifier,
        snapshot=arguments.vm_snapshot,
        hostname=arguments.vm_hostname,
        ssh_port=arguments.vm_ssh_port)

    with instance:
        instance.start()

        logger.debug("Upgrading guest OS ...")

        update_output = instance.execute(
            ["sudo", "DEBIAN_FRONTEND=noninteractive",
             "apt-get", "-q", "-y", "update"])

        logger.debug("Output from 'apt-get -q -y update':\n" + update_output)

        upgrade_output = instance.execute(
            ["sudo", "DEBIAN_FRONTEND=noninteractive",
             "apt-get", "-q", "-y", "upgrade"])

        logger.debug("Output from 'apt-get -q -y upgrade':\n" + upgrade_output)

        if "The following packages will be upgraded:" in upgrade_output.splitlines():
            logger.info("Upgraded guest OS")
            logger.debug("Retaking snapshot ...")

            instance.retake_snapshot(arguments.vm_snapshot)

            logger.info("Snapshot '%s' upgraded!" % arguments.vm_snapshot)
        else:
            logger.info("No packages upgraded in guest OS")
示例#2
0
def main():
    parser = argparse.ArgumentParser(description="Critic testing framework")
    parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
    parser.add_argument("--debug-mails", help="Log every mail sent by the tested system", action="store_true")
    parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
    parser.add_argument("--coverage", help="Enable coverage measurement mode", action="store_true")
    parser.add_argument("--commit", help="Commit (symbolic ref or SHA-1) to test [default=HEAD]", default="HEAD")
    parser.add_argument("--upgrade-from", help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
    parser.add_argument("--strict-fs-permissions", help="Set strict file-system permissions in guest OS", action="store_true")
    parser.add_argument("--vbox-host", help="Host that's running VirtualBox [default=host]", default="host")
    parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
    parser.add_argument("--vm-hostname", help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
    parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to restore [default=clean]", default="clean")
    parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
    parser.add_argument("--vm-http-port", help="VirtualBox instance HTTP port [default=80]", type=int, default=80)
    parser.add_argument("--git-daemon-port", help="Port to tell 'git daemon' to bind to", type=int)
    parser.add_argument("--pause-before", help="Pause testing before specified test(s)", action="append")
    parser.add_argument("--pause-after", help="Pause testing before specified test(s)", action="append")
    parser.add_argument("--pause-on-failure", help="Pause testing after each failed test", action="store_true")
    parser.add_argument("--pause-upgrade-loop", help="Support upgrading the tested system while paused", action="store_true")
    parser.add_argument("--pause-upgrade-hook", help="Command to run (locally) before upgrading", action="append")
    parser.add_argument("test", help="Specific tests to run [default=all]", nargs="*")

    arguments = parser.parse_args()

    class Counters:
        def __init__(self):
            self.tests_run = 0
            self.tests_failed = 0
            self.errors_logged = 0
            self.warnings_logged = 0

    counters = Counters()

    class CountingLogger(object):
        def __init__(self, real, counters):
            self.real = real
            self.counters = counters

        def log(self, level, message):
            if level == logging.ERROR:
                self.counters.errors_logged += 1
            elif level == logging.WARNING:
                self.counters.warnings_logged += 1
            for line in message.splitlines() or [""]:
                self.real.log(level, line)
        def debug(self, message):
            self.log(logging.DEBUG, message)
        def info(self, message):
            self.log(logging.INFO, message)
        def warning(self, message):
            self.log(logging.WARNING, message)
        def error(self, message):
            self.log(logging.ERROR, message)
        def exception(self, message):
            self.log(logging.ERROR, message + "\n" + traceback.format_exc())

    logger = testing.configureLogging(
        arguments, wrap=lambda logger: CountingLogger(logger, counters))
    logger.info("Critic Testing Framework")

    import_errors = False

    try:
        import requests
    except ImportError:
        logger.error("Failed to import 'requests'!")
        import_errors = True

    try:
        import BeautifulSoup
    except ImportError:
        logger.error("Failed to import 'BeautifulSoup'!")
        import_errors = True

    git_version = subprocess.check_output(["git", "--version"]).strip()
    m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
    if not m:
        logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
    else:
        version_tuple = tuple(map(int, m.groups()))
        if version_tuple >= (1, 8, 5):
            logger.debug("Using Git version %s on host." % git_version)
        else:
            logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
            logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
            import_errors = True

    if import_errors:
        logger.error("Required software missing; see testing/USAGE.md for details.")
        return

    # Note: we are not ignoring typical temporary editor files such as the
    # ".#<name>" files created by Emacs when a buffer has unsaved changes.  This
    # is because unsaved changes in an editor is probably also something you
    # don't want to test with.

    locally_modified_paths = []

    status_output = subprocess.check_output(
        ["git", "status", "--porcelain"])

    for line in status_output.splitlines():
        locally_modified_paths.extend(line[3:].split(" -> "))

    tests_modified = []
    input_modified = []
    other_modified = []

    for path in locally_modified_paths:
        if path.startswith("testing/input/"):
            input_modified.append(path)
        elif path.startswith("testing/"):
            tests_modified.append(path)
        else:
            other_modified.append(path)

    if input_modified:
        logger.error("Test input files locally modified:\n  " + "\n  ".join(input_modified))
    if other_modified:
        logger.error("Critic files locally modified:\n  " + "\n  ".join(other_modified))
    if input_modified or other_modified:
        logger.error("Please commit or stash local modifications before running tests.")
        return

    if tests_modified:
        logger.warning("Running tests using locally modified files:\n  " + "\n  ".join(tests_modified))

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    try:
        frontend = testing.frontend.Frontend(
            hostname=arguments.vm_hostname or arguments.vm_identifier,
            http_port=arguments.vm_http_port)

        instance = testing.virtualbox.Instance(
            vboxhost=arguments.vbox_host,
            identifier=arguments.vm_identifier,
            snapshot=arguments.vm_snapshot,
            hostname=arguments.vm_hostname,
            ssh_port=arguments.vm_ssh_port,
            install_commit=(install_commit, install_commit_description),
            upgrade_commit=(upgrade_commit, upgrade_commit_description),
            frontend=frontend,
            strict_fs_permissions=arguments.strict_fs_permissions,
            coverage=arguments.coverage)
    except testing.Error as error:
        logger.error(error.message)
        return

    if arguments.test:
        tests = set(test.strip("/") for test in arguments.test)
        groups = {}

        for test in sorted(tests):
            components = test.split("/")
            if test.endswith(".py"):
                del components[-1]
            else:
                test += "/"
            for index in range(len(components)):
                groups.setdefault("/".join(components[:index + 1]), set()).add(test)

        def directory_enabled(path):
            if path in groups:
                # Directory (or sub-directory of or file in it) was directly named.
                return True
            # Check if an ancestor directory was directly name:
            directory = path.rpartition("/")[0]
            while directory:
                if directory in groups and directory + "/" in groups[directory]:
                    return True
                directory = directory.rpartition("/")[0]
            return False

        def file_enabled(path):
            if path in tests:
                # File was directly named.
                return True
            directory, _, name = path.rpartition("/")
            if directory in groups:
                # Loop over all enabled items in the same directory as 'path':
                for item in groups[directory]:
                    if item != directory:
                        local, slash, _ = item[len(directory + "/"):].partition("/")
                        if local > name and (slash or "/" not in directory):
                            # A file in a later sub-directory is enabled; this
                            # means 'path' is a dependency of the other file.
                            return "dependency"
            # Check if the file's directory or an ancestor directory thereof was
            # directly name:
            while directory:
                if directory in groups and directory + "/" in groups[directory]:
                    return True
                directory = directory.rpartition("/")[0]
            return False
    else:
        def directory_enabled(path):
            return True
        def file_enabled(path):
            return True

    def pause():
        if arguments.pause_upgrade_loop:
            print "Testing paused."

            while True:
                testing.pause("Press ENTER to upgrade (to HEAD), CTRL-c to stop: ")

                for command in arguments.pause_upgrade_hook:
                    subprocess.check_call(command, shell=True)

                repository.push("HEAD")

                instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
                instance.upgrade_commit = "FETCH_HEAD"
                instance.upgrade()
        else:
            testing.pause("Testing paused.  Press ENTER to continue: ")

    pause_before = set(arguments.pause_before or [])
    pause_after = set(arguments.pause_after or [])

    def run_group(group_name):
        try:
            instance.start()
            instance.mailbox = mailbox

            def run_tests(directory):
                has_failed = False

                if directory in pause_before:
                    pause()

                for name in sorted(os.listdir(os.path.join("testing/tests", directory))):
                    if not re.match("\d{3}-", name):
                        continue

                    path = os.path.join(directory, name)

                    if os.path.isdir(os.path.join("testing/tests", path)):
                        if not directory_enabled(path):
                            logger.debug("Skipping: %s/" % path)
                        elif has_failed:
                            logger.info("Skipping: %s/ (failed dependency)" % path)
                        else:
                            run_tests(path)
                        continue
                    elif not re.search("\\.py$", name):
                        continue

                    enabled = file_enabled(path)
                    if not enabled:
                        logger.debug("Skipping: %s" % path)
                        continue

                    if path in pause_before:
                        pause()
                    if enabled is True:
                        mode = ""
                    else:
                        mode = " (%s)" % enabled
                    logger.info("Running: %s%s" % (path, mode))
                    counters.tests_run += 1
                    try:
                        execfile(os.path.join("testing/tests", path),
                                 { "testing": testing,
                                   "logger": logger,
                                   "instance": instance,
                                   "frontend": frontend,
                                   "repository": repository,
                                   "mailbox": mailbox })
                    except testing.TestFailure as failure:
                        counters.tests_failed += 1
                        if failure.message:
                            logger.error(failure.message)
                        while True:
                            mail = mailbox.pop(
                                accept=testing.mailbox.to_recipient("*****@*****.**"),
                                timeout=1)
                            if not mail:
                                break
                            logger.error("Administrator message: %s\n  %s"
                                         % (mail.header("Subject"),
                                            "\n  ".join(mail.lines)))
                        if arguments.pause_on_failure:
                            pause()
                        if "/" in directory:
                            has_failed = True
                        else:
                            return
                    else:
                        if path in pause_after:
                            pause()

                if directory in pause_after:
                    pause()

            run_tests(group_name)
        except KeyboardInterrupt:
            logger.error("Testing aborted.")
            return False
        except testing.Error as error:
            if error.message:
                logger.exception(error.message)
            if arguments.pause_on_failure:
                pause()
            return False
        except Exception:
            logger.exception("Unexpected exception!")
            if arguments.pause_on_failure:
                pause()
            return False
        else:
            return True

    for group_name in sorted(os.listdir("testing/tests")):
        if not re.match("\d{3}-", group_name):
            continue

        if not directory_enabled(group_name):
            logger.debug("Skipping: %s/" % group_name)
            continue

        repository = testing.repository.Repository(
            arguments.vbox_host,
            arguments.git_daemon_port,
            tested_commit,
            arguments.vm_hostname)
        mailbox = testing.mailbox.Mailbox({ "username": "******",
                                            "password": "******" },
                                          arguments.debug_mails)

        with repository:
            with mailbox:
                if not repository.export():
                    return

                with instance:
                    if run_group(group_name):
                        instance.finish()

                mailbox.check_empty()

    logger.info("""
Test summary
============
Tests run:       %3d
Tests failed:    %3d
Errors logged:   %3d
Warnings logged: %3d
""" % (counters.tests_run,
       counters.tests_failed,
       counters.errors_logged,
       counters.warnings_logged))
示例#3
0
文件: install.py 项目: Aessy/critic
def main():
    parser = argparse.ArgumentParser(
        description="Critic testing framework: Quick install utility")

    parser.add_argument("--debug",
                        help="Enable DEBUG level logging", action="store_true")
    parser.add_argument("--quiet",
                        help="Disable INFO level logging", action="store_true")

    parser.add_argument("--commit", default="HEAD",
                        help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
    parser.add_argument("--upgrade-from",
                        help="Commit (symbolic ref or SHA-1) to install first and upgrade from")

    parser.add_argument("--vm-identifier", required=True,
                        help="VirtualBox instance name or UUID")
    parser.add_argument("--vm-hostname",
                        help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
    parser.add_argument("--vm-snapshot", default="clean",
                        help="VirtualBox snapshot (name or UUID) to upgrade [default=clean]")
    parser.add_argument("--vm-ssh-port", type=int, default=22,
                        help="VirtualBox instance SSH port [default=22]")
    parser.add_argument("--git-daemon-port", type=int,
                        help="Port to tell 'git daemon' to bind to")

    parser.add_argument("--interactive", "-i", action="store_true",
                        help="Install interactively (without arguments)")

    arguments = parser.parse_args()

    logger = testing.configureLogging(arguments)
    logger.info("Critic testing framework: Quick install")

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    instance = testing.virtualbox.Instance(
        arguments,
        install_commit=(install_commit, install_commit_description),
        upgrade_commit=(upgrade_commit, upgrade_commit_description))

    repository = testing.repository.Repository(
        arguments.git_daemon_port,
        install_commit,
        arguments.vm_hostname)

    mailbox = testing.mailbox.Mailbox()

    with repository, mailbox, instance:
        if not repository.export():
            return

        instance.mailbox = mailbox
        instance.start()

        if arguments.interactive:
            print """
Note: To use the simple SMTP server built into the Critic testing framework,
      enter "host" as the SMTP host and "%d" as the SMTP port.

Also note: The administrator user's password will be "testing" (password
           input doesn't work over this channel.)""" % mailbox.port

        instance.install(repository, quick=True,
                         interactive=arguments.interactive)
        instance.upgrade(interactive=arguments.interactive)

        testing.pause("Press ENTER to stop VM: ")

        try:
            while True:
                mail = mailbox.pop()
                logger.info("Mail to <%s>:\n%s" % (mail.recipient, mail))
        except testing.mailbox.MissingMail:
            pass
示例#4
0
def main():
    parser = argparse.ArgumentParser(
        description="Critic testing framework: instance upgrade utility")
    parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
    parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
    parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
    parser.add_argument("--vm-hostname", help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
    parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to upgrade", default="clean")
    parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
    parser.add_argument("--pause-before-upgrade", help="Pause before upgrading", action="store_true")
    parser.add_argument("--pause-after-upgrade", help="Pause after upgrading", action="store_true")
    parser.add_argument("--no-upgrade", action="store_true", help="Do not upgrade installed packages")
    parser.add_argument("--install", action="append", help="Install named package")
    parser.add_argument("--custom", action="store_true",
                        help="Stop for custom maintenance, and always retake snapshot")
    parser.add_argument("--reboot", action="store_true",
                        help="Reboot VM before retaking snapshot")

    arguments = parser.parse_args()

    logger = testing.configureLogging(arguments)
    logger.info("Critic Testing Framework: Instance Upgrade")

    instance = testing.virtualbox.Instance(
        identifier=arguments.vm_identifier,
        snapshot=arguments.vm_snapshot,
        hostname=arguments.vm_hostname,
        ssh_port=arguments.vm_ssh_port)

    with instance:
        instance.start()

        if not arguments.no_upgrade:
            logger.debug("Upgrading guest OS ...")

            update_output = instance.execute(
                ["sudo", "DEBIAN_FRONTEND=noninteractive",
                 "apt-get", "-q", "-y", "update"])

            logger.debug("Output from 'apt-get -q -y update':\n" + update_output)

            upgrade_output = instance.execute(
                ["sudo", "DEBIAN_FRONTEND=noninteractive",
                 "apt-get", "-q", "-y", "upgrade"])

            logger.debug("Output from 'apt-get -q -y upgrade':\n" + upgrade_output)

            retake_snapshot = False

            if "The following packages will be upgraded:" in upgrade_output.splitlines():
                retake_snapshot = True

        if arguments.install:
            install_output = instance.execute(
                ["sudo", "DEBIAN_FRONTEND=noninteractive",
                 "apt-get", "-q", "-y", "install"] + arguments.install)

            logger.debug("Output from 'apt-get -q -y install':\n" +
                         install_output)

            retake_snapshot = True

        if arguments.custom:
            testing.pause()
            retake_snapshot = True

        if retake_snapshot:
            if arguments.reboot:
                instance.execute(["sudo", "reboot"])

                logger.debug("Sleeping 10 seconds ...")
                time.sleep(10)

                instance.wait()

                logger.debug("Sleeping 10 seconds ...")
                time.sleep(10)

                logger.info("Rebooted VM: %s" % arguments.vm_identifier)

            logger.info("Upgraded guest OS")
            logger.debug("Retaking snapshot ...")

            instance.retake_snapshot(arguments.vm_snapshot)

            logger.info("Snapshot '%s' upgraded!" % arguments.vm_snapshot)
        else:
            logger.info("No packages upgraded in guest OS")
示例#5
0
文件: main.py 项目: ryfow/critic
def main():
    parser = argparse.ArgumentParser(description="Critic testing framework")
    parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
    parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
    parser.add_argument("--commit", help="Commit (symbolic ref or SHA-1) to test [default=HEAD]", default="HEAD")
    parser.add_argument("--upgrade-from", help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
    parser.add_argument("--strict-fs-permissions", help="Set strict file-system permissions in guest OS", action="store_true")
    parser.add_argument("--vbox-host", help="Host that's running VirtualBox [default=host]", default="host")
    parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
    parser.add_argument("--vm-hostname", help="VirtualBox instance hostname", required=True)
    parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to restore", default="clean")
    parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
    parser.add_argument("--vm-http-port", help="VirtualBox instance HTTP port [default=80]", type=int, default=80)
    parser.add_argument("--git-daemon-port", help="Port to tell 'git daemon' to bind to", type=int)
    parser.add_argument("--pause-before", help="Pause testing before specified test(s)", action="append")
    parser.add_argument("--pause-after", help="Pause testing before specified test(s)", action="append")
    parser.add_argument("--pause-on-failure", help="Pause testing after each failed test", action="store_true")
    parser.add_argument("test", help="Specific tests to run [default=all]", nargs="*")

    arguments = parser.parse_args()

    logger = testing.configureLogging(arguments)
    logger.info("Critic Testing Framework")

    import_errors = False

    try:
        import requests
    except ImportError:
        logger.error("Failed to import 'requests'!")
        import_errors = True

    try:
        import BeautifulSoup
    except ImportError:
        logger.error("Failed to import 'BeautifulSoup'!")
        import_errors = True

    if import_errors:
        logger.error("Required software missing; see testing/USAGE.md for details.")
        return

    locally_modified_paths = subprocess.check_output(
        ["git", "diff", "--name-only"])

    tests_modified = []
    input_modified = []
    other_modified = []

    for path in locally_modified_paths.splitlines():
        if path.startswith("testing/input/"):
            input_modified.append(path)
        elif path.startswith("testing/"):
            tests_modified.append(path)
        else:
            other_modified.append(path)

    if input_modified:
        logger.error("Test input files locally modified:\n  " + "\n  ".join(input_modified))
    if other_modified:
        logger.error("Critic files locally modified:\n  " + "\n  ".join(other_modified))
    if input_modified or other_modified:
        logger.error("Please commit or stash local modifications before running tests.")
        return

    if tests_modified:
        logger.warning("Running tests using locally modified files:\n  " + "\n  ".join(tests_modified))

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    try:
        frontend = testing.frontend.Frontend(
            hostname=arguments.vm_hostname,
            http_port=arguments.vm_http_port)

        instance = testing.virtualbox.Instance(
            vboxhost=arguments.vbox_host,
            identifier=arguments.vm_identifier,
            snapshot=arguments.vm_snapshot,
            hostname=arguments.vm_hostname,
            ssh_port=arguments.vm_ssh_port,
            install_commit=(install_commit, install_commit_description),
            upgrade_commit=(upgrade_commit, upgrade_commit_description),
            frontend=frontend,
            strict_fs_permissions=arguments.strict_fs_permissions)
    except testing.Error as error:
        logger.error(error.message)
        return

    if arguments.test:
        tests = set(test.strip("/") for test in arguments.test)
        groups = {}

        for test in sorted(tests):
            components = test.split("/")
            if test.endswith(".py"):
                del components[-1]
            else:
                test += "/"
            for index in range(len(components)):
                groups.setdefault("/".join(components[:index + 1]), set()).add(test)

        def directory_enabled(path):
            if path in groups:
                # Directory (or sub-directory of or file in it) was directly named.
                return True
            # Check if an ancestor directory was directly name:
            directory = path.rpartition("/")[0]
            while directory:
                if directory in groups and directory + "/" in groups[directory]:
                    return True
                directory = directory.rpartition("/")[0]
            return False

        def file_enabled(path):
            if path in tests:
                # File was directly named.
                return True
            directory, _, name = path.rpartition("/")
            if directory in groups:
                # Loop over all enabled items in the same directory as 'path':
                for item in groups[directory]:
                    if item != directory:
                        local, slash, _ = item[len(directory + "/"):].partition("/")
                        if local > name and (slash or "/" not in directory):
                            # A file in a later sub-directory is enabled; this
                            # means 'path' is a dependency of the other file.
                            return "dependency"
            # Check if the file's directory or an ancestor directory thereof was
            # directly name:
            while directory:
                if directory in groups and directory + "/" in groups[directory]:
                    return True
                directory = directory.rpartition("/")[0]
            return False
    else:
        def directory_enabled(path):
            return True
        def file_enabled(path):
            return True

    def pause():
        print
        try:
            raw_input("Testing paused.  Press ENTER to continue: ")
        except KeyboardInterrupt:
            print
            print
            raise
        print

    pause_before = set(arguments.pause_before or [])
    pause_after = set(arguments.pause_after or [])

    def run_group(group_name):
        try:
            instance.start()
            instance.mailbox = mailbox

            def run_tests(directory):
                has_failed = False

                for name in sorted(os.listdir(os.path.join("testing/tests", directory))):
                    if not re.match("\d{3}-", name):
                        continue

                    path = os.path.join(directory, name)

                    if os.path.isdir(os.path.join("testing/tests", path)):
                        if not directory_enabled(path):
                            logger.debug("Skipping: %s/" % path)
                        elif has_failed:
                            logger.info("Skipping: %s/ (failed dependency)" % path)
                        else:
                            run_tests(path)
                        continue
                    elif not re.search("\\.py$", name):
                        continue

                    enabled = file_enabled(path)
                    if not enabled:
                        logger.debug("Skipping: %s" % path)
                        continue

                    if path in pause_before:
                        pause()
                    if enabled is True:
                        mode = ""
                    else:
                        mode = " (%s)" % enabled
                    logger.info("Running: %s%s" % (path, mode))
                    try:
                        execfile(os.path.join("testing/tests", path),
                                 { "testing": testing,
                                   "logger": logger,
                                   "instance": instance,
                                   "frontend": frontend,
                                   "repository": repository,
                                   "mailbox": mailbox })
                    except testing.TestFailure as failure:
                        if failure.message:
                            logger.error(failure.message)
                        while True:
                            mail = mailbox.pop(
                                accept=testing.mailbox.to_recipient("*****@*****.**"),
                                timeout=1)
                            if not mail:
                                break
                            logger.error("Administrator message: %s\n  %s"
                                         % (mail.header("Subject"),
                                            "\n  ".join(mail.lines)))
                        if arguments.pause_on_failure:
                            pause()
                        if "/" in directory:
                            has_failed = True
                        else:
                            return
                    else:
                        if path in pause_after:
                            pause()

            run_tests(group_name)
        except KeyboardInterrupt:
            logger.error("Testing aborted.")
        except testing.Error as error:
            if error.message:
                logger.exception(error.message)
            if arguments.pause_on_failure:
                pause()
        except Exception:
            logger.exception("Unexpected exception!")
            if arguments.pause_on_failure:
                pause()

    for group_name in sorted(os.listdir("testing/tests")):
        if not re.match("\d{3}-", group_name):
            continue

        if not directory_enabled(group_name):
            logger.debug("Skipping: %s/" % group_name)
            continue

        repository = testing.repository.Repository(
            arguments.vbox_host,
            arguments.git_daemon_port,
            tested_commit,
            arguments.vm_hostname)
        mailbox = testing.mailbox.Mailbox()

        with repository:
            with mailbox:
                if not repository.export():
                    return

                with instance:
                    run_group(group_name)

                mailbox.check_empty()
示例#6
0
文件: main.py 项目: dbratell/critic
def run():
    global logger

    parser = argparse.ArgumentParser(description="Critic testing framework")

    parser.add_argument("--debug", action="store_true",
                        help="Enable DEBUG level logging")
    parser.add_argument("--debug-mails", action="store_true",
                        help="Log every mail sent by the tested system")
    parser.add_argument("--quiet", action="store_true",
                        help="Disable INFO level logging")

    parser.add_argument("--coverage", action="store_true",
                        help="Enable coverage measurement mode")
    parser.add_argument("--commit",
                        help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
    parser.add_argument("--upgrade-from",
                        help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
    parser.add_argument("--strict-fs-permissions", action="store_true",
                        help="Set strict file-system permissions in guest OS")
    parser.add_argument("--test-extensions", action="store_true",
                        help="Test extensions")

    parser.add_argument("--local", action="store_true",
                        help="Run local standalone tests only")

    parser.add_argument("--vbox-host", default="host",
                        help="Host that's running VirtualBox [default=host]")
    parser.add_argument("--vm-identifier",
                        help="VirtualBox instance name or UUID")
    parser.add_argument("--vm-hostname",
                        help="VirtualBox instance hostname [default=VM_IDENTIFIER")
    parser.add_argument("--vm-snapshot", default="clean",
                        help="VirtualBox snapshot (name or UUID) to restore [default=clean]")
    parser.add_argument("--vm-ssh-port", type=int, default=22,
                        help="VirtualBox instance SSH port [default=22]")
    parser.add_argument("--vm-http-port", type=int, default=80,
                        help="VirtualBox instance HTTP port [default=80]")

    parser.add_argument("--git-daemon-port", type=int,
                        help="Port to tell 'git daemon' to bind to")
    parser.add_argument("--cache-dir", default="testing/cache",
                        help="Directory where cache files are stored")

    parser.add_argument("--pause-before", action="append",
                        help="Pause testing before specified test(s)")
    parser.add_argument("--pause-after", action="append",
                        help="Pause testing before specified test(s)")
    parser.add_argument("--pause-on-failure", action="store_true",
                        help="Pause testing after each failed test")
    parser.add_argument("--pause-upgrade-loop", action="store_true",
                        help="Support upgrading the tested system while paused")
    parser.add_argument("--pause-upgrade-hook", action="append",
                        help="Command to run (locally) before upgrading")

    parser.add_argument("test", nargs="*",
                        help="Specific tests to run [default=all]")

    arguments = parser.parse_args()

    class CountingLogger(object):
        def __init__(self, real, counters):
            self.real = real
            self.counters = counters

        def log(self, level, message):
            if level == logging.ERROR:
                self.counters.errors_logged += 1
            elif level == logging.WARNING:
                self.counters.warnings_logged += 1
            for line in message.splitlines() or [""]:
                self.real.log(level, line)
        def debug(self, message):
            self.log(logging.DEBUG, message)
        def info(self, message):
            self.log(logging.INFO, message)
        def warning(self, message):
            self.log(logging.WARNING, message)
        def error(self, message):
            self.log(logging.ERROR, message)
        def exception(self, message):
            self.log(logging.ERROR, message + "\n" + traceback.format_exc())

    logger = testing.configureLogging(
        arguments, wrap=lambda logger: CountingLogger(logger, counters))
    logger.info("""\
Critic Testing Framework
========================

""")

    if not arguments.local and not arguments.vm_identifier:
        logger.error("Must specify one of --local and --vm-identifier!")
        return

    if arguments.local:
        incompatible_arguments = []

        # This is not a complete list; just those that are most significantly
        # incompatible or irrelevant with --local.
        if arguments.commit:
            incompatible_arguments.append("--commit")
        if arguments.upgrade_from:
            incompatible_arguments.append("--upgrade-from")
        if arguments.coverage:
            incompatible_arguments.append("--coverage")
        if arguments.test_extensions:
            incompatible_arguments.append("--strict-fs-permissions")
        if arguments.test_extensions:
            incompatible_arguments.append("--test-extensions")
        if arguments.vm_identifier:
            incompatible_arguments.append("--vm-identifier")

        if incompatible_arguments:
            logger.error("These arguments can't be combined with --local:\n  " +
                         "\n  ".join(incompatible_arguments))
            return

    import_errors = False

    try:
        import requests
    except ImportError:
        logger.error("Failed to import 'requests'!")
        import_errors = True

    try:
        import BeautifulSoup
    except ImportError:
        logger.error("Failed to import 'BeautifulSoup'!")
        import_errors = True

    git_version = subprocess.check_output(["git", "--version"]).strip()
    m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
    if not m:
        logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
    else:
        version_tuple = tuple(map(int, m.groups()))
        if version_tuple >= (1, 8, 5):
            logger.debug("Using Git version %s on host." % git_version)
        else:
            logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
            logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
            import_errors = True

    if import_errors:
        logger.error("Required software missing; see testing/USAGE.md for details.")
        return

    if arguments.test_extensions:
        # Check that the v8-jsshell submodule is checked out if extension
        # testing was requested.
        output = subprocess.check_output(["git", "submodule", "status",
                                          "installation/externals/v8-jsshell"])
        if output.startswith("-"):
            logger.error("""\
The v8-jsshell submodule must be checked for extension testing.  Please run
  git submodule update --init installation/externals/v8-jsshell
first or run this script without --test-extensions.""")
            return

    if not arguments.local:
        # Note: we are not ignoring typical temporary editor files such as the
        # ".#<name>" files created by Emacs when a buffer has unsaved changes.
        # This is because unsaved changes in an editor is probably also
        # something you don't want to test with.

        locally_modified_paths = []

        status_output = subprocess.check_output(
            ["git", "status", "--porcelain"])

        for line in status_output.splitlines():
            locally_modified_paths.extend(line[3:].split(" -> "))

        tests_modified = []
        input_modified = []
        other_modified = []

        for path in locally_modified_paths:
            if path.startswith("testing/input/"):
                input_modified.append(path)
            elif path.startswith("testing/"):
                tests_modified.append(path)
            else:
                other_modified.append(path)

        if input_modified:
            logger.error("Test input files locally modified:\n  " +
                         "\n  ".join(input_modified))
        if other_modified:
            logger.error("Critic files locally modified:\n  " +
                         "\n  ".join(other_modified))
        if input_modified or other_modified:
            logger.error("Please commit or stash local modifications before "
                         "running tests.")
            return

        if tests_modified:
            logger.warning("Running tests using locally modified files:\n  " +
                           "\n  ".join(tests_modified))

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit or "HEAD"]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    flags_on = set()
    flags_off = set()

    try:
        if arguments.local:
            frontend = None
            instance = testing.local.Instance()
            flags_on.add("local")
        else:
            frontend = testing.frontend.Frontend(
                hostname=arguments.vm_hostname or arguments.vm_identifier,
                http_port=arguments.vm_http_port)

            instance = testing.virtualbox.Instance(
                arguments,
                install_commit=(install_commit, install_commit_description),
                upgrade_commit=(upgrade_commit, upgrade_commit_description),
                frontend=frontend)
    except testing.Error as error:
        logger.error(error.message)
        return

    tests, dependencies = testing.findtests.selectTests(
        arguments.test, strict=False, flags_on=flags_on, flags_off=flags_off)

    if not tests:
        logger.error("No tests selected!")
        return

    def pause():
        if arguments.pause_upgrade_loop:
            print "Testing paused."

            while True:
                testing.pause("Press ENTER to upgrade (to HEAD), CTRL-c to stop: ")

                for command in arguments.pause_upgrade_hook:
                    subprocess.check_call(command, shell=True)

                repository.push("HEAD")

                instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
                instance.upgrade_commit = "FETCH_HEAD"
                instance.upgrade()
        else:
            testing.pause("Testing paused.  Press ENTER to continue: ")

    pause_before = pause_after = set()

    if arguments.pause_before:
        pause_before = testing.findtests.filterPatterns(arguments.pause_before)
        pause_before_tests, _ = testing.findtests.selectTests(pause_before,
                                                              strict=True)
        pause_before_tests = set(pause_before_tests)
        pause_before_groups = set(pause_before)

        def maybe_pause_before(test):
            def do_pause(what):
                logger.info("Pausing before: %s" % what)
                pause()

            if test in pause_before_tests:
                do_pause(test)
            else:
                for group in test.groups:
                    if group in pause_before_groups \
                            and test == all_groups[group][0]:
                        do_pause(group)
                        break
    else:
        def maybe_pause_before(test):
            pass

    if arguments.pause_after:
        pause_after = testing.findtests.filterPatterns(arguments.pause_after)
        pause_after_tests, _ = testing.findtests.selectTests(pause_after,
                                                             strict=True)
        pause_after_tests = set(pause_after_tests)
        pause_after_groups = set(pause_after)

        def maybe_pause_after(test):
            def do_pause(what):
                logger.info("Pausing after: %s" % what)
                pause()

            if test in pause_after_tests:
                do_pause(test)
            else:
                for group in test.groups:
                    if group in pause_after_groups \
                            and test == all_groups[group][-1]:
                        do_pause(group)
                        break
    else:
        def maybe_pause_after(test):
            pass

    root_groups = {}
    all_groups = {}

    for test in tests:
        for group in test.groups:
            all_groups.setdefault(group, []).append(test)
        root_groups.setdefault(test.groups[0], []).append(test)

    failed_tests = set()

    def run_group(group_name, tests):
        scope = { "testing": testing,
                  "logger": logger,
                  "instance": instance }

        if not arguments.local:
            scope.update({ "frontend": frontend,
                           "repository": repository,
                           "mailbox": mailbox })

        try:
            for test in tests:
                if test.dependencies & failed_tests:
                    logger.info("Skipping %s (failed dependency)" % test)
                    continue

                maybe_pause_before(test)

                if test in dependencies:
                    logger.info("Running: %s (dependency)" % test)
                else:
                    logger.info("Running: %s" % test)

                counters.tests_run += 1

                try:
                    errors_before = counters.errors_logged
                    execfile(os.path.join("testing/tests", test.filename),
                             scope.copy())
                    if errors_before < counters.errors_logged:
                        raise testing.TestFailure
                except testing.TestFailure as failure:
                    counters.tests_failed += 1

                    failed_tests.add(test)

                    if failure.message:
                        logger.error(failure.message)

                    if mailbox:
                        while True:
                            try:
                                mail = mailbox.pop(
                                    accept=testing.mailbox.ToRecipient(
                                        "*****@*****.**"),
                                    timeout=1)
                            except testing.mailbox.MissingMail:
                                break
                            else:
                                logger.error("System message: %s\n  %s"
                                             % (mail.header("Subject"),
                                                "\n  ".join(mail.lines)))

                    if arguments.pause_on_failure:
                        pause()
                except testing.NotSupported as not_supported:
                    failed_tests.add(test)
                    logger.info("Test not supported: %s" % not_supported.message)
                else:
                    maybe_pause_after(test)
        except KeyboardInterrupt:
            logger.error("Testing aborted.")
            return False
        except testing.Error as error:
            if error.message:
                logger.exception(error.message)
            if arguments.pause_on_failure:
                pause()
            return False
        except Exception:
            logger.exception("Unexpected exception!")
            if arguments.pause_on_failure:
                pause()
            return False
        else:
            return True

    for group_name in sorted(root_groups.keys()):
        if arguments.local:
            repository = None
            mailbox = None

            run_group(group_name, all_groups[group_name])
        else:
            repository = testing.repository.Repository(
                arguments.vbox_host,
                arguments.git_daemon_port,
                tested_commit,
                arguments.vm_hostname)
            mailbox = testing.mailbox.Mailbox({ "username": "******",
                                                "password": "******" },
                                              arguments.debug_mails)

            with repository:
                with mailbox:
                    if not repository.export():
                        return

                    with instance:
                        instance.mailbox = mailbox

                        if run_group(group_name, all_groups[group_name]):
                            instance.finish()

                    mailbox.check_empty()
示例#7
0
def run():
    global logger

    parser = argparse.ArgumentParser(description="Critic testing framework")

    parser.add_argument("--debug", action="store_true",
                        help="Enable DEBUG level logging")
    parser.add_argument("--debug-mails", action="store_true",
                        help="Log every mail sent by the tested system")
    parser.add_argument("--quiet", action="store_true",
                        help="Disable INFO level logging")

    parser.add_argument("--quickstart", action="store_true",
                        help="Test against a quick-start instance")
    parser.add_argument("--coverage", action="store_true",
                        help="Enable coverage measurement mode")
    parser.add_argument("--commit",
                        help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
    parser.add_argument("--upgrade-from",
                        help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
    parser.add_argument("--strict-fs-permissions", action="store_true",
                        help="Set strict file-system permissions in guest OS")
    parser.add_argument("--test-extensions", action="store_true",
                        help="Test extensions")

    parser.add_argument("--local", action="store_true",
                        help="Run local standalone tests only")

    parser.add_argument("--vbox-host", default="host",
                        help="Host that's running VirtualBox [default=host]")
    parser.add_argument("--vm-identifier",
                        help="VirtualBox instance name or UUID")
    parser.add_argument("--vm-hostname",
                        help="VirtualBox instance hostname [default=VM_IDENTIFIER")
    parser.add_argument("--vm-snapshot", default="clean",
                        help="VirtualBox snapshot (name or UUID) to restore [default=clean]")
    parser.add_argument("--vm-ssh-port", type=int, default=22,
                        help="VirtualBox instance SSH port [default=22]")
    parser.add_argument("--vm-http-port", type=int, default=80,
                        help="VirtualBox instance HTTP port [default=80]")

    parser.add_argument("--git-daemon-port", type=int,
                        help="Port to tell 'git daemon' to bind to")
    parser.add_argument("--cache-dir", default="testing/cache",
                        help="Directory where cache files are stored")

    parser.add_argument("--upgrade-after",
                        help="Upgrade after specified test")
    parser.add_argument("--pause-before", action="append",
                        help="Pause testing before specified test(s)")
    parser.add_argument("--pause-after", action="append",
                        help="Pause testing before specified test(s)")
    parser.add_argument("--pause-on-failure", action="store_true",
                        help="Pause testing after each failed test")
    parser.add_argument("--pause-upgrade-loop", action="store_true",
                        help="Support upgrading the tested system while paused")
    parser.add_argument("--pause-upgrade-retry", action="store_true",
                        help=("Support upgrading the tested system while paused "
                              "after a failed test, and retrying the failed test"))
    parser.add_argument("--pause-upgrade-hook", action="append",
                        help="Command to run (locally) before upgrading")

    parser.add_argument("test", nargs="*",
                        help="Specific tests to run [default=all]")

    arguments = parser.parse_args()

    class CountingLogger(object):
        def __init__(self, real, counters):
            self.real = real
            self.counters = counters

        def log(self, level, message):
            if level == logging.ERROR:
                self.counters.errors_logged += 1
            elif level == logging.WARNING:
                self.counters.warnings_logged += 1
            for line in message.splitlines() or [""]:
                self.real.log(level, line)
        def debug(self, message):
            self.log(logging.DEBUG, message)
        def info(self, message):
            self.log(logging.INFO, message)
        def warning(self, message):
            self.log(logging.WARNING, message)
        def error(self, message):
            self.log(logging.ERROR, message)
        def exception(self, message):
            self.log(logging.ERROR, message + "\n" + traceback.format_exc())

    logger = testing.configureLogging(
        arguments, wrap=lambda logger: CountingLogger(logger, counters))
    logger.info("""\
Critic Testing Framework
========================

""")

    key_arguments = [arguments.local,
                     arguments.quickstart,
                     arguments.vm_identifier]

    if len(filter(None, key_arguments)) != 1:
        logger.error("Must specify exactly one of --local, --quickstart and "
                     "--vm-identifier!")
        return

    if arguments.local or arguments.quickstart:
        incompatible_arguments = []

        # This is not a complete list; just those that are most significantly
        # incompatible or irrelevant with --local/--quickstart.
        if arguments.commit:
            incompatible_arguments.append("--commit")
        if arguments.upgrade_from:
            incompatible_arguments.append("--upgrade-from")
        if arguments.coverage:
            incompatible_arguments.append("--coverage")
        if arguments.test_extensions:
            incompatible_arguments.append("--strict-fs-permissions")
        if arguments.test_extensions:
            incompatible_arguments.append("--test-extensions")
        if arguments.vm_identifier:
            incompatible_arguments.append("--vm-identifier")

        if incompatible_arguments:
            logger.error("These arguments can't be combined with "
                         "--local/--quickstart:\n  " +
                         "\n  ".join(incompatible_arguments))
            return

    import_errors = False

    try:
        import requests
    except ImportError:
        logger.error("Failed to import 'requests'!")
        import_errors = True

    try:
        import BeautifulSoup
    except ImportError:
        logger.error("Failed to import 'BeautifulSoup'!")
        import_errors = True

    git_version = subprocess.check_output(["git", "--version"]).strip()
    m = re.search("(\d+)\.(\d+)\.(\d+)(?:[^\d]+|$)", git_version)
    if not m:
        logger.warning("Failed to parse host-side git version number: '%s'" % git_version)
    else:
        version_tuple = tuple(map(int, m.groups()))
        if version_tuple >= (1, 8, 5):
            logger.debug("Using Git version %s on host." % git_version)
        else:
            logger.error("Git version on host machine must be version 1.8.5 or above (detected version %s)." % git_version)
            logger.error("Earlier Git versions crashed with SIGBUS causing test suite flakiness.")
            import_errors = True

    if import_errors:
        logger.error("Required software missing; see testing/USAGE.md for details.")
        return

    if arguments.test_extensions:
        # Check that the v8-jsshell submodule is checked out if extension
        # testing was requested.
        output = subprocess.check_output(["git", "submodule", "status",
                                          "installation/externals/v8-jsshell"])
        if output.startswith("-"):
            logger.error("""\
The v8-jsshell submodule must be checked for extension testing.  Please run
  git submodule update --init installation/externals/v8-jsshell
first or run this script without --test-extensions.""")
            return

    if arguments.vm_identifier:
        # Note: we are not ignoring typical temporary editor files such as the
        # ".#<name>" files created by Emacs when a buffer has unsaved changes.
        # This is because unsaved changes in an editor is probably also
        # something you don't want to test with.

        locally_modified_paths = []

        status_output = subprocess.check_output(
            ["git", "status", "--porcelain"])

        for line in status_output.splitlines():
            locally_modified_paths.extend(line[3:].split(" -> "))

        tests_modified = []
        input_modified = []
        other_modified = []

        for path in locally_modified_paths:
            if path.startswith("testing/input/"):
                input_modified.append(path)
            elif path.startswith("testing/"):
                tests_modified.append(path)
            else:
                other_modified.append(path)

        if input_modified:
            logger.error("Test input files locally modified:\n  " +
                         "\n  ".join(input_modified))
        if other_modified:
            logger.error("Critic files locally modified:\n  " +
                         "\n  ".join(other_modified))
        if input_modified or other_modified:
            logger.error("Please commit or stash local modifications before "
                         "running tests.")
            return

        if tests_modified:
            logger.warning("Running tests using locally modified files:\n  " +
                           "\n  ".join(tests_modified))

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit or "HEAD"]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    flags_on = set()
    flags_off = set()

    try:
        if arguments.local:
            frontend = None
            instance = testing.local.Instance()
        else:
            frontend = testing.frontend.Frontend(
                hostname=arguments.vm_hostname or arguments.vm_identifier,
                http_port=arguments.vm_http_port)

            if arguments.quickstart:
                instance = testing.quickstart.Instance(
                    frontend=frontend)
            else:
                instance = testing.virtualbox.Instance(
                    arguments,
                    install_commit=(install_commit, install_commit_description),
                    upgrade_commit=(upgrade_commit, upgrade_commit_description),
                    frontend=frontend)
    except testing.Error as error:
        logger.error(error.message)
        return

    if not arguments.test_extensions:
        flags_off.add("extensions")

    flags_on.update(instance.flags_on)
    flags_off.update(instance.flags_off)

    tests, dependencies = testing.findtests.selectTests(
        arguments.test, strict=False, flags_on=flags_on, flags_off=flags_off)

    if not tests:
        logger.error("No tests selected!")
        return

    if arguments.upgrade_after:
        upgrade_after = testing.findtests.filterPatterns([arguments.upgrade_after])
        upgrade_after_tests, _ = testing.findtests.selectTests(upgrade_after,
                                                               strict=True)
        upgrade_after_tests = set(upgrade_after_tests)
        upgrade_after_groups = set(upgrade_after)

        def maybe_upgrade_after(test):
            def do_upgrade(what):
                logger.info("Upgrading after: %s" % what)
                instance.upgrade(is_after_test=True)

            if test in upgrade_after_tests:
                do_upgrade(test)
            else:
                for group in test.groups:
                    if group in upgrade_after_groups \
                            and test == all_groups[group][-1]:
                        do_upgrade(group)
                        break
    else:
        def maybe_upgrade_after(test):
            pass

    def pause(failed_test=None):
        if arguments.pause_upgrade_loop \
                or (failed_test and arguments.pause_upgrade_retry):
            print "Testing paused."

            while True:
                if failed_test and arguments.pause_upgrade_retry:
                    testing.pause("Press ENTER to upgrade (to HEAD) and "
                                  "retry %s, CTRL-c to stop: "
                                  % os.path.basename(failed_test))
                else:
                    testing.pause("Press ENTER to upgrade (to HEAD), "
                                  "CTRL-c to stop: ")

                for command in arguments.pause_upgrade_hook:
                    subprocess.check_call(command, shell=True)

                if isinstance(instance, testing.virtualbox.Instance):
                    repository.push("HEAD")

                    instance.execute(["git", "fetch", "origin", "master"], cwd="critic")
                    instance.upgrade_commit = "FETCH_HEAD"
                    instance.upgrade()

                if failed_test and arguments.pause_upgrade_retry:
                    return "retry"
        else:
            testing.pause("Testing paused.  Press ENTER to continue: ")

    if arguments.pause_before:
        pause_before = testing.findtests.filterPatterns(arguments.pause_before)
        pause_before_tests, _ = testing.findtests.selectTests(pause_before,
                                                              strict=True)
        pause_before_tests = set(pause_before_tests)
        pause_before_groups = set(pause_before)

        def maybe_pause_before(test):
            def do_pause(what):
                logger.info("Pausing before: %s" % what)
                pause()

            if test in pause_before_tests:
                do_pause(test)
            else:
                for group in test.groups:
                    if group in pause_before_groups \
                            and test == all_groups[group][0]:
                        do_pause(group)
                        break
    else:
        def maybe_pause_before(test):
            pass

    if arguments.pause_after:
        pause_after = testing.findtests.filterPatterns(arguments.pause_after)
        pause_after_tests, _ = testing.findtests.selectTests(pause_after,
                                                             strict=True)
        pause_after_tests = set(pause_after_tests)
        pause_after_groups = set(pause_after)

        def maybe_pause_after(test):
            def do_pause(what):
                logger.info("Pausing after: %s" % what)
                pause()

            if test in pause_after_tests:
                do_pause(test)
            else:
                for group in test.groups:
                    if group in pause_after_groups \
                            and test == all_groups[group][-1]:
                        do_pause(group)
                        break
    else:
        def maybe_pause_after(test):
            pass

    root_groups = {}
    all_groups = {}

    for test in tests:
        for group in test.groups:
            all_groups.setdefault(group, []).append(test)
        root_groups.setdefault(test.groups[0], []).append(test)

    failed_tests = set()

    def run_test(test, scope):
        prefix = "testing/tests"
        path = ""
        for component in test.filename.split("/")[:-1]:
            path = os.path.join(path, component)
            init_filename = os.path.join(path, "__init__.py")
            if os.path.isfile(os.path.join(prefix, init_filename)):
                logger.debug("Including: %s" % init_filename)
                execfile(os.path.join(prefix, init_filename), scope)
        execfile(os.path.join(prefix, test.filename), scope)

    def run_group(group_name, tests):
        scope = { "testing": testing,
                  "logger": logger,
                  "instance": instance }

        if not arguments.local:
            scope.update({ "frontend": frontend,
                           "repository": repository,
                           "mailbox": mailbox })

        try:
            for test in tests:
                if test.dependencies & failed_tests:
                    logger.info("Skipping %s (failed dependency)" % test)
                    continue

                maybe_pause_before(test)

                if test in dependencies:
                    logger.info("Running: %s (dependency)" % test)
                else:
                    logger.info("Running: %s" % test)

                counters.tests_run += 1

                while True:
                    try:
                        errors_before = counters.errors_logged
                        run_test(test, scope.copy())
                        if mailbox:
                            mailbox.check_empty()
                        instance.check_service_logs()
                        if errors_before < counters.errors_logged:
                            raise testing.TestFailure
                    except testing.TestFailure as failure:
                        counters.tests_failed += 1

                        failed_tests.add(test)

                        if failure.message:
                            logger.error(failure.message)

                        if mailbox:
                            try:
                                while True:
                                    mail = mailbox.pop(
                                        accept=testing.mailbox.ToRecipient(
                                            "*****@*****.**"))
                                    logger.error("System message: %s\n  %s"
                                                 % (mail.header("Subject"),
                                                    "\n  ".join(mail.lines)))
                            except testing.mailbox.MissingMail:
                                pass

                        instance.check_service_logs()

                        if arguments.pause_on_failure \
                                or arguments.pause_upgrade_retry:
                            if pause(test.filename) == "retry":
                                # Re-run test due to --pause-upgrade-retry.
                                continue
                    except testing.NotSupported as not_supported:
                        failed_tests.add(test)
                        logger.info("Test not supported: %s"
                                    % not_supported.message)
                    else:
                        maybe_upgrade_after(test)
                        maybe_pause_after(test)
                    break
        except KeyboardInterrupt:
            raise TestingAborted
        except testing.Error as error:
            if error.message:
                logger.exception(error.message)
            if arguments.pause_on_failure:
                pause()
            return False
        except Exception:
            logger.exception("Unexpected exception!")
            if arguments.pause_on_failure:
                pause()
            return False
        else:
            return True

    for group_name in sorted(root_groups.keys()):
        if arguments.local:
            repository = None
            mailbox = None

            run_group(group_name, all_groups[group_name])
        else:
            repository = testing.repository.Repository(
                "localhost" if arguments.quickstart else arguments.vbox_host,
                arguments.git_daemon_port,
                tested_commit,
                instance)
            mailbox = testing.mailbox.Mailbox(instance,
                                              { "username": "******",
                                                "password": "******" },
                                              arguments.debug_mails)

            with repository:
                with mailbox:
                    if not repository.export():
                        return

                    with instance:
                        instance.mailbox = mailbox

                        testing.utils.instance = instance
                        testing.utils.frontend = frontend

                        if run_group(group_name, all_groups[group_name]):
                            instance.finish()

                    mailbox.instance = None
                    mailbox.check_empty()
示例#8
0
def main():
    parser = argparse.ArgumentParser(
        description="Critic testing framework: Quick install utility")

    parser.add_argument("--debug",
                        help="Enable DEBUG level logging",
                        action="store_true")
    parser.add_argument("--quiet",
                        help="Disable INFO level logging",
                        action="store_true")

    parser.add_argument(
        "--commit",
        default="HEAD",
        help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
    parser.add_argument(
        "--upgrade-from",
        help="Commit (symbolic ref or SHA-1) to install first and upgrade from"
    )

    parser.add_argument("--vm-identifier",
                        required=True,
                        help="VirtualBox instance name or UUID")
    parser.add_argument(
        "--vm-hostname",
        help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
    parser.add_argument(
        "--vm-snapshot",
        default="clean",
        help="VirtualBox snapshot (name or UUID) to upgrade [default=clean]")
    parser.add_argument("--vm-ssh-port",
                        type=int,
                        default=22,
                        help="VirtualBox instance SSH port [default=22]")
    parser.add_argument("--git-daemon-port",
                        type=int,
                        help="Port to tell 'git daemon' to bind to")

    parser.add_argument("--interactive",
                        "-i",
                        action="store_true",
                        help="Install interactively (without arguments)")

    arguments = parser.parse_args()

    logger = testing.configureLogging(arguments)
    logger.info("Critic testing framework: Quick install")

    tested_commit = subprocess.check_output(
        ["git", "rev-parse", "--verify", arguments.commit]).strip()

    if arguments.upgrade_from:
        install_commit = subprocess.check_output(
            ["git", "rev-parse", "--verify", arguments.upgrade_from]).strip()
        upgrade_commit = tested_commit
    else:
        install_commit = tested_commit
        upgrade_commit = None

    install_commit_description = subprocess.check_output(
        ["git", "log", "--oneline", "-1", install_commit]).strip()

    if upgrade_commit:
        upgrade_commit_description = subprocess.check_output(
            ["git", "log", "--oneline", "-1", upgrade_commit]).strip()
    else:
        upgrade_commit_description = None

    instance = testing.virtualbox.Instance(
        arguments,
        install_commit=(install_commit, install_commit_description),
        upgrade_commit=(upgrade_commit, upgrade_commit_description))

    repository = testing.repository.Repository(arguments.git_daemon_port,
                                               install_commit,
                                               arguments.vm_hostname)

    mailbox = testing.mailbox.Mailbox()

    with repository, mailbox, instance:
        if not repository.export():
            return

        instance.mailbox = mailbox
        instance.start()

        if arguments.interactive:
            print """
Note: To use the simple SMTP server built into the Critic testing framework,
      enter "host" as the SMTP host and "%d" as the SMTP port.

Also note: The administrator user's password will be "testing" (password
           input doesn't work over this channel.)""" % mailbox.port

        instance.install(repository,
                         quick=True,
                         interactive=arguments.interactive)
        instance.upgrade(interactive=arguments.interactive)

        testing.pause("Press ENTER to stop VM: ")

        try:
            while True:
                mail = mailbox.pop()
                logger.info("Mail to <%s>:\n%s" % (mail.recipient, mail))
        except testing.mailbox.MissingMail:
            pass