release.py

"""%prog RELEASE_AREA [action ...]

Perform needed actions to release mechanize, doing the work in directory
RELEASE_AREA.

If no actions are given, print the tree of actions and do nothing.

This is only intended to work on Unix (unlike mechanize itself).  Some of it
only works on Ubuntu 10.04 (lucid).

Warning:

 * Many actions do rm -rf on RELEASE_AREA or subdirectories of RELEASE_AREA.

 * The install_deps action installs some debian packages system-wide.  The
clean action doesn't uninstall them.

 * The install_deps action adds a PPA.

 * The install_deps action downloads and installs software to RELEASE_AREA.
The clean action uninstalls (by rm -rf).
"""

# This script depends on the code from this git repository:
# git://github.com/jjlee/mechanize-build-tools.git

# TODO

#  * 0install package?
#  * test in a Windows VM

import glob
import optparse
import os
import re
import shutil
import smtplib
import subprocess
import sys
import tempfile
import time
import unittest

# Stop the test runner from reporting import failure if these modules aren't
# available or not running under Python >= 2.6.  AttributeError occurs if run
# with Python < 2.6, due to lack of collections.namedtuple
try:
    import email.mime.text

    import action_tree
    import cmd_env

    import buildtools.release as release
except (ImportError, AttributeError):
    # fake module
    class action_tree(object):
        @staticmethod
        def action_node(func):
            return func

# based on Mark Seaborn's plash build-tools (action_tree) and Cmed's in-chroot
# (cmd_env) -- which is also Mark's idea


class WrongVersionError(Exception):

    def __init__(self, version):
        Exception.__init__(self, version)
        self.version = version

    def __str__(self):
        return str(self.version)


class MissingVersionError(Exception):

    def __init__(self, path, release_version):
        Exception.__init__(self, path, release_version)
        self.path = path
        self.release_version = release_version

    def __str__(self):
        return ("Release version string not found in %s: should be %s" %
                (self.path, self.release_version))


class CSSValidationError(Exception):

    def __init__(self, path, details):
        Exception.__init__(self, path, details)
        self.path = path
        self.details = details

    def __str__(self):
        return ("CSS validation of %s failed:\n%s" % 
                (self.path, self.details))


def run_performance_tests(path):
    # TODO: use a better/standard test runner
    sys.path.insert(0, os.path.join(path, "test"))
    test_runner = unittest.TextTestRunner(verbosity=1)
    test_loader = unittest.defaultTestLoader
    modules = []
    for module_name in ["test_performance"]:
        module = __import__(module_name)
        for part in module_name.split('.')[1:]:
            module = getattr(module, part)
        modules.append(module)
    suite = unittest.TestSuite()
    for module in modules:
        test = test_loader.loadTestsFromModule(module)
        suite.addTest(test)
    result = test_runner.run(test)
    return result


def send_email(from_address, to_address, subject, body):
    msg = email.mime.text.MIMEText(body)
    msg['Subject'] = subject
    msg['From'] = from_address
    msg['To'] = to_address
    # print "from_address %r" % from_address
    # print "to_address %r" % to_address
    # print "msg.as_string():\n%s" % msg.as_string()
    s = smtplib.SMTP()
    s.connect()
    s.sendmail(from_address, [to_address], msg.as_string())
    s.quit()


def is_git_repository(path):
    return os.path.exists(os.path.join(path, ".git"))


def ensure_unmodified(env, path):
    # raise if working tree differs from HEAD
    release.CwdEnv(env, path).cmd(["git", "diff", "--exit-code", "HEAD"])


def add_to_path_cmd(value):
    set_path_script = """\
if [ -n "$PATH" ]
  then
    export PATH="$PATH":%(value)s
  else
    export PATH=%(value)s
fi
exec "$@"
""" % dict(value=value)
    return ["sh", "-c", set_path_script, "inline_script"]


def clean_environ_env(env):
    return cmd_env.PrefixCmdEnv(
        ["sh", "-c", 'env -i HOME="$HOME" PATH="$PATH" "$@"',
         "clean_environ_env"], env)


def ensure_trailing_slash(path):
    return path.rstrip("/") + "/"


def clean_dir(env, path):
    env.cmd(release.rm_rf_cmd(path))
    env.cmd(["mkdir", "-p", path])


def check_version_equals(env, version, python):
    try:
        output = release.get_cmd_stdout(
            env,
            [python, "-c",
             "import mechanize; print mechanize.__version__"],
            stderr=subprocess.PIPE)
    except cmd_env.CommandFailedError:
        raise WrongVersionError(None)
    else:
        version_tuple_string = output.strip()
        assert len(version.tuple) == 6, len(version.tuple)
        if not(version_tuple_string == str(version.tuple) or
               version_tuple_string == str(version.tuple[:-1])):
            raise WrongVersionError(version_tuple_string)


def check_not_installed(env, python):
    bogus_version = release.Version(release.parse_version("0.0.0"))
    try:
        check_version_equals(env, bogus_version, python)
    except WrongVersionError, exc:
        if exc.version is not None:
            raise
    else:
        raise WrongVersionError(bogus_version)


class EasyInstallTester(object):

    def __init__(self, env, install_dir, project_name,
                 test_cmd, expected_version,
                 easy_install_cmd=("easy_install",),
                 python="python"):
        self._env = env
        self._install_dir = install_dir
        self._project_name = project_name
        self._test_cmd = test_cmd
        self._expected_version = expected_version
        self._easy_install_cmd = list(easy_install_cmd)
        self._python = python
        self._install_dir_on_pythonpath = cmd_env.set_environ_vars_env(
            [("PYTHONPATH", self._install_dir)], env)

    def easy_install(self, log):
        clean_dir(self._env, self._install_dir)
        check_not_installed(self._install_dir_on_pythonpath, self._python)
        output = release.get_cmd_stdout(
            self._install_dir_on_pythonpath,
            self._easy_install_cmd + ["-d", self._install_dir,
                                      self._project_name])
        # easy_install doesn't fail properly :-(
        if "SyntaxError" in output:
            raise Exception(output)
        check_version_equals(self._install_dir_on_pythonpath,
                             self._expected_version,
                             self._python)

    def test(self, log):
        self._install_dir_on_pythonpath.cmd(self._test_cmd)

    @action_tree.action_node
    def easy_install_test(self):
        return [
            self.easy_install,
            self.test,
            ]


def make_source_dist_easy_install_test_step(env, install_dir,
                                            source_dir,
                                            test_cmd, expected_version,
                                            python_version):
    python = "python%d.%d" % python_version
    tester = EasyInstallTester(
        env,
        install_dir,
        project_name=".",
        test_cmd=test_cmd,
        expected_version=expected_version,
        easy_install_cmd=(cmd_env.in_dir(source_dir) +
                          [python, "setup.py", "easy_install"]),
        python=python)
    return tester.easy_install_test


def make_pypi_easy_install_test_step(env, install_dir,
                                     test_cmd, expected_version,
                                     python_version):
    easy_install = "easy_install-%d.%d" % python_version
    python = "python%d.%d" % python_version
    tester = EasyInstallTester(
        env,
        install_dir,
        project_name="mechanize",
        test_cmd=test_cmd,
        expected_version=expected_version,
        easy_install_cmd=[easy_install],
        python=python)
    return tester.easy_install_test


def make_tarball_easy_install_test_step(env, install_dir,
                                        tarball_path,
                                        test_cmd, expected_version,
                                        python_version):
    easy_install = "easy_install-%d.%d" % python_version
    python = "python%d.%d" % python_version
    tester = EasyInstallTester(
        env,
        install_dir,
        project_name=tarball_path,
        test_cmd=test_cmd,
        expected_version=expected_version,
        easy_install_cmd=[easy_install],
        python=python)
    return tester.easy_install_test


class Releaser(object):

    def __init__(self, env, git_repository_path, release_area, mirror_path,
                 build_tools_repo_path=None, run_in_repository=False,
                 tag_name=None, test_uri=None):
        self._release_area = release_area
        self._release_dir = release_dir = os.path.join(release_area, "release")
        self._opt_dir = os.path.join(release_dir, "opt")
        self._bin_dir = os.path.join(self._opt_dir, "bin")
        AddToPathEnv = release.make_env_maker(add_to_path_cmd)
        self._env = AddToPathEnv(release.GitPagerWrapper(env), self._bin_dir)
        self._source_repo_path = git_repository_path
        self._in_source_repo = release.CwdEnv(self._env,
                                              self._source_repo_path)
        self._previous_version, self._release_version = \
            self._get_next_release_version()
        if tag_name is not None:
            self._release_version = release.parse_version(tag_name)
        self._source_distributions = self._get_source_distributions(
            self._release_version)
        self._clone_path = os.path.join(release_dir, "clone")
        self._in_clone = release.CwdEnv(self._env, self._clone_path)
        if run_in_repository:
            self._in_repo = self._in_source_repo
            self._repo_path = self._source_repo_path
        else:
            self._in_repo = self._in_clone
            self._repo_path = self._clone_path
        self._docs_dir = os.path.join(self._repo_path, "docs")
        self._in_docs_dir = release.CwdEnv(self._env, self._docs_dir)
        self._in_release_dir = release.CwdEnv(self._env, self._release_dir)
        self._build_tools_path = build_tools_repo_path
        if self._build_tools_path is not None:
            self._website_source_path = os.path.join(self._build_tools_path,
                                                     "website")
        self._mirror_path = mirror_path
        self._in_mirror = release.CwdEnv(self._env, self._mirror_path)
        self._css_validator_path = "css_validator"
        self._test_uri = test_uri
        self._test_deps_dir = os.path.join(release_dir, "test_deps")
        self._easy_install_test_dir = os.path.join(release_dir,
                                                   "easy_install_test")
        self._in_easy_install_dir = release.CwdEnv(self._env,
                                                   self._easy_install_test_dir)
        # prevent anything other than functional test dependencies being on
        # sys.path due to cwd or PYTHONPATH
        self._easy_install_env = clean_environ_env(
            release.CwdEnv(env, self._test_deps_dir))
        self._zope_testbrowser_dir = os.path.join(release_dir,
                                                  "zope_testbrowser_test")

    def _get_next_release_version(self):
        # --pretend / git not installed
        most_recent, next = "dummy version", "dummy version"
        try:
            tags = release.get_cmd_stdout(self._in_source_repo,
                                          ["git", "tag", "-l"]).split()
        except cmd_env.CommandFailedError:
            pass
        else:
            versions = [release.parse_version(tag) for tag in tags]
            if versions:
                most_recent = max(versions)
                next = most_recent.next_version()
        return most_recent, next

    def _get_source_distributions(self, version):
        def dist_basename(version, format):
            return "mechanize-%s.%s" % (version, format)
        return set([dist_basename(version, "zip"),
                    dist_basename(version, "tar.gz")])

    def print_next_tag(self, log):
        print self._release_version

    def _verify_version(self, path):
        if str(self._release_version) not in \
                release.read_file_from_env(self._in_repo, path):
            raise MissingVersionError(path, self._release_version)

    def _verify_versions(self):
        for path in ["ChangeLog", "mechanize/_version.py"]:
            self._verify_version(path)

    def clone(self, log):
        self._env.cmd(["git", "clone",
                       self._source_repo_path, self._clone_path])

    def checks(self, log):
        self._verify_versions()

    def _ensure_installed(self, package_name, ppa):
        release.ensure_installed(self._env,
                                 cmd_env.PrefixCmdEnv(["sudo"], self._env),
                                 package_name,
                                 ppa=ppa)

    def install_css_validator_in_release_area(self, log):
        jar_dir = os.path.join(self._release_area, self._css_validator_path)
        clean_dir(self._env, jar_dir)
        in_jar_dir = release.CwdEnv(self._env, jar_dir)
        in_jar_dir.cmd([
                "wget",
                "http://www.w3.org/QA/Tools/css-validator/css-validator.jar"])
        in_jar_dir.cmd(["wget",
                        "http://jigsaw.w3.org/Distrib/jigsaw_2.2.6.tar.bz2"])
        in_jar_dir.cmd(["sh", "-c", "tar xf jigsaw_*.tar.bz2"])
        in_jar_dir.cmd(["ln", "-s", "Jigsaw/classes/jigsaw.jar"])

    @action_tree.action_node
    def install_deps(self):
        dependency_actions = []
        standard_dependency_actions = []
        def add_dependency(package_name, ppa=None):
            if ppa is None:
                actions = standard_dependency_actions
            else:
                actions = dependency_actions
            actions.append(
                (package_name.replace(".", ""),
                 lambda log: self._ensure_installed(package_name, ppa)))
        add_dependency("python2.6")
        # required, but ubuntu doesn't have them any more :-( I installed these
        # (and zope.interface and twisted SVN trunk) by hand
        # add_dependency("python2.4"),
        # add_dependency("python2.5")
        # add_dependency("python2.7")
        add_dependency("python-setuptools")
        add_dependency("git-core")
        # for running zope_testbrowser tests
        add_dependency("python-virtualenv")
        add_dependency("python2.6-dev")
        # for deployment to SF and local collation of files for release
        add_dependency("rsync")
        # for running functional tests against local web server
        add_dependency("python-twisted-web2")
        # for generating .html docs from .txt markdown files
        add_dependency("pandoc")
        # for generating docs from .in templates
        add_dependency("python-empy")
        # for post-processing generated HTML
        add_dependency("python-lxml")
        # for the validate command
        add_dependency("wdg-html-validator")
        # for collecting code coverage data and generating coverage reports
        # no 64 bit .deb ATM
        # add_dependency("python-figleaf", ppa="jjl/figleaf")

        # for css validator
        add_dependency("default-jre")
        add_dependency("libcommons-collections3-java")
        add_dependency("libcommons-lang-java")
        add_dependency("libxerces2-java")
        add_dependency("libtagsoup-java")
        # OMG, it depends on piles of java web server stuff, even for local
        # command-line validation.  You're doing it wrong!
        add_dependency("velocity")
        dependency_actions.append(self.install_css_validator_in_release_area)

        dependency_actions.insert(0, action_tree.make_node(
                standard_dependency_actions, "standard_dependencies"))
        return dependency_actions

    def copy_test_dependencies(self, log):
        # so test.py can be run without the mechanize alongside it being on
        # sys.path
        # TODO: move mechanize package into a top-level directory, so it's not
        # automatically on sys.path
        def copy_in(src):
            self._env.cmd(["cp", "-r", src, self._test_deps_dir])
        clean_dir(self._env, self._test_deps_dir)
        copy_in(os.path.join(self._repo_path, "test.py"))
        copy_in(os.path.join(self._repo_path, "test"))
        copy_in(os.path.join(self._repo_path, "test-tools"))
        copy_in(os.path.join(self._repo_path, "examples"))

    def _make_test_cmd(self, python_version,
                       local_server=True,
                       uri=None,
                       coverage=False):
        python = "python%d.%d" % python_version
        if coverage:
            # python-figleaf only supports Python 2.6 ATM
            assert python_version == (2, 6), python_version
            python = "figleaf"
        test_cmd = [python, "test.py"]
        if not local_server:
            test_cmd.append("--no-local-server")
            # running against wwwsearch.sourceforge.net is slow, want to
            # see where it failed
            test_cmd.append("-v")
        if coverage:
            # TODO: Fix figleaf traceback with doctests
            test_cmd.append("--skip-doctests")
        if uri is not None:
            test_cmd.extend(["--uri", uri])
        return test_cmd

    def performance_test(self, log):
        result = run_performance_tests(self._repo_path)
        if not result.wasSuccessful():
            raise Exception("performance tests failed")

    def clean_coverage(self, log):
        self._in_repo.cmd(["rm", "-f", ".figleaf"])
        self._in_repo.cmd(release.rm_rf_cmd("html"))

    def _make_test_step(self, env, **kwds):
        test_cmd = self._make_test_cmd(**kwds)
        def test_step(log):
            env.cmd(test_cmd)
        return test_step

    def _make_easy_install_test_cmd(self, **kwds):
        test_cmd = self._make_test_cmd(**kwds)
        test_cmd.extend(["discover", "--start-directory", self._test_deps_dir])
        return test_cmd

    def _make_source_dist_easy_install_test_step(self, env, **kwds):
        test_cmd = self._make_easy_install_test_cmd(**kwds)
        return make_source_dist_easy_install_test_step(
            self._easy_install_env, self._easy_install_test_dir,
            self._repo_path, test_cmd, self._release_version,
            kwds["python_version"])

    def _make_pypi_easy_install_test_step(self, env, **kwds):
        test_cmd = self._make_easy_install_test_cmd(**kwds)
        return make_pypi_easy_install_test_step(
            self._easy_install_env, self._easy_install_test_dir,
            test_cmd, self._release_version, kwds["python_version"])

    def _make_tarball_easy_install_test_step(self, env, **kwds):
        test_cmd = self._make_easy_install_test_cmd(**kwds)
        [tarball] = list(d for d in self._source_distributions if
                         d.endswith(".tar.gz"))
        return make_tarball_easy_install_test_step(
            self._easy_install_env, self._easy_install_test_dir,
            os.path.abspath(os.path.join(self._repo_path, "dist", tarball)),
            test_cmd, self._release_version, kwds["python_version"])

    @action_tree.action_node
    def test(self):
        r = []
        r.append(("python27_test",
                  self._make_test_step(self._in_repo, python_version=(2, 7))))
        r.append(("python27_easy_install_test",
                  self._make_source_dist_easy_install_test_step(
                    self._in_repo, python_version=(2, 7))))
        r.append(("python26_test",
                  self._make_test_step(self._in_repo, python_version=(2, 6))))
        # disabled for the moment -- think I probably built the launchpad .deb
        # from wrong branch, without bug fixes
        # r.append(("python26_coverage",
        #           self._make_test_step(self._in_repo, python_version=(2, 6),
        #                                coverage=True)))
        r.append(("python25_easy_install_test",
                  self._make_source_dist_easy_install_test_step(
                    self._in_repo, python_version=(2, 5))))
        r.append(("python24_easy_install_test",
                  self._make_source_dist_easy_install_test_step(
                    self._in_repo, python_version=(2, 4))))
        r.append(self.performance_test)
        return r

    def make_coverage_html(self, log):
        self._in_repo.cmd(["figleaf2html"])

    def tag(self, log):
        self._in_repo.cmd(["git", "checkout", "master"])
        self._in_repo.cmd(["git", "tag",
                           "-m", "Tagging release %s" % self._release_version,
                           str(self._release_version)])

    def clean_docs(self, log):
        self._in_docs_dir.cmd(release.rm_rf_cmd("html"))

    def make_docs(self, log):
        self._in_docs_dir.cmd(["mkdir", "-p", "html"])
        site_map = release.site_map()
        def pandoc(filename, source_filename):
            last_modified = release.last_modified(source_filename,
                                                  self._in_docs_dir)
            variables = [
                ("last_modified_iso",
                 time.strftime("%Y-%m-%d", last_modified)),
                ("last_modified_month_year",
                 time.strftime("%B %Y", last_modified))]
            page_name = os.path.splitext(os.path.basename(filename))[0]
            variables.append(("nav", release.nav_html(site_map, page_name)))
            variables.append(("subnav", release.subnav_html(site_map,
                                                            page_name)))
            release.pandoc(self._in_docs_dir, filename, variables=variables)
        release.empy(self._in_docs_dir, "forms.txt.in")
        release.empy(self._in_docs_dir, "download.txt.in",
                     defines=["version=%r" % str(self._release_version)])
        for page in site_map.iter_pages():
            if page.name in ["Root", "Changelog"]:
                continue
            source_filename = filename = page.name + ".txt"
            if page.name in ["forms", "download"]:
                source_filename += ".in"
            pandoc(filename, source_filename)
        self._in_repo.cmd(["cp", "-r", "ChangeLog", "docs/html/ChangeLog.txt"])
        if self._build_tools_path is not None:
            styles = ensure_trailing_slash(
                os.path.join(self._website_source_path, "styles"))
            self._env.cmd(["rsync", "-a", styles,
                           os.path.join(self._docs_dir, "styles")])

    def setup_py_sdist(self, log):
        self._in_repo.cmd(release.rm_rf_cmd("dist"))
        # write empty setup.cfg so source distribution is built using a version
        # number without ".dev" and today's date appended
        self._in_repo.cmd(cmd_env.write_file_cmd("setup.cfg", ""))
        self._in_repo.cmd(["python", "setup.py", "sdist",
                           "--formats=gztar,zip"])
        archives = set(os.listdir(os.path.join(self._repo_path, "dist")))
        assert archives == self._source_distributions, \
            (archives, self._source_distributions)

    @action_tree.action_node
    def build_sdist(self):
        return [
            self.clean_docs,
            self.make_docs,
            self.setup_py_sdist,
            ]

    def _stage(self, path, dest_dir, dest_basename=None,
               source_base_path=None):
        # IIRC not using rsync because didn't see easy way to avoid updating
        # timestamp of unchanged files, which was upsetting git
        # note: files in the website repository that are no longer generated
        # must be manually deleted from the repository
        if source_base_path is None:
            source_base_path = self._repo_path
        full_path = os.path.join(source_base_path, path)
        try:
            self._env.cmd(["readlink", "-e", full_path],
                          stdout=open(os.devnull, "w"))
        except cmd_env.CommandFailedError:
            print "not staging (does not exist):", full_path
            return
        if dest_basename is None:
            dest_basename = os.path.basename(path)
        dest = os.path.join(self._mirror_path, dest_dir, dest_basename)
        try:
            self._env.cmd(["cmp", full_path, dest])
        except cmd_env.CommandFailedError:
            print "staging: %s -> %s" % (full_path, dest)
            self._env.cmd(["cp", full_path, dest])
        else:
            print "not staging (unchanged): %s -> %s" % (full_path, dest)

    def ensure_unmodified(self, log):
        if self._build_tools_path:
            ensure_unmodified(self._env, self._website_source_path)
        ensure_unmodified(self._env, self._mirror_path)

    def _stage_flat_dir(self, path, dest):
        self._env.cmd(["mkdir", "-p", os.path.join(self._mirror_path, dest)])
        for filename in os.listdir(path):
            self._stage(os.path.join(path, filename), dest)

    def _symlink_flat_dir(self, path, exclude):
        for filename in os.listdir(path):
            if filename in exclude:
                continue
            link_dir = os.path.dirname(path)
            target = os.path.relpath(os.path.join(path, filename), link_dir)
            link_path = os.path.join(link_dir, filename)
            if not os.path.islink(link_path) or \
                    os.path.realpath(link_path) != target:
                self._env.cmd(["ln", "-f", "-s", "-t", link_dir, target])

    def collate_from_mechanize(self, log):
        html_dir = os.path.join(self._docs_dir, "html")
        self._stage_flat_dir(html_dir, "htdocs/mechanize/docs")
        self._symlink_flat_dir(
            os.path.join(self._mirror_path, "htdocs/mechanize/docs"),
            exclude=[".git", ".htaccess", ".svn", "CVS"])
        self._stage("test-tools/cookietest.cgi", "cgi-bin")
        self._stage("examples/forms/echo.cgi", "cgi-bin")
        self._stage("examples/forms/example.html", "htdocs/mechanize")
        for archive in self._source_distributions:
            placeholder = os.path.join("htdocs/mechanize/src", archive)
            self._in_mirror.cmd(["touch", placeholder])

    def collate_from_build_tools(self, log):
        self._stage(os.path.join(self._website_source_path, "frontpage.html"),
                    "htdocs", "index.html")
        self._stage_flat_dir(
            os.path.join(self._website_source_path, "styles"), "htdocs/styles")

    @action_tree.action_node
    def collate(self):
        r = [self.collate_from_mechanize]
        if self._build_tools_path is not None:
            r.append(self.collate_from_build_tools)
        return r

    def collate_pypi_upload_built_items(self, log):
        for archive in self._source_distributions:
            self._stage(os.path.join("dist", archive), "htdocs/mechanize/src")

    def commit_staging_website(self, log):
        self._in_mirror.cmd(["git", "add", "--all"])
        self._in_mirror.cmd(
            ["git", "commit",
             "-m", "Automated update for release %s" % self._release_version])

    def validate_html(self, log):
        exclusions = set(f for f in """\
./cookietest.html
htdocs/basic_auth/index.html
htdocs/digest_auth/index.html
htdocs/mechanize/example.html
htdocs/test_fixtures/index.html
htdocs/test_fixtures/mechanize_reload_test.html
htdocs/test_fixtures/referertest.html
""".splitlines() if not f.startswith("#"))
        for dirpath, dirnames, filenames in os.walk(self._mirror_path):
            try:
                # archived website
                dirnames.remove("old")
            except ValueError:
                pass
            for filename in filenames:
                if filename.endswith(".html"):
                    page_path = os.path.join(
                        os.path.relpath(dirpath, self._mirror_path), filename)
                    if page_path not in exclusions:
                        self._in_mirror.cmd(["validate", page_path])

    def _classpath_cmd(self):
        from_packages = ["/usr/share/java/commons-collections3.jar",
                         "/usr/share/java/commons-lang.jar",
                         "/usr/share/java/xercesImpl.jar",
                         "/usr/share/java/tagsoup.jar",
                         "/usr/share/java/velocity.jar",
                         ]
        jar_dir = os.path.join(self._release_dir, self._css_validator_path)
        local = glob.glob(os.path.join(jar_dir, "*.jar"))
        path = ":".join(local + from_packages)
        return ["env", "CLASSPATH=%s" % path]

    def _sanitise_css(self, path):
        temp_dir = tempfile.mkdtemp(prefix="tmp-%s-" % self.__class__.__name__)
        def tear_down():
            shutil.rmtree(temp_dir)
        temp_path = os.path.join(temp_dir, os.path.basename(path))
        temp = open(temp_path, "w")
        try:
            for line in open(path):
                if line.rstrip().endswith("/*novalidate*/"):
                    # temp.write("/*%s*/\n" % line.rstrip())
                    temp.write("/*sanitised*/\n")
                else:
                    temp.write(line)
        finally:
            temp.close()
        return temp_path, tear_down

    def validate_css(self, log):
        env = cmd_env.PrefixCmdEnv(self._classpath_cmd(), self._in_release_dir)
        # env.cmd(["java", "org.w3c.css.css.CssValidator", "--help"])
        """
Usage: java org.w3c.css.css.CssValidator  [OPTIONS] | [URL]*
OPTIONS
	-p, --printCSS
		Prints the validated CSS (only with text output, the CSS is printed with other outputs)
	-profile PROFILE, --profile=PROFILE
		Checks the Stylesheet against PROFILE
		Possible values for PROFILE are css1, css2, css21 (default), css3, svg, svgbasic, svgtiny, atsc-tv, mobile, tv
	-medium MEDIUM, --medium=MEDIUM
		Checks the Stylesheet using the medium MEDIUM
		Possible values for MEDIUM are all (default), aural, braille, embossed, handheld, print, projection, screen, tty, tv, presentation
	-output OUTPUT, --output=OUTPUT
		Prints the result in the selected format
		Possible values for OUTPUT are text (default), xhtml, html (same result as xhtml), soap12
	-lang LANG, --lang=LANG
		Prints the result in the specified language
		Possible values for LANG are de, en (default), es, fr, ja, ko, nl, zh-cn, pl, it
	-warning WARN, --warning=WARN
		Warnings verbosity level
		Possible values for WARN are -1 (no warning), 0, 1, 2 (default, all the warnings

URL
	URL can either represent a distant web resource (http://) or a local file (file:/)
"""
        validate_cmd = ["java", "org.w3c.css.css.CssValidator"]
        for dirpath, dirnames, filenames in os.walk(self._mirror_path):
            for filename in filenames:
                if filename.endswith(".css"):
                    path = os.path.join(dirpath, filename)
                    temp_path, tear_down = self._sanitise_css(path)
                    try:
                        page_url = "file://" + temp_path
                        output = release.get_cmd_stdout(
                            env, validate_cmd + [page_url])
                        # the validator doesn't fail properly: it exits
                        # successfully on validation failure
                        if "Sorry! We found the following errors" in output:
                            raise CSSValidationError(path, output)
                    finally:
                        tear_down()

    def fetch_zope_testbrowser(self, log):
        clean_dir(self._env, self._zope_testbrowser_dir)
        in_testbrowser = release.CwdEnv(self._env, self._zope_testbrowser_dir)
        in_testbrowser.cmd(["easy_install", "--editable",
                            "--build-directory", ".",
                            "zope.testbrowser[test]"])
        in_testbrowser.cmd(
            ["virtualenv", "--no-site-packages", "zope.testbrowser"])
        project_dir = os.path.join(self._zope_testbrowser_dir,
                                   "zope.testbrowser")
        in_project_dir = clean_environ_env(
            release.CwdEnv(self._env, project_dir))
        check_not_installed(in_project_dir, "bin/python")
        in_project_dir.cmd(
            ["sed", "-i", "-e", "s/mechanize[^\"']*/mechanize/", "setup.py"])
        in_project_dir.cmd(["bin/easy_install", "zc.buildout"])
        in_project_dir.cmd(["bin/buildout", "init"])
        [mechanize_tarball] = list(d for d in self._source_distributions if
                                   d.endswith(".tar.gz"))
        tarball_path = os.path.join(self._repo_path, "dist", mechanize_tarball)
        in_project_dir.cmd(["bin/easy_install", tarball_path])
        in_project_dir.cmd(["bin/buildout", "install"])

    def test_zope_testbrowser(self, log):
        project_dir = os.path.join(self._zope_testbrowser_dir,
                                   "zope.testbrowser")
        env = clean_environ_env(
            release.CwdEnv(self._env, project_dir))
        check_version_equals(env, self._release_version, "bin/python")
        env.cmd(["bin/test"])

    @action_tree.action_node
    def zope_testbrowser(self):
        return [self.fetch_zope_testbrowser,
                self.test_zope_testbrowser,
                ]

    def upload_to_pypi(self, log):
        self._in_repo.cmd(["python", "setup.py", "sdist",
                           "--formats=gztar,zip", "upload"])

    def sync_to_sf(self, log):
        assert os.path.isdir(
            os.path.join(self._mirror_path, "htdocs/mechanize"))
        self._env.cmd(["rsync", "-rlptvuz", "--exclude", "*~", "--delete",
                       ensure_trailing_slash(self._mirror_path),
                       "jjlee,wwwsearch@web.sourceforge.net:"])

    @action_tree.action_node
    def upload(self):
        r = []
        r.append(self.upload_to_pypi)
        # setup.py upload requires sdist command to upload zip files, and the
        # sdist comment insists on rebuilding source distributions, so it's not
        # possible to use the upload command to upload the already-built zip
        # file.  Work around that by copying the rebuilt source distributions
        # into website repository only now (rather than at build/test time), so
        # don't end up with two different sets of source distributions with
        # different md5 sums due to timestamps in the archives.
        r.append(self.collate_pypi_upload_built_items)
        r.append(self.commit_staging_website)

        if self._mirror_path is not None:
            r.append(self.sync_to_sf)
        return r

    def clean(self, log):
        clean_dir(self._env, self._release_area)

    def clean_most(self, log):
        # not dependencies installed in release area (css validator)
        clean_dir(self._env, self._release_dir)

    def write_email(self, log):
        log = release.get_cmd_stdout(self._in_repo,
                                     ["git", "log", '--pretty=format: * %s',
                                      "%s..HEAD" % self._previous_version])
        # filter out some uninteresting commits
        log = "".join(line for line in log.splitlines(True) if not
                      re.match("^ \* Update (?:changelog|version)$", line,
                               re.I))
        self._in_release_dir.cmd(cmd_env.write_file_cmd(
                "announce_email.txt", u"""\
ANN: mechanize {version} released

http://wwwsearch.sourceforge.net/mechanize/

This is a stable bugfix release.

Changes since {previous_version}:

{log}

About mechanize
=============================================

Requires Python 2.4, 2.5, 2.6, or 2.7.


Stateful programmatic web browsing, after Andy Lester's Perl module
WWW::Mechanize.

Example:

import re
from mechanize import Browser

b = Browser()
b.open("http://www.example.com/")
# follow second link with element text matching regular expression
response = b.follow_link(text_regex=re.compile(r"cheese\s*shop"), nr=1)

b.select_form(name="order")
# Browser passes through unknown attributes (including methods)
# to the selected HTMLForm
b["cheeses"] = ["mozzarella", "caerphilly"]  # (the method here is __setitem__)
response2 = b.submit()  # submit current form

response3 = b.back()  # back to cheese shop
response4 = b.reload()

for link in b.forms():
       print form
# .links() optionally accepts the keyword args of .follow_/.find_link()
for link in b.links(url_regex=re.compile("python.org")):
       print link
       b.follow_link(link)  # can be EITHER Link instance OR keyword args
       b.back()


John
""".format(log=log,
           version=self._release_version,
           previous_version=self._previous_version)))

    def push_tag(self, log):
        self._in_repo.cmd(["git", "push", "git@github.com:jjlee/mechanize.git",
                           "tag", str(self._release_version)])

    def send_email(self, log):
        text = release.read_file_from_env(self._in_release_dir,
                                          "announce_email.txt")
        print "text %r" % text
        subject, sep, body = text.partition("\n")
        body = body.lstrip()
        assert len(body) > 0, body
        send_email(from_address="jjl@pobox.com",
                   to_address="wwwsearch-general@lists.sourceforge.net",
                   subject=subject,
                   body=body)

    @action_tree.action_node
    def build(self):
        return [
            self.clean,
            self.clean_most,
            self.install_deps,
            self.print_next_tag,
            self.clone,
            self.checks,
            # self.clean_coverage,
            self.copy_test_dependencies,
            self.test,
            # self.make_coverage_html,
            self.tag,
            self.build_sdist,
            ("easy_install_test", self._make_tarball_easy_install_test_step(
                    self._in_repo, python_version=(2, 6),
                    local_server=False, uri=self._test_uri)),
            self.zope_testbrowser,
            self.write_email,
            ]

    def update_version(self, log):
        version_path = "mechanize/_version.py"
        template = """\
"%(text)s"
__version__ = %(tuple)s
"""
        old_text = release.read_file_from_env(self._in_source_repo,
                                              version_path)
        old_version = old_text.splitlines()[0].strip(' "')
        assert old_version == str(self._release_version), \
            (old_version, str(self._release_version))
        def version_text(version):
            return template % {"text": str(version),
                               "tuple": repr(tuple(version.tuple[:-1]))}
        assert old_text == version_text(release.parse_version(old_version)), \
            (old_text, version_text(release.parse_version(old_version)))
        self._in_source_repo.cmd(cmd_env.write_file_cmd(
                version_path,
                version_text(self._release_version.next_version())))
        self._in_source_repo.cmd(["git", "commit", "-m", "Update version",
                                  version_path])

    @action_tree.action_node
    def update_staging_website(self):
        if self._mirror_path is None:
            return []

        return [
            self.ensure_unmodified,
            self.collate,
            self.validate_html,
            self.validate_css,
            self.commit_staging_website,
            ]

    @action_tree.action_node
    def tell_the_world(self):
        return [
            self.push_tag,
            self.upload,
            ("easy_install_test_internet",
             self._make_pypi_easy_install_test_step(
                    self._in_repo, python_version=(2, 6),
                    local_server=False,
                    # XXX seems to be ignoring this
                    uri="http://wwwsearch.sourceforge.net/")),
            self.send_email,
            ]

    @action_tree.action_node
    def all(self):
        return [
            self.build,
            self.update_staging_website,
            self.update_version,
            self.tell_the_world,
            ]


def parse_options(args):
    parser = optparse.OptionParser(usage=__doc__.strip())
    release.add_basic_env_options(parser)
    parser.add_option("--mechanize-repository", metavar="DIRECTORY",
                      dest="git_repository_path",
                      help="path to mechanize git repository (default is cwd)")
    parser.add_option("--build-tools-repository", metavar="DIRECTORY",
                      help=("path of mechanize-build-tools git repository, "
                            "from which to get other website source files "
                            "(default is not to build those files)"))
    parser.add_option("--website-repository", metavar="DIRECTORY",
                      dest="mirror_path",
                      help=("path of local website mirror git repository into "
                            "which built files will be copied (default is not "
                            "to copy the files)"))
    parser.add_option("--in-source-repository", action="store_true",
                      dest="in_repository",
                      help=("run all commands in original repository "
                            "(specified by --git-repository), rather than in "
                            "the clone of it in the release area"))
    parser.add_option("--tag-name", metavar="TAG_NAME")
    parser.add_option("--uri", default="http://wwwsearch.sourceforge.net/",
                      help=("base URI to run tests against when not using a "
                            "built-in web server"))
    options, remaining_args = parser.parse_args(args)
    nr_args = len(remaining_args)
    try:
        options.release_area = remaining_args.pop(0)
    except IndexError:
        parser.error("Expected at least 1 argument, got %d" % nr_args)
    if options.git_repository_path is None:
        options.git_repository_path = os.getcwd()
    if not is_git_repository(options.git_repository_path):
        parser.error("incorrect git repository path")
    if options.build_tools_repository is not None and \
            not is_git_repository(options.build_tools_repository):
        parser.error("incorrect mechanize-build-tools repository path")
    mirror_path = options.mirror_path
    if mirror_path is not None:
        if not is_git_repository(options.mirror_path):
            parser.error("mirror path is not a git reporsitory")
        mirror_path = os.path.join(mirror_path, "mirror")
        if not os.path.isdir(mirror_path):
            parser.error("%r does not exist" % mirror_path)
    options.mirror_path = mirror_path
    return options, remaining_args


def main(argv):
    if not hasattr(action_tree, "action_main"):
        sys.exit("failed to import required modules")

    options, action_tree_args = parse_options(argv[1:])
    env = release.get_env_from_options(options)
    releaser = Releaser(env, options.git_repository_path, options.release_area,
                        options.mirror_path, options.build_tools_repository,
                        options.in_repository, options.tag_name, options.uri)
    action_tree.action_main(releaser.all, action_tree_args)


if __name__ == "__main__":
    main(sys.argv)