def update_repo(path, bareflag):
"""Pull the latest changes to the given repo and update. The bareflag
parameter indicates whether the updated repo must be a bare git repo."""
if bareflag:
execute("git fetch origin master:master", working_dir=path)
else:
execute("git pull", working_dir=path)
def copy_release_dir(path_to_dev_releases, path_to_live_releases,
release_version):
"""Copy a release directory with the given release version from the dev
site to the live site. For example,
/cse/www2/types/dev/checker-framework/releases/2.0.0 ->
/cse/www2/types/checker-framework/releases/2.0.0"""
source_location = os.path.join(path_to_dev_releases, release_version)
dest_location = os.path.join(path_to_live_releases, release_version)
if os.path.exists(dest_location):
delete_path(dest_location)
if os.path.exists(dest_location):
raise Exception("Destination location exists: " + dest_location)
# The / at the end of the source location is necessary so that
# rsync copies the files in the source directory to the destination directory
# rather than a subdirectory of the destination directory.
cmd = "rsync --omit-dir-times --recursive --links --quiet %s/ %s" % (
source_location,
dest_location,
)
execute(cmd)
return dest_location
def clone(src_repo, dst_repo, bareflag):
"""Clone the given git or Mercurial repo from scratch into the filesystem
path specified by dst_repo. The bareflag parameter indicates whether the
cloned repo must be a bare git repo."""
flags = ""
if bareflag:
flags = "--bare"
execute("git clone --quiet %s %s %s" % (flags, src_repo, dst_repo))
def promote_release(path_to_releases, release_version):
"""Copy a release directory to the top level. For example,
/cse/www2/types/checker-framework/releases/2.0.0/* ->
/cse/www2/types/checker-framework/*"""
from_dir = os.path.join(path_to_releases, release_version)
to_dir = os.path.join(path_to_releases, "..")
# Trailing slash is crucial.
cmd = "rsync -aJ --omit-dir-times %s/ %s" % (from_dir, to_dir)
execute(cmd)
def create_dev_website_release_version_dir(project_name, version):
"""Create the directory for the given version of the given project under
the releases directory of the dev web site."""
if project_name in (None, "checker-framework"):
interm_dir = os.path.join(DEV_SITE_DIR, "releases", version)
else:
interm_dir = os.path.join(DEV_SITE_DIR, project_name, "releases", version)
delete_path_if_exists(interm_dir)
execute("mkdir -p %s" % interm_dir, True, False)
return interm_dir
def check_command(command):
"""Executes the UNIX \"which\" command to determine whether the given command
is installed and on the PATH."""
p = execute(["which", command], False)
if p:
raise AssertionError("command not found: %s" % command)
print("")
def get_commit_for_tag(revision, repo_file_path, tag_prefixes):
"""Get the commit hash for the tag matching the given project revision of
the Git repository at the given filesystem path. Uses the given array of
tag prefix strings if provided. For example, given an array of tag prefixes
[\"checker-framework-\", \"checkers-\"] and project revision \"2.0.0\", the
tags named \"checker-framework-2.0.0\" and \"checkers-2.0.0\" are sought."""
# assume the first is the most recent
tags = execute(
"git rev-list " + tag_prefixes[0] + revision,
True,
True,
working_dir=repo_file_path,
)
lines = tags.splitlines()
commit = lines[0]
if commit is None:
msg = "Could not find revision %s in repo %s using tags %s " % (
revision,
repo_file_path,
",".join(tag_prefixes),
)
raise Exception(msg)
return commit
def stage_maven_artifacts_in_maven_central(new_cf_version):
"""Stages the Checker Framework artifacts on Maven Central. After the
artifacts are staged, the user can then close them, which makes them
available for testing purposes but does not yet release them on Maven
Central. This is a reversible step, since artifacts that have not been
released can be dropped, which for our purposes is equivalent to never
having staged them."""
gnupgPassphrase = read_first_line(
"/projects/swlab1/checker-framework/hosting-info/release-private.password"
)
# When bufalo uses gpg2 version 2.2+, then remove signing.gnupg.useLegacyGpg=true
execute(
"./gradlew publish -Prelease=true --no-parallel -Psigning.gnupg.useLegacyGpg=true [email protected] -Psigning.gnupg.passphrase=%s"
% gnupgPassphrase,
working_dir=CHECKER_FRAMEWORK,
)
def add_repo_information(pom, repo_url):
"""Adds development maven repo to pom file so that the artifacts used are
the development artifacts"""
to_insert = """
<repositories>
<repository>
<id>checker-framework-repo</id>
<url>%s</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>checker-framework-repo</id>
<url>%s</url>
</pluginRepository>
</pluginRepositories>
""" % (
repo_url,
repo_url,
)
result_str = execute('grep -nm 1 "<build>" %s' % pom, True, True).decode()
line_no_str = result_str.split(":")[0]
line_no = int(line_no_str)
print(" LINE_NO: " + line_no_str)
insert_before_line(to_insert, pom, line_no)
def is_repo_cleaned_and_updated(repo):
"""IMPORTANT: this function is not known to be fully reliable in ensuring
that a repo is fully clean of all changes, such as committed tags. To be
certain of success throughout the release_build and release_push process,
the best option is to clone repositories from scratch.
Returns whether the repository at the given filesystem path is clean (i.e.
there are no committed changes and no untracked files in the working tree)
and up-to-date with respect to the repository it was cloned from."""
# The idiom "not execute(..., capture_output=True)" evaluates to True when the captured output is empty.
if git_bare_repo_exists_at_path(repo):
execute("git fetch origin", working_dir=repo)
is_updated = not execute(
"git diff master..FETCH_HEAD", working_dir=repo, capture_output=True
)
return is_updated
else:
# Could add "--untracked-files=no" to this command
is_clean = not execute(
"git status --porcelain", working_dir=repo, capture_output=True
)
execute("git fetch origin", working_dir=repo)
is_updated = not execute(
"git diff origin/master..master", working_dir=repo, capture_output=True
)
return is_clean and is_updated
def maven_sanity_check(sub_sanity_dir_name, repo_url, release_version):
"""
Run the Maven sanity check with the local artifacts or from the repo at
repo_url.
"""
checker_dir = os.path.join(CHECKER_FRAMEWORK, "checker")
maven_sanity_dir = os.path.join(SANITY_DIR, sub_sanity_dir_name)
if os.path.isdir(maven_sanity_dir):
delete_path(maven_sanity_dir)
execute("mkdir -p " + maven_sanity_dir)
maven_example_dir = os.path.join(maven_sanity_dir, "MavenExample")
output_log = os.path.join(maven_example_dir, "output.log")
ant_release_script = os.path.join(CHECKER_FRAMEWORK_RELEASE, "release.xml")
get_example_dir_cmd = (
"ant -f %s update-and-copy-maven-example -Dchecker=%s -Dversion=%s -Ddest.dir=%s"
% (ant_release_script, checker_dir, release_version, maven_sanity_dir))
execute(get_example_dir_cmd)
path_to_artifacts = os.path.join(os.path.expanduser("~"), ".m2",
"repository", "org", "checkerframework")
if repo_url != "":
print((
"This script will now delete your Maven Checker Framework artifacts.\n"
+
"See README-release-process.html#Maven-Plugin dependencies. These artifacts "
+
"will need to be re-downloaded the next time you need them. This will be "
+ "done automatically by Maven next time you use the plugin."))
if os.path.isdir(path_to_artifacts):
delete_path(path_to_artifacts)
maven_example_pom = os.path.join(maven_example_dir, "pom.xml")
add_repo_information(maven_example_pom, repo_url)
os.environ["JAVA_HOME"] = os.environ["JAVA_8_HOME"]
execute_write_to_file("mvn compile", output_log, False, maven_example_dir)
if repo_url != "":
delete_path(path_to_artifacts)
def build_annotation_tools_release(version, afu_interm_dir):
"""Build the Annotation File Utilities project's artifacts and place them
in the development web site."""
execute("java -version", True)
date = get_current_date()
build = os.path.join(ANNO_FILE_UTILITIES, "build.xml")
ant_cmd = (
'ant %s -buildfile %s -e update-versions -Drelease.ver="%s" -Drelease.date="%s"'
% (ant_debug, build, version, date))
execute(ant_cmd)
# Deploy to intermediate site
gradle_cmd = "./gradlew releaseBuild -Pafu.version=%s -Pdeploy-dir=%s" % (
version,
afu_interm_dir,
)
execute(gradle_cmd, True, False, ANNO_FILE_UTILITIES)
update_project_dev_website("annotation-file-utilities", version)
def ensure_user_access(path):
"Give the user access to all files and directories under the specified path"
execute("chmod -f -R u+rwx %s" % path, halt_if_fail=True)
def javac_sanity_check(checker_framework_website, release_version):
"""
Download the release of the Checker Framework from the development website
and NullnessExampleWithWarnings.java from the GitHub repository.
Run the Nullness Checker on NullnessExampleWithWarnings and verify the output
Fails if the expected errors are not found in the output.
"""
new_checkers_release_zip = os.path.join(
checker_framework_website,
"releases",
release_version,
"checker-framework-" + release_version + ".zip",
)
javac_sanity_dir = os.path.join(SANITY_DIR, "javac")
if os.path.isdir(javac_sanity_dir):
delete_path(javac_sanity_dir)
execute("mkdir -p " + javac_sanity_dir)
javac_sanity_zip = os.path.join(
javac_sanity_dir, "checker-framework-%s.zip" % release_version)
print("Attempting to download %s to %s" %
(new_checkers_release_zip, javac_sanity_zip))
download_binary(new_checkers_release_zip, javac_sanity_zip)
nullness_example_url = "https://raw.githubusercontent.com/eisop/checker-framework/master/docs/examples/NullnessExampleWithWarnings.java"
nullness_example = os.path.join(javac_sanity_dir,
"NullnessExampleWithWarnings.java")
if os.path.isfile(nullness_example):
delete(nullness_example)
wget_file(nullness_example_url, javac_sanity_dir)
deploy_dir = os.path.join(javac_sanity_dir,
"checker-framework-" + release_version)
if os.path.exists(deploy_dir):
print("Deleting existing path: " + deploy_dir)
delete_path(deploy_dir)
with zipfile.ZipFile(javac_sanity_zip, "r") as z:
z.extractall(javac_sanity_dir)
ensure_user_access(deploy_dir)
sanity_javac = os.path.join(deploy_dir, "checker", "bin", "javac")
nullness_output = os.path.join(deploy_dir, "output.log")
cmd = (
sanity_javac +
" -processor org.checkerframework.checker.nullness.NullnessChecker " +
nullness_example + " -Anomsgtext")
execute_write_to_file(cmd, nullness_output, False)
check_results(
"Javac sanity check",
nullness_output,
[
"NullnessExampleWithWarnings.java:23: error: (assignment.type.incompatible)",
"NullnessExampleWithWarnings.java:33: error: (argument.type.incompatible)",
],
)
# this is a smoke test for the built-in checker shorthand feature
# https://checkerframework.org/manual/#shorthand-for-checkers
nullness_shorthand_output = os.path.join(deploy_dir,
"output_shorthand.log")
cmd = (sanity_javac + " -processor NullnessChecker " + nullness_example +
" -Anomsgtext")
execute_write_to_file(cmd, nullness_shorthand_output, False)
check_results(
"Javac Shorthand Sanity Check",
nullness_shorthand_output,
[
"NullnessExampleWithWarnings.java:23: error: (assignment.type.incompatible)",
"NullnessExampleWithWarnings.java:33: error: (argument.type.incompatible)",
],
)
def main(argv):
"""The release_push script is mainly responsible for copying the artifacts
(for the AFU and the Checker Framework) from the
development web site to Maven Central and to
the live site. It also performs link checking on the live site, pushes
the release to GitHub repositories, and guides the user to
perform manual steps such as sending the
release announcement e-mail."""
# MANUAL Indicates a manual step
# AUTO Indicates the step is fully automated.
set_umask()
validate_args(argv)
test_mode = not read_command_line_option(argv, "release")
m2_settings = expanduser("~") + "/.m2/settings.xml"
if not os.path.exists(m2_settings):
raise Exception("File does not exist: " + m2_settings)
if test_mode:
msg = (
"You have chosen test_mode.\n" +
"This means that this script will execute all build steps that " +
"do not have side effects. That is, this is a test run of the script. All checks and user prompts "
+
"will be shown but no steps will be executed that will cause the release to be deployed or partially "
+ "deployed.\n" +
'If you meant to do an actual release, re-run this script with one argument, "release".'
)
else:
msg = "You have chosen release_mode. Please follow the prompts to run a full Checker Framework release."
continue_or_exit(msg + "\n")
if test_mode:
print("Continuing in test mode.")
else:
print("Continuing in release mode.")
if not os.path.exists(RELEASE_BUILD_COMPLETED_FLAG_FILE):
continue_or_exit(
"It appears that release_build.py has not been run since the last push to "
+
"the AFU or Checker Framework repositories. Please ensure it has "
+ "been run.")
# The release script checks that the new release version is greater than the previous release version.
print_step("Push Step 1: Checking release versions") # SEMIAUTO
dev_afu_website = os.path.join(DEV_SITE_URL, "annotation-file-utilities")
live_afu_website = os.path.join(LIVE_SITE_URL, "annotation-file-utilities")
dev_checker_website = DEV_SITE_URL
live_checker_website = LIVE_SITE_URL
current_cf_version = current_distribution_by_website(live_checker_website)
new_cf_version = CF_VERSION
check_release_version(current_cf_version, new_cf_version)
print("Checker Framework and AFU: current-version=%s new-version=%s" %
(current_cf_version, new_cf_version))
# Runs the link the checker on all websites at:
# https://checkerframework.org/dev/
# The output of the link checker is written to files in the /scratch/$USER/cf-release directory
# whose locations will be output at the command prompt if the link checker reported errors.
# In rare instances (such as when a link is correct but the link checker is
# unable to check it), you may add a suppression to the checklink-args.txt file.
# In extremely rare instances (such as when a website happens to be down at the
# time you ran the link checker), you may ignore an error.
print_step("Push Step 2: Check links on development site") # SEMIAUTO
if prompt_yes_no("Run link checker on DEV site?", True):
check_all_links(dev_afu_website, dev_checker_website, "dev", test_mode,
new_cf_version)
# Runs sanity tests on the development release. Later, we will run a smaller set of sanity
# tests on the live release to ensure no errors occurred when promoting the release.
print_step("Push Step 3: Run development sanity tests") # SEMIAUTO
if prompt_yes_no("Perform this step?", True):
print_step("3a: Run javac sanity test on development release.")
if prompt_yes_no("Run javac sanity test on development release?",
True):
javac_sanity_check(dev_checker_website, new_cf_version)
print_step("3b: Run Maven sanity test on development release.")
if prompt_yes_no("Run Maven sanity test on development repo?", True):
maven_sanity_check("maven-dev", "", new_cf_version)
# The Central repository is a repository of build artifacts for build programs like Maven and Ivy.
# This step stages (but doesn't release) the Checker Framework's Maven artifacts in the Sonatypes
# Central Repository.
# Once staging is complete, there are manual steps to log into Sonatypes Central and "close" the
# staging repository. Closing allows us to test the artifacts.
# This step deploys the artifacts to the Central repository and prompts the user to close the
# artifacts. Later, you will be prompted to release the staged artifacts after we push the
# release to our GitHub repositories.
# For more information on deploying to the Central Repository see:
# https://docs.sonatype.org/display/Repository/Sonatype+OSS+Maven+Repository+Usage+Guide
print_step("Push Step 4: Stage Maven artifacts in Central") # SEMIAUTO
print_step("4a: Stage the artifacts at Maven central.")
if (not test_mode) or prompt_yes_no(
"Stage Maven artifacts in Maven Central?", not test_mode):
stage_maven_artifacts_in_maven_central(new_cf_version)
print_step("4b: Close staged artifacts at Maven central.")
continue_or_exit(
"Maven artifacts have been staged! Please 'close' (but don't release) the artifacts.\n"
+ " * Browse to https://oss.sonatype.org/#stagingRepositories\n" +
" * Log in using your Sonatype credentials\n" +
' * In the search box at upper right, type "checker"\n' +
" * In the top pane, click on orgcheckerframework-XXXX\n" +
' * Click "close" at the top\n' +
" * For the close message, enter: Checker Framework release " +
new_cf_version + "\n" +
" * Click the Refresh button near the top of the page until the bottom pane has:\n"
+ ' "Activity Last operation completed successfully".\n' +
" * Copy the URL of the closed artifacts (in the bottom pane) for use in the next step\n"
"(You can also see the instructions at: http://central.sonatype.org/pages/releasing-the-deployment.html)\n"
)
print_step("4c: Run Maven sanity test on Maven central artifacts.")
if prompt_yes_no("Run Maven sanity test on Maven central artifacts?",
True):
repo_url = input(
"Please enter the repo URL of the closed artifacts:\n")
maven_sanity_check("maven-staging", repo_url, new_cf_version)
# This step copies the development release directories to the live release directories.
# It then adds the appropriate permissions to the release. Symlinks need to be updated to point
# to the live website rather than the development website. A straight copy of the directory
# will NOT update the symlinks.
print_step("Push Step 5. Copy dev current release website to live website"
) # SEMIAUTO
if not test_mode:
if prompt_yes_no("Copy release to the live website?"):
print("Copying to live site")
copy_releases_to_live_site(new_cf_version)
copy_htaccess()
ensure_group_access_to_releases()
else:
print("Test mode: Skipping copy to live site!")
# This step downloads the checker-framework-X.Y.Z.zip file of the newly live release and ensures we
# can run the Nullness Checker. If this step fails, you should backout the release.
print_step(
"Push Step 6: Run javac sanity tests on the live release.") # SEMIAUTO
if not test_mode:
if prompt_yes_no("Run javac sanity test on live release?", True):
javac_sanity_check(live_checker_website, new_cf_version)
SANITY_TEST_CHECKER_FRAMEWORK_DIR = SANITY_DIR + "/test-checker-framework"
if not os.path.isdir(SANITY_TEST_CHECKER_FRAMEWORK_DIR):
execute("mkdir -p " + SANITY_TEST_CHECKER_FRAMEWORK_DIR)
sanity_test_script = os.path.join(SCRIPTS_DIR,
"test-checker-framework.sh")
execute(
"sh " + sanity_test_script + " " + new_cf_version,
True,
False,
SANITY_TEST_CHECKER_FRAMEWORK_DIR,
)
else:
print("Test mode: Skipping javac sanity tests on the live release.")
# Runs the link the checker on all websites at:
# https://checkerframework.org/
# The output of the link checker is written to files in the /scratch/$USER/cf-release directory whose locations
# will be output at the command prompt. Review the link checker output.
# The set of broken links that is displayed by this check will differ from those in push
# step 2 because the Checker Framework manual and website uses a mix of absolute and
# relative links. Therefore, some links from the development site actually point to the
# live site (the previous release). After step 5, these links point to the current
# release and may be broken.
print_step("Push Step 7. Check live site links") # SEMIAUTO
if not test_mode:
if prompt_yes_no("Run link checker on LIVE site?", True):
check_all_links(live_afu_website, live_checker_website, "live",
test_mode)
else:
print("Test mode: Skipping checking of live site links.")
# This step pushes the changes committed to the interm repositories to the GitHub
# repositories. This is the first irreversible change. After this point, you can no longer
# backout changes and should do another release in case of critical errors.
print_step("Push Step 8. Push changes to repositories") # SEMIAUTO
# This step could be performed without asking for user input but I think we should err on the side of caution.
if not test_mode:
if prompt_yes_no(
"Push the release to GitHub repositories? This is irreversible.",
True):
push_interm_to_release_repos()
print("Pushed to repos")
else:
print("Test mode: Skipping push to GitHub!")
# This is a manual step that releases the staged Maven artifacts to the actual Central repository.
# This is also an irreversible step. Once you have released these artifacts they will be forever
# available to the Java community through the Central repository. Follow the prompts. The Maven
# artifacts (such as checker-qual.jar) are still needed, but the Maven plug-in is no longer maintained.
print_step("Push Step 9. Release staged artifacts in Central repository."
) # MANUAL
if test_mode:
msg = (
"Test Mode: You are in test_mode. Please 'DROP' the artifacts. " +
"To drop, log into https://oss.sonatype.org using your " +
"Sonatype credentials and follow the 'DROP' instructions at: " +
"http://central.sonatype.org/pages/releasing-the-deployment.html")
else:
msg = (
"Please 'release' the artifacts.\n" +
"First log into https://oss.sonatype.org using your Sonatype credentials. Go to Staging Repositories and "
+ "locate the orgcheckerframework repository and click on it.\n" +
"If you have a permissions problem, try logging out and back in.\n"
+
"Finally, click on the Release button at the top of the page. In the dialog box that pops up, "
+
'leave the "Automatically drop" box checked. For the description, write '
+ "Checker Framework release " + new_cf_version + "\n\n")
print(msg)
prompt_to_continue()
if test_mode:
print("Test complete")
else:
# A prompt describes the email you should send to all relevant mailing lists.
# Please fill out the email and announce the release.
print_step(
"Push Step 10. Post the Checker Framework and Annotation File Utilities releases on GitHub."
) # MANUAL
msg = (
"\n" + "* Download the following files to your local machine." +
"\n" + "https://checkerframework.org/checker-framework-" +
new_cf_version + ".zip\n" +
"https://checkerframework.org/annotation-file-utilities/annotation-tools-"
+ new_cf_version + ".zip\n" + "\n" +
"To post the Checker Framework release on GitHub:\n" + "\n" +
"* Go to https://github.com/typetools/checker-framework/releases/new?tag=checker-framework-"
+ new_cf_version + "\n" +
"* For the release title, enter: Checker Framework " +
new_cf_version + "\n" +
"* For the description, insert the latest Checker Framework changelog entry (available at https://checkerframework.org/CHANGELOG.md). Please include the first line with the release version and date.\n"
+
'* Find the link below "Attach binaries by dropping them here or selecting them." Click on "selecting them" and upload checker-framework-'
+ new_cf_version + ".zip from your machine.\n" +
'* Click on the green "Publish release" button.\n' + "\n" +
"To post the Annotation File Utilities release on GitHub:\n" +
"\n" +
"* Go to https://github.com/typetools/annotation-tools/releases/new?tag="
+ new_cf_version + "\n" +
"* For the release title, enter: Annotation File Utilities " +
new_cf_version + "\n" +
"* For the description, insert the latest Annotation File Utilities changelog entry (available at https://checkerframework.org/annotation-file-utilities/changelog.html). Please include the first line with the release version and date. For bullet points, use the * Markdown character.\n"
+
'* Find the link below "Attach binaries by dropping them here or selecting them." Click on "selecting them" and upload annotation-tools-'
+ new_cf_version + ".zip from your machine.\n" +
'* Click on the green "Publish release" button.\n')
print(msg)
print_step("Push Step 11. Announce the release.") # MANUAL
continue_or_exit(
"Please announce the release using the email structure below.\n" +
get_announcement_email(new_cf_version))
print_step("Push Step 12. Update the Checker Framework Gradle plugin."
) # MANUAL
continue_or_exit(
"Please update the Checker Framework Gradle plugin:\n" +
"https://github.com/kelloggm/checkerframework-gradle-plugin/blob/master/RELEASE.md#updating-the-checker-framework-version\n"
)
print_step(
"Push Step 13. Prep for next Checker Framework release.") # MANUAL
continue_or_exit(
"Change the patch level (last number) of the Checker Framework version\nin build.gradle: increment it and add -SNAPSHOT\n"
)
delete_if_exists(RELEASE_BUILD_COMPLETED_FLAG_FILE)
print("Done with release_push.py")
def push_changes(repo_root):
"""Pushes changes, including tags, that were committed to the repository at
the given filesystem path."""
execute("git push --tags", working_dir=repo_root)
execute("git push origin master", working_dir=repo_root)
def copy_htaccess():
"Copy the .htaccess file from the dev site to the live site."
LIVE_HTACCESS = os.path.join(LIVE_SITE_DIR, ".htaccess")
execute("rsync --times %s %s" %
(os.path.join(DEV_SITE_DIR, ".htaccess"), LIVE_HTACCESS))
ensure_group_access(LIVE_HTACCESS)
def main(argv):
"""The release_build script is responsible for building the release
artifacts for the AFU and the Checker Framework projects
and placing them in the development web site. It can also be used to review
the documentation and changelogs for the three projects."""
# MANUAL Indicates a manual step
# AUTO Indicates the step is fully automated.
delete_if_exists(RELEASE_BUILD_COMPLETED_FLAG_FILE)
set_umask()
global debug
global ant_debug
debug = read_command_line_option(argv, "--debug")
if debug:
ant_debug = "-debug"
global notest
notest = read_command_line_option(argv, "--notest")
afu_date = get_afu_date()
# For each project, build what is necessary but don't push
print(
"Building a new release of Annotation Tools and the Checker Framework!"
)
print("\nPATH:\n" + os.environ["PATH"] + "\n")
print_step("Build Step 1: Clone the build and intermediate repositories."
) # MANUAL
# Recall that there are 3 relevant sets of repositories for the release:
# * build repository - repository where the project is built for release
# * intermediate repository - repository to which release related changes are pushed after the project is built
# * release repository - GitHub repositories, the central repository.
# Every time we run release_build, changes are committed to the intermediate repository from build but NOT to
# the release repositories. If we are running the build script multiple times without actually committing the
# release then these changes need to be cleaned before we run the release_build script again.
# The "Clone/update repositories" step updates the repositories with respect to the live repositories on
# GitHub, but it is the "Verify repositories" step that ensures that they are clean,
# i.e. indistinguishable from a freshly cloned repository.
# check we are cloning LIVE -> INTERM, INTERM -> RELEASE
print_step("\n1a: Clone/update repositories.") # MANUAL
clone_or_update_repos()
# This step ensures the previous step worked. It checks to see if we have any modified files, untracked files,
# or outgoing changesets. If so, it fails.
print_step("1b: Verify repositories.") # MANUAL
check_repos(INTERM_REPOS, True, True)
check_repos(BUILD_REPOS, True, False)
# The release script requires a number of common tools (Ant, Maven, make, etc...). This step checks
# to make sure all tools are available on the command line in order to avoid wasting time in the
# event a tool is missing late in execution.
print_step("Build Step 2: Check tools.") # AUTO
check_tools(TOOLS)
# Usually we increment the release by 0.0.1 per release unless there is a major change.
# The release script will read the current version of the Checker Framework/Annotation File Utilities
# from the release website and then suggest the next release version 0.0.1 higher than the current
# version. You can also manually specify a version higher than the current version. Lower or equivalent
# versions are not possible and will be rejected when you try to push the release.
print_step("Build Step 3: Determine release versions.") # MANUAL
old_cf_version = current_distribution_by_website(LIVE_SITE_URL)
cf_version = CF_VERSION
print("Version: " + cf_version + "\n")
if old_cf_version == cf_version:
print((
"It is *strongly discouraged* to not update the release version numbers for the Checker Framework "
+
"even if no changes were made to these in a month. This would break so much "
+
"in the release scripts that they would become unusable. Update the version number in checker-framework/build.gradle\n"
))
prompt_to_continue()
print_step(
"Build Step 4: Create directories for the current release on the dev site."
) # AUTO
(
afu_interm_dir,
checker_framework_interm_dir,
) = create_dirs_for_dev_website_release_versions(cf_version)
# The projects are built in the following order:
# Annotation File Utilities and Checker Framework. Furthermore, their
# manuals and websites are also built and placed in their relevant locations
# at https://checkerframework.org/dev/ . This is the most time-consuming
# piece of the release. There are no prompts from this step forward; you
# might want to get a cup of coffee and do something else until it is done.
print_step("Build Step 5: Build projects and websites.") # AUTO
print_step("5a: Build Annotation File Utilities.")
build_annotation_tools_release(cf_version, afu_interm_dir)
print_step("5b: Build Checker Framework.")
build_checker_framework_release(
cf_version,
old_cf_version,
afu_date,
checker_framework_interm_dir,
)
print_step("Build Step 6: Overwrite .htaccess and CFLogo.png .") # AUTO
# Not "cp -p" because that does not work across filesystems whereas rsync does
CFLOGO = os.path.join(CHECKER_FRAMEWORK, "docs", "logo", "Logo",
"CFLogo.png")
execute("rsync --times %s %s" % (CFLOGO, checker_framework_interm_dir))
# Each project has a set of files that are updated for release. Usually these updates include new
# release date and version information. All changed files are committed and pushed to the intermediate
# repositories. Keep this in mind if you have any changed files from steps 1d, 4, or 5. Edits to the
# scripts in the cf-release/scripts directory will never be checked in.
print_step("Build Step 7: Commit projects to intermediate repos.") # AUTO
commit_to_interm_projects(cf_version)
# Adds read/write/execute group permissions to all of the new dev website directories
# under https://checkerframework.org/dev/ These directories need group read/execute
# permissions in order for them to be served.
print_step("\n\nBuild Step 8: Add group permissions to repos.")
for build in BUILD_REPOS:
ensure_group_access(build)
for interm in INTERM_REPOS:
ensure_group_access(interm)
# At the moment, this will lead to output error messages because some metadata in some of the
# dirs I think is owned by Mike or Werner. We should identify these and have them fix it.
# But as long as the processes return a zero exit status, we should be ok.
print_step("\n\nBuild Step 9: Add group permissions to websites.") # AUTO
ensure_group_access(DEV_SITE_DIR)
create_empty_file(RELEASE_BUILD_COMPLETED_FLAG_FILE)
def build_checker_framework_release(version, old_cf_version, afu_release_date,
checker_framework_interm_dir):
"""Build the release files for the Checker Framework project, including the
manual and the zip file, and run tests on the build."""
checker_dir = os.path.join(CHECKER_FRAMEWORK, "checker")
afu_build_properties = os.path.join(ANNO_FILE_UTILITIES,
"build.properties")
# build stubparser
execute("mvn package -Dmaven.test.skip=true", True, False, STUBPARSER)
# build annotation-tools
execute("./gradlew assemble -Prelease=true", True, False,
ANNO_FILE_UTILITIES)
# update versions
ant_props = (
'-Dchecker=%s -Drelease.ver=%s -Dafu.version=%s -Dafu.properties=%s -Dafu.release.date="%s"'
% (checker_dir, version, version, afu_build_properties,
afu_release_date))
# IMPORTANT: The release.xml in the directory where the Checker Framework is being built is used. Not the release.xml in the directory you ran release_build.py from.
ant_cmd = "ant %s -f release.xml %s update-checker-framework-versions " % (
ant_debug,
ant_props,
)
execute(ant_cmd, True, False, CHECKER_FRAMEWORK_RELEASE)
# Check that updating versions didn't overlook anything.
print("Here are occurrences of the old version number, " + old_cf_version)
grep_cmd = "grep -n -r --exclude-dir=build --exclude-dir=.git -F %s" % old_cf_version
execute(grep_cmd, False, False, CHECKER_FRAMEWORK)
continue_or_exit(
'If any occurrence is not acceptable, then stop the release, update target "update-checker-framework-versions" in file release.xml, and start over.'
)
# build the checker framework binaries and documents. Tests are run by release_push.py
gradle_cmd = "./gradlew releaseBuild"
execute(gradle_cmd, True, False, CHECKER_FRAMEWORK)
# make the Checker Framework Manual
checker_manual_dir = os.path.join(CHECKER_FRAMEWORK, "docs", "manual")
execute("make manual.pdf manual.html", True, False, checker_manual_dir)
# make the dataflow manual
dataflow_manual_dir = os.path.join(CHECKER_FRAMEWORK, "dataflow", "manual")
execute("make", True, False, dataflow_manual_dir)
# make the checker framework tutorial
checker_tutorial_dir = os.path.join(CHECKER_FRAMEWORK, "docs", "tutorial")
execute("make", True, False, checker_tutorial_dir)
cfZipName = "checker-framework-%s.zip" % version
# Create checker-framework-X.Y.Z.zip and put it in checker_framework_interm_dir
ant_props = "-Dchecker=%s -Ddest.dir=%s -Dfile.name=%s -Dversion=%s" % (
checker_dir,
checker_framework_interm_dir,
cfZipName,
version,
)
# IMPORTANT: The release.xml in the directory where the Checker Framework is being built is used. Not the release.xml in the directory you ran release_build.py from.
ant_cmd = "ant %s -f release.xml %s zip-checker-framework " % (ant_debug,
ant_props)
execute(ant_cmd, True, False, CHECKER_FRAMEWORK_RELEASE)
ant_props = "-Dchecker=%s -Ddest.dir=%s -Dfile.name=%s -Dversion=%s" % (
checker_dir,
checker_framework_interm_dir,
"mvn-examples.zip",
version,
)
# IMPORTANT: The release.xml in the directory where the Checker Framework is being built is used. Not the release.xml in the directory you ran release_build.py from.
ant_cmd = "ant %s -f release.xml %s zip-maven-examples " % (ant_debug,
ant_props)
execute(ant_cmd, True, False, CHECKER_FRAMEWORK_RELEASE)
# copy the remaining checker-framework website files to checker_framework_interm_dir
ant_props = "-Dchecker=%s -Ddest.dir=%s -Dmanual.name=%s -Ddataflow.manual.name=%s -Dchecker.webpage=%s" % (
checker_dir,
checker_framework_interm_dir,
"checker-framework-manual",
"checker-framework-dataflow-manual",
"checker-framework-webpage.html",
)
# IMPORTANT: The release.xml in the directory where the Checker Framework is being built is used. Not the release.xml in the directory you ran release_build.py from.
ant_cmd = "ant %s -f release.xml %s checker-framework-website-docs " % (
ant_debug,
ant_props,
)
execute(ant_cmd, True, False, CHECKER_FRAMEWORK_RELEASE)
# clean no longer necessary files left over from building the checker framework tutorial
checker_tutorial_dir = os.path.join(CHECKER_FRAMEWORK, "docs", "tutorial")
execute("make clean", True, False, checker_tutorial_dir)
build_and_locally_deploy_maven(version)
update_project_dev_website("checker-framework", version)
return
def build_and_locally_deploy_maven(version):
execute("./gradlew publishToMavenLocal", working_dir=CHECKER_FRAMEWORK)
def commit_tag_and_push(version, path, tag_prefix):
"""Commit the changes made for this release, add a tag for this release, and
push these changes."""
execute('git commit -a -m "new release %s"' % (version), working_dir=path)
execute("git tag %s%s" % (tag_prefix, version), working_dir=path)
push_changes(path)
def wget_file(source_url, destination_dir):
"""Download a file from the source URL to the given destination directory.
Useful since download_binary does not seem to work on source files."""
print("DEST DIR: " + destination_dir)
execute("wget %s" % source_url, True, False, destination_dir)
def ensure_group_access(path):
"Give group access to all files and directories under the specified path"
# Errs for any file not owned by this user.
# But, the point is to set group writeability of any *new* files.
execute("chmod -f -R g+rw %s" % path, halt_if_fail=False)
