def run_link_checker(site, output, additional_param=""):
"""Runs the link checker on the given web site and saves the output to the
given file. Additional parameters (if given) are passed directly to the
link checker script."""
delete_if_exists(output)
check_links_script = os.path.join(SCRIPTS_DIR, "checkLinks.sh")
if additional_param == "":
cmd = ["sh", check_links_script, site]
else:
cmd = ["sh", check_links_script, additional_param, site]
env = {"CHECKLINK": CHECKLINK}
out_file = open(output, "w+")
print(("Executing: " + " ".join("%s=%r" % (key2, val2)
for (key2, val2) in list(env.items())) +
" " + " ".join(cmd)))
process = subprocess.Popen(cmd, env=env, stdout=out_file, stderr=out_file)
process.communicate()
process.wait()
out_file.close()
if process.returncode != 0:
raise Exception(
"Non-zero return code (%s; see output in %s) while executing %s" %
(process.returncode, output, cmd))
return output
def main(argv):
"""The release_push script is mainly responsible for copying the artifacts
(for the AFU and the Checker Framework) from the
development web site to Maven Central and to
the live site. It also performs link checking on the live site, pushes
the release to GitHub repositories, and guides the user to
perform manual steps such as sending the
release announcement e-mail."""
# MANUAL Indicates a manual step
# AUTO Indicates the step is fully automated.
set_umask()
validate_args(argv)
test_mode = not read_command_line_option(argv, "release")
m2_settings = expanduser("~") + "/.m2/settings.xml"
if not os.path.exists(m2_settings):
raise Exception("File does not exist: " + m2_settings)
if test_mode:
msg = (
"You have chosen test_mode.\n" +
"This means that this script will execute all build steps that " +
"do not have side effects. That is, this is a test run of the script. All checks and user prompts "
+
"will be shown but no steps will be executed that will cause the release to be deployed or partially "
+ "deployed.\n" +
'If you meant to do an actual release, re-run this script with one argument, "release".'
)
else:
msg = "You have chosen release_mode. Please follow the prompts to run a full Checker Framework release."
continue_or_exit(msg + "\n")
if test_mode:
print("Continuing in test mode.")
else:
print("Continuing in release mode.")
if not os.path.exists(RELEASE_BUILD_COMPLETED_FLAG_FILE):
continue_or_exit(
"It appears that release_build.py has not been run since the last push to "
+
"the AFU or Checker Framework repositories. Please ensure it has "
+ "been run.")
# The release script checks that the new release version is greater than the previous release version.
print_step("Push Step 1: Checking release versions") # SEMIAUTO
dev_afu_website = os.path.join(DEV_SITE_URL, "annotation-file-utilities")
live_afu_website = os.path.join(LIVE_SITE_URL, "annotation-file-utilities")
dev_checker_website = DEV_SITE_URL
live_checker_website = LIVE_SITE_URL
current_cf_version = current_distribution_by_website(live_checker_website)
new_cf_version = CF_VERSION
check_release_version(current_cf_version, new_cf_version)
print("Checker Framework and AFU: current-version=%s new-version=%s" %
(current_cf_version, new_cf_version))
# Runs the link the checker on all websites at:
# https://checkerframework.org/dev/
# The output of the link checker is written to files in the /scratch/$USER/cf-release directory
# whose locations will be output at the command prompt if the link checker reported errors.
# In rare instances (such as when a link is correct but the link checker is
# unable to check it), you may add a suppression to the checklink-args.txt file.
# In extremely rare instances (such as when a website happens to be down at the
# time you ran the link checker), you may ignore an error.
print_step("Push Step 2: Check links on development site") # SEMIAUTO
if prompt_yes_no("Run link checker on DEV site?", True):
check_all_links(dev_afu_website, dev_checker_website, "dev", test_mode,
new_cf_version)
# Runs sanity tests on the development release. Later, we will run a smaller set of sanity
# tests on the live release to ensure no errors occurred when promoting the release.
print_step("Push Step 3: Run development sanity tests") # SEMIAUTO
if prompt_yes_no("Perform this step?", True):
print_step("3a: Run javac sanity test on development release.")
if prompt_yes_no("Run javac sanity test on development release?",
True):
javac_sanity_check(dev_checker_website, new_cf_version)
print_step("3b: Run Maven sanity test on development release.")
if prompt_yes_no("Run Maven sanity test on development repo?", True):
maven_sanity_check("maven-dev", "", new_cf_version)
# The Central repository is a repository of build artifacts for build programs like Maven and Ivy.
# This step stages (but doesn't release) the Checker Framework's Maven artifacts in the Sonatypes
# Central Repository.
# Once staging is complete, there are manual steps to log into Sonatypes Central and "close" the
# staging repository. Closing allows us to test the artifacts.
# This step deploys the artifacts to the Central repository and prompts the user to close the
# artifacts. Later, you will be prompted to release the staged artifacts after we push the
# release to our GitHub repositories.
# For more information on deploying to the Central Repository see:
# https://docs.sonatype.org/display/Repository/Sonatype+OSS+Maven+Repository+Usage+Guide
print_step("Push Step 4: Stage Maven artifacts in Central") # SEMIAUTO
print_step("4a: Stage the artifacts at Maven central.")
if (not test_mode) or prompt_yes_no(
"Stage Maven artifacts in Maven Central?", not test_mode):
stage_maven_artifacts_in_maven_central(new_cf_version)
print_step("4b: Close staged artifacts at Maven central.")
continue_or_exit(
"Maven artifacts have been staged! Please 'close' (but don't release) the artifacts.\n"
+ " * Browse to https://oss.sonatype.org/#stagingRepositories\n" +
" * Log in using your Sonatype credentials\n" +
' * In the search box at upper right, type "checker"\n' +
" * In the top pane, click on orgcheckerframework-XXXX\n" +
' * Click "close" at the top\n' +
" * For the close message, enter: Checker Framework release " +
new_cf_version + "\n" +
" * Click the Refresh button near the top of the page until the bottom pane has:\n"
+ ' "Activity Last operation completed successfully".\n' +
" * Copy the URL of the closed artifacts (in the bottom pane) for use in the next step\n"
"(You can also see the instructions at: http://central.sonatype.org/pages/releasing-the-deployment.html)\n"
)
print_step("4c: Run Maven sanity test on Maven central artifacts.")
if prompt_yes_no("Run Maven sanity test on Maven central artifacts?",
True):
repo_url = input(
"Please enter the repo URL of the closed artifacts:\n")
maven_sanity_check("maven-staging", repo_url, new_cf_version)
# This step copies the development release directories to the live release directories.
# It then adds the appropriate permissions to the release. Symlinks need to be updated to point
# to the live website rather than the development website. A straight copy of the directory
# will NOT update the symlinks.
print_step("Push Step 5. Copy dev current release website to live website"
) # SEMIAUTO
if not test_mode:
if prompt_yes_no("Copy release to the live website?"):
print("Copying to live site")
copy_releases_to_live_site(new_cf_version)
copy_htaccess()
ensure_group_access_to_releases()
else:
print("Test mode: Skipping copy to live site!")
# This step downloads the checker-framework-X.Y.Z.zip file of the newly live release and ensures we
# can run the Nullness Checker. If this step fails, you should backout the release.
print_step(
"Push Step 6: Run javac sanity tests on the live release.") # SEMIAUTO
if not test_mode:
if prompt_yes_no("Run javac sanity test on live release?", True):
javac_sanity_check(live_checker_website, new_cf_version)
SANITY_TEST_CHECKER_FRAMEWORK_DIR = SANITY_DIR + "/test-checker-framework"
if not os.path.isdir(SANITY_TEST_CHECKER_FRAMEWORK_DIR):
execute("mkdir -p " + SANITY_TEST_CHECKER_FRAMEWORK_DIR)
sanity_test_script = os.path.join(SCRIPTS_DIR,
"test-checker-framework.sh")
execute(
"sh " + sanity_test_script + " " + new_cf_version,
True,
False,
SANITY_TEST_CHECKER_FRAMEWORK_DIR,
)
else:
print("Test mode: Skipping javac sanity tests on the live release.")
# Runs the link the checker on all websites at:
# https://checkerframework.org/
# The output of the link checker is written to files in the /scratch/$USER/cf-release directory whose locations
# will be output at the command prompt. Review the link checker output.
# The set of broken links that is displayed by this check will differ from those in push
# step 2 because the Checker Framework manual and website uses a mix of absolute and
# relative links. Therefore, some links from the development site actually point to the
# live site (the previous release). After step 5, these links point to the current
# release and may be broken.
print_step("Push Step 7. Check live site links") # SEMIAUTO
if not test_mode:
if prompt_yes_no("Run link checker on LIVE site?", True):
check_all_links(live_afu_website, live_checker_website, "live",
test_mode)
else:
print("Test mode: Skipping checking of live site links.")
# This step pushes the changes committed to the interm repositories to the GitHub
# repositories. This is the first irreversible change. After this point, you can no longer
# backout changes and should do another release in case of critical errors.
print_step("Push Step 8. Push changes to repositories") # SEMIAUTO
# This step could be performed without asking for user input but I think we should err on the side of caution.
if not test_mode:
if prompt_yes_no(
"Push the release to GitHub repositories? This is irreversible.",
True):
push_interm_to_release_repos()
print("Pushed to repos")
else:
print("Test mode: Skipping push to GitHub!")
# This is a manual step that releases the staged Maven artifacts to the actual Central repository.
# This is also an irreversible step. Once you have released these artifacts they will be forever
# available to the Java community through the Central repository. Follow the prompts. The Maven
# artifacts (such as checker-qual.jar) are still needed, but the Maven plug-in is no longer maintained.
print_step("Push Step 9. Release staged artifacts in Central repository."
) # MANUAL
if test_mode:
msg = (
"Test Mode: You are in test_mode. Please 'DROP' the artifacts. " +
"To drop, log into https://oss.sonatype.org using your " +
"Sonatype credentials and follow the 'DROP' instructions at: " +
"http://central.sonatype.org/pages/releasing-the-deployment.html")
else:
msg = (
"Please 'release' the artifacts.\n" +
"First log into https://oss.sonatype.org using your Sonatype credentials. Go to Staging Repositories and "
+ "locate the orgcheckerframework repository and click on it.\n" +
"If you have a permissions problem, try logging out and back in.\n"
+
"Finally, click on the Release button at the top of the page. In the dialog box that pops up, "
+
'leave the "Automatically drop" box checked. For the description, write '
+ "Checker Framework release " + new_cf_version + "\n\n")
print(msg)
prompt_to_continue()
if test_mode:
print("Test complete")
else:
# A prompt describes the email you should send to all relevant mailing lists.
# Please fill out the email and announce the release.
print_step(
"Push Step 10. Post the Checker Framework and Annotation File Utilities releases on GitHub."
) # MANUAL
msg = (
"\n" + "* Download the following files to your local machine." +
"\n" + "https://checkerframework.org/checker-framework-" +
new_cf_version + ".zip\n" +
"https://checkerframework.org/annotation-file-utilities/annotation-tools-"
+ new_cf_version + ".zip\n" + "\n" +
"To post the Checker Framework release on GitHub:\n" + "\n" +
"* Go to https://github.com/typetools/checker-framework/releases/new?tag=checker-framework-"
+ new_cf_version + "\n" +
"* For the release title, enter: Checker Framework " +
new_cf_version + "\n" +
"* For the description, insert the latest Checker Framework changelog entry (available at https://checkerframework.org/CHANGELOG.md). Please include the first line with the release version and date.\n"
+
'* Find the link below "Attach binaries by dropping them here or selecting them." Click on "selecting them" and upload checker-framework-'
+ new_cf_version + ".zip from your machine.\n" +
'* Click on the green "Publish release" button.\n' + "\n" +
"To post the Annotation File Utilities release on GitHub:\n" +
"\n" +
"* Go to https://github.com/typetools/annotation-tools/releases/new?tag="
+ new_cf_version + "\n" +
"* For the release title, enter: Annotation File Utilities " +
new_cf_version + "\n" +
"* For the description, insert the latest Annotation File Utilities changelog entry (available at https://checkerframework.org/annotation-file-utilities/changelog.html). Please include the first line with the release version and date. For bullet points, use the * Markdown character.\n"
+
'* Find the link below "Attach binaries by dropping them here or selecting them." Click on "selecting them" and upload annotation-tools-'
+ new_cf_version + ".zip from your machine.\n" +
'* Click on the green "Publish release" button.\n')
print(msg)
print_step("Push Step 11. Announce the release.") # MANUAL
continue_or_exit(
"Please announce the release using the email structure below.\n" +
get_announcement_email(new_cf_version))
print_step("Push Step 12. Update the Checker Framework Gradle plugin."
) # MANUAL
continue_or_exit(
"Please update the Checker Framework Gradle plugin:\n" +
"https://github.com/kelloggm/checkerframework-gradle-plugin/blob/master/RELEASE.md#updating-the-checker-framework-version\n"
)
print_step(
"Push Step 13. Prep for next Checker Framework release.") # MANUAL
continue_or_exit(
"Change the patch level (last number) of the Checker Framework version\nin build.gradle: increment it and add -SNAPSHOT\n"
)
delete_if_exists(RELEASE_BUILD_COMPLETED_FLAG_FILE)
print("Done with release_push.py")
def main(argv):
"""The release_build script is responsible for building the release
artifacts for the AFU and the Checker Framework projects
and placing them in the development web site. It can also be used to review
the documentation and changelogs for the three projects."""
# MANUAL Indicates a manual step
# AUTO Indicates the step is fully automated.
delete_if_exists(RELEASE_BUILD_COMPLETED_FLAG_FILE)
set_umask()
global debug
global ant_debug
debug = read_command_line_option(argv, "--debug")
if debug:
ant_debug = "-debug"
global notest
notest = read_command_line_option(argv, "--notest")
afu_date = get_afu_date()
# For each project, build what is necessary but don't push
print(
"Building a new release of Annotation Tools and the Checker Framework!"
)
print("\nPATH:\n" + os.environ["PATH"] + "\n")
print_step("Build Step 1: Clone the build and intermediate repositories."
) # MANUAL
# Recall that there are 3 relevant sets of repositories for the release:
# * build repository - repository where the project is built for release
# * intermediate repository - repository to which release related changes are pushed after the project is built
# * release repository - GitHub repositories, the central repository.
# Every time we run release_build, changes are committed to the intermediate repository from build but NOT to
# the release repositories. If we are running the build script multiple times without actually committing the
# release then these changes need to be cleaned before we run the release_build script again.
# The "Clone/update repositories" step updates the repositories with respect to the live repositories on
# GitHub, but it is the "Verify repositories" step that ensures that they are clean,
# i.e. indistinguishable from a freshly cloned repository.
# check we are cloning LIVE -> INTERM, INTERM -> RELEASE
print_step("\n1a: Clone/update repositories.") # MANUAL
clone_or_update_repos()
# This step ensures the previous step worked. It checks to see if we have any modified files, untracked files,
# or outgoing changesets. If so, it fails.
print_step("1b: Verify repositories.") # MANUAL
check_repos(INTERM_REPOS, True, True)
check_repos(BUILD_REPOS, True, False)
# The release script requires a number of common tools (Ant, Maven, make, etc...). This step checks
# to make sure all tools are available on the command line in order to avoid wasting time in the
# event a tool is missing late in execution.
print_step("Build Step 2: Check tools.") # AUTO
check_tools(TOOLS)
# Usually we increment the release by 0.0.1 per release unless there is a major change.
# The release script will read the current version of the Checker Framework/Annotation File Utilities
# from the release website and then suggest the next release version 0.0.1 higher than the current
# version. You can also manually specify a version higher than the current version. Lower or equivalent
# versions are not possible and will be rejected when you try to push the release.
print_step("Build Step 3: Determine release versions.") # MANUAL
old_cf_version = current_distribution_by_website(LIVE_SITE_URL)
cf_version = CF_VERSION
print("Version: " + cf_version + "\n")
if old_cf_version == cf_version:
print((
"It is *strongly discouraged* to not update the release version numbers for the Checker Framework "
+
"even if no changes were made to these in a month. This would break so much "
+
"in the release scripts that they would become unusable. Update the version number in checker-framework/build.gradle\n"
))
prompt_to_continue()
print_step(
"Build Step 4: Create directories for the current release on the dev site."
) # AUTO
(
afu_interm_dir,
checker_framework_interm_dir,
) = create_dirs_for_dev_website_release_versions(cf_version)
# The projects are built in the following order:
# Annotation File Utilities and Checker Framework. Furthermore, their
# manuals and websites are also built and placed in their relevant locations
# at https://checkerframework.org/dev/ . This is the most time-consuming
# piece of the release. There are no prompts from this step forward; you
# might want to get a cup of coffee and do something else until it is done.
print_step("Build Step 5: Build projects and websites.") # AUTO
print_step("5a: Build Annotation File Utilities.")
build_annotation_tools_release(cf_version, afu_interm_dir)
print_step("5b: Build Checker Framework.")
build_checker_framework_release(
cf_version,
old_cf_version,
afu_date,
checker_framework_interm_dir,
)
print_step("Build Step 6: Overwrite .htaccess and CFLogo.png .") # AUTO
# Not "cp -p" because that does not work across filesystems whereas rsync does
CFLOGO = os.path.join(CHECKER_FRAMEWORK, "docs", "logo", "Logo",
"CFLogo.png")
execute("rsync --times %s %s" % (CFLOGO, checker_framework_interm_dir))
# Each project has a set of files that are updated for release. Usually these updates include new
# release date and version information. All changed files are committed and pushed to the intermediate
# repositories. Keep this in mind if you have any changed files from steps 1d, 4, or 5. Edits to the
# scripts in the cf-release/scripts directory will never be checked in.
print_step("Build Step 7: Commit projects to intermediate repos.") # AUTO
commit_to_interm_projects(cf_version)
# Adds read/write/execute group permissions to all of the new dev website directories
# under https://checkerframework.org/dev/ These directories need group read/execute
# permissions in order for them to be served.
print_step("\n\nBuild Step 8: Add group permissions to repos.")
for build in BUILD_REPOS:
ensure_group_access(build)
for interm in INTERM_REPOS:
ensure_group_access(interm)
# At the moment, this will lead to output error messages because some metadata in some of the
# dirs I think is owned by Mike or Werner. We should identify these and have them fix it.
# But as long as the processes return a zero exit status, we should be ok.
print_step("\n\nBuild Step 9: Add group permissions to websites.") # AUTO
ensure_group_access(DEV_SITE_DIR)
create_empty_file(RELEASE_BUILD_COMPLETED_FLAG_FILE)