def main(personal_access_token):
"""Updates the files corresponding to LOCAL_FECONF_PATH and
LOCAL_CONSTANTS_PATH after doing the prerequisite checks.
Args:
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.is_current_branch_a_release_branch()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
try:
python_utils.url_open(TERMS_PAGE_URL)
except Exception:
raise Exception('Terms mainpage does not exist on Github.')
try:
check_updates_to_terms_of_service(personal_access_token)
add_mailgun_api_key()
apply_changes_based_on_config(LOCAL_FECONF_PATH, FECONF_CONFIG_PATH,
FECONF_REGEX)
apply_changes_based_on_config(LOCAL_CONSTANTS_PATH,
CONSTANTS_CONFIG_PATH, CONSTANTS_REGEX)
except Exception as e:
common.run_cmd(
['git', 'checkout', '--', LOCAL_FECONF_PATH, LOCAL_CONSTANTS_PATH])
raise Exception(e)
common.ask_user_to_confirm(
'Done! Please check manually to ensure all the changes are correct.')
def main():
"""Performs task to generate message for release announcement."""
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
if not os.path.exists(release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception('Release summary file %s is missing. Please run the '
'release_info.py script and re-run this script.' %
(release_constants.RELEASE_SUMMARY_FILEPATH))
draft_new_release()
prompt_user_to_send_announcement_email()
prepare_for_next_release()
def main():
"""Performs task to wrap up the release."""
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
if not os.path.exists(release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception('Release summary file %s is missing. Please run the '
'release_info.py script and re-run this script.' %
(release_constants.RELEASE_SUMMARY_FILEPATH))
personal_access_token = common.get_personal_access_token()
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
remove_blocking_bugs_milestone_from_issues(repo)
remove_release_labels(repo)
def update_indexes():
"""Updates production indexes after doing the prerequisite checks."""
parsed_args = _PARSER.parse_args()
if parsed_args.app_name:
app_name = parsed_args.app_name
else:
raise Exception('No app name specified.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
gcloud_adapter.require_gcloud_to_be_available()
if not common.is_current_branch_a_release_branch():
raise Exception(
'Indexes should only be updated from a release branch.')
# Update the indexes.
gcloud_adapter.update_indexes(INDEX_YAML_PATH, app_name)
def main():
"""Performs task to generate message for release announcement."""
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
if not os.path.exists(release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception('Release summary file %s is missing. Please run the '
'release_info.py script and re-run this script.' %
(release_constants.RELEASE_SUMMARY_FILEPATH))
try:
create_new_file_with_release_message_template()
validate_release_message()
prompt_user_to_send_announcement_email()
finally:
if os.path.exists(RELEASE_MAIL_MESSAGE_FILEPATH):
os.remove(RELEASE_MAIL_MESSAGE_FILEPATH)
prepare_for_next_release()
def main(personal_access_token):
"""Updates the files corresponding to LOCAL_FECONF_PATH and
LOCAL_CONSTANTS_PATH after doing the prerequisite checks.
Args:
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.is_current_branch_a_release_branch(), (
'Current branch is not a release branch_name')
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
try:
python_utils.url_open(TERMS_PAGE_URL)
except Exception:
raise Exception('Terms mainpage does not exist on Github.')
try:
check_updates_to_terms_of_service(personal_access_token)
add_mailgun_api_key()
apply_changes_based_on_config(
LOCAL_FECONF_PATH, FECONF_CONFIG_PATH, FECONF_REGEX)
apply_changes_based_on_config(
LOCAL_CONSTANTS_PATH, CONSTANTS_CONFIG_PATH, CONSTANTS_REGEX)
except Exception as e:
common.run_cmd([
'git', 'checkout', '--', LOCAL_FECONF_PATH, LOCAL_CONSTANTS_PATH])
raise Exception(e)
common.ask_user_to_confirm(
'Done! Please check feconf.py and assets/constants.ts to ensure that '
'the changes made are correct. Specifically verify that the '
'MAILGUN_API_KEY and REDISHOST are updated correctly and '
'other config changes are corresponding to %s and %s.\n' % (
FECONF_CONFIG_PATH, CONSTANTS_CONFIG_PATH))
def execute_deployment():
"""Executes the deployment process after doing the prerequisite checks.
Raises:
Exception. App name is invalid.
Exception. Custom version is used with production app.
Exception. App name is not specified.
Exception. The deployment script is not run from a release or test
branch.
Exception. The deployment script is run for prod server from a test
branch.
Exception. Current release version has '.' character.
Exception. Last commit message is invalid.
Exception. The mailgun API key is not added before deployment.
Exception. Could not find third party directory.
Exception. Invalid directory accessed during deployment.
"""
parsed_args = _PARSER.parse_args()
custom_version = None
if parsed_args.app_name:
app_name = parsed_args.app_name
if app_name not in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
] and ('migration' not in app_name):
raise Exception('Invalid app name: %s' % app_name)
if parsed_args.version and app_name == APP_NAME_OPPIASERVER:
raise Exception('Cannot use custom version with production app.')
# Note that custom_version may be None.
custom_version = parsed_args.version
else:
raise Exception('No app name specified.')
current_branch_name = common.get_current_branch_name()
release_dir_name = 'deploy-%s-%s-%s' % (
'-'.join('-'.join(app_name.split('.')).split(':')),
current_branch_name, CURRENT_DATETIME.strftime('%Y%m%d-%H%M%S'))
release_dir_path = os.path.join(os.getcwd(), '..', release_dir_name)
deploy_data_path = os.path.join(os.getcwd(), os.pardir, 'release-scripts',
'deploy_data', app_name)
install_third_party_libs.main()
if not (common.is_current_branch_a_release_branch() or
(common.is_current_branch_a_test_branch())):
raise Exception(
'The deployment script must be run from a release or test branch.')
if common.is_current_branch_a_test_branch() and (app_name in [
APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
]):
raise Exception('Test branch can only be deployed to backup server.')
if custom_version is not None:
current_release_version = custom_version.replace(DOT_CHAR, HYPHEN_CHAR)
else:
current_release_version = current_branch_name[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace(
DOT_CHAR, HYPHEN_CHAR)
# This is required to compose the release_version_library_url
# (defined in switch_version function) correctly.
if '.' in current_release_version:
raise Exception('Current release version has \'.\' character.')
assert len(current_release_version) <= 25, (
'The length of the "version" arg should be less than or '
'equal to 25 characters.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
try:
if app_name == APP_NAME_OPPIASERVER:
check_release_doc()
release_version_number = common.get_current_release_version_number(
current_branch_name)
last_commit_message = subprocess.check_output(
'git log -1 --pretty=%B'.split())
personal_access_token = common.get_personal_access_token()
if not common.is_current_branch_a_hotfix_branch():
if not last_commit_message.startswith(
'Update authors and changelog for v%s' %
(release_version_number)):
raise Exception('Invalid last commit message: %s.' %
(last_commit_message))
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
check_travis_and_circleci_tests(current_branch_name)
update_configs.main(personal_access_token)
with python_utils.open_file(common.FECONF_PATH, 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
update_configs.add_redishost()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run '
'install_third_party_libs.py prior to running this script.' %
THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
python_utils.PRINT('Ensuring that the release directory parent exists')
common.ensure_directory_exists(os.path.dirname(release_dir_path))
# Copy files to the release directory. Omits the .git subfolder.
python_utils.PRINT('Copying files to the release directory')
shutil.copytree(os.getcwd(),
release_dir_path,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(release_dir_path):
if not os.getcwd().endswith(release_dir_name):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
python_utils.PRINT('Changing directory to %s' % os.getcwd())
python_utils.PRINT('Preprocessing release...')
preprocess_release(app_name, deploy_data_path)
update_and_check_indexes(app_name)
build_scripts(parsed_args.maintenance_mode)
deploy_application_and_write_log_entry(app_name,
current_release_version,
current_git_revision)
python_utils.PRINT('Returning to oppia/ root directory.')
switch_version(app_name, current_release_version)
flush_memcache(app_name)
check_breakage(app_name, current_release_version)
python_utils.PRINT('Done!')
finally:
subprocess.check_output([
'git', 'checkout', '--', update_configs.LOCAL_FECONF_PATH,
update_configs.LOCAL_CONSTANTS_PATH, APP_DEV_YAML_PATH
])
def main():
"""Collects necessary info and dumps it to disk."""
branch_name = common.get_current_branch_name()
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
parsed_args = _PARSER.parse_args()
if parsed_args.github_username is None:
raise Exception('No GitHub username provided. Please re-run the '
'script specifying a username using '
'--github_username=<Your username>')
github_username = parsed_args.github_username
personal_access_token = common.get_personal_access_token()
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
repo_fork = g.get_repo('%s/oppia' % github_username)
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
if not os.path.exists(
constants.release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception('Release summary file %s is missing. Please re-run '
'this script.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
current_release_version_number = common.get_current_release_version_number(
branch_name)
target_branch = 'update-changelog-for-releasev%s' % (
current_release_version_number)
remove_updates_and_delete_branch(repo_fork, target_branch)
# Opens Credit Form.
python_utils.PRINT(
'Note: Make following changes directly to %s and make sure to '
'save the file after making these changes.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Check emails and names for new authors and new contributors in the '
'file: %s and verify that the emails are '
'correct through welcome emails sent from [email protected] '
'(confirm with Sean in case of doubt). Please ensure that you correct '
'the emails of the form: %s.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH,
constants.release_constants.INVALID_EMAIL_SUFFIX))
common.open_new_tab_in_browser_if_possible(
constants.release_constants.CREDITS_FORM_URL)
common.ask_user_to_confirm(
'Check the credits form and add any additional contributors '
'to the contributor list in the file: %s.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Categorize the PR titles in the Uncategorized section of the '
'changelog in the file: %s, and arrange the changelog '
'to have user-facing categories on top.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Verify each item is in the correct section in the '
'file: %s and remove trivial changes like "Fix lint errors" '
'from the changelog.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Ensure that all items in changelog in the file: %s '
'start with a verb in simple present tense.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Please save the file: %s with all the changes that '
'you have made.' %
(constants.release_constants.RELEASE_SUMMARY_FILEPATH))
release_summary_lines = get_release_summary_lines()
update_changelog(branch_name, release_summary_lines,
current_release_version_number)
update_authors(release_summary_lines)
update_contributors(release_summary_lines)
update_developer_names(release_summary_lines)
update_version_in_config_files()
list_of_numbered_files = []
for i, filepath in enumerate(LIST_OF_FILEPATHS_TO_MODIFY, start=1):
list_of_numbered_files.append('%s. %s' % (i, filepath))
message = ('Please check the changes and make updates if required in the '
'following files:\n%s\n' % '\n'.join(list_of_numbered_files))
common.ask_user_to_confirm(message)
create_branch(repo, repo_fork, target_branch, github_username,
current_release_version_number)
def main(personal_access_token):
"""Collects necessary info and dumps it to disk.
Args:
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
current_release = get_current_version_tag(repo)
current_release_tag = current_release.name
base_commit = current_release.commit.sha
new_commits = get_extra_commits_in_new_release(base_commit, repo)
new_release_logs = gather_logs(base_commit)
for index, log in enumerate(new_release_logs):
is_cherrypicked = all(
[log.sha1 != commit.sha for commit in new_commits])
if is_cherrypicked:
del new_release_logs[index]
past_logs = gather_logs(FIRST_OPPIA_COMMIT, stop=base_commit)
issue_links = extract_issues(new_release_logs)
feconf_version_changes = check_versions(current_release_tag)
setup_changes = check_setup_scripts(current_release_tag)
storage_changes = check_storage_models(current_release_tag)
pr_numbers = extract_pr_numbers(new_release_logs)
prs = get_prs_from_pr_numbers(pr_numbers, repo)
categorized_pr_titles = get_changelog_categories(prs)
with python_utils.open_file(release_constants.RELEASE_SUMMARY_FILEPATH,
'w') as out:
out.write('## Collected release information\n')
if feconf_version_changes:
out.write('\n### Feconf version changes:\nThis indicates that a '
'migration may be needed\n\n')
for var in feconf_version_changes:
out.write('* %s\n' % var)
if setup_changes:
out.write('\n### Changed setup scripts:\n')
for var in setup_changes.keys():
out.write('* %s\n' % var)
if storage_changes:
out.write('\n### Changed storage models:\n')
for item in storage_changes:
out.write('* %s\n' % item)
past_authors = {log.email: log.author for log in past_logs}
release_authors = {(log.author, log.email) for log in new_release_logs}
new_authors = sorted(
set([(name, email) for name, email in release_authors
if email not in past_authors]))
existing_authors = sorted(
set([(name, email) for name, email in release_authors
if email in past_authors]))
new_author_names = [name for name, _ in new_authors]
existing_author_names = [name for name, _ in existing_authors]
# TODO(apb7): duplicate author handling due to email changes.
out.write('\n### New Authors:\n')
for name, email in new_authors:
out.write('* %s <%s>\n' % (name, email))
out.write('\n### Existing Authors:\n')
for name, email in existing_authors:
out.write('* %s <%s>\n' % (name, email))
out.write('\n### New Contributors:\n')
for name, email in new_authors:
out.write('* %s <%s>\n' % (name, email))
# Generate the author sections of the email.
out.write('\n### Email C&P Blurbs about authors:\n')
new_author_comma_list = (
'%s, and %s' %
(', '.join(new_author_names[:-1]), new_author_names[-1]))
existing_author_comma_list = (
'%s, and %s' %
(', '.join(existing_author_names[:-1]), existing_author_names[-1]))
out.write(
'``Please welcome %s for whom this release marks their first '
'contribution to Oppia!``\n\n' % new_author_comma_list)
out.write(
'``Thanks to %s, our returning contributors who made this release '
'possible.``\n' % existing_author_comma_list)
if personal_access_token:
out.write('\n### Changelog:\n')
for category in categorized_pr_titles:
out.write('%s\n' % category)
for pr_title in categorized_pr_titles[category]:
out.write('* %s\n' % pr_title)
out.write('\n')
out.write('\n### Commit History:\n')
for name, title in [(log.author, log.message.split('\n\n')[0])
for log in new_release_logs]:
out.write('* %s\n' % title)
if issue_links:
out.write('\n### Issues mentioned in commits:\n')
for link in issue_links:
out.write('* [%s](%s)\n' % (link, link))
python_utils.PRINT('Done. Summary file generated in %s' %
(release_constants.RELEASE_SUMMARY_FILEPATH))
def main():
"""Collects necessary info and dumps it to disk."""
branch_name = common.get_current_branch_name()
if not common.is_current_branch_a_release_branch():
raise Exception(
'This script should only be run from the latest release branch.')
parsed_args = _PARSER.parse_args()
if parsed_args.github_username is None:
raise Exception(
'No GitHub username provided. Please re-run the '
'script specifying a username using '
'--github_username=<Your username>')
github_username = parsed_args.github_username
personal_access_token = common.get_personal_access_token()
python_utils.PRINT('Generating release summary...')
generate_release_info.main(personal_access_token)
if not os.path.exists(release_constants.RELEASE_SUMMARY_FILEPATH):
raise Exception(
'Release summary file %s is missing. Please re-run '
'this script.' % release_constants.RELEASE_SUMMARY_FILEPATH)
g = github.Github(personal_access_token)
repo_fork = g.get_repo('%s/oppia' % github_username)
current_release_version_number = common.get_current_release_version_number(
branch_name)
target_branch = 'update-changelog-for-releasev%s' % (
current_release_version_number)
remove_updates_and_delete_branch(repo_fork, target_branch)
# Opens Credit Form.
python_utils.PRINT(
'Note: Make following changes directly to %s and make sure to '
'save the file after making these changes.' % (
release_constants.RELEASE_SUMMARY_FILEPATH))
common.ask_user_to_confirm(
'Check emails and names for authors and contributors in the release '
'summary file and verify that the emails are '
'correct through welcome emails sent from [email protected] '
'(confirm with Sean in case of doubt).')
common.open_new_tab_in_browser_if_possible(
release_constants.CREDITS_FORM_URL)
common.ask_user_to_confirm(
'Check the credits form and add any additional contributors '
'to the contributor list in the release summary file.')
common.ask_user_to_confirm(
'Categorize the PR titles in the Uncategorized section of the '
'changelog in the release summary file, and arrange the changelog '
'to have user-facing categories on top.')
common.ask_user_to_confirm(
'Verify each item is in the correct section in the release summary '
'file and remove trivial changes like "Fix lint errors" '
'from the changelog.')
common.ask_user_to_confirm(
'Ensure that all items in changelog in the release summary file '
'start with a verb in simple present tense.')
common.ask_user_to_confirm(
'Please save the release summary file with all the changes that '
'you have made.')
release_summary_lines = []
with python_utils.open_file(
release_constants.RELEASE_SUMMARY_FILEPATH, 'r'
) as release_summary_file:
release_summary_lines = release_summary_file.readlines()
check_ordering_of_sections(release_summary_lines)
update_changelog(
branch_name, release_summary_lines, current_release_version_number)
update_authors(release_summary_lines)
update_contributors(release_summary_lines)
update_developer_names(release_summary_lines)
message = (
'Please check the changes and make updates if required in the '
'following files:\n1. %s\n2. %s\n3. %s\n4. %s\n' % (
CHANGELOG_FILEPATH, AUTHORS_FILEPATH, CONTRIBUTORS_FILEPATH,
ABOUT_PAGE_FILEPATH))
common.ask_user_to_confirm(message)
create_branch(
repo_fork, target_branch, github_username,
current_release_version_number)
