def _execute_branch_cut():
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.verify_local_repo_is_clean()
common.verify_current_branch_name('develop')
# Update the local repo.
remote_alias = common.get_remote_alias('https://github.com/oppia/oppia')
subprocess.call(['git', 'pull', remote_alias])
_verify_target_branch_does_not_already_exist(remote_alias)
_verify_target_version_is_consistent_with_latest_released_version()
# The release coordinator should verify that tests are passing on develop
# before checking out the release branch.
common.open_new_tab_in_browser_if_possible(
'https://github.com/oppia/oppia#oppia---')
while True:
print (
'Please confirm: are Travis checks passing on develop? (y/n) ')
answer = raw_input().lower()
if answer in ['y', 'ye', 'yes']:
break
elif answer:
print (
'Tests should pass on develop before this script is run. '
'Exiting.')
sys.exit()
# Cut a new release branch.
print 'Cutting a new release branch: %s' % NEW_BRANCH_NAME
subprocess.call(['git', 'checkout', '-b', NEW_BRANCH_NAME])
# Update the version in app.yaml.
print 'Updating the version number in app.yaml ...'
with open('app.yaml', 'r') as f:
content = f.read()
assert content.count('version: default') == 1
os.remove('app.yaml')
content = content.replace(
'version: default', 'version: %s' % NEW_APP_YAML_VERSION)
with open('app.yaml', 'w+') as f:
f.write(content)
print 'Version number updated.'
# Make a commit.
print 'Committing the change.'
subprocess.call([
'git', 'commit', '-a', '-m',
'"Update version number to %s"' % TARGET_VERSION])
# Push the new release branch to GitHub.
print 'Pushing new release branch to GitHub.'
subprocess.call(['git', 'push', remote_alias, NEW_BRANCH_NAME])
print ''
print (
'New release branch successfully cut. You are now on branch %s' %
NEW_BRANCH_NAME)
print 'Done!'
def _update_feconf():
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.get_current_branch_name().startswith('release-')
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
_apply_changes_to_feconf()
print 'Done! Please check manually to ensure all the changes are correct.'
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
print ''
print 'Starting deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(os.getcwd(),
RELEASE_DIR_PATH,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
update_cache_slug()
# Do a build; ensure there are no errors.
print 'Building and minifying scripts...'
subprocess.check_output(['python', 'scripts/build.py'])
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' %
(APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
print 'Done!'
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
print ''
print 'Starting deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(
os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s'
% os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
update_cache_slug()
# Do a build; ensure there are no errors.
print 'Building and minifying scripts...'
subprocess.check_output(['python', 'scripts/build.py'])
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' % (
APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
print 'Done!'
def _update_indexes():
"""Updates production indexes after doing the prerequisite checks."""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
gcloud_adapter.require_gcloud_to_be_available()
if not common.is_current_branch_a_release_branch():
raise Exception(
'Indexes should only be updated from a release branch.')
# Update the indexes.
gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME)
def _update_configs():
"""Updates the 'feconf.py' and 'constants.js' files after doing the
prerequisite checks.
"""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.get_current_branch_name().startswith('release-')
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
_apply_changes_based_on_config(
LOCAL_FECONF_PATH, FECONF_CONFIG_PATH, '^([A-Z_]+ = ).*$')
_apply_changes_based_on_config(
LOCAL_CONSTANTS_PATH, CONSTANTS_CONFIG_PATH, '^( "[A-Z_]+": ).*$')
print 'Done! Please check manually to ensure all the changes are correct.'
def _execute_branch_cut():
"""Pushes the new release branch to Github."""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.verify_local_repo_is_clean()
common.verify_current_branch_name('develop')
# Update the local repo.
remote_alias = common.get_remote_alias('[email protected]:oppia/oppia.git')
subprocess.call(['git', 'pull', remote_alias])
_verify_target_branch_does_not_already_exist(remote_alias)
_verify_target_version_is_consistent_with_latest_released_version()
# The release coordinator should verify that tests are passing on develop
# before checking out the release branch.
common.open_new_tab_in_browser_if_possible(
'https://github.com/oppia/oppia#oppia---')
while True:
print (
'Please confirm: are Travis checks passing on develop? (y/n) ')
answer = raw_input().lower()
if answer in ['y', 'ye', 'yes']:
break
elif answer:
print (
'Tests should pass on develop before this script is run. '
'Exiting.')
sys.exit()
# Cut a new release branch.
print 'Cutting a new release branch: %s' % NEW_BRANCH_NAME
subprocess.call(['git', 'checkout', '-b', NEW_BRANCH_NAME])
# Push the new release branch to GitHub.
print 'Pushing new release branch to GitHub.'
subprocess.call(['git', 'push', remote_alias, NEW_BRANCH_NAME])
print ''
print (
'New release branch successfully cut. You are now on branch %s' %
NEW_BRANCH_NAME)
print 'Done!'
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
"""
# Change the app name in app.yaml.
f = open('app.yaml', 'r')
content = f.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
d = open('app.yaml', 'w+')
d.write(content)
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
CURRENT_GIT_VERSION = subprocess.check_output(['git', 'rev-parse',
'HEAD']).strip()
print ''
print 'Starting experimental deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
#[Errno -2] Name or service not known" error
# in urllib.urlretrieve, if the user is behind a proxy.
if 'VAGRANT' in os.environ:
os.environ['http_proxy'] = ''
TOOLS_DIR = os.path.join('..', 'oppia_tools')
THIRD_PARTY_DIR = os.path.join('.', 'third_party')
THIRD_PARTY_STATIC_DIR = os.path.join(THIRD_PARTY_DIR, 'static')
MANIFEST_FILE_PATH = os.path.join(os.getcwd(), 'manifest.json')
# Place to download zip files for temporary storage.
TMP_UNZIP_PATH = os.path.join('.', 'tmp_unzip.zip')
# Check that the current directory is correct.
common.require_cwd_to_be_oppia(allow_deploy_dir=True)
TARGET_DOWNLOAD_DIRS = {
'frontend': THIRD_PARTY_STATIC_DIR,
'backend': THIRD_PARTY_DIR,
'oppiaTools': TOOLS_DIR
}
_DOWNLOAD_FORMAT_ZIP = 'zip'
_DOWNLOAD_FORMAT_TAR = 'tar'
_DOWNLOAD_FORMAT_FILES = 'files'
DOWNLOAD_FORMATS_TO_MANIFEST_KEYS = {
'zip': {
'mandatory_keys': ['version', 'url', 'downloadFormat'],
'optional_key_pairs': [
def _execute_deployment():
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(
os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s'
% os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
# Do a build, while outputting to the terminal.
print 'Building and minifying scripts...'
build_process = subprocess.Popen(
['python', 'scripts/build.py', '--prod_env'],
stdout=subprocess.PIPE)
while True:
line = build_process.stdout.readline().strip()
if not line:
break
print line
# Wait for process to terminate, then check return code.
build_process.communicate()
if build_process.returncode > 0:
raise Exception('Build failed.')
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' % (
APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
# If this is a test server deployment and the current release version is
# already serving, open the library page (for sanity checking) and the GAE
# error logs.
if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and (
_get_served_version() == _get_current_release_version()):
common.open_new_tab_in_browser_if_possible(
'https://console.cloud.google.com/logs/viewer?'
'project=%s&key1=default&minLogLevel=500'
% APP_NAME_OPPIATESTSERVER)
common.open_new_tab_in_browser_if_possible(
'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER)
print 'Done!'
import tarfile
import urllib
import zipfile
import common
TOOLS_DIR = os.path.join('..', 'oppia_tools')
THIRD_PARTY_DIR = os.path.join('.', 'third_party')
THIRD_PARTY_STATIC_DIR = os.path.join(THIRD_PARTY_DIR, 'static')
# Place to download zip files for temporary storage.
TMP_UNZIP_PATH = os.path.join('.', 'tmp_unzip.zip')
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
def download_files(source_url_root, target_dir, source_filenames):
"""Downloads a group of files and saves them to a given directory.
Each file is downloaded only if it does not already exist.
Args:
source_url_root: the URL to prepend to all the filenames.
target_dir: the directory to save the files to.
source_filenames: a list of filenames. Each filename is appended to the
end of the source_url_root in order to give the URL from which to
download the file. The downloaded file is then placed in target_dir,
and retains the same filename.
"""
def _execute_deployment():
"""Executes the deployment process after doing the prerequisite checks."""
if not common.is_current_branch_a_release_branch():
raise Exception(
'The deployment script must be run from a release branch.')
current_release_version = CURRENT_BRANCH_NAME[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace('.', '-')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
if APP_NAME in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER]:
if not common.is_current_branch_a_release_branch():
raise Exception(
'The deployment script must be run from a release branch.')
if APP_NAME == APP_NAME_OPPIASERVER:
with open('./feconf.py', 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(os.getcwd(),
RELEASE_DIR_PATH,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
# Update indexes, then prompt for a check that they are all serving
# before continuing with the deployment.
# NOTE: This assumes that the build process does not modify the
# index.yaml file or create a different version of it to use in
# production.
gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME)
datastore_indexes_url = (
'https://console.cloud.google.com/datastore/indexes?project=%s' %
APP_NAME)
common.open_new_tab_in_browser_if_possible(datastore_indexes_url)
while True:
print '******************************************************'
print(
'PLEASE CONFIRM: are all datastore indexes serving? See %s '
'(y/n)' % datastore_indexes_url)
answer = raw_input().lower()
if answer in ['y', 'ye', 'yes']:
break
elif answer:
raise Exception(
'Please wait for all indexes to serve, then run this '
'script again to complete the deployment. Exiting.')
# Do a build, while outputting to the terminal.
print 'Building and minifying scripts...'
build_process = subprocess.Popen(
['python', 'scripts/build.py', '--prod_env'],
stdout=subprocess.PIPE)
while True:
line = build_process.stdout.readline().strip()
if not line:
break
print line
# Wait for process to terminate, then check return code.
build_process.communicate()
if build_process.returncode > 0:
raise Exception('Build failed.')
# Deploy export service to GAE.
gcloud_adapter.deploy_application('export/app.yaml', APP_NAME)
# Deploy app to GAE.
gcloud_adapter.deploy_application(
'./app.yaml',
APP_NAME,
version=(CUSTOM_VERSION
if CUSTOM_VERSION else current_release_version))
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' %
(APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
# If this is a test server deployment and the current release version is
# already serving, open the library page (for sanity checking) and the GAE
# error logs.
currently_served_version = (
gcloud_adapter.get_currently_served_version(APP_NAME))
if (APP_NAME == APP_NAME_OPPIATESTSERVER
or 'migration' in APP_NAME) and (currently_served_version
== current_release_version):
common.open_new_tab_in_browser_if_possible(
'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER)
common.open_new_tab_in_browser_if_possible(
'https://console.cloud.google.com/logs/viewer?'
'project=%s&key1=default&minLogLevel=500' %
APP_NAME_OPPIATESTSERVER)
print 'Done!'
#[Errno -2] Name or service not known" error
# in urllib.urlretrieve, if the user is behind a proxy.
if 'VAGRANT' in os.environ:
os.environ['http_proxy'] = ''
TOOLS_DIR = os.path.join('..', 'oppia_tools')
THIRD_PARTY_DIR = os.path.join('.', 'third_party')
THIRD_PARTY_STATIC_DIR = os.path.join(THIRD_PARTY_DIR, 'static')
MANIFEST_FILE_PATH = os.path.join(os.getcwd(), 'manifest.json')
# Place to download zip files for temporary storage.
TMP_UNZIP_PATH = os.path.join('.', 'tmp_unzip.zip')
# Check that the current directory is correct.
common.require_cwd_to_be_oppia(allow_deploy_dir=True)
TARGET_DOWNLOAD_DIRS = {
'frontend': THIRD_PARTY_STATIC_DIR,
'backend': THIRD_PARTY_DIR,
'oppiaTools': TOOLS_DIR
}
_DOWNLOAD_FORMAT_ZIP = 'zip'
_DOWNLOAD_FORMAT_TAR = 'tar'
_DOWNLOAD_FORMAT_FILES = 'files'
DOWNLOAD_FORMATS_TO_MANIFEST_KEYS = {
'zip': {
'mandatory_keys': ['version', 'url', 'downloadFormat'],
'optional_key_pairs': [
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
print ""
print "Starting deployment process."
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
"Could not find third_party directory at %s. Please run start.sh "
"prior to running this script." % THIRD_PARTY_DIR
)
# Create a folder in which to save the release candidate.
print "Ensuring that the release directory parent exists"
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print "Copying files to the release directory"
shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns(".git"))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception("Invalid directory accessed during deployment: %s" % os.getcwd())
print "Changing directory to %s" % os.getcwd()
print "Preprocessing release..."
preprocess_release()
# Do a build; ensure there are no errors.
print "Building and minifying scripts..."
subprocess.check_output(["python", "scripts/build.py"])
# Run the tests; ensure there are no errors.
print "Running tests..."
tests_proc = subprocess.Popen(["bash", os.path.join("scripts", "run_tests.sh")], stdout=subprocess.PIPE)
tests_stdout, tests_stderr = tests_proc.communicate()
print tests_stdout
print tests_stderr
if tests_proc.returncode != 0:
raise Exception("Tests failed. Halting deployment.")
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, "update", ".", "--oauth2"])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, "a") as log_file:
log_file.write(
"Successfully deployed to %s at %s (version %s)\n"
% (APP_NAME, CURRENT_DATETIME.strftime("%Y-%m-%d %H:%M:%S"), current_git_revision)
)
print "Returning to oppia/ root directory."
print "Done!"
