def deploy_application_and_write_log_entry(
app_name, version_to_deploy_to, current_git_revision):
"""Deploys the app and writes the log entry.
Args:
app_name: str. The name of the app to deploy.
version_to_deploy_to: str. The version to deploy to.
current_git_revision: str. The current git revision.
"""
# Deploy export service to GAE.
gcloud_adapter.deploy_application('export/app.yaml', app_name)
# Deploy app to GAE.
gcloud_adapter.deploy_application(
'./app.yaml', app_name, version=version_to_deploy_to)
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with python_utils.open_file(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' % (
app_name, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
def preprocess_release(app_name, deploy_data_path):
"""Pre-processes release files.
This function should be called from within release_dir_name defined
in execute_deployment function. Currently it does the following:
(1) Substitutes files from the per-app deployment data.
(2) Changes GCS_RESOURCE_BUCKET in assets/constants.ts.
(3) Updates project id for vpc_access_connector in app_dev.yaml.
(4) Updates REDISHOST in feconf.py
Args:
app_name: str. Name of the app to deploy.
deploy_data_path: str. Path for deploy data directory.
Raises:
Exception. Could not find deploy data directory.
Exception. Could not find source path.
Exception. Could not find destination path.
Exception. Constants file has invalid GCS_RESOURCE_BUCKET.
Exception. The vpc_access_connector line is missing in app_dev.yaml.
"""
if not os.path.exists(deploy_data_path):
raise Exception('Could not find deploy_data directory at %s' %
deploy_data_path)
# Copies files in root folder to assets/.
for filename in FILES_AT_ROOT:
src = os.path.join(deploy_data_path, filename)
dst = os.path.join(os.getcwd(), 'assets', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images.
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(deploy_data_path, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
with python_utils.open_file(os.path.join(common.CONSTANTS_FILE_PATH),
'r') as assets_file:
common_content = assets_file.read()
with python_utils.open_file(os.path.join(APP_DEV_YAML_PATH),
'r') as app_dev_file:
app_dev_content = app_dev_file.read()
assert '"DEV_MODE": true' in common_content, 'Invalid DEV_MODE'
assert '"GCS_RESOURCE_BUCKET_NAME": "None-resources",' in common_content, (
'Invalid value for GCS_RESOURCE_BUCKET_NAME in %s' %
(common.CONSTANTS_FILE_PATH))
assert 'vpc_access_connector:\n name: projects/PROJECT_ID' in (
app_dev_content), (
'"name: projects/PROJECT_ID" string is missing in app_dev.yaml')
bucket_name = app_name + BUCKET_NAME_SUFFIX
common.inplace_replace_file(
common.CONSTANTS_FILE_PATH,
r'"GCS_RESOURCE_BUCKET_NAME": "None-resources",',
'"GCS_RESOURCE_BUCKET_NAME": "%s",' % bucket_name)
updated_app_dev_content = app_dev_content.replace(
'vpc_access_connector:\n name: projects/PROJECT_ID',
'vpc_access_connector:\n name: projects/%s' % app_name)
with python_utils.open_file(os.path.join(APP_DEV_YAML_PATH),
'w') as app_dev_file:
app_dev_file.write(updated_app_dev_content)
def execute_deployment():
"""Executes the deployment process after doing the prerequisite checks.
Raises:
Exception. App name is invalid.
Exception. Custom version is used with production app.
Exception. App name is not specified.
Exception. The deployment script is not run from a release or test
branch.
Exception. The deployment script is run for prod server from a test
branch.
Exception. Current release version has '.' character.
Exception. Last commit message is invalid.
Exception. The mailgun API key is not added before deployment.
Exception. Could not find third party directory.
Exception. Invalid directory accessed during deployment.
"""
parsed_args = _PARSER.parse_args()
custom_version = None
if parsed_args.app_name:
app_name = parsed_args.app_name
if app_name not in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
] and ('migration' not in app_name):
raise Exception('Invalid app name: %s' % app_name)
if parsed_args.version and app_name == APP_NAME_OPPIASERVER:
raise Exception('Cannot use custom version with production app.')
# Note that custom_version may be None.
custom_version = parsed_args.version
else:
raise Exception('No app name specified.')
current_branch_name = common.get_current_branch_name()
release_dir_name = 'deploy-%s-%s-%s' % (
'-'.join('-'.join(app_name.split('.')).split(':')),
current_branch_name, CURRENT_DATETIME.strftime('%Y%m%d-%H%M%S'))
release_dir_path = os.path.join(os.getcwd(), '..', release_dir_name)
deploy_data_path = os.path.join(os.getcwd(), os.pardir, 'release-scripts',
'deploy_data', app_name)
install_third_party_libs.main()
if not (common.is_current_branch_a_release_branch() or
(common.is_current_branch_a_test_branch())):
raise Exception(
'The deployment script must be run from a release or test branch.')
if common.is_current_branch_a_test_branch() and (app_name in [
APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
]):
raise Exception('Test branch can only be deployed to backup server.')
if custom_version is not None:
current_release_version = custom_version.replace(DOT_CHAR, HYPHEN_CHAR)
else:
current_release_version = current_branch_name[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace(
DOT_CHAR, HYPHEN_CHAR)
# This is required to compose the release_version_library_url
# (defined in switch_version function) correctly.
if '.' in current_release_version:
raise Exception('Current release version has \'.\' character.')
assert len(current_release_version) <= 25, (
'The length of the "version" arg should be less than or '
'equal to 25 characters.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
try:
if app_name == APP_NAME_OPPIASERVER:
check_release_doc()
release_version_number = common.get_current_release_version_number(
current_branch_name)
last_commit_message = subprocess.check_output(
'git log -1 --pretty=%B'.split())
personal_access_token = common.get_personal_access_token()
if not common.is_current_branch_a_hotfix_branch():
if not last_commit_message.startswith(
'Update authors and changelog for v%s' %
(release_version_number)):
raise Exception('Invalid last commit message: %s.' %
(last_commit_message))
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
check_travis_and_circleci_tests(current_branch_name)
update_configs.main(personal_access_token)
with python_utils.open_file(common.FECONF_PATH, 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
update_configs.add_redishost()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run '
'install_third_party_libs.py prior to running this script.' %
THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
python_utils.PRINT('Ensuring that the release directory parent exists')
common.ensure_directory_exists(os.path.dirname(release_dir_path))
# Copy files to the release directory. Omits the .git subfolder.
python_utils.PRINT('Copying files to the release directory')
shutil.copytree(os.getcwd(),
release_dir_path,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(release_dir_path):
if not os.getcwd().endswith(release_dir_name):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
python_utils.PRINT('Changing directory to %s' % os.getcwd())
python_utils.PRINT('Preprocessing release...')
preprocess_release(app_name, deploy_data_path)
update_and_check_indexes(app_name)
build_scripts(parsed_args.maintenance_mode)
deploy_application_and_write_log_entry(app_name,
current_release_version,
current_git_revision)
python_utils.PRINT('Returning to oppia/ root directory.')
switch_version(app_name, current_release_version)
flush_memcache(app_name)
check_breakage(app_name, current_release_version)
python_utils.PRINT('Done!')
finally:
subprocess.check_output([
'git', 'checkout', '--', update_configs.LOCAL_FECONF_PATH,
update_configs.LOCAL_CONSTANTS_PATH, APP_DEV_YAML_PATH
])
def preprocess_release(app_name, deploy_data_path):
"""Pre-processes release files.
This function should be called from within release_dir_name defined
in execute_deployment function. Currently it does the following:
(1) Substitutes files from the per-app deployment data.
(2) Change the DEV_MODE constant in assets/constants.ts.
(3) Change GCS_RESOURCE_BUCKET in assets/constants.ts.
(4) Removes the "version" field from app.yaml, since gcloud does not like
it (when deploying).
Args:
app_name: str. Name of the app to deploy.
deploy_data_path: str. Path for deploy data directory.
Raises:
Exception: Could not find deploy data directory.
Exception: Could not find source path.
Exception: Could not find destination path.
"""
if not os.path.exists(deploy_data_path):
raise Exception('Could not find deploy_data directory at %s' %
deploy_data_path)
# Copies files in root folder to assets/.
for filename in FILES_AT_ROOT:
src = os.path.join(deploy_data_path, filename)
dst = os.path.join(os.getcwd(), 'assets', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images.
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(deploy_data_path, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
# Changes the DEV_MODE constant in assets/constants.ts.
with python_utils.open_file(os.path.join(CONSTANTS_FILE_PATH),
'r') as assets_file:
content = assets_file.read()
bucket_name = app_name + BUCKET_NAME_SUFFIX
assert '"DEV_MODE": true' in content
assert '"GCS_RESOURCE_BUCKET_NAME": "None-resources",' in content
os.remove(os.path.join(CONSTANTS_FILE_PATH))
content = content.replace('"DEV_MODE": true', '"DEV_MODE": false')
content = content.replace(
'"GCS_RESOURCE_BUCKET_NAME": "None-resources",',
'"GCS_RESOURCE_BUCKET_NAME": "%s",' % bucket_name)
with python_utils.open_file(os.path.join(CONSTANTS_FILE_PATH),
'w+') as new_assets_file:
new_assets_file.write(content)
