def test_addition_of_redishost(self):
def mock_getpass(prompt): # pylint: disable=unused-argument
return '192.123.23.1'
getpass_swap = self.swap(getpass, 'getpass', mock_getpass)
temp_feconf_path = tempfile.NamedTemporaryFile().name
feconf_text = (
'REDISHOST = \'localhost\'\n'
'# When the site terms were last updated, in UTC.\n'
'REGISTRATION_PAGE_LAST_UPDATED_UTC = '
'datetime.datetime(2015, 10, 14, 2, 40, 0)\n'
'# Format of string for dashboard statistics logs.\n'
'# NOTE TO DEVELOPERS: This format should not be changed, '
'since it is used in\n'
'# the existing storage models for UserStatsModel.\n'
'DASHBOARD_STATS_DATETIME_STRING_FORMAT = \'YY-mm-dd\'\n')
expected_feconf_text = (
'REDISHOST = \'192.123.23.1\'\n'
'# When the site terms were last updated, in UTC.\n'
'REGISTRATION_PAGE_LAST_UPDATED_UTC = '
'datetime.datetime(2015, 10, 14, 2, 40, 0)\n'
'# Format of string for dashboard statistics logs.\n'
'# NOTE TO DEVELOPERS: This format should not be changed, '
'since it is used in\n'
'# the existing storage models for UserStatsModel.\n'
'DASHBOARD_STATS_DATETIME_STRING_FORMAT = \'YY-mm-dd\'\n')
with python_utils.open_file(temp_feconf_path, 'w') as f:
f.write(feconf_text)
feconf_swap = self.swap(
update_configs, 'LOCAL_FECONF_PATH', temp_feconf_path)
with getpass_swap, feconf_swap:
update_configs.add_redishost()
with python_utils.open_file(temp_feconf_path, 'r') as f:
self.assertEqual(f.read(), expected_feconf_text)
def test_invalid_redishost(self):
check_prompts = {
'Enter REDISHOST from the release process doc.':
False,
'You have entered an invalid IP Address: 192.123.23.11235, '
'please retry.':
False
}
expected_check_prompts = {
'Enter REDISHOST from the release process doc.':
True,
'You have entered an invalid IP Address: 192.123.23.11235, '
'please retry.':
True
}
def mock_getpass(prompt):
check_prompts[prompt] = True
if 'invalid' in prompt:
return '192.123.23.1'
return '192.123.23.11235'
getpass_swap = self.swap(getpass, 'getpass', mock_getpass)
temp_feconf_path = tempfile.NamedTemporaryFile().name
feconf_text = (
'REDISHOST = \'localhost\'\n'
'# When the site terms were last updated, in UTC.\n'
'REGISTRATION_PAGE_LAST_UPDATED_UTC = '
'datetime.datetime(2015, 10, 14, 2, 40, 0)\n'
'# Format of string for dashboard statistics logs.\n'
'# NOTE TO DEVELOPERS: This format should not be changed, '
'since it is used in\n'
'# the existing storage models for UserStatsModel.\n'
'DASHBOARD_STATS_DATETIME_STRING_FORMAT = \'YY-mm-dd\'\n')
expected_feconf_text = (
'REDISHOST = \'192.123.23.1\'\n'
'# When the site terms were last updated, in UTC.\n'
'REGISTRATION_PAGE_LAST_UPDATED_UTC = '
'datetime.datetime(2015, 10, 14, 2, 40, 0)\n'
'# Format of string for dashboard statistics logs.\n'
'# NOTE TO DEVELOPERS: This format should not be changed, '
'since it is used in\n'
'# the existing storage models for UserStatsModel.\n'
'DASHBOARD_STATS_DATETIME_STRING_FORMAT = \'YY-mm-dd\'\n')
with python_utils.open_file(temp_feconf_path, 'w') as f:
f.write(feconf_text)
feconf_swap = self.swap(update_configs, 'LOCAL_FECONF_PATH',
temp_feconf_path)
with getpass_swap, feconf_swap:
update_configs.add_redishost()
self.assertEqual(check_prompts, expected_check_prompts)
with python_utils.open_file(temp_feconf_path, 'r') as f:
self.assertEqual(f.read(), expected_feconf_text)
def test_error_is_raised_if_redishost_not_updated(self):
def mock_getpass(prompt): # pylint: disable=unused-argument
return '192.123.23.1'
getpass_swap = self.swap(getpass, 'getpass', mock_getpass)
feconf_lines = [
'REDISHOST = \'localhost\'\n',
'# When the site terms were last updated, in UTC.\n',
'REGISTRATION_PAGE_LAST_UPDATED_UTC = '
'datetime.datetime(2015, 10, 14, 2, 40, 0)\n',
'# Format of string for dashboard statistics logs.\n',
'# NOTE TO DEVELOPERS: This format should not be changed, '
'since it is used in\n',
'# the existing storage models for UserStatsModel.\n',
'DASHBOARD_STATS_DATETIME_STRING_FORMAT = \'YY-mm-dd\'\n'
]
class MockFile(python_utils.OBJECT):
def readlines(self):
"""Mock method to read lines of the file object."""
return mock_readlines()
def write(self, unused_line):
"""Mock method to write lines."""
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
pass
def mock_readlines():
return feconf_lines
def mock_open_file(unused_path, unused_mode):
return MockFile()
open_file_swap = self.swap(python_utils, 'open_file', mock_open_file)
with getpass_swap, open_file_swap, self.assertRaisesRegexp(
AssertionError,
'REDISHOST not updated correctly in feconf.py'):
update_configs.add_redishost()
def execute_deployment():
"""Executes the deployment process after doing the prerequisite checks.
Raises:
Exception. App name is invalid.
Exception. Custom version is used with production app.
Exception. App name is not specified.
Exception. The deployment script is not run from a release or test
branch.
Exception. The deployment script is run for prod server from a test
branch.
Exception. Current release version has '.' character.
Exception. Last commit message is invalid.
Exception. The mailgun API key is not added before deployment.
Exception. Could not find third party directory.
Exception. Invalid directory accessed during deployment.
"""
parsed_args = _PARSER.parse_args()
custom_version = None
if parsed_args.app_name:
app_name = parsed_args.app_name
if app_name not in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
] and ('migration' not in app_name):
raise Exception('Invalid app name: %s' % app_name)
if parsed_args.version and app_name == APP_NAME_OPPIASERVER:
raise Exception('Cannot use custom version with production app.')
# Note that custom_version may be None.
custom_version = parsed_args.version
else:
raise Exception('No app name specified.')
current_branch_name = common.get_current_branch_name()
release_dir_name = 'deploy-%s-%s-%s' % (
'-'.join('-'.join(app_name.split('.')).split(':')),
current_branch_name, CURRENT_DATETIME.strftime('%Y%m%d-%H%M%S'))
release_dir_path = os.path.join(os.getcwd(), '..', release_dir_name)
deploy_data_path = os.path.join(os.getcwd(), os.pardir, 'release-scripts',
'deploy_data', app_name)
install_third_party_libs.main()
if not (common.is_current_branch_a_release_branch() or
(common.is_current_branch_a_test_branch())):
raise Exception(
'The deployment script must be run from a release or test branch.')
if common.is_current_branch_a_test_branch() and (app_name in [
APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
]):
raise Exception('Test branch can only be deployed to backup server.')
if custom_version is not None:
current_release_version = custom_version.replace(DOT_CHAR, HYPHEN_CHAR)
else:
current_release_version = current_branch_name[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace(
DOT_CHAR, HYPHEN_CHAR)
# This is required to compose the release_version_library_url
# (defined in switch_version function) correctly.
if '.' in current_release_version:
raise Exception('Current release version has \'.\' character.')
assert len(current_release_version) <= 25, (
'The length of the "version" arg should be less than or '
'equal to 25 characters.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
try:
if app_name == APP_NAME_OPPIASERVER:
check_release_doc()
release_version_number = common.get_current_release_version_number(
current_branch_name)
last_commit_message = subprocess.check_output(
'git log -1 --pretty=%B'.split())
personal_access_token = common.get_personal_access_token()
if not common.is_current_branch_a_hotfix_branch():
if not last_commit_message.startswith(
'Update authors and changelog for v%s' %
(release_version_number)):
raise Exception('Invalid last commit message: %s.' %
(last_commit_message))
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
check_travis_and_circleci_tests(current_branch_name)
update_configs.main(personal_access_token)
with python_utils.open_file(common.FECONF_PATH, 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
update_configs.add_redishost()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run '
'install_third_party_libs.py prior to running this script.' %
THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
python_utils.PRINT('Ensuring that the release directory parent exists')
common.ensure_directory_exists(os.path.dirname(release_dir_path))
# Copy files to the release directory. Omits the .git subfolder.
python_utils.PRINT('Copying files to the release directory')
shutil.copytree(os.getcwd(),
release_dir_path,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(release_dir_path):
if not os.getcwd().endswith(release_dir_name):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
python_utils.PRINT('Changing directory to %s' % os.getcwd())
python_utils.PRINT('Preprocessing release...')
preprocess_release(app_name, deploy_data_path)
update_and_check_indexes(app_name)
build_scripts(parsed_args.maintenance_mode)
deploy_application_and_write_log_entry(app_name,
current_release_version,
current_git_revision)
python_utils.PRINT('Returning to oppia/ root directory.')
switch_version(app_name, current_release_version)
flush_memcache(app_name)
check_breakage(app_name, current_release_version)
python_utils.PRINT('Done!')
finally:
subprocess.check_output([
'git', 'checkout', '--', update_configs.LOCAL_FECONF_PATH,
update_configs.LOCAL_CONSTANTS_PATH, APP_DEV_YAML_PATH
])
def preprocess_release(app_name, deploy_data_path):
"""Pre-processes release files.
This function should be called from within release_dir_name defined
in execute_deployment function. Currently it does the following:
(1) Substitutes files from the per-app deployment data.
(2) Changes GCS_RESOURCE_BUCKET in assets/constants.ts.
(3) Updates project id for vpc_access_connector in app_dev.yaml.
(4) Updates REDISHOST in feconf.py
Args:
app_name: str. Name of the app to deploy.
deploy_data_path: str. Path for deploy data directory.
Raises:
Exception. Could not find deploy data directory.
Exception. Could not find source path.
Exception. Could not find destination path.
Exception. Constants file has invalid GCS_RESOURCE_BUCKET.
Exception. The vpc_access_connector line is missing in app_dev.yaml.
"""
if not os.path.exists(deploy_data_path):
raise Exception('Could not find deploy_data directory at %s' %
deploy_data_path)
# Copies files in root folder to assets/.
for filename in FILES_AT_ROOT:
src = os.path.join(deploy_data_path, filename)
dst = os.path.join(os.getcwd(), 'assets', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images.
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(deploy_data_path, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
with python_utils.open_file(os.path.join(common.CONSTANTS_FILE_PATH),
'r') as assets_file:
common_content = assets_file.read()
with python_utils.open_file(os.path.join(APP_DEV_YAML_PATH),
'r') as app_dev_file:
app_dev_content = app_dev_file.read()
assert '"DEV_MODE": true' in common_content, 'Invalid DEV_MODE'
assert '"GCS_RESOURCE_BUCKET_NAME": "None-resources",' in common_content, (
'Invalid value for GCS_RESOURCE_BUCKET_NAME in %s' %
(common.CONSTANTS_FILE_PATH))
assert 'vpc_access_connector:\n name: projects/PROJECT_ID' in (
app_dev_content), 'Missing vpc_access_connector'
bucket_name = app_name + BUCKET_NAME_SUFFIX
common.inplace_replace_file(
common.CONSTANTS_FILE_PATH,
r'"GCS_RESOURCE_BUCKET_NAME": "None-resources",',
'"GCS_RESOURCE_BUCKET_NAME": "%s",' % bucket_name)
updated_app_dev_content = app_dev_content.replace(
'vpc_access_connector:\n name: projects/PROJECT_ID',
'vpc_access_connector:\n name: projects/%s' % app_name)
with python_utils.open_file(os.path.join(APP_DEV_YAML_PATH),
'w') as app_dev_file:
app_dev_file.write(updated_app_dev_content)
update_configs.add_redishost()