def test_function_calls_without_prompt_for_feconf_and_terms_update(self):
check_function_calls = {
'apply_changes_based_on_config_gets_called': False,
'ask_user_to_confirm_gets_called': False
}
expected_check_function_calls = {
'apply_changes_based_on_config_gets_called': True,
'ask_user_to_confirm_gets_called': True
}
def mock_apply_changes(unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_ask_user_to_confirm(unused_msg):
check_function_calls['ask_user_to_confirm_gets_called'] = True
apply_changes_swap = self.swap(update_configs,
'apply_changes_based_on_config',
mock_apply_changes)
ask_user_swap = self.swap(common, 'ask_user_to_confirm',
mock_ask_user_to_confirm)
with ask_user_swap, apply_changes_swap:
update_configs.main('test-release-dir', 'test-deploy-dir',
'test-token', False)
self.assertEqual(check_function_calls, expected_check_function_calls)
def test_checkout_is_run_in_case_of_exception(self):
check_function_calls = {
'check_updates_to_terms_of_service_gets_called': False,
'run_cmd_gets_called': False
}
expected_check_function_calls = {
'check_updates_to_terms_of_service_gets_called': True,
'run_cmd_gets_called': True
}
def mock_check_updates(unused_personal_access_token):
check_function_calls[
'check_updates_to_terms_of_service_gets_called'] = True
raise Exception('Testing')
def mock_run_cmd(unused_cmd_tokens):
check_function_calls['run_cmd_gets_called'] = True
check_updates_swap = self.swap(update_configs,
'check_updates_to_terms_of_service',
mock_check_updates)
run_cmd_swap = self.swap(common, 'run_cmd', mock_run_cmd)
with self.branch_name_swap, self.release_scripts_exist_swap:
with self.url_open_swap, check_updates_swap, run_cmd_swap:
with self.assertRaisesRegexp(Exception, 'Testing'):
update_configs.main('test-token')
self.assertEqual(check_function_calls, expected_check_function_calls)
def test_function_calls_without_prompt_for_feconf_and_terms_update(self):
check_function_calls = {
'apply_changes_based_on_config_gets_called': False,
'verify_feconf_gets_called': False,
'mailgun_api_key_is_to_be_verified': False,
'mailchimp_api_key_is_to_be_verified': False
}
expected_check_function_calls = {
'apply_changes_based_on_config_gets_called': True,
'verify_feconf_gets_called': True,
'mailgun_api_key_is_to_be_verified': False,
'mailchimp_api_key_is_to_be_verified': False
}
def mock_apply_changes(
unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_verify_feconf(
unused_release_feconf_path, verify_email_api_keys):
check_function_calls['verify_feconf_gets_called'] = True
check_function_calls['mailgun_api_key_is_to_be_verified'] = (
verify_email_api_keys)
check_function_calls['mailchimp_api_key_is_to_be_verified'] = (
verify_email_api_keys)
apply_changes_swap = self.swap(
update_configs, 'apply_changes_based_on_config', mock_apply_changes)
verify_feconf_swap = self.swap(
update_configs, 'verify_feconf', mock_verify_feconf)
with apply_changes_swap, verify_feconf_swap:
update_configs.main(
'test-release-dir', 'test-deploy-dir', 'test-token', False)
self.assertEqual(check_function_calls, expected_check_function_calls)
def test_missing_terms_page(self):
def mock_url_open(unused_url):
raise Exception('Not found.')
url_open_swap = self.swap(python_utils, 'url_open', mock_url_open)
with url_open_swap, self.assertRaisesRegexp(
Exception, 'Terms mainpage does not exist on Github.'):
update_configs.main(
'test-release-dir', 'test-deploy-dir', 'test-token', True)
def test_invalid_branch_name(self):
def mock_get_current_branch_name():
return 'invalid'
branch_name_swap = self.swap(common, 'get_current_branch_name',
mock_get_current_branch_name)
with branch_name_swap, self.assertRaises(AssertionError):
update_configs.main('test-token')
def test_missing_terms_page(self):
def mock_url_open(unused_url):
raise Exception('Not found.')
url_open_swap = self.swap(python_utils, 'url_open', mock_url_open)
with self.branch_name_swap, self.release_scripts_exist_swap:
with url_open_swap, self.assertRaisesRegexp(
Exception, 'Terms mainpage does not exist on Github.'):
update_configs.main('test-token')
def test_missing_terms_page(self):
def mock_url_open(unused_url):
raise Exception('Not found.')
url_open_swap = self.swap(python_utils, 'url_open', mock_url_open)
with url_open_swap, self.assertRaisesRegexp(
Exception, 'Terms mainpage does not exist on Github.'):
update_configs.main(args=[
'--release_dir_path', 'test-release-dir', '--deploy_data_path',
'test-deploy-dir', '--personal_access_token', 'test-token',
'--prompt_for_mailgun_and_terms_update'
])
def test_function_calls_without_prompt_for_feconf_and_terms_update(self):
check_function_calls = {
'apply_changes_based_on_config_gets_called': False,
'verify_config_files_gets_called': False,
'update_app_yaml_gets_called': False,
'mailgun_api_key_is_to_be_verified': False,
'mailchimp_api_key_is_to_be_verified': False
}
expected_check_function_calls = {
'apply_changes_based_on_config_gets_called': True,
'verify_config_files_gets_called': True,
'update_app_yaml_gets_called': True,
'mailgun_api_key_is_to_be_verified': False,
'mailchimp_api_key_is_to_be_verified': False
}
def mock_apply_changes(unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_verify_config_files(unused_release_feconf_path,
unused_release_app_dev_yaml_path,
verify_email_api_keys):
check_function_calls['verify_config_files_gets_called'] = True
check_function_calls['mailgun_api_key_is_to_be_verified'] = (
verify_email_api_keys)
check_function_calls['mailchimp_api_key_is_to_be_verified'] = (
verify_email_api_keys)
def mock_update_app_yaml(unused_release_app_dev_yaml_path,
unused_feconf_config_path):
check_function_calls['update_app_yaml_gets_called'] = True
apply_changes_swap = self.swap(update_configs,
'apply_changes_based_on_config',
mock_apply_changes)
verify_config_files_swap = self.swap(update_configs,
'verify_config_files',
mock_verify_config_files)
update_app_yaml_swap = self.swap(update_configs, 'update_app_yaml',
mock_update_app_yaml)
with apply_changes_swap, verify_config_files_swap, update_app_yaml_swap:
update_configs.main(args=[
'--release_dir_path', 'test-release-dir', '--deploy_data_path',
'test-deploy-dir', '--personal_access_token', 'test-token'
])
self.assertEqual(check_function_calls, expected_check_function_calls)
def test_function_calls_with_prompt_for_feconf_and_terms_update(self):
check_function_calls = {
'check_updates_to_terms_of_service_gets_called': False,
'add_mailgun_api_key_gets_called': False,
'apply_changes_based_on_config_gets_called': False,
'ask_user_to_confirm_gets_called': False
}
expected_check_function_calls = {
'check_updates_to_terms_of_service_gets_called': True,
'add_mailgun_api_key_gets_called': True,
'apply_changes_based_on_config_gets_called': True,
'ask_user_to_confirm_gets_called': True
}
def mock_check_updates(unused_release_feconf_path,
unused_personal_access_token):
check_function_calls[
'check_updates_to_terms_of_service_gets_called'] = True
def mock_add_mailgun_api_key(unused_release_feconf_path):
check_function_calls['add_mailgun_api_key_gets_called'] = True
def mock_apply_changes(unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_ask_user_to_confirm(unused_msg):
check_function_calls['ask_user_to_confirm_gets_called'] = True
check_updates_swap = self.swap(update_configs,
'check_updates_to_terms_of_service',
mock_check_updates)
add_mailgun_api_key_swap = self.swap(update_configs,
'add_mailgun_api_key',
mock_add_mailgun_api_key)
apply_changes_swap = self.swap(update_configs,
'apply_changes_based_on_config',
mock_apply_changes)
ask_user_swap = self.swap(common, 'ask_user_to_confirm',
mock_ask_user_to_confirm)
with self.url_open_swap, check_updates_swap, add_mailgun_api_key_swap:
with apply_changes_swap, ask_user_swap:
update_configs.main('test-release-dir', 'test-deploy-dir',
'test-token', True)
self.assertEqual(check_function_calls, expected_check_function_calls)
def test_function_calls(self):
check_function_calls = {
'check_updates_to_terms_of_service_gets_called': False,
'add_mailgun_api_key_gets_called': False,
'apply_changes_based_on_config_gets_called': False,
'ask_user_to_confirm_gets_called': False
}
expected_check_function_calls = {
'check_updates_to_terms_of_service_gets_called': True,
'add_mailgun_api_key_gets_called': True,
'apply_changes_based_on_config_gets_called': True,
'ask_user_to_confirm_gets_called': True
}
def mock_check_updates(unused_personal_access_token):
check_function_calls[
'check_updates_to_terms_of_service_gets_called'] = True
def mock_add_mailgun_api_key():
check_function_calls['add_mailgun_api_key_gets_called'] = True
def mock_apply_changes(unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_ask_user_to_confirm(unused_msg):
check_function_calls['ask_user_to_confirm_gets_called'] = True
check_updates_swap = self.swap(update_configs,
'check_updates_to_terms_of_service',
mock_check_updates)
add_mailgun_api_key_swap = self.swap(update_configs,
'add_mailgun_api_key',
mock_add_mailgun_api_key)
apply_changes_swap = self.swap(update_configs,
'apply_changes_based_on_config',
mock_apply_changes)
ask_user_swap = self.swap(common, 'ask_user_to_confirm',
mock_ask_user_to_confirm)
with self.branch_name_swap, self.release_scripts_exist_swap:
with self.url_open_swap, check_updates_swap, ask_user_swap:
with add_mailgun_api_key_swap, apply_changes_swap:
update_configs.main('test-token')
self.assertEqual(check_function_calls, expected_check_function_calls)
def update_configs_in_deploy_data(
app_name, release_dir_path, deploy_data_path, personal_access_token):
"""Updates feconf and constants file in deploy data to match the config
files in release scripts.
Args:
app_name: str. The name of the app to deploy.
release_dir_path: str. Path of directory where all files are copied
for release.
deploy_data_path: str. Path for deploy data directory.
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
if app_name == APP_NAME_OPPIASERVER:
update_configs.main(
release_dir_path, deploy_data_path, personal_access_token, True)
else:
update_configs.main(
release_dir_path, deploy_data_path, personal_access_token, False)
def execute_deployment():
"""Executes the deployment process after doing the prerequisite checks.
Raises:
Exception. App name is invalid.
Exception. Custom version is used with production app.
Exception. App name is not specified.
Exception. The deployment script is not run from a release or test
branch.
Exception. The deployment script is run for prod server from a test
branch.
Exception. Current release version has '.' character.
Exception. Last commit message is invalid.
Exception. The mailgun API key is not added before deployment.
Exception. Could not find third party directory.
Exception. Invalid directory accessed during deployment.
"""
parsed_args = _PARSER.parse_args()
custom_version = None
if parsed_args.app_name:
app_name = parsed_args.app_name
if app_name not in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
] and ('migration' not in app_name):
raise Exception('Invalid app name: %s' % app_name)
if parsed_args.version and app_name == APP_NAME_OPPIASERVER:
raise Exception('Cannot use custom version with production app.')
# Note that custom_version may be None.
custom_version = parsed_args.version
else:
raise Exception('No app name specified.')
current_branch_name = common.get_current_branch_name()
release_dir_name = 'deploy-%s-%s-%s' % (
'-'.join('-'.join(app_name.split('.')).split(':')),
current_branch_name, CURRENT_DATETIME.strftime('%Y%m%d-%H%M%S'))
release_dir_path = os.path.join(os.getcwd(), '..', release_dir_name)
deploy_data_path = os.path.join(os.getcwd(), os.pardir, 'release-scripts',
'deploy_data', app_name)
install_third_party_libs.main()
if not (common.is_current_branch_a_release_branch() or
(common.is_current_branch_a_test_branch())):
raise Exception(
'The deployment script must be run from a release or test branch.')
if common.is_current_branch_a_test_branch() and (app_name in [
APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
]):
raise Exception('Test branch can only be deployed to backup server.')
if custom_version is not None:
current_release_version = custom_version.replace(DOT_CHAR, HYPHEN_CHAR)
else:
current_release_version = current_branch_name[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace(
DOT_CHAR, HYPHEN_CHAR)
# This is required to compose the release_version_library_url
# (defined in switch_version function) correctly.
if '.' in current_release_version:
raise Exception('Current release version has \'.\' character.')
assert len(current_release_version) <= 25, (
'The length of the "version" arg should be less than or '
'equal to 25 characters.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
try:
if app_name == APP_NAME_OPPIASERVER:
check_release_doc()
release_version_number = common.get_current_release_version_number(
current_branch_name)
last_commit_message = subprocess.check_output(
'git log -1 --pretty=%B'.split())
personal_access_token = common.get_personal_access_token()
if not common.is_current_branch_a_hotfix_branch():
if not last_commit_message.startswith(
'Update authors and changelog for v%s' %
(release_version_number)):
raise Exception('Invalid last commit message: %s.' %
(last_commit_message))
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
check_travis_and_circleci_tests(current_branch_name)
update_configs.main(personal_access_token)
with python_utils.open_file(common.FECONF_PATH, 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
update_configs.add_redishost()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run '
'install_third_party_libs.py prior to running this script.' %
THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
python_utils.PRINT('Ensuring that the release directory parent exists')
common.ensure_directory_exists(os.path.dirname(release_dir_path))
# Copy files to the release directory. Omits the .git subfolder.
python_utils.PRINT('Copying files to the release directory')
shutil.copytree(os.getcwd(),
release_dir_path,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(release_dir_path):
if not os.getcwd().endswith(release_dir_name):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
python_utils.PRINT('Changing directory to %s' % os.getcwd())
python_utils.PRINT('Preprocessing release...')
preprocess_release(app_name, deploy_data_path)
update_and_check_indexes(app_name)
build_scripts(parsed_args.maintenance_mode)
deploy_application_and_write_log_entry(app_name,
current_release_version,
current_git_revision)
python_utils.PRINT('Returning to oppia/ root directory.')
switch_version(app_name, current_release_version)
flush_memcache(app_name)
check_breakage(app_name, current_release_version)
python_utils.PRINT('Done!')
finally:
subprocess.check_output([
'git', 'checkout', '--', update_configs.LOCAL_FECONF_PATH,
update_configs.LOCAL_CONSTANTS_PATH, APP_DEV_YAML_PATH
])
def test_function_calls_with_prompt_for_feconf_and_terms_update(self):
check_function_calls = {
'check_updates_to_terms_of_service_gets_called': False,
'add_mailgun_api_key_gets_called': False,
'add_mailchimp_api_key_gets_called': False,
'apply_changes_based_on_config_gets_called': False,
'verify_feconf_gets_called': False,
'mailgun_api_key_is_to_be_verified': False,
'mailchimp_api_key_is_to_be_verified': False
}
expected_check_function_calls = {
'check_updates_to_terms_of_service_gets_called': True,
'add_mailgun_api_key_gets_called': True,
'add_mailchimp_api_key_gets_called': True,
'apply_changes_based_on_config_gets_called': True,
'verify_feconf_gets_called': True,
'mailgun_api_key_is_to_be_verified': True,
'mailchimp_api_key_is_to_be_verified': True
}
def mock_check_updates(unused_release_feconf_path,
unused_personal_access_token):
check_function_calls[
'check_updates_to_terms_of_service_gets_called'] = True
def mock_add_mailgun_api_key(unused_release_feconf_path):
check_function_calls['add_mailgun_api_key_gets_called'] = True
def mock_add_mailchimp_api_key(unused_release_feconf_path):
check_function_calls['add_mailchimp_api_key_gets_called'] = True
def mock_apply_changes(unused_local_filepath, unused_config_filepath,
unused_expected_config_line_regex):
check_function_calls[
'apply_changes_based_on_config_gets_called'] = True
def mock_verify_feconf(unused_release_feconf_path,
verify_email_api_keys):
check_function_calls['verify_feconf_gets_called'] = True
check_function_calls['mailgun_api_key_is_to_be_verified'] = (
verify_email_api_keys)
check_function_calls['mailchimp_api_key_is_to_be_verified'] = (
verify_email_api_keys)
check_updates_swap = self.swap(update_configs,
'check_updates_to_terms_of_service',
mock_check_updates)
add_mailgun_api_key_swap = self.swap(update_configs,
'add_mailgun_api_key',
mock_add_mailgun_api_key)
add_mailchimp_api_key_swap = self.swap(update_configs,
'add_mailchimp_api_key',
mock_add_mailchimp_api_key)
apply_changes_swap = self.swap(update_configs,
'apply_changes_based_on_config',
mock_apply_changes)
verify_feconf_swap = self.swap(update_configs, 'verify_feconf',
mock_verify_feconf)
with self.url_open_swap, check_updates_swap, add_mailgun_api_key_swap:
with apply_changes_swap, verify_feconf_swap:
with add_mailchimp_api_key_swap:
update_configs.main(args=[
'--release_dir_path',
'test-release-dir',
'--deploy_data_path',
'test-deploy-dir',
'--personal_access_token',
'test-token',
'--prompt_for_mailgun_and_terms_update',
])
self.assertEqual(check_function_calls, expected_check_function_calls)
