def main():
""" db_uninstaller main loop """
# handle parser option
options = get_options()
notify = lambda n: raw_input('Uninstall Trafodion on [%s] [N]: ' % n)
format_output('Trafodion Uninstall Start')
if options.pwd:
pwd = getpass.getpass('Input remote host SSH Password: '******''
node_list = ''
# parse node list from trafodion_config
if os.path.exists(TRAF_CFG_FILE):
with open(TRAF_CFG_FILE, 'r') as f:
traf_cfgs = f.readlines()
try:
line = [l for l in traf_cfgs if 'NODE_LIST' in l][0]
node_list = re.search(r'NODE_LIST="(.*)"', line).groups()[0]
except Exception as e:
err_m('Cannot find node list info from %s: %s' % (TRAF_CFG_FILE, e))
# parse node list from installation config file
elif options.cfgfile:
if not os.path.exists(options.cfgfile):
err_m('Cannot find config file \'%s\'' % options.cfgfile)
config_file = options.cfgfile
p = ParseInI(config_file, 'dbconfigs')
cfgs = p.load()
node_list = cfgs['node_list']
# user input
else:
node_lists = raw_input('Enter Trafodion node list to uninstall(separated by comma): ')
if not node_lists: err_m('Empty value')
node_list = ' '.join(expNumRe(node_lists))
if not options.silent:
rc = notify(node_list)
if rc.lower() != 'y': sys.exit(1)
nodes = node_list.split()
first_node = nodes[0]
remotes = [Remote(node, pwd=pwd) for node in nodes]
sudo_prefix = get_sudo_prefix()
# remove trafodion userid and group on all trafodion nodes, together with folders
for remote in remotes:
info('Remove Trafodion on node [%s] ...' % remote.host)
remote.execute('ps -f -u %s|awk \'{print $2}\'|xargs %s kill -9' % (TRAF_USER, sudo_prefix), chkerr=False)
remote.execute('trafid=`getent passwd %s|awk -F: \'{print $3}\'`; if [[ -n $trafid ]]; then ps -f -u $trafid|awk \'{print $2}\'|xargs %s kill -9; fi' % (TRAF_USER, sudo_prefix), chkerr=False)
remote.execute('%s /usr/sbin/userdel -rf %s' % (sudo_prefix, TRAF_USER), chkerr=False)
remote.execute('%s /usr/sbin/groupdel %s' % (sudo_prefix, TRAF_USER), chkerr=False)
remote.execute('%s rm -rf /etc/security/limits.d/%s.conf %s /tmp/hsperfdata_%s 2>/dev/null' % (sudo_prefix, TRAF_USER, TRAF_CFG_DIR, TRAF_USER), chkerr=False)
run_cmd('rm -f %s/*.status %s' % (INSTALLER_LOC, DBCFG_FILE))
format_output('Trafodion Uninstall Completed')
def set_project(project_name):
"""Sets the project to the given project name.
Args:
project_name: str. The name of the project.
"""
common.run_cmd([GCLOUD_PATH, 'config', 'set', 'project', project_name])
def main(personal_access_token):
"""Updates the files corresponding to LOCAL_FECONF_PATH and
LOCAL_CONSTANTS_PATH after doing the prerequisite checks.
Args:
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.is_current_branch_a_release_branch()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
try:
python_utils.url_open(TERMS_PAGE_URL)
except Exception:
raise Exception('Terms mainpage does not exist on Github.')
try:
check_updates_to_terms_of_service(personal_access_token)
add_mailgun_api_key()
apply_changes_based_on_config(LOCAL_FECONF_PATH, FECONF_CONFIG_PATH,
FECONF_REGEX)
apply_changes_based_on_config(LOCAL_CONSTANTS_PATH,
CONSTANTS_CONFIG_PATH, CONSTANTS_REGEX)
except Exception as e:
common.run_cmd(
['git', 'checkout', '--', LOCAL_FECONF_PATH, LOCAL_CONSTANTS_PATH])
raise Exception(e)
common.ask_user_to_confirm(
'Done! Please check manually to ensure all the changes are correct.')
def main(tokens=None):
"""Check if a PR is low-risk."""
parser = argparse.ArgumentParser()
parser.add_argument('pr_url', help='The URL of the pull request.')
args = parser.parse_args(args=tokens)
parsed_url = parse_pr_url(args.pr_url)
if not parsed_url:
raise RuntimeError('Failed to parse PR URL %s' % args.pr_url)
owner, repo, number = parsed_url
pr = lookup_pr(owner, repo, number)
if not pr:
raise RuntimeError('Failed to load PR from GitHub API')
base_repo_url = pr['base']['repo']['clone_url']
common.run_cmd(['git', 'remote', 'add', UPSTREAM_REMOTE, base_repo_url])
base_branch = pr['base']['ref']
common.run_cmd(['git', 'fetch', UPSTREAM_REMOTE, base_branch])
diff_files, file_diffs = load_diff(pr['base']['ref'])
if not diff_files:
raise RuntimeError('Failed to load PR diff')
for low_risk_type, low_risk_checker in LOW_RISK_CHECKERS:
reason_not_low_risk = low_risk_checker(pr, diff_files, file_diffs)
if reason_not_low_risk:
python_utils.PRINT(
'PR is not a low-risk PR of type %s because: %s' %
(low_risk_type, reason_not_low_risk))
else:
python_utils.PRINT('PR is low-risk. Skipping some CI checks.')
return 0
python_utils.PRINT('PR is not low-risk. Running all CI checks.')
return 1
def copy_files():
# package trafodion binary into a tar file
if not os.path.exists(TRAF_PKG_FILE):
info('Creating trafodion packages of %s, this will take a while ...' % cfgs['traf_home'])
run_cmd_as_user(cfgs['traf_user'], 'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*' % (cfgs['traf_home'], TRAF_PKG_FILE))
else:
info('Using existing trafodion package %s' % TRAF_PKG_FILE)
info('Copying trafodion files to new nodes, this will take a while ...')
run_cmd('%s cp -rf %s/../.ssh /tmp' % (get_sudo_prefix(), cfgs['traf_home']))
run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
traf_ssh_folder = '/tmp/.ssh'
hbase_trx_file = cmd_output('ls %s/hbase-trx-*' % cfgs['hbase_lib_path'])
trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' % cfgs['hbase_lib_path'])
files = [TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file, trafodion_utility_file]
remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
for thread in threads: thread.start()
for thread in threads: thread.join()
for r in remote_insts:
if r.rc != 0: err_m('Failed to copy files to %s' % r.host)
def main():
""" db_uninstaller main loop """
# handle parser option
options = get_options()
notify = lambda n: raw_input('Uninstall Trafodion on [%s] [N]: ' % n)
format_output('Trafodion Uninstall Start')
if options.pwd:
pwd = getpass.getpass('Input remote host SSH Password: '******''
try:
traf_var = run_cmd_as_user(TRAF_USER,"echo $TRAF_VAR")
node_list = run_cmd_as_user(TRAF_USER,"trafconf -name")
except:
if options.cfgfile:
if not os.path.exists(options.cfgfile):
err_m('Cannot find config file \'%s\'' % options.cfgfile)
config_file = options.cfgfile
p = ParseInI(config_file, 'dbconfigs')
cfgs = p.load()
traf_var = cfgs['traf_var']
node_list = cfgs['node_list']
# user input
else:
traf_var = '/var/lib/trafodion'
node_lists = raw_input('Enter Trafodion node list to uninstall(separated by comma): ')
if not node_lists: err_m('Empty value')
node_list = ' '.join(expNumRe(node_lists))
if not options.silent:
rc = notify(node_list)
if rc.lower() != 'y': sys.exit(1)
nodes = node_list.split()
first_node = nodes[0]
remotes = [Remote(node, pwd=pwd) for node in nodes]
sudo_prefix = get_sudo_prefix()
# remove trafodion userid and group on all trafodion nodes, together with folders
for remote in remotes:
info('Remove Trafodion on node [%s] ...' % remote.host)
remote.execute('ps -f -u %s|awk \'{print $2}\'|xargs %s kill -9' % (TRAF_USER, sudo_prefix), chkerr=False)
remote.execute('trafid=`getent passwd %s|awk -F: \'{print $3}\'`; if [[ -n $trafid ]]; then ps -f -u $trafid|awk \'{print $2}\'|xargs %s kill -9; fi' % (TRAF_USER, sudo_prefix), chkerr=False)
remote.execute('%s /usr/sbin/userdel -rf %s' % (sudo_prefix, TRAF_USER), chkerr=False)
remote.execute('%s /usr/sbin/groupdel %s' % (sudo_prefix, TRAF_USER), chkerr=False)
remote.execute('%s rm -rf /etc/security/limits.d/%s.conf %s %s /tmp/hsperfdata_%s 2>/dev/null' % (sudo_prefix, TRAF_USER, TRAF_CFG_DIR, traf_var, TRAF_USER), chkerr=False)
run_cmd('rm -f %s/*.status %s' % (INSTALLER_LOC, DBCFG_FILE))
format_output('Trafodion Uninstall Completed')
def get_extra_jobs_due_to_schema_changes(
remote_alias, previous_release_version):
"""Finds additional jobs which should be run based on
schema changes in feconf.
Args:
remote_alias: str. The alias for Oppia repo.
previous_release_version: str. The version of the previous release.
Returns:
list(str). The list of jobs to run based on schema changes.
"""
schema_versions_to_jobs_mapping = {
'CURRENT_COLLECTION_SCHEMA_VERSION': 'CollectionMigrationOneOffJob',
'CURRENT_STATE_SCHEMA_VERSION': 'ExplorationMigrationJobManager',
'CURRENT_SKILL_CONTENTS_SCHEMA_VERSION': 'SkillMigrationOneOffJob',
'CURRENT_MISCONCEPTIONS_SCHEMA_VERSION': 'SkillMigrationOneOffJob',
'CURRENT_RUBRIC_SCHEMA_VERSION': 'SkillMigrationOneOffJob',
'CURRENT_STORY_CONTENTS_SCHEMA_VERSION': 'StoryMigrationOneOffJob',
'CURRENT_SUBTOPIC_SCHEMA_VERSION': 'TopicMigrationOneOffJob',
'CURRENT_STORY_REFERENCE_SCHEMA_VERSION': 'TopicMigrationOneOffJob'}
diff_output = common.run_cmd([
'git', 'diff', '%s/develop' % remote_alias,
'%s/release-%s' % (remote_alias, previous_release_version),
'feconf.py'])
feconf_diff = diff_output[:-1].split('\n')
jobs_to_run = []
for version_key in schema_versions_to_jobs_mapping:
for line in feconf_diff:
if line.startswith(('+%s' % version_key, '-%s' % version_key)):
jobs_to_run.append(schema_versions_to_jobs_mapping[version_key])
return list(set(jobs_to_run))
def install_chrome(version):
"""Install Chrome from the URL in URL_TEMPLATE.
Args:
version: str. The version of Chrome to install. This must be one
of the versions available from
github.com/webnicer/chrome-downloads.
"""
_ = common.run_cmd(['sudo', 'apt-get', 'update'])
_ = common.run_cmd(['sudo', 'apt-get', 'install', 'libappindicator3-1'])
_ = common.run_cmd([
'curl', '-L', '-o', CHROME_DEB_FILE,
URL_TEMPLATE.format(version)])
_ = common.run_cmd([
'sudo', 'sed', '-i',
's|HERE/chrome\\"|HERE/chrome\\" --disable-setuid-sandbox|g',
'/opt/google/chrome/google-chrome'])
_ = common.run_cmd(['sudo', 'dpkg', '-i', CHROME_DEB_FILE])
def notify_user(self):
""" show the final configs to user """
format_output('Final Configs')
title = ['config type', 'value']
pt = PrettyTable(title)
for item in title:
pt.align[item] = 'l'
for key, value in sorted(cfgs.items()):
# only notify user input value
if self.in_data.has_key(key) and value:
if self.in_data[key].has_key('ispasswd'): continue
pt.add_row([key, value])
print pt
confirm = self.get_confirm()
if confirm != 'Y':
if os.path.exists(DBCFG_FILE): os.remove(DBCFG_FILE)
run_cmd('rm -rf %s/*.status' % INSTALLER_LOC)
log_err('User quit')
def notify_user(self):
""" show the final configs to user """
format_output('Final Configs')
title = ['config type', 'value']
pt = PrettyTable(title)
for item in title:
pt.align[item] = 'l'
for key, value in sorted(cfgs.items()):
# only notify user input value
if self.in_data.has_key(key) and value:
if self.in_data[key].has_key('ispasswd'): continue
pt.add_row([key, value])
print pt
confirm = self.get_confirm()
if confirm != 'Y':
if os.path.exists(DBCFG_FILE): os.remove(DBCFG_FILE)
run_cmd('rm -rf %s/*.status' % INSTALLER_LOC)
log_err('User quit')
def create_branch(repo, repo_fork, target_branch, github_username,
current_release_version_number):
"""Creates a new branch with updates to AUTHORS, CHANGELOG,
CONTRIBUTORS and about-page.
Args:
repo: github.Repository.Repository. The PyGithub object for the
original repo.
repo_fork: github.Repository.Repository. The PyGithub object for the
forked repo.
target_branch: str. The name of the target branch.
github_username: str. The github username of the user.
current_release_version_number: str. The version of current release.
"""
python_utils.PRINT(
'Creating new branch with updates to AUTHORS, CONTRIBUTORS, '
'CHANGELOG and about-page...')
sb = repo.get_branch('develop')
repo_fork.create_git_ref(ref='refs/heads/%s' % target_branch,
sha=sb.commit.sha)
for filepath in [
CHANGELOG_FILEPATH, AUTHORS_FILEPATH, CONTRIBUTORS_FILEPATH,
ABOUT_PAGE_CONSTANTS_FILEPATH
]:
contents = repo_fork.get_contents(filepath, ref=target_branch)
with python_utils.open_file(filepath, 'r') as f:
repo_fork.update_file(contents.path,
'Update %s' % filepath,
f.read(),
contents.sha,
branch=target_branch)
common.run_cmd(GIT_CMD_CHECKOUT.split(' '))
common.open_new_tab_in_browser_if_possible(
'https://github.com/oppia/oppia/compare/develop...%s:%s?'
'expand=1&title=Update authors and changelog for v%s' %
(github_username, target_branch, current_release_version_number))
python_utils.PRINT('Pushed changes to Github. '
'Please create a pull request from the %s branch\n\n'
'Note: PR title should be exactly: '
'"Update authors and changelog for v%s" '
'otherwise deployment will fail.' %
(target_branch, current_release_version_number))
def run_webdriver_manager(parameters):
"""Run commands of webdriver manager.
Args:
parameters: list(str). A list of parameters to pass to webdriver
manager.
"""
web_driver_command = [common.NODE_BIN_PATH, WEBDRIVER_MANAGER_BIN_PATH]
web_driver_command.extend(parameters)
python_utils.PRINT(common.run_cmd(web_driver_command))
def remove_updates_and_delete_branch(repo_fork, target_branch):
"""Remove changes made to AUTHORS, CHANGELOG, CONTRIBUTORS
and about-page and delete the branch created with these changes.
Args:
repo_fork: github.Repository.Repository. The PyGithub object for the
forked repo.
target_branch: str. The name of the target branch.
"""
common.run_cmd(GIT_CMD_CHECKOUT.split(' '))
# The get_git_ref code is wrapped in try except block since the
# function raises an exception if the target branch is not found.
try:
repo_fork.get_git_ref('heads/%s' % target_branch).delete()
except github.UnknownObjectException:
pass
except Exception:
raise Exception('Please ensure that %s branch is deleted before '
're-running the script' % target_branch)
def initiate_backup_restoration_process():
"""Initiate the backup restoration process on backup migration server."""
common.open_new_tab_in_browser_if_possible(LIST_OF_BUCKETS_URL)
python_utils.PRINT('Navigate into the newest backup folder. \n'
'There should be a file here of the form '
'<date_time>.overall_export_metadata. \n'
'For example, "<folder-name>/20200213-090001/'
'20200213-090001.overall_export_metadata". '
'This is the file you want to import.\n'
'Please copy and enter the full path of this file\n')
export_metadata_filepath = python_utils.INPUT()
if not re.match(
r'^oppia-export-backups/(\d{8}-\d{6})/\1\.overall_export_metadata$',
export_metadata_filepath):
raise Exception('Invalid export metadata filepath: %s' %
(export_metadata_filepath))
common.run_cmd([
GCLOUD_PATH, 'datastore', 'import',
'gs://%s' % export_metadata_filepath, '--async'
])
def cancel_operation():
"""Cancels a datastore operation."""
python_utils.PRINT('Cancellation of operation may corrupt the datastore. '
'Refer: https://cloud.google.com/datastore/docs/'
'export-import-entities#cancel_an_operation\n'
'Do you want to continue?\n')
execute_cancellation = python_utils.INPUT().lower()
if execute_cancellation not in release_constants.AFFIRMATIVE_CONFIRMATIONS:
python_utils.PRINT('Aborting Cancellation.')
return
python_utils.PRINT('List of operations in progress:\n')
check_backup_restoration_status()
python_utils.PRINT(
'Enter the name of the operation to cancel from the above list. '
'The name of an operation is listed in the field called "name". '
'Check the example here: https://stackoverflow.com/a/53630367 for '
'details.\n')
operation_name = python_utils.INPUT()
common.run_cmd(
[GCLOUD_PATH, 'datastore', 'operations', 'cancel', operation_name])
def _git_diff_names_only(left, right='HEAD'):
"""Get names of changed files from git.
Args:
left: str. Lefthand timepoint.
right: str. rightand timepoint.
Returns:
list(str): List of files that are different between the two points.
"""
diff_cmd = (GIT_CMD_DIFF_NAMES_ONLY_FORMAT_STRING % (left, right))
return common.run_cmd(diff_cmd.split(' ')).splitlines()
def get_chrome_version():
"""Get the current version of Chrome.
Note that this only works on Linux systems. On macOS, for example,
the `google-chrome` command may not work.
Returns:
str. The version of Chrome we found.
"""
output = str(common.run_cmd(['google-chrome', '--version']))
chrome_version = ''.join(re.findall(r'([0-9]|\.)', output))
return chrome_version
def did_supported_audio_languages_change(remote_alias,
previous_release_version):
"""Checks changes in constants.SUPPORTED_AUDIO_LANGUAGES between
the current and previous release.
Args:
remote_alias: str. The alias for Oppia repo.
previous_release_version: str. The version of the previous release.
Returns:
bool. Whether supported audio languages have changed.
"""
try:
from constants import constants
supported_audio_language_ids_for_current_release = [
lang_dict['id']
for lang_dict in constants['SUPPORTED_AUDIO_LANGUAGES']
]
common.run_cmd([
'git', 'checkout',
'%s/release-%s' % (remote_alias, previous_release_version), '--',
'assets/constants.ts'
])
from constants import constants
supported_audio_language_ids_for_previous_release = [
lang_dict['id']
for lang_dict in constants['SUPPORTED_AUDIO_LANGUAGES']
]
finally:
common.run_cmd(['git', 'reset', 'assets/constants.ts'])
common.run_cmd(['git', 'checkout', '--', 'assets/constants.ts'])
return (sorted(supported_audio_language_ids_for_current_release) !=
sorted(supported_audio_language_ids_for_previous_release))
def _get_changed_filenames_since_tag(release_tag_to_diff_against):
"""Get names of changed files from git since a given release.
Args:
release_tag_to_diff_against: str. The release tag to diff against.
Returns:
list(str). List of filenames for files that have been modified since
the release against which diff is being checked.
"""
diff_cmd = (GIT_CMD_DIFF_NAMES_ONLY_FORMAT_STRING %
(release_tag_to_diff_against, 'HEAD'))
return common.run_cmd(diff_cmd.split(' ')).splitlines()
def main(personal_access_token):
"""Updates the files corresponding to LOCAL_FECONF_PATH and
LOCAL_CONSTANTS_PATH after doing the prerequisite checks.
Args:
personal_access_token: str. The personal access token for the
GitHub id of user.
"""
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
assert common.is_current_branch_a_release_branch(), (
'Current branch is not a release branch_name')
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
try:
python_utils.url_open(TERMS_PAGE_URL)
except Exception:
raise Exception('Terms mainpage does not exist on Github.')
try:
check_updates_to_terms_of_service(personal_access_token)
add_mailgun_api_key()
apply_changes_based_on_config(
LOCAL_FECONF_PATH, FECONF_CONFIG_PATH, FECONF_REGEX)
apply_changes_based_on_config(
LOCAL_CONSTANTS_PATH, CONSTANTS_CONFIG_PATH, CONSTANTS_REGEX)
except Exception as e:
common.run_cmd([
'git', 'checkout', '--', LOCAL_FECONF_PATH, LOCAL_CONSTANTS_PATH])
raise Exception(e)
common.ask_user_to_confirm(
'Done! Please check feconf.py and assets/constants.ts to ensure that '
'the changes made are correct. Specifically verify that the '
'MAILGUN_API_KEY and REDISHOST are updated correctly and '
'other config changes are corresponding to %s and %s.\n' % (
FECONF_CONFIG_PATH, CONSTANTS_CONFIG_PATH))
def build_js_files(dev_mode_setting):
"""Build the javascript files.
Args:
dev_mode_setting: bool. Represents whether to run the related commands
in dev mode.
"""
update_dev_mode_in_constants_js(CONSTANT_FILE_PATH, dev_mode_setting)
if not dev_mode_setting:
python_utils.PRINT(' Generating files for production mode...')
else:
# The 'hashes.json' file is used by the `url-interpolation` service.
if not os.path.isfile(HASHES_FILE_PATH):
with python_utils.open_file(HASHES_FILE_PATH, 'w') as hash_file:
hash_file.write('{}')
try:
common.run_cmd([
common.NODE_BIN_PATH, WEBPACK_BIN_PATH, '--config',
'webpack.dev.config.ts'
])
except subprocess.CalledProcessError as error:
python_utils.PRINT(error.output)
sys.exit(error.returncode)
build.main(args=(['--prod_env'] if not dev_mode_setting else []))
def gather_logs(start, stop='HEAD'):
"""Gathers the logs between the start and endpoint.
Args:
start: str. Tag, Branch or SHA1 of start point
stop: str. Tag, Branch or SHA1 of end point, defaults to HEAD
Returns:
list(Log): List of Logs.
"""
get_logs_cmd = GIT_CMD_GET_LOGS_FORMAT_STRING.format(
GROUP_SEP, start, stop)
# The unicode conversion is required because there can be non-ascii
# characters in the logs and it can result in breaking the flow
# of release summary generation.
out = python_utils.UNICODE(common.run_cmd(get_logs_cmd.split(' ')),
'utf-8').split('\x00')
if len(out) == 1 and out[0] == '':
return []
else:
return [Log(*line.strip().split(GROUP_SEP)) for line in out]
def check_versions(current_release):
"""Checks if the versions for the exploration or collection schemas have
changed.
Args:
current_release: str. The current release tag to diff against.
Returns:
List of variable names that changed.
"""
feconf_changed_version = []
git_show_cmd = (GIT_CMD_SHOW_FORMAT_STRING % current_release)
old_feconf = common.run_cmd(git_show_cmd.split(' '))
with python_utils.open_file(FECONF_FILEPATH, 'r') as feconf_file:
new_feconf = feconf_file.read()
for variable in FECONF_VAR_NAMES:
old_version = re.findall(VERSION_RE_FORMAT_STRING % variable,
old_feconf)[0]
new_version = re.findall(VERSION_RE_FORMAT_STRING % variable,
new_feconf)[0]
if old_version != new_version:
feconf_changed_version.append(variable)
return feconf_changed_version
def get_extra_commits_in_new_release(base_commit, repo):
"""Gets extra commits in the new release.
Args:
base_commit: str. The base commit common between current branch and the
latest release.
repo: github.Repository.Repository. The PyGithub object for the repo.
Returns:
list(github.Commit.Commit). List of commits from the base commit up to
the current commit, which haven't been cherrypicked already.
"""
get_commits_cmd = GIT_CMD_TEMPLATE_GET_NEW_COMMITS % base_commit
out = common.run_cmd(get_commits_cmd.split(' ')).split('\n')
commits = []
for line in out:
# Lines that start with a - are already cherrypicked. The commits of
# interest are on lines that start with +.
if line[0] == '+':
line = line[2:]
commit = repo.get_commit(line[:line.find(' ')])
commits.append(commit)
return commits
def get_changed_schema_version_constant_names(release_tag_to_diff_against):
"""Returns a list of schema version constant names in feconf that have
changed since the release against which diff is being checked.
Args:
release_tag_to_diff_against: str. The release tag to diff against.
Returns:
list(str). List of version constant names in feconf that changed.
"""
changed_version_constants_in_feconf = []
git_show_cmd = (GIT_CMD_SHOW_FORMAT_STRING % release_tag_to_diff_against)
old_feconf = common.run_cmd(git_show_cmd.split(' '))
with python_utils.open_file(FECONF_FILEPATH, 'r') as feconf_file:
new_feconf = feconf_file.read()
for version_constant in FECONF_SCHEMA_VERSION_CONSTANT_NAMES:
old_version = re.findall(VERSION_RE_FORMAT_STRING % version_constant,
old_feconf)[0]
new_version = re.findall(VERSION_RE_FORMAT_STRING % version_constant,
new_feconf)[0]
if old_version != new_version:
changed_version_constants_in_feconf.append(version_constant)
return changed_version_constants_in_feconf
def check_backup_restoration_status():
"""Checks the status of backup restoration process."""
python_utils.PRINT(
common.run_cmd([GCLOUD_PATH, 'datastore', 'operations', 'list']))
def execute_deployment():
"""Executes the deployment process after doing the prerequisite checks.
Raises:
Exception: App name is invalid.
Exception: Custom version is used with production app.
Exception: App name is not specified.
Exception: The deployment script is not run from a release or test
branch.
Exception: The deployment script is run for prod server from a test
branch.
Exception: Current release version has '.' character.
Exception: Last commit message is invalid.
Exception: The mailgun API key is not added before deployment.
Exception: Could not find third party directory.
Exception: Invalid directory accessed during deployment.
"""
parsed_args = _PARSER.parse_args()
custom_version = None
if parsed_args.app_name:
app_name = parsed_args.app_name
if app_name not in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
] and ('migration' not in app_name):
raise Exception('Invalid app name: %s' % app_name)
if parsed_args.version and app_name == APP_NAME_OPPIASERVER:
raise Exception('Cannot use custom version with production app.')
# Note that custom_version may be None.
custom_version = parsed_args.version
else:
raise Exception('No app name specified.')
current_branch_name = common.get_current_branch_name()
release_dir_name = 'deploy-%s-%s-%s' % (
'-'.join('-'.join(app_name.split('.')).split(':')),
current_branch_name, CURRENT_DATETIME.strftime('%Y%m%d-%H%M%S'))
release_dir_path = os.path.join(os.getcwd(), '..', release_dir_name)
deploy_data_path = os.path.join(os.getcwd(), os.pardir, 'release-scripts',
'deploy_data', app_name)
install_third_party_libs.main()
if not (common.is_current_branch_a_release_branch() or
(common.is_current_branch_a_test_branch())):
raise Exception(
'The deployment script must be run from a release or test branch.')
if common.is_current_branch_a_test_branch() and (app_name in [
APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER
]):
raise Exception('Test branch can only be deployed to backup server.')
if custom_version is not None:
current_release_version = custom_version.replace(DOT_CHAR, HYPHEN_CHAR)
else:
current_release_version = current_branch_name[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace(
DOT_CHAR, HYPHEN_CHAR)
# This is required to compose the release_version_library_url
# (defined in switch_version function) correctly.
if '.' in current_release_version:
raise Exception('Current release version has \'.\' character.')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
try:
if app_name == APP_NAME_OPPIASERVER:
create_release_doc()
release_version_number = common.get_current_release_version_number(
current_branch_name)
last_commit_message = subprocess.check_output(
'git log -1 --pretty=%B'.split())
if not last_commit_message.startswith(
'Update authors and changelog for v%s' %
(release_version_number)):
raise Exception('Invalid last commit message: %s.' %
last_commit_message)
check_travis_and_circleci_tests(current_branch_name)
personal_access_token = common.get_personal_access_token()
g = github.Github(personal_access_token)
repo = g.get_organization('oppia').get_repo('oppia')
common.check_blocking_bug_issue_count(repo)
common.check_prs_for_current_release_are_released(repo)
update_configs.main(personal_access_token)
with python_utils.open_file(FECONF_PATH, 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run '
'install_third_party_libs.py prior to running this script.' %
THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
python_utils.PRINT('Ensuring that the release directory parent exists')
common.ensure_directory_exists(os.path.dirname(release_dir_path))
# Copy files to the release directory. Omits the .git subfolder.
python_utils.PRINT('Copying files to the release directory')
shutil.copytree(os.getcwd(),
release_dir_path,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(release_dir_path):
if not os.getcwd().endswith(release_dir_name):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
python_utils.PRINT('Changing directory to %s' % os.getcwd())
python_utils.PRINT('Preprocessing release...')
preprocess_release(app_name, deploy_data_path)
update_and_check_indexes(app_name)
build_scripts()
deploy_application_and_write_log_entry(app_name,
current_release_version,
current_git_revision)
python_utils.PRINT('Returning to oppia/ root directory.')
switch_version(app_name, current_release_version)
flush_memcache(app_name)
check_breakage(app_name, current_release_version)
python_utils.PRINT('Done!')
finally:
common.run_cmd([
'git', 'checkout', '--', update_configs.LOCAL_FECONF_PATH,
update_configs.LOCAL_CONSTANTS_PATH
])
def main():
""" add_nodes main loop """
cfgs = defaultdict(str)
# handle parser option
options = get_options()
if not options.nodes:
err_m('Must specifiy the node names using \'--nodes\' option')
# get node list from user input
new_nodes = expNumRe(options.nodes)
if not new_nodes:
err_m('Incorrect format')
if options.pwd:
pwd = getpass.getpass('Input remote host SSH Password: '******''
u = UserInput(options, pwd)
g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)
format_output('Trafodion Elastic Add Nodes Script')
### read configs from current trafodion_config and save it to cfgs
if os.path.exists(TRAF_CFG_FILE):
with open(TRAF_CFG_FILE, 'r') as f:
traf_cfgs = f.readlines()
for traf_cfg in traf_cfgs:
if not traf_cfg.strip(): continue
key, value = traf_cfg.replace('export ', '').split('=')
value = value.replace('"', '')
value = value.replace('\n', '')
cfgs[key.lower()] = value
else:
err_m(
'Cannot find %s, be sure to run this script on one of trafodion nodes'
% TRAF_CFG_FILE)
### config check
if not cfgs['hbase_lib_path'] or not cfgs['traf_version']:
err_m('Missing parameters in Trafodion config file')
if not cfgs['traf_home'] or not cmd_output(
'%s ls %s' % (get_sudo_prefix(), cfgs['traf_home'])):
err_m('Cannot find trafodion binary folder')
# get trafodion user from traf_home path
cfgs['traf_user'] = cfgs['traf_home'].split('/')[-2]
if not cfgs['traf_user']:
err_m('Cannot detect trafodion user')
### parse trafodion user's password
cfgs['traf_shadow'] = cmd_output(
"%s grep %s /etc/shadow |awk -F: '{print $2}'" %
(get_sudo_prefix(), cfgs['traf_user']))
def copy_files():
# package trafodion binary into a tar file
if not os.path.exists(TRAF_PKG_FILE):
info(
'Creating trafodion packages of %s, this will take a while ...'
% cfgs['traf_home'])
run_cmd_as_user(
cfgs['traf_user'],
'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*'
% (cfgs['traf_home'], TRAF_PKG_FILE))
else:
info('Using existing trafodion package %s' % TRAF_PKG_FILE)
info(
'Copying trafodion files to new nodes, this will take a while ...')
run_cmd('%s cp -rf %s/../.ssh /tmp' %
(get_sudo_prefix(), cfgs['traf_home']))
run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
traf_ssh_folder = '/tmp/.ssh'
hbase_trx_file = cmd_output('ls %s/hbase-trx-*' %
cfgs['hbase_lib_path'])
trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' %
cfgs['hbase_lib_path'])
files = [
TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file,
trafodion_utility_file
]
remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
threads = [
Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for r in remote_insts:
if r.rc != 0: err_m('Failed to copy files to %s' % r.host)
### copy trafodion_config/trafodion-package/hbase-trx to the new nodes
copy_files()
### set parameters
if cfgs['enable_ha'].upper() == 'true':
g('dcs_backup_nodes')
cfgs['dcs_ha'] = 'Y'
else:
cfgs['dcs_ha'] = 'N'
if cfgs['trafodion_enable_authentication'] == 'YES':
cfgs['ldap_security'] = 'Y'
else:
cfgs['ldap_security'] = 'N'
if cfgs['secure_hadoop'].upper() == 'Y':
g('kdc_server')
g('admin_principal')
g('kdcadmin_pwd')
#TODO: offline support
cfgs['offline_mode'] = 'N'
format_output('AddNode sub scripts Start')
### run addNode script on new nodes ###
cfgs['node_list'] = ','.join(new_nodes)
info('Running add node setup on new node(s) [%s] ...' % cfgs['node_list'])
wrapper.run(cfgs, options, mode='addnodes_new', pwd=pwd)
### run dcs setup script on all nodes ###
# get current trafodion node list
current_nodes = cmd_output('%s su - %s -c "trafconf -name 2>/dev/null"' %
(get_sudo_prefix(), cfgs['traf_user'])).split()
all_nodes = list(set(new_nodes + current_nodes))
cfgs['node_list'] = ','.join(all_nodes)
info('Running dcs setup on all node(s) [%s] ...' % cfgs['node_list'])
wrapper.run(cfgs, options, mode='addnodes_all', pwd=pwd)
### do sqshell node add/up, sqregen
# check if trafodion is running
mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
if int(mon_process) > 0:
info('Trafodion instance is up, adding node in sqshell ...')
# cores=0-1;processors=2;roles=connection,aggregation,storage
sqconfig_ptr = cmd_output(
'%s su - %s -c "trafconf -node|sed -n 2p|cut -d\\\";\\\" -f3-5"' %
(get_sudo_prefix(), cfgs['traf_user']))
for node in new_nodes:
info('adding node [%s] in sqshell ...' % node)
run_cmd_as_user(
cfgs['traf_user'],
'echo "node add {node-name %s,%s}" | sqshell -a' %
(node, sqconfig_ptr))
run_cmd_as_user(cfgs['traf_user'],
'echo "node up %s" | sqshell -a' % node)
ok('Node [%s] added!' % node)
info('Starting DCS on new nodes ...')
run_cmd_as_user(cfgs['traf_user'], 'dcsstart')
else:
info('Trafodion instance is not up, do sqgen ...')
run_cmd_as_user(cfgs['traf_user'],
'rm %s/sqconfig.db' % cfgs['traf_var'])
run_cmd_as_user(cfgs['traf_user'], 'sqgen')
ok('Setup completed. You need to start trafodion manually')
### clean up
run_cmd('%s rm -rf /tmp/.ssh' % get_sudo_prefix())
run_cmd('%s rm -rf %s' % (get_sudo_prefix(), TRAF_PKG_FILE))
format_output('AddNode Complete')
info(
'NOTICE: You need to manually restart RegionServer on newly added nodes to take effect'
)
def load_diff(base_branch):
"""Load the diff between the head and base.
Only determine the diffs for files in FILES_THAT_NEED_DIFFS. Other
files will be listed as having changed, but their diff lines will
not be returned.
Args:
base_branch: str. Base branch of PR.
Returns:
tuple(list(tuple(str, str)), dict(str, list(str)). Tuple of a
list of changed files (each a tuple of before, after) and a
dictionary mapping from file names to list of diff lines for
each file. In the event of a parsing error, a tuple of an empty
list and empty dictionary is returned.
"""
diff_name_status = common.run_cmd([
'git',
'diff',
'--name-status',
'{}/{}'.format(UPSTREAM_REMOTE, base_branch),
])
diff_files = []
for line in diff_name_status.split('\n'):
if not line:
continue
split = line.split()
if len(split) < 2 or len(split) > 3:
python_utils.PRINT('Failed to parse diff --name-status line "%s"' %
line)
return [], {}
elif len(split) == 2:
diff_files.append((split[1], split[1]))
elif len(split) == 3:
diff_files.append((split[1], split[2]))
file_diffs = {}
for file_tuple in diff_files:
for filename in file_tuple:
if filename in file_diffs:
# Don't re-generate a diff we already have.
continue
if filename not in FILES_THAT_NEED_DIFFS:
continue
file_diff = common.run_cmd([
'git',
'diff',
'-U0',
'{}/{}'.format(UPSTREAM_REMOTE, base_branch),
'--',
filename,
])
file_diff_split = file_diff.rstrip().split('\n')
i = 0
# Find the end of the diff header. See
# https://git-scm.com/docs/diff-format for details on the
# git diff format.
for line in file_diff_split:
i += 1
if line.startswith('@@'):
break
if i == len(file_diff_split):
# We reached the end of the diff without finding the
# header, or the header consumes the entire diff.
python_utils.PRINT(
'Failed to find end of header in "%s" diff' % filename)
return [], {}
file_diffs[filename] = file_diff_split[i:]
return diff_files, file_diffs
def main():
""" add_nodes main loop """
cfgs = defaultdict(str)
# handle parser option
options = get_options()
if not options.nodes:
err_m('Must specifiy the node names using \'--nodes\' option')
# get node list from user input
new_nodes = expNumRe(options.nodes)
if not new_nodes:
err_m('Incorrect format')
if options.pwd:
pwd = getpass.getpass('Input remote host SSH Password: '******''
u = UserInput(options, pwd)
g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)
format_output('Trafodion Elastic Add Nodes Script')
### read configs from current trafodion_config and save it to cfgs
if os.path.exists(TRAF_CFG_FILE):
with open(TRAF_CFG_FILE, 'r') as f:
traf_cfgs = f.readlines()
for traf_cfg in traf_cfgs:
if not traf_cfg.strip(): continue
key, value = traf_cfg.replace('export ', '').split('=')
value = value.replace('"','')
value = value.replace('\n','')
cfgs[key.lower()] = value
else:
err_m('Cannot find %s, be sure to run this script on one of trafodion nodes' % TRAF_CFG_FILE)
### config check
if not cfgs['hbase_lib_path'] or not cfgs['traf_version']:
err_m('Missing parameters in Trafodion config file')
if not cfgs['traf_home'] or not cmd_output('%s ls %s' % (get_sudo_prefix(), cfgs['traf_home'])):
err_m('Cannot find trafodion binary folder')
# get trafodion user from traf_home path
cfgs['traf_user'] = cfgs['traf_home'].split('/')[-2]
if not cfgs['traf_user']:
err_m('Cannot detect trafodion user')
### parse trafodion user's password
cfgs['traf_shadow'] = cmd_output("%s grep %s /etc/shadow |awk -F: '{print $2}'" % (get_sudo_prefix(), cfgs['traf_user']))
def copy_files():
# package trafodion binary into a tar file
if not os.path.exists(TRAF_PKG_FILE):
info('Creating trafodion packages of %s, this will take a while ...' % cfgs['traf_home'])
run_cmd_as_user(cfgs['traf_user'], 'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*' % (cfgs['traf_home'], TRAF_PKG_FILE))
else:
info('Using existing trafodion package %s' % TRAF_PKG_FILE)
info('Copying trafodion files to new nodes, this will take a while ...')
run_cmd('%s cp -rf %s/../.ssh /tmp' % (get_sudo_prefix(), cfgs['traf_home']))
run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
traf_ssh_folder = '/tmp/.ssh'
hbase_trx_file = cmd_output('ls %s/hbase-trx-*' % cfgs['hbase_lib_path'])
trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' % cfgs['hbase_lib_path'])
files = [TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file, trafodion_utility_file]
remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
for thread in threads: thread.start()
for thread in threads: thread.join()
for r in remote_insts:
if r.rc != 0: err_m('Failed to copy files to %s' % r.host)
### copy trafodion_config/trafodion-package/hbase-trx to the new nodes
copy_files()
### set parameters
if cfgs['enable_ha'].upper() == 'true':
g('dcs_backup_nodes')
cfgs['dcs_ha'] = 'Y'
else:
cfgs['dcs_ha'] = 'N'
if cfgs['trafodion_enable_authentication'] == 'YES':
cfgs['ldap_security'] = 'Y'
else:
cfgs['ldap_security'] = 'N'
if cfgs['secure_hadoop'].upper() == 'Y':
g('kdc_server')
g('admin_principal')
g('kdcadmin_pwd')
#TODO: offline support
cfgs['offline_mode'] = 'N'
format_output('AddNode sub scripts Start')
### run addNode script on new nodes ###
cfgs['node_list'] = ','.join(new_nodes)
info('Running add node setup on new node(s) [%s] ...' % cfgs['node_list'])
wrapper.run(cfgs, options, mode='addnodes_new', pwd=pwd)
### run dcs setup script on all nodes ###
# get current trafodion node list
current_nodes = cmd_output('%s su - %s -c "trafconf -name 2>/dev/null"' % (get_sudo_prefix(), cfgs['traf_user'])).split()
all_nodes = list(set(new_nodes + current_nodes))
cfgs['node_list'] = ','.join(all_nodes)
info('Running dcs setup on all node(s) [%s] ...' % cfgs['node_list'])
wrapper.run(cfgs, options, mode='addnodes_all', pwd=pwd)
### do sqshell node add/up, sqregen
# check if trafodion is running
mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
if int(mon_process) > 0:
info('Trafodion instance is up, adding node in sqshell ...')
# cores=0-1;processors=2;roles=connection,aggregation,storage
sqconfig_ptr = cmd_output('%s su - %s -c "trafconf -node|sed -n 2p|cut -d\\\";\\\" -f3-5"' % (get_sudo_prefix(), cfgs['traf_user']))
for node in new_nodes:
info('adding node [%s] in sqshell ...' % node)
run_cmd_as_user(cfgs['traf_user'], 'echo "node add {node-name %s,%s}" | sqshell -a' % (node, sqconfig_ptr))
run_cmd_as_user(cfgs['traf_user'], 'echo "node up %s" | sqshell -a' % node)
ok('Node [%s] added!' % node)
info('Starting DCS on new nodes ...')
run_cmd_as_user(cfgs['traf_user'], 'dcsstart')
else:
info('Trafodion instance is not up, do sqgen ...')
run_cmd_as_user(cfgs['traf_user'], 'rm %s/sql/scripts/sqconfig.db' % cfgs['traf_home'])
run_cmd_as_user(cfgs['traf_user'], 'sqgen')
ok('Setup completed. You need to start trafodion manually')
### clean up
run_cmd('%s rm -rf /tmp/.ssh' % get_sudo_prefix())
run_cmd('%s rm -rf %s' % (get_sudo_prefix(), TRAF_PKG_FILE))
format_output('AddNode Complete')
info('NOTICE: You need to manually restart RegionServer on newly added nodes to take effect')
