def _execute_branch_cut(): # Do prerequisite checks. common.require_cwd_to_be_oppia() common.verify_local_repo_is_clean() common.verify_current_branch_name('develop') # Update the local repo. remote_alias = common.get_remote_alias('https://github.com/oppia/oppia') subprocess.call(['git', 'pull', remote_alias]) _verify_target_branch_does_not_already_exist(remote_alias) _verify_target_version_is_consistent_with_latest_released_version() # The release coordinator should verify that tests are passing on develop # before checking out the release branch. common.open_new_tab_in_browser_if_possible( 'https://github.com/oppia/oppia#oppia---') while True: print ( 'Please confirm: are Travis checks passing on develop? (y/n) ') answer = raw_input().lower() if answer in ['y', 'ye', 'yes']: break elif answer: print ( 'Tests should pass on develop before this script is run. ' 'Exiting.') sys.exit() # Cut a new release branch. print 'Cutting a new release branch: %s' % NEW_BRANCH_NAME subprocess.call(['git', 'checkout', '-b', NEW_BRANCH_NAME]) # Update the version in app.yaml. print 'Updating the version number in app.yaml ...' with open('app.yaml', 'r') as f: content = f.read() assert content.count('version: default') == 1 os.remove('app.yaml') content = content.replace( 'version: default', 'version: %s' % NEW_APP_YAML_VERSION) with open('app.yaml', 'w+') as f: f.write(content) print 'Version number updated.' # Make a commit. print 'Committing the change.' subprocess.call([ 'git', 'commit', '-a', '-m', '"Update version number to %s"' % TARGET_VERSION]) # Push the new release branch to GitHub. print 'Pushing new release branch to GitHub.' subprocess.call(['git', 'push', remote_alias, NEW_BRANCH_NAME]) print '' print ( 'New release branch successfully cut. You are now on branch %s' % NEW_BRANCH_NAME) print 'Done!'
def _update_feconf(): # Do prerequisite checks. common.require_cwd_to_be_oppia() assert common.get_current_branch_name().startswith('release-') common.ensure_release_scripts_folder_exists_and_is_up_to_date() _apply_changes_to_feconf() print 'Done! Please check manually to ensure all the changes are correct.'
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() update_cache_slug() # Do a build; ensure there are no errors. print 'Building and minifying scripts...' subprocess.check_output(['python', 'scripts/build.py']) # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % (APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' print 'Done!'
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree( os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() update_cache_slug() # Do a build; ensure there are no errors. print 'Building and minifying scripts...' subprocess.check_output(['python', 'scripts/build.py']) # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % ( APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' print 'Done!'
def _update_indexes(): """Updates production indexes after doing the prerequisite checks.""" # Do prerequisite checks. common.require_cwd_to_be_oppia() gcloud_adapter.require_gcloud_to_be_available() if not common.is_current_branch_a_release_branch(): raise Exception( 'Indexes should only be updated from a release branch.') # Update the indexes. gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME)
def _update_configs(): """Updates the 'feconf.py' and 'constants.js' files after doing the prerequisite checks. """ # Do prerequisite checks. common.require_cwd_to_be_oppia() assert common.get_current_branch_name().startswith('release-') common.ensure_release_scripts_folder_exists_and_is_up_to_date() _apply_changes_based_on_config( LOCAL_FECONF_PATH, FECONF_CONFIG_PATH, '^([A-Z_]+ = ).*$') _apply_changes_based_on_config( LOCAL_CONSTANTS_PATH, CONSTANTS_CONFIG_PATH, '^( "[A-Z_]+": ).*$') print 'Done! Please check manually to ensure all the changes are correct.'
def _execute_branch_cut(): """Pushes the new release branch to Github.""" # Do prerequisite checks. common.require_cwd_to_be_oppia() common.verify_local_repo_is_clean() common.verify_current_branch_name('develop') # Update the local repo. remote_alias = common.get_remote_alias('[email protected]:oppia/oppia.git') subprocess.call(['git', 'pull', remote_alias]) _verify_target_branch_does_not_already_exist(remote_alias) _verify_target_version_is_consistent_with_latest_released_version() # The release coordinator should verify that tests are passing on develop # before checking out the release branch. common.open_new_tab_in_browser_if_possible( 'https://github.com/oppia/oppia#oppia---') while True: print ( 'Please confirm: are Travis checks passing on develop? (y/n) ') answer = raw_input().lower() if answer in ['y', 'ye', 'yes']: break elif answer: print ( 'Tests should pass on develop before this script is run. ' 'Exiting.') sys.exit() # Cut a new release branch. print 'Cutting a new release branch: %s' % NEW_BRANCH_NAME subprocess.call(['git', 'checkout', '-b', NEW_BRANCH_NAME]) # Push the new release branch to GitHub. print 'Pushing new release branch to GitHub.' subprocess.call(['git', 'push', remote_alias, NEW_BRANCH_NAME]) print '' print ( 'New release branch successfully cut. You are now on branch %s' % NEW_BRANCH_NAME) print 'Done!'
This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. """ # Change the app name in app.yaml. f = open('app.yaml', 'r') content = f.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) d = open('app.yaml', 'w+') d.write(content) # Check that the current directory is correct. common.require_cwd_to_be_oppia() CURRENT_GIT_VERSION = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting experimental deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
#[Errno -2] Name or service not known" error # in urllib.urlretrieve, if the user is behind a proxy. if 'VAGRANT' in os.environ: os.environ['http_proxy'] = '' TOOLS_DIR = os.path.join('..', 'oppia_tools') THIRD_PARTY_DIR = os.path.join('.', 'third_party') THIRD_PARTY_STATIC_DIR = os.path.join(THIRD_PARTY_DIR, 'static') MANIFEST_FILE_PATH = os.path.join(os.getcwd(), 'manifest.json') # Place to download zip files for temporary storage. TMP_UNZIP_PATH = os.path.join('.', 'tmp_unzip.zip') # Check that the current directory is correct. common.require_cwd_to_be_oppia(allow_deploy_dir=True) TARGET_DOWNLOAD_DIRS = { 'frontend': THIRD_PARTY_STATIC_DIR, 'backend': THIRD_PARTY_DIR, 'oppiaTools': TOOLS_DIR } _DOWNLOAD_FORMAT_ZIP = 'zip' _DOWNLOAD_FORMAT_TAR = 'tar' _DOWNLOAD_FORMAT_FILES = 'files' DOWNLOAD_FORMATS_TO_MANIFEST_KEYS = { 'zip': { 'mandatory_keys': ['version', 'url', 'downloadFormat'], 'optional_key_pairs': [
def _execute_deployment(): # Do prerequisite checks. common.require_cwd_to_be_oppia() common.ensure_release_scripts_folder_exists_and_is_up_to_date() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree( os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() # Do a build, while outputting to the terminal. print 'Building and minifying scripts...' build_process = subprocess.Popen( ['python', 'scripts/build.py', '--prod_env'], stdout=subprocess.PIPE) while True: line = build_process.stdout.readline().strip() if not line: break print line # Wait for process to terminate, then check return code. build_process.communicate() if build_process.returncode > 0: raise Exception('Build failed.') # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % ( APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' # If this is a test server deployment and the current release version is # already serving, open the library page (for sanity checking) and the GAE # error logs. if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and ( _get_served_version() == _get_current_release_version()): common.open_new_tab_in_browser_if_possible( 'https://console.cloud.google.com/logs/viewer?' 'project=%s&key1=default&minLogLevel=500' % APP_NAME_OPPIATESTSERVER) common.open_new_tab_in_browser_if_possible( 'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER) print 'Done!'
import tarfile import urllib import zipfile import common TOOLS_DIR = os.path.join('..', 'oppia_tools') THIRD_PARTY_DIR = os.path.join('.', 'third_party') THIRD_PARTY_STATIC_DIR = os.path.join(THIRD_PARTY_DIR, 'static') # Place to download zip files for temporary storage. TMP_UNZIP_PATH = os.path.join('.', 'tmp_unzip.zip') # Check that the current directory is correct. common.require_cwd_to_be_oppia() def download_files(source_url_root, target_dir, source_filenames): """Downloads a group of files and saves them to a given directory. Each file is downloaded only if it does not already exist. Args: source_url_root: the URL to prepend to all the filenames. target_dir: the directory to save the files to. source_filenames: a list of filenames. Each filename is appended to the end of the source_url_root in order to give the URL from which to download the file. The downloaded file is then placed in target_dir, and retains the same filename. """
def _execute_deployment(): """Executes the deployment process after doing the prerequisite checks.""" if not common.is_current_branch_a_release_branch(): raise Exception( 'The deployment script must be run from a release branch.') current_release_version = CURRENT_BRANCH_NAME[ len(common.RELEASE_BRANCH_NAME_PREFIX):].replace('.', '-') # Do prerequisite checks. common.require_cwd_to_be_oppia() common.ensure_release_scripts_folder_exists_and_is_up_to_date() gcloud_adapter.require_gcloud_to_be_available() if APP_NAME in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER]: if not common.is_current_branch_a_release_branch(): raise Exception( 'The deployment script must be run from a release branch.') if APP_NAME == APP_NAME_OPPIASERVER: with open('./feconf.py', 'r') as f: feconf_contents = f.read() if ('MAILGUN_API_KEY' not in feconf_contents or 'MAILGUN_API_KEY = None' in feconf_contents): raise Exception( 'The mailgun API key must be added before deployment.') if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() # Update indexes, then prompt for a check that they are all serving # before continuing with the deployment. # NOTE: This assumes that the build process does not modify the # index.yaml file or create a different version of it to use in # production. gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME) datastore_indexes_url = ( 'https://console.cloud.google.com/datastore/indexes?project=%s' % APP_NAME) common.open_new_tab_in_browser_if_possible(datastore_indexes_url) while True: print '******************************************************' print( 'PLEASE CONFIRM: are all datastore indexes serving? See %s ' '(y/n)' % datastore_indexes_url) answer = raw_input().lower() if answer in ['y', 'ye', 'yes']: break elif answer: raise Exception( 'Please wait for all indexes to serve, then run this ' 'script again to complete the deployment. Exiting.') # Do a build, while outputting to the terminal. print 'Building and minifying scripts...' build_process = subprocess.Popen( ['python', 'scripts/build.py', '--prod_env'], stdout=subprocess.PIPE) while True: line = build_process.stdout.readline().strip() if not line: break print line # Wait for process to terminate, then check return code. build_process.communicate() if build_process.returncode > 0: raise Exception('Build failed.') # Deploy export service to GAE. gcloud_adapter.deploy_application('export/app.yaml', APP_NAME) # Deploy app to GAE. gcloud_adapter.deploy_application( './app.yaml', APP_NAME, version=(CUSTOM_VERSION if CUSTOM_VERSION else current_release_version)) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % (APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' # If this is a test server deployment and the current release version is # already serving, open the library page (for sanity checking) and the GAE # error logs. currently_served_version = ( gcloud_adapter.get_currently_served_version(APP_NAME)) if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and (currently_served_version == current_release_version): common.open_new_tab_in_browser_if_possible( 'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER) common.open_new_tab_in_browser_if_possible( 'https://console.cloud.google.com/logs/viewer?' 'project=%s&key1=default&minLogLevel=500' % APP_NAME_OPPIATESTSERVER) print 'Done!'
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip() print "" print "Starting deployment process." if not os.path.exists(THIRD_PARTY_DIR): raise Exception( "Could not find third_party directory at %s. Please run start.sh " "prior to running this script." % THIRD_PARTY_DIR ) # Create a folder in which to save the release candidate. print "Ensuring that the release directory parent exists" common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print "Copying files to the release directory" shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns(".git")) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception("Invalid directory accessed during deployment: %s" % os.getcwd()) print "Changing directory to %s" % os.getcwd() print "Preprocessing release..." preprocess_release() # Do a build; ensure there are no errors. print "Building and minifying scripts..." subprocess.check_output(["python", "scripts/build.py"]) # Run the tests; ensure there are no errors. print "Running tests..." tests_proc = subprocess.Popen(["bash", os.path.join("scripts", "run_tests.sh")], stdout=subprocess.PIPE) tests_stdout, tests_stderr = tests_proc.communicate() print tests_stdout print tests_stderr if tests_proc.returncode != 0: raise Exception("Tests failed. Halting deployment.") # Deploy to GAE. subprocess.check_output([APPCFG_PATH, "update", ".", "--oauth2"]) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, "a") as log_file: log_file.write( "Successfully deployed to %s at %s (version %s)\n" % (APP_NAME, CURRENT_DATETIME.strftime("%Y-%m-%d %H:%M:%S"), current_git_revision) ) print "Returning to oppia/ root directory." print "Done!"