def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes files from the per-app deployment data. (3) Change the DEV_MODE constant in assets/constants.js. """ # Change the app name in app.yaml. with open('app.yaml', 'r') as app_yaml_file: content = app_yaml_file.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) with open('app.yaml', 'w+') as new_app_yaml_file: new_app_yaml_file.write(content) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception( 'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) # Copies files in root folder to assets/. for filename in FILES_AT_ROOT: src = os.path.join(DEPLOY_DATA_PATH, filename) dst = os.path.join(os.getcwd(), 'assets', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) # Copies files in images to /assets/images. for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst) # Changes the DEV_MODE constant in assets/constants.js. with open(os.path.join('assets', 'constants.js'), 'r') as assets_file: content = assets_file.read() assert '"DEV_MODE": true' in content os.remove(os.path.join('assets', 'constants.js')) content = content.replace('"DEV_MODE": true', '"DEV_MODE": false') with open(os.path.join('assets', 'constants.js'), 'w+') as new_assets_file: new_assets_file.write(content)
def download_and_untar_files( source_url, target_parent_dir, tar_root_name, target_root_name): """Downloads a tar file, untars it, and saves the result in a given dir. The download occurs only if the target directory that the tar file untars to does not exist. NB: This function assumes that the root level of the tar file has exactly one folder. Args: source_url: the URL from which to download the tar file. target_parent_dir: the directory to save the contents of the tar file to. tar_root_name: the name of the top-level folder in the tar directory. target_root_name: the name that the top-level folder should be renamed to in the local directory. """ if not os.path.exists(os.path.join(target_parent_dir, target_root_name)): print 'Downloading and untarring file %s to %s' % ( tar_root_name, target_parent_dir) common.ensure_directory_exists(target_parent_dir) urllib.urlretrieve(source_url, TMP_UNZIP_PATH) with contextlib.closing(tarfile.open(TMP_UNZIP_PATH, 'r:gz')) as tfile: tfile.extractall(target_parent_dir) os.remove(TMP_UNZIP_PATH) # Rename the target directory. os.rename( os.path.join(target_parent_dir, tar_root_name), os.path.join(target_parent_dir, target_root_name))
def download_and_untar_files( source_url, target_parent_dir, tar_root_name, target_root_name): """Downloads a tar file, untars it, and saves the result in a given dir. The download occurs only if the target directory that the tar file untars to does not exist. NB: This function assumes that the root level of the tar file has exactly one folder. Args: source_url: the URL from which to download the tar file. target_parent_dir: the directory to save the contents of the tar file to. tar_root_name: the name of the top-level folder in the tar directory. target_root_name: the name that the top-level folder should be renamed to in the local directory. """ if not os.path.exists(os.path.join(target_parent_dir, target_root_name)): print 'Downloading and untarring file %s to %s' % ( tar_root_name, target_parent_dir) common.ensure_directory_exists(target_parent_dir) urllib.urlretrieve(source_url, TMP_UNZIP_PATH) with tarfile.open(TMP_UNZIP_PATH, 'r:gz') as t: t.extractall(target_parent_dir) os.remove(TMP_UNZIP_PATH) # Rename the target directory. os.rename( os.path.join(target_parent_dir, tar_root_name), os.path.join(target_parent_dir, target_root_name))
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Substitutes files from the per-app deployment data. (2) Change the DEV_MODE constant in assets/constants.js. (3) Change GCS_RESOURCE_BUCKET in assets/constants.js. (4) Removes the "version" field from app.yaml, since gcloud does not like it (when deploying). """ if not os.path.exists(DEPLOY_DATA_PATH): raise Exception('Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) # Copies files in root folder to assets/. for filename in FILES_AT_ROOT: src = os.path.join(DEPLOY_DATA_PATH, filename) dst = os.path.join(os.getcwd(), 'assets', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) # Copies files in images to /assets/images. for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst) # Changes the DEV_MODE constant in assets/constants.js. with open(os.path.join('assets', 'constants.js'), 'r') as assets_file: content = assets_file.read() bucket_name = APP_NAME + BUCKET_NAME_SUFFIX assert '"DEV_MODE": true' in content assert '"GCS_RESOURCE_BUCKET_NAME": "None-resources",' in content os.remove(os.path.join('assets', 'constants.js')) content = content.replace('"DEV_MODE": true', '"DEV_MODE": false') content = content.replace( '"GCS_RESOURCE_BUCKET_NAME": "None-resources",', '"GCS_RESOURCE_BUCKET_NAME": "%s",' % bucket_name) with open(os.path.join('assets', 'constants.js'), 'w+') as new_assets_file: new_assets_file.write(content)
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() update_cache_slug() # Do a build; ensure there are no errors. print 'Building and minifying scripts...' subprocess.check_output(['python', 'scripts/build.py']) # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % (APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' print 'Done!'
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree( os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() update_cache_slug() # Do a build; ensure there are no errors. print 'Building and minifying scripts...' subprocess.check_output(['python', 'scripts/build.py']) # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % ( APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' print 'Done!'
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes image files in the images/ directory. """ # Change the app name in app.yaml. f = open('app.yaml', 'r') content = f.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) d = open('app.yaml', 'w+') d.write(content) # Substitute image files for the splash page. SPLASH_PAGE_FILES = ['favicon.ico'] DEPLOY_DATA_PATH = os.path.join( os.getcwd(), '..', 'deploy_data', APP_NAME) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception( 'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) for filename in SPLASH_PAGE_FILES: src = os.path.join(DEPLOY_DATA_PATH, 'images', filename) dst = os.path.join(os.getcwd(), 'static', 'images', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) IMAGE_DIRS = ['avatar', 'splash', 'sidebar', 'logo'] for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'static', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst)
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes image files for the splash page. """ # Change the app name in app.yaml. f = open('app.yaml', 'r') content = f.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) d = open('app.yaml', 'w+') d.write(content) # Substitute image files for the splash page. SPLASH_PAGE_FILES = ['favicon.ico'] DEPLOY_DATA_PATH = os.path.join(os.getcwd(), '..', 'deploy_data', APP_NAME) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception('Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) for filename in SPLASH_PAGE_FILES: src = os.path.join(DEPLOY_DATA_PATH, 'images', filename) dst = os.path.join(os.getcwd(), 'static', 'images', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) IMAGE_DIRS = ['splash', 'sidebar'] for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'static', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst)
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes image files in the images/ directory. """ # Change the app name in app.yaml. f = open('app.yaml', 'r') content = f.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) d = open('app.yaml', 'w+') d.write(content) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception('Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) # Copies files in common folder to assets/common. for filename in FILES_AT_ROOT_IN_COMMON: src = os.path.join(DEPLOY_DATA_PATH, 'common', filename) dst = os.path.join(os.getcwd(), 'assets', 'common', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) # Copies files in images to /assets/images for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst)
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes image files in the images/ directory. """ # Change the app name in app.yaml. f = open('app.yaml', 'r') content = f.read() os.remove('app.yaml') content = content.replace('oppiaserver', APP_NAME) d = open('app.yaml', 'w+') d.write(content) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception( 'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH) # Copies files in common folder to assets/common. for filename in FILES_AT_ROOT_IN_COMMON: src = os.path.join(DEPLOY_DATA_PATH, 'common', filename) dst = os.path.join(os.getcwd(), 'assets', 'common', filename) if not os.path.exists(src): raise Exception( 'Could not find source path %s. Please check your deploy_data ' 'folder.' % src) if not os.path.exists(dst): raise Exception( 'Could not find destination path %s. Has the code been ' 'updated in the meantime?' % dst) shutil.copyfile(src, dst) # Copies files in images to /assets/images for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name) dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name) if not os.path.exists(src_dir): raise Exception( 'Could not find source dir %s. Please check your deploy_data ' 'folder.' % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst)
def download_and_unzip_files( source_url, target_parent_dir, zip_root_name, target_root_name): """Downloads a zip file, unzips it, and saves the result in a given dir. The download occurs only if the target directory that the zip file unzips to does not exist. NB: This function assumes that the root level of the zip file has exactly one folder. Args: source_url: the URL from which to download the zip file. target_parent_dir: the directory to save the contents of the zip file to. zip_root_name: the name of the top-level folder in the zip directory. target_root_name: the name that the top-level folder should be renamed to in the local directory. """ if not os.path.exists(os.path.join(target_parent_dir, target_root_name)): print 'Downloading and unzipping file %s to %s ...' % ( zip_root_name, target_parent_dir) common.ensure_directory_exists(target_parent_dir) urllib.urlretrieve(source_url, filename=TMP_UNZIP_PATH) try: with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile: zfile.extractall(path=target_parent_dir) os.remove(TMP_UNZIP_PATH) except Exception: if os.path.exists(TMP_UNZIP_PATH): os.remove(TMP_UNZIP_PATH) # Some downloads (like jqueryui-themes) may require a user-agent. req = urllib2.Request(source_url) req.add_header('User-agent', 'python') # This is needed to get a seekable filestream that can be used # by zipfile.ZipFile. file_stream = StringIO.StringIO(urllib2.urlopen(req).read()) with zipfile.ZipFile(file_stream, 'r') as zfile: zfile.extractall(path=target_parent_dir) # Rename the target directory. os.rename( os.path.join(target_parent_dir, zip_root_name), os.path.join(target_parent_dir, target_root_name)) print 'Download of %s succeeded.' % zip_root_name
def preprocess_release(): """Pre-processes release files. This function should be called from within RELEASE_DIR_NAME. Currently it does the following: (1) Changes the app name in app.yaml to APP_NAME. (2) Substitutes image files in the images/ directory. """ # Change the app name in app.yaml. f = open("app.yaml", "r") content = f.read() os.remove("app.yaml") content = content.replace("oppiaserver", APP_NAME) d = open("app.yaml", "w+") d.write(content) # Substitute image files for the splash page. SPLASH_PAGE_FILES = ["favicon.ico"] DEPLOY_DATA_PATH = os.path.join(os.getcwd(), "..", "deploy_data", APP_NAME) if not os.path.exists(DEPLOY_DATA_PATH): raise Exception("Could not find deploy_data directory at %s" % DEPLOY_DATA_PATH) for filename in SPLASH_PAGE_FILES: src = os.path.join(DEPLOY_DATA_PATH, "images", filename) dst = os.path.join(os.getcwd(), "static", "images", filename) if not os.path.exists(src): raise Exception("Could not find source path %s. Please check your deploy_data " "folder." % src) if not os.path.exists(dst): raise Exception("Could not find destination path %s. Has the code been " "updated in the meantime?" % dst) shutil.copyfile(src, dst) IMAGE_DIRS = ["splash", "sidebar", "logo"] for dir_name in IMAGE_DIRS: src_dir = os.path.join(DEPLOY_DATA_PATH, "images", dir_name) dst_dir = os.path.join(os.getcwd(), "static", "images", dir_name) if not os.path.exists(src_dir): raise Exception("Could not find source dir %s. Please check your deploy_data " "folder." % src_dir) common.ensure_directory_exists(dst_dir) for filename in os.listdir(src_dir): src = os.path.join(src_dir, filename) dst = os.path.join(dst_dir, filename) shutil.copyfile(src, dst)
def download_and_unzip_files( source_url, target_parent_dir, zip_root_name, target_root_name): """Downloads a zip file, unzips it, and saves the result in a given dir. The download occurs only if the target directory that the zip file unzips to does not exist. NB: This function assumes that the root level of the zip file has exactly one folder. Args: source_url: the URL from which to download the zip file. target_parent_dir: the directory to save the contents of the zip file to. zip_root_name: the name of the top-level folder in the zip directory. target_root_name: the name that the top-level folder should be renamed to in the local directory. """ if not os.path.exists(os.path.join(target_parent_dir, target_root_name)): print 'Downloading and unzipping file %s to %s' % ( zip_root_name, target_parent_dir) common.ensure_directory_exists(target_parent_dir) urllib.urlretrieve(source_url, TMP_UNZIP_PATH) try: with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile: zfile.extractall(target_parent_dir) os.remove(TMP_UNZIP_PATH) except Exception: if os.path.exists(TMP_UNZIP_PATH): os.remove(TMP_UNZIP_PATH) # Some downloads (like jqueryui-themes) may require a user-agent. req = urllib2.Request(source_url) req.add_header('User-agent', 'python') # This is needed to get a seekable filestream that can be used # by zipfile.ZipFile. file_stream = StringIO.StringIO(urllib2.urlopen(req).read()) with zipfile.ZipFile(file_stream, 'r') as zfile: zfile.extractall(target_parent_dir) # Rename the target directory. os.rename( os.path.join(target_parent_dir, zip_root_name), os.path.join(target_parent_dir, target_root_name))
def download_files(source_url_root, target_dir, source_filenames): """Downloads a group of files and saves them to a given directory. Each file is downloaded only if it does not already exist. Args: source_url_root: the URL to prepend to all the filenames. target_dir: the directory to save the files to. source_filenames: a list of filenames. Each filename is appended to the end of the source_url_root in order to give the URL from which to download the file. The downloaded file is then placed in target_dir, and retains the same filename. """ assert isinstance(source_filenames, list) common.ensure_directory_exists(target_dir) for filename in source_filenames: if not os.path.exists(os.path.join(target_dir, filename)): print "Downloading file %s to %s" % (filename, target_dir) urllib.urlretrieve("%s/%s" % (source_url_root, filename), os.path.join(target_dir, filename))
def download_files(source_url_root, target_dir, source_filenames): """Downloads a group of files and saves them to a given directory. Each file is downloaded only if it does not already exist. Args: source_url_root: the URL to prepend to all the filenames. target_dir: the directory to save the files to. source_filenames: a list of filenames. Each filename is appended to the end of the source_url_root in order to give the URL from which to download the file. The downloaded file is then placed in target_dir, and retains the same filename. """ assert isinstance(source_filenames, list) common.ensure_directory_exists(target_dir) for filename in source_filenames: if not os.path.exists(os.path.join(target_dir, filename)): print 'Downloading file %s to %s' % (filename, target_dir) urllib.urlretrieve('%s/%s' % (source_url_root, filename), os.path.join(target_dir, filename))
common.require_cwd_to_be_oppia() CURRENT_GIT_VERSION = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting experimental deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception('Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd()
def _execute_deployment(): # Check that the current directory is correct. common.require_cwd_to_be_oppia() current_git_revision = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip() print "" print "Starting deployment process." if not os.path.exists(THIRD_PARTY_DIR): raise Exception( "Could not find third_party directory at %s. Please run start.sh " "prior to running this script." % THIRD_PARTY_DIR ) # Create a folder in which to save the release candidate. print "Ensuring that the release directory parent exists" common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print "Copying files to the release directory" shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns(".git")) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception("Invalid directory accessed during deployment: %s" % os.getcwd()) print "Changing directory to %s" % os.getcwd() print "Preprocessing release..." preprocess_release() # Do a build; ensure there are no errors. print "Building and minifying scripts..." subprocess.check_output(["python", "scripts/build.py"]) # Run the tests; ensure there are no errors. print "Running tests..." tests_proc = subprocess.Popen(["bash", os.path.join("scripts", "run_tests.sh")], stdout=subprocess.PIPE) tests_stdout, tests_stderr = tests_proc.communicate() print tests_stdout print tests_stderr if tests_proc.returncode != 0: raise Exception("Tests failed. Halting deployment.") # Deploy to GAE. subprocess.check_output([APPCFG_PATH, "update", ".", "--oauth2"]) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, "a") as log_file: log_file.write( "Successfully deployed to %s at %s (version %s)\n" % (APP_NAME, CURRENT_DATETIME.strftime("%Y-%m-%d %H:%M:%S"), current_git_revision) ) print "Returning to oppia/ root directory." print "Done!"
common.require_cwd_to_be_oppia() CURRENT_GIT_VERSION = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() print '' print 'Starting experimental deployment process.' if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree( os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...'
def _execute_deployment(): # Do prerequisite checks. common.require_cwd_to_be_oppia() common.ensure_release_scripts_folder_exists_and_is_up_to_date() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree( os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() # Do a build, while outputting to the terminal. print 'Building and minifying scripts...' build_process = subprocess.Popen( ['python', 'scripts/build.py', '--prod_env'], stdout=subprocess.PIPE) while True: line = build_process.stdout.readline().strip() if not line: break print line # Wait for process to terminate, then check return code. build_process.communicate() if build_process.returncode > 0: raise Exception('Build failed.') # Deploy to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.']) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % ( APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' # If this is a test server deployment and the current release version is # already serving, open the library page (for sanity checking) and the GAE # error logs. if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and ( _get_served_version() == _get_current_release_version()): common.open_new_tab_in_browser_if_possible( 'https://console.cloud.google.com/logs/viewer?' 'project=%s&key1=default&minLogLevel=500' % APP_NAME_OPPIATESTSERVER) common.open_new_tab_in_browser_if_possible( 'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER) print 'Done!'
def _execute_deployment(): """Executes the deployment process after doing the prerequisite checks.""" if not common.is_current_branch_a_release_branch(): raise Exception( 'The deployment script must be run from a release branch.') current_release_version = CURRENT_BRANCH_NAME[ len(common.RELEASE_BRANCH_NAME_PREFIX):].replace('.', '-') # Do prerequisite checks. common.require_cwd_to_be_oppia() common.ensure_release_scripts_folder_exists_and_is_up_to_date() gcloud_adapter.require_gcloud_to_be_available() if APP_NAME in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER]: if not common.is_current_branch_a_release_branch(): raise Exception( 'The deployment script must be run from a release branch.') if APP_NAME == APP_NAME_OPPIASERVER: with open('./feconf.py', 'r') as f: feconf_contents = f.read() if ('MAILGUN_API_KEY' not in feconf_contents or 'MAILGUN_API_KEY = None' in feconf_contents): raise Exception( 'The mailgun API key must be added before deployment.') if not os.path.exists(THIRD_PARTY_DIR): raise Exception( 'Could not find third_party directory at %s. Please run start.sh ' 'prior to running this script.' % THIRD_PARTY_DIR) current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() # Create a folder in which to save the release candidate. print 'Ensuring that the release directory parent exists' common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH)) # Copy files to the release directory. Omits the .git subfolder. print 'Copying files to the release directory' shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git')) # Change the current directory to the release candidate folder. with common.CD(RELEASE_DIR_PATH): if not os.getcwd().endswith(RELEASE_DIR_NAME): raise Exception( 'Invalid directory accessed during deployment: %s' % os.getcwd()) print 'Changing directory to %s' % os.getcwd() print 'Preprocessing release...' preprocess_release() # Update indexes, then prompt for a check that they are all serving # before continuing with the deployment. # NOTE: This assumes that the build process does not modify the # index.yaml file or create a different version of it to use in # production. gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME) datastore_indexes_url = ( 'https://console.cloud.google.com/datastore/indexes?project=%s' % APP_NAME) common.open_new_tab_in_browser_if_possible(datastore_indexes_url) while True: print '******************************************************' print( 'PLEASE CONFIRM: are all datastore indexes serving? See %s ' '(y/n)' % datastore_indexes_url) answer = raw_input().lower() if answer in ['y', 'ye', 'yes']: break elif answer: raise Exception( 'Please wait for all indexes to serve, then run this ' 'script again to complete the deployment. Exiting.') # Do a build, while outputting to the terminal. print 'Building and minifying scripts...' build_process = subprocess.Popen( ['python', 'scripts/build.py', '--prod_env'], stdout=subprocess.PIPE) while True: line = build_process.stdout.readline().strip() if not line: break print line # Wait for process to terminate, then check return code. build_process.communicate() if build_process.returncode > 0: raise Exception('Build failed.') # Deploy export service to GAE. gcloud_adapter.deploy_application('export/app.yaml', APP_NAME) # Deploy app to GAE. gcloud_adapter.deploy_application( './app.yaml', APP_NAME, version=(CUSTOM_VERSION if CUSTOM_VERSION else current_release_version)) # Writing log entry. common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH)) with open(LOG_FILE_PATH, 'a') as log_file: log_file.write( 'Successfully deployed to %s at %s (version %s)\n' % (APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'), current_git_revision)) print 'Returning to oppia/ root directory.' # If this is a test server deployment and the current release version is # already serving, open the library page (for sanity checking) and the GAE # error logs. currently_served_version = ( gcloud_adapter.get_currently_served_version(APP_NAME)) if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and (currently_served_version == current_release_version): common.open_new_tab_in_browser_if_possible( 'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER) common.open_new_tab_in_browser_if_possible( 'https://console.cloud.google.com/logs/viewer?' 'project=%s&key1=default&minLogLevel=500' % APP_NAME_OPPIATESTSERVER) print 'Done!'