def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes files from the per-app deployment data.
(3) Change the DEV_MODE constant in assets/constants.js.
"""
# Change the app name in app.yaml.
with open('app.yaml', 'r') as app_yaml_file:
content = app_yaml_file.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
with open('app.yaml', 'w+') as new_app_yaml_file:
new_app_yaml_file.write(content)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception(
'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH)
# Copies files in root folder to assets/.
for filename in FILES_AT_ROOT:
src = os.path.join(DEPLOY_DATA_PATH, filename)
dst = os.path.join(os.getcwd(), 'assets', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images.
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
# Changes the DEV_MODE constant in assets/constants.js.
with open(os.path.join('assets', 'constants.js'), 'r') as assets_file:
content = assets_file.read()
assert '"DEV_MODE": true' in content
os.remove(os.path.join('assets', 'constants.js'))
content = content.replace('"DEV_MODE": true', '"DEV_MODE": false')
with open(os.path.join('assets', 'constants.js'), 'w+') as new_assets_file:
new_assets_file.write(content)
def download_and_untar_files(
source_url, target_parent_dir, tar_root_name, target_root_name):
"""Downloads a tar file, untars it, and saves the result in a given dir.
The download occurs only if the target directory that the tar file untars
to does not exist.
NB: This function assumes that the root level of the tar file has exactly
one folder.
Args:
source_url: the URL from which to download the tar file.
target_parent_dir: the directory to save the contents of the tar file to.
tar_root_name: the name of the top-level folder in the tar directory.
target_root_name: the name that the top-level folder should be renamed to
in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
print 'Downloading and untarring file %s to %s' % (
tar_root_name, target_parent_dir)
common.ensure_directory_exists(target_parent_dir)
urllib.urlretrieve(source_url, TMP_UNZIP_PATH)
with contextlib.closing(tarfile.open(TMP_UNZIP_PATH, 'r:gz')) as tfile:
tfile.extractall(target_parent_dir)
os.remove(TMP_UNZIP_PATH)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, tar_root_name),
os.path.join(target_parent_dir, target_root_name))
def download_and_untar_files(
source_url, target_parent_dir, tar_root_name, target_root_name):
"""Downloads a tar file, untars it, and saves the result in a given dir.
The download occurs only if the target directory that the tar file untars
to does not exist.
NB: This function assumes that the root level of the tar file has exactly
one folder.
Args:
source_url: the URL from which to download the tar file.
target_parent_dir: the directory to save the contents of the tar file to.
tar_root_name: the name of the top-level folder in the tar directory.
target_root_name: the name that the top-level folder should be renamed to
in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
print 'Downloading and untarring file %s to %s' % (
tar_root_name, target_parent_dir)
common.ensure_directory_exists(target_parent_dir)
urllib.urlretrieve(source_url, TMP_UNZIP_PATH)
with tarfile.open(TMP_UNZIP_PATH, 'r:gz') as t:
t.extractall(target_parent_dir)
os.remove(TMP_UNZIP_PATH)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, tar_root_name),
os.path.join(target_parent_dir, target_root_name))
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Substitutes files from the per-app deployment data.
(2) Change the DEV_MODE constant in assets/constants.js.
(3) Change GCS_RESOURCE_BUCKET in assets/constants.js.
(4) Removes the "version" field from app.yaml, since gcloud does not like
it (when deploying).
"""
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception('Could not find deploy_data directory at %s' %
DEPLOY_DATA_PATH)
# Copies files in root folder to assets/.
for filename in FILES_AT_ROOT:
src = os.path.join(DEPLOY_DATA_PATH, filename)
dst = os.path.join(os.getcwd(), 'assets', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images.
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
# Changes the DEV_MODE constant in assets/constants.js.
with open(os.path.join('assets', 'constants.js'), 'r') as assets_file:
content = assets_file.read()
bucket_name = APP_NAME + BUCKET_NAME_SUFFIX
assert '"DEV_MODE": true' in content
assert '"GCS_RESOURCE_BUCKET_NAME": "None-resources",' in content
os.remove(os.path.join('assets', 'constants.js'))
content = content.replace('"DEV_MODE": true', '"DEV_MODE": false')
content = content.replace(
'"GCS_RESOURCE_BUCKET_NAME": "None-resources",',
'"GCS_RESOURCE_BUCKET_NAME": "%s",' % bucket_name)
with open(os.path.join('assets', 'constants.js'), 'w+') as new_assets_file:
new_assets_file.write(content)
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
print ''
print 'Starting deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(os.getcwd(),
RELEASE_DIR_PATH,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
update_cache_slug()
# Do a build; ensure there are no errors.
print 'Building and minifying scripts...'
subprocess.check_output(['python', 'scripts/build.py'])
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' %
(APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
print 'Done!'
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
print ''
print 'Starting deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(
os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s'
% os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
update_cache_slug()
# Do a build; ensure there are no errors.
print 'Building and minifying scripts...'
subprocess.check_output(['python', 'scripts/build.py'])
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.', '--oauth2'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' % (
APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
print 'Done!'
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes image files in the images/ directory.
"""
# Change the app name in app.yaml.
f = open('app.yaml', 'r')
content = f.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
d = open('app.yaml', 'w+')
d.write(content)
# Substitute image files for the splash page.
SPLASH_PAGE_FILES = ['favicon.ico']
DEPLOY_DATA_PATH = os.path.join(
os.getcwd(), '..', 'deploy_data', APP_NAME)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception(
'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH)
for filename in SPLASH_PAGE_FILES:
src = os.path.join(DEPLOY_DATA_PATH, 'images', filename)
dst = os.path.join(os.getcwd(), 'static', 'images', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
IMAGE_DIRS = ['avatar', 'splash', 'sidebar', 'logo']
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'static', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes image files for the splash page.
"""
# Change the app name in app.yaml.
f = open('app.yaml', 'r')
content = f.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
d = open('app.yaml', 'w+')
d.write(content)
# Substitute image files for the splash page.
SPLASH_PAGE_FILES = ['favicon.ico']
DEPLOY_DATA_PATH = os.path.join(os.getcwd(), '..', 'deploy_data', APP_NAME)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception('Could not find deploy_data directory at %s' %
DEPLOY_DATA_PATH)
for filename in SPLASH_PAGE_FILES:
src = os.path.join(DEPLOY_DATA_PATH, 'images', filename)
dst = os.path.join(os.getcwd(), 'static', 'images', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
IMAGE_DIRS = ['splash', 'sidebar']
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'static', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes image files in the images/ directory.
"""
# Change the app name in app.yaml.
f = open('app.yaml', 'r')
content = f.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
d = open('app.yaml', 'w+')
d.write(content)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception('Could not find deploy_data directory at %s' %
DEPLOY_DATA_PATH)
# Copies files in common folder to assets/common.
for filename in FILES_AT_ROOT_IN_COMMON:
src = os.path.join(DEPLOY_DATA_PATH, 'common', filename)
dst = os.path.join(os.getcwd(), 'assets', 'common', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes image files in the images/ directory.
"""
# Change the app name in app.yaml.
f = open('app.yaml', 'r')
content = f.read()
os.remove('app.yaml')
content = content.replace('oppiaserver', APP_NAME)
d = open('app.yaml', 'w+')
d.write(content)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception(
'Could not find deploy_data directory at %s' % DEPLOY_DATA_PATH)
# Copies files in common folder to assets/common.
for filename in FILES_AT_ROOT_IN_COMMON:
src = os.path.join(DEPLOY_DATA_PATH, 'common', filename)
dst = os.path.join(os.getcwd(), 'assets', 'common', filename)
if not os.path.exists(src):
raise Exception(
'Could not find source path %s. Please check your deploy_data '
'folder.' % src)
if not os.path.exists(dst):
raise Exception(
'Could not find destination path %s. Has the code been '
'updated in the meantime?' % dst)
shutil.copyfile(src, dst)
# Copies files in images to /assets/images
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, 'images', dir_name)
dst_dir = os.path.join(os.getcwd(), 'assets', 'images', dir_name)
if not os.path.exists(src_dir):
raise Exception(
'Could not find source dir %s. Please check your deploy_data '
'folder.' % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
def download_and_unzip_files(
source_url, target_parent_dir, zip_root_name, target_root_name):
"""Downloads a zip file, unzips it, and saves the result in a given dir.
The download occurs only if the target directory that the zip file unzips
to does not exist.
NB: This function assumes that the root level of the zip file has exactly
one folder.
Args:
source_url: the URL from which to download the zip file.
target_parent_dir: the directory to save the contents of the zip file to.
zip_root_name: the name of the top-level folder in the zip directory.
target_root_name: the name that the top-level folder should be renamed to
in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
print 'Downloading and unzipping file %s to %s ...' % (
zip_root_name, target_parent_dir)
common.ensure_directory_exists(target_parent_dir)
urllib.urlretrieve(source_url, filename=TMP_UNZIP_PATH)
try:
with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
os.remove(TMP_UNZIP_PATH)
except Exception:
if os.path.exists(TMP_UNZIP_PATH):
os.remove(TMP_UNZIP_PATH)
# Some downloads (like jqueryui-themes) may require a user-agent.
req = urllib2.Request(source_url)
req.add_header('User-agent', 'python')
# This is needed to get a seekable filestream that can be used
# by zipfile.ZipFile.
file_stream = StringIO.StringIO(urllib2.urlopen(req).read())
with zipfile.ZipFile(file_stream, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, zip_root_name),
os.path.join(target_parent_dir, target_root_name))
print 'Download of %s succeeded.' % zip_root_name
def preprocess_release():
"""Pre-processes release files.
This function should be called from within RELEASE_DIR_NAME. Currently it
does the following:
(1) Changes the app name in app.yaml to APP_NAME.
(2) Substitutes image files in the images/ directory.
"""
# Change the app name in app.yaml.
f = open("app.yaml", "r")
content = f.read()
os.remove("app.yaml")
content = content.replace("oppiaserver", APP_NAME)
d = open("app.yaml", "w+")
d.write(content)
# Substitute image files for the splash page.
SPLASH_PAGE_FILES = ["favicon.ico"]
DEPLOY_DATA_PATH = os.path.join(os.getcwd(), "..", "deploy_data", APP_NAME)
if not os.path.exists(DEPLOY_DATA_PATH):
raise Exception("Could not find deploy_data directory at %s" % DEPLOY_DATA_PATH)
for filename in SPLASH_PAGE_FILES:
src = os.path.join(DEPLOY_DATA_PATH, "images", filename)
dst = os.path.join(os.getcwd(), "static", "images", filename)
if not os.path.exists(src):
raise Exception("Could not find source path %s. Please check your deploy_data " "folder." % src)
if not os.path.exists(dst):
raise Exception("Could not find destination path %s. Has the code been " "updated in the meantime?" % dst)
shutil.copyfile(src, dst)
IMAGE_DIRS = ["splash", "sidebar", "logo"]
for dir_name in IMAGE_DIRS:
src_dir = os.path.join(DEPLOY_DATA_PATH, "images", dir_name)
dst_dir = os.path.join(os.getcwd(), "static", "images", dir_name)
if not os.path.exists(src_dir):
raise Exception("Could not find source dir %s. Please check your deploy_data " "folder." % src_dir)
common.ensure_directory_exists(dst_dir)
for filename in os.listdir(src_dir):
src = os.path.join(src_dir, filename)
dst = os.path.join(dst_dir, filename)
shutil.copyfile(src, dst)
def download_and_unzip_files(
source_url, target_parent_dir, zip_root_name, target_root_name):
"""Downloads a zip file, unzips it, and saves the result in a given dir.
The download occurs only if the target directory that the zip file unzips
to does not exist.
NB: This function assumes that the root level of the zip file has exactly
one folder.
Args:
source_url: the URL from which to download the zip file.
target_parent_dir: the directory to save the contents of the zip file to.
zip_root_name: the name of the top-level folder in the zip directory.
target_root_name: the name that the top-level folder should be renamed to
in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
print 'Downloading and unzipping file %s to %s' % (
zip_root_name, target_parent_dir)
common.ensure_directory_exists(target_parent_dir)
urllib.urlretrieve(source_url, TMP_UNZIP_PATH)
try:
with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile:
zfile.extractall(target_parent_dir)
os.remove(TMP_UNZIP_PATH)
except Exception:
if os.path.exists(TMP_UNZIP_PATH):
os.remove(TMP_UNZIP_PATH)
# Some downloads (like jqueryui-themes) may require a user-agent.
req = urllib2.Request(source_url)
req.add_header('User-agent', 'python')
# This is needed to get a seekable filestream that can be used
# by zipfile.ZipFile.
file_stream = StringIO.StringIO(urllib2.urlopen(req).read())
with zipfile.ZipFile(file_stream, 'r') as zfile:
zfile.extractall(target_parent_dir)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, zip_root_name),
os.path.join(target_parent_dir, target_root_name))
def download_files(source_url_root, target_dir, source_filenames):
"""Downloads a group of files and saves them to a given directory.
Each file is downloaded only if it does not already exist.
Args:
source_url_root: the URL to prepend to all the filenames.
target_dir: the directory to save the files to.
source_filenames: a list of filenames. Each filename is appended to the
end of the source_url_root in order to give the URL from which to
download the file. The downloaded file is then placed in target_dir,
and retains the same filename.
"""
assert isinstance(source_filenames, list)
common.ensure_directory_exists(target_dir)
for filename in source_filenames:
if not os.path.exists(os.path.join(target_dir, filename)):
print "Downloading file %s to %s" % (filename, target_dir)
urllib.urlretrieve("%s/%s" % (source_url_root, filename), os.path.join(target_dir, filename))
def download_files(source_url_root, target_dir, source_filenames):
"""Downloads a group of files and saves them to a given directory.
Each file is downloaded only if it does not already exist.
Args:
source_url_root: the URL to prepend to all the filenames.
target_dir: the directory to save the files to.
source_filenames: a list of filenames. Each filename is appended to the
end of the source_url_root in order to give the URL from which to
download the file. The downloaded file is then placed in target_dir,
and retains the same filename.
"""
assert isinstance(source_filenames, list)
common.ensure_directory_exists(target_dir)
for filename in source_filenames:
if not os.path.exists(os.path.join(target_dir, filename)):
print 'Downloading file %s to %s' % (filename, target_dir)
urllib.urlretrieve('%s/%s' % (source_url_root, filename),
os.path.join(target_dir, filename))
common.require_cwd_to_be_oppia()
CURRENT_GIT_VERSION = subprocess.check_output(['git', 'rev-parse',
'HEAD']).strip()
print ''
print 'Starting experimental deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(os.getcwd(),
RELEASE_DIR_PATH,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception('Invalid directory accessed during deployment: %s' %
os.getcwd())
print 'Changing directory to %s' % os.getcwd()
def _execute_deployment():
# Check that the current directory is correct.
common.require_cwd_to_be_oppia()
current_git_revision = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
print ""
print "Starting deployment process."
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
"Could not find third_party directory at %s. Please run start.sh "
"prior to running this script." % THIRD_PARTY_DIR
)
# Create a folder in which to save the release candidate.
print "Ensuring that the release directory parent exists"
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print "Copying files to the release directory"
shutil.copytree(os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns(".git"))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception("Invalid directory accessed during deployment: %s" % os.getcwd())
print "Changing directory to %s" % os.getcwd()
print "Preprocessing release..."
preprocess_release()
# Do a build; ensure there are no errors.
print "Building and minifying scripts..."
subprocess.check_output(["python", "scripts/build.py"])
# Run the tests; ensure there are no errors.
print "Running tests..."
tests_proc = subprocess.Popen(["bash", os.path.join("scripts", "run_tests.sh")], stdout=subprocess.PIPE)
tests_stdout, tests_stderr = tests_proc.communicate()
print tests_stdout
print tests_stderr
if tests_proc.returncode != 0:
raise Exception("Tests failed. Halting deployment.")
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, "update", ".", "--oauth2"])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, "a") as log_file:
log_file.write(
"Successfully deployed to %s at %s (version %s)\n"
% (APP_NAME, CURRENT_DATETIME.strftime("%Y-%m-%d %H:%M:%S"), current_git_revision)
)
print "Returning to oppia/ root directory."
print "Done!"
common.require_cwd_to_be_oppia()
CURRENT_GIT_VERSION = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
print ''
print 'Starting experimental deployment process.'
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(
os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s' % os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
def _execute_deployment():
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(
os.getcwd(), RELEASE_DIR_PATH, ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s'
% os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
# Do a build, while outputting to the terminal.
print 'Building and minifying scripts...'
build_process = subprocess.Popen(
['python', 'scripts/build.py', '--prod_env'],
stdout=subprocess.PIPE)
while True:
line = build_process.stdout.readline().strip()
if not line:
break
print line
# Wait for process to terminate, then check return code.
build_process.communicate()
if build_process.returncode > 0:
raise Exception('Build failed.')
# Deploy to GAE.
subprocess.check_output([APPCFG_PATH, 'update', '.'])
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' % (
APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
# If this is a test server deployment and the current release version is
# already serving, open the library page (for sanity checking) and the GAE
# error logs.
if (APP_NAME == APP_NAME_OPPIATESTSERVER or 'migration' in APP_NAME) and (
_get_served_version() == _get_current_release_version()):
common.open_new_tab_in_browser_if_possible(
'https://console.cloud.google.com/logs/viewer?'
'project=%s&key1=default&minLogLevel=500'
% APP_NAME_OPPIATESTSERVER)
common.open_new_tab_in_browser_if_possible(
'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER)
print 'Done!'
def _execute_deployment():
"""Executes the deployment process after doing the prerequisite checks."""
if not common.is_current_branch_a_release_branch():
raise Exception(
'The deployment script must be run from a release branch.')
current_release_version = CURRENT_BRANCH_NAME[
len(common.RELEASE_BRANCH_NAME_PREFIX):].replace('.', '-')
# Do prerequisite checks.
common.require_cwd_to_be_oppia()
common.ensure_release_scripts_folder_exists_and_is_up_to_date()
gcloud_adapter.require_gcloud_to_be_available()
if APP_NAME in [APP_NAME_OPPIASERVER, APP_NAME_OPPIATESTSERVER]:
if not common.is_current_branch_a_release_branch():
raise Exception(
'The deployment script must be run from a release branch.')
if APP_NAME == APP_NAME_OPPIASERVER:
with open('./feconf.py', 'r') as f:
feconf_contents = f.read()
if ('MAILGUN_API_KEY' not in feconf_contents
or 'MAILGUN_API_KEY = None' in feconf_contents):
raise Exception(
'The mailgun API key must be added before deployment.')
if not os.path.exists(THIRD_PARTY_DIR):
raise Exception(
'Could not find third_party directory at %s. Please run start.sh '
'prior to running this script.' % THIRD_PARTY_DIR)
current_git_revision = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
# Create a folder in which to save the release candidate.
print 'Ensuring that the release directory parent exists'
common.ensure_directory_exists(os.path.dirname(RELEASE_DIR_PATH))
# Copy files to the release directory. Omits the .git subfolder.
print 'Copying files to the release directory'
shutil.copytree(os.getcwd(),
RELEASE_DIR_PATH,
ignore=shutil.ignore_patterns('.git'))
# Change the current directory to the release candidate folder.
with common.CD(RELEASE_DIR_PATH):
if not os.getcwd().endswith(RELEASE_DIR_NAME):
raise Exception(
'Invalid directory accessed during deployment: %s' %
os.getcwd())
print 'Changing directory to %s' % os.getcwd()
print 'Preprocessing release...'
preprocess_release()
# Update indexes, then prompt for a check that they are all serving
# before continuing with the deployment.
# NOTE: This assumes that the build process does not modify the
# index.yaml file or create a different version of it to use in
# production.
gcloud_adapter.update_indexes(INDEX_YAML_PATH, APP_NAME)
datastore_indexes_url = (
'https://console.cloud.google.com/datastore/indexes?project=%s' %
APP_NAME)
common.open_new_tab_in_browser_if_possible(datastore_indexes_url)
while True:
print '******************************************************'
print(
'PLEASE CONFIRM: are all datastore indexes serving? See %s '
'(y/n)' % datastore_indexes_url)
answer = raw_input().lower()
if answer in ['y', 'ye', 'yes']:
break
elif answer:
raise Exception(
'Please wait for all indexes to serve, then run this '
'script again to complete the deployment. Exiting.')
# Do a build, while outputting to the terminal.
print 'Building and minifying scripts...'
build_process = subprocess.Popen(
['python', 'scripts/build.py', '--prod_env'],
stdout=subprocess.PIPE)
while True:
line = build_process.stdout.readline().strip()
if not line:
break
print line
# Wait for process to terminate, then check return code.
build_process.communicate()
if build_process.returncode > 0:
raise Exception('Build failed.')
# Deploy export service to GAE.
gcloud_adapter.deploy_application('export/app.yaml', APP_NAME)
# Deploy app to GAE.
gcloud_adapter.deploy_application(
'./app.yaml',
APP_NAME,
version=(CUSTOM_VERSION
if CUSTOM_VERSION else current_release_version))
# Writing log entry.
common.ensure_directory_exists(os.path.dirname(LOG_FILE_PATH))
with open(LOG_FILE_PATH, 'a') as log_file:
log_file.write(
'Successfully deployed to %s at %s (version %s)\n' %
(APP_NAME, CURRENT_DATETIME.strftime('%Y-%m-%d %H:%M:%S'),
current_git_revision))
print 'Returning to oppia/ root directory.'
# If this is a test server deployment and the current release version is
# already serving, open the library page (for sanity checking) and the GAE
# error logs.
currently_served_version = (
gcloud_adapter.get_currently_served_version(APP_NAME))
if (APP_NAME == APP_NAME_OPPIATESTSERVER
or 'migration' in APP_NAME) and (currently_served_version
== current_release_version):
common.open_new_tab_in_browser_if_possible(
'https://%s.appspot.com/library' % APP_NAME_OPPIATESTSERVER)
common.open_new_tab_in_browser_if_possible(
'https://console.cloud.google.com/logs/viewer?'
'project=%s&key1=default&minLogLevel=500' %
APP_NAME_OPPIATESTSERVER)
print 'Done!'
