def main(args=None):
"""Installs and provides a constant version of Chrome."""
unused_parsed_args = _PARSER.parse_args(args=args)
home_directory = os.environ.get('HOME')
oppia_dir = os.getcwd()
# CHROME_SOURCE_URL is an environment variable set in Oppia's Travis repo
# settings. It can be found under 'Environment Variables' header here:
# https://travis-ci.org/oppia/oppia/settings.
chrome_source_url = os.environ.get('CHROME_SOURCE_URL')
travis_chrome_path = os.path.join(home_directory, '.cache/TravisChrome/',
os.path.basename(chrome_source_url))
# Caching Chrome's Debian package after download to prevent connection
# problem.
if not os.path.isdir(os.path.join(home_directory, '.cache/TravisChrome/')):
os.makedirs(os.path.join(home_directory, '.cache/TravisChrome/'))
if not os.path.isfile(travis_chrome_path):
os.chdir(os.path.join(home_directory, '.cache/TravisChrome/'))
python_utils.url_retrieve(chrome_source_url,
filename=os.path.basename(chrome_source_url))
os.chdir(oppia_dir)
python_utils.PRINT('Installing %s' % travis_chrome_path)
subprocess.check_call(['sudo', 'dpkg', '-i', travis_chrome_path])
def download_and_untar_files(source_url, target_parent_dir, tar_root_name,
target_root_name):
"""Downloads a tar file, untars it, and saves the result in a given dir.
The download occurs only if the target directory that the tar file untars
to does not exist.
NB: This function assumes that the root level of the tar file has exactly
one folder.
Args:
source_url: str. The URL from which to download the tar file.
target_parent_dir: str. The directory to save the contents of the tar
file to.
tar_root_name: str. The name of the top-level folder in the tar
directory.
target_root_name: str. The name that the top-level folder should be
renamed to in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
python_utils.PRINT('Downloading and untarring file %s to %s ...' %
(tar_root_name, target_parent_dir))
common.ensure_directory_exists(target_parent_dir)
python_utils.url_retrieve(source_url, filename=TMP_UNZIP_PATH)
with contextlib.closing(tarfile.open(name=TMP_UNZIP_PATH,
mode='r:gz')) as tfile:
tfile.extractall(target_parent_dir)
os.remove(TMP_UNZIP_PATH)
# Rename the target directory.
os.rename(os.path.join(target_parent_dir, tar_root_name),
os.path.join(target_parent_dir, target_root_name))
python_utils.PRINT('Download of %s succeeded.' % tar_root_name)
def download_files(source_url_root, target_dir, source_filenames):
"""Downloads a group of files and saves them to a given directory.
Each file is downloaded only if it does not already exist.
Args:
source_url_root: str. The URL to prepend to all the filenames.
target_dir: str. The directory to save the files to.
source_filenames: list(str). Each filename is appended to the
end of the source_url_root in order to give the URL from which to
download the file. The downloaded file is then placed in target_dir,
and retains the same filename.
"""
assert isinstance(source_filenames, list), (
'Expected list of filenames, got \'%s\'' % source_filenames)
common.ensure_directory_exists(target_dir)
for filename in source_filenames:
if not os.path.exists(os.path.join(target_dir, filename)):
python_utils.PRINT(
'Downloading file %s to %s ...' % (filename, target_dir))
python_utils.url_retrieve(
'%s/%s' % (source_url_root, filename),
filename=os.path.join(target_dir, filename))
python_utils.PRINT('Download of %s succeeded.' % filename)
def install_buf_and_protoc():
"""Installs buf and protoc for Linux or Darwin, depending upon the
platform.
"""
buf_files = BUF_DARWIN_FILES if common.is_mac_os() else BUF_LINUX_FILES
protoc_file = (PROTOC_DARWIN_FILE
if common.is_mac_os() else PROTOC_LINUX_FILE)
buf_path = os.path.join(BUF_DIR, buf_files[0])
protoc_path = os.path.join(PROTOC_DIR, 'bin', 'protoc')
if os.path.isfile(buf_path) and os.path.isfile(protoc_path):
return
common.ensure_directory_exists(BUF_DIR)
for bin_file in buf_files:
python_utils.url_retrieve('%s/%s' % (BUF_BASE_URL, bin_file),
filename=os.path.join(BUF_DIR, bin_file))
python_utils.url_retrieve('%s/%s' % (PROTOC_URL, protoc_file),
filename=os.path.join(BUF_DIR, protoc_file))
try:
with zipfile.ZipFile(os.path.join(BUF_DIR, protoc_file), 'r') as zfile:
zfile.extractall(path=PROTOC_DIR)
os.remove(os.path.join(BUF_DIR, protoc_file))
except Exception:
raise Exception('Error installing protoc binary')
common.recursive_chmod(buf_path, 0o744)
common.recursive_chmod(protoc_path, 0o744)
def main(args=None):
"""Runs the script to setup GAE."""
unused_parsed_args = _PARSER.parse_args(args=args)
coverage_home = os.path.join(common.OPPIA_TOOLS_DIR,
'coverage-%s' % common.COVERAGE_VERSION)
# Note that if the following line is changed so that it uses webob_1_1_1,
# PUT requests from the frontend fail.
sys.path.append('.')
sys.path.append(coverage_home)
sys.path.append(common.GOOGLE_APP_ENGINE_SDK_HOME)
sys.path.append(
os.path.join(common.GOOGLE_APP_ENGINE_SDK_HOME, 'lib', 'webob_0_9'))
sys.path.append(os.path.join(common.OPPIA_TOOLS_DIR, 'webtest-2.0.33'))
# Delete old *.pyc files.
for directory, _, files in os.walk('.'):
for file_name in files:
if file_name.endswith('.pyc'):
filepath = os.path.join(directory, file_name)
os.remove(filepath)
python_utils.PRINT('Checking whether google-cloud-sdk is installed in %s' %
common.GOOGLE_CLOUD_SDK_HOME)
if not os.path.exists(common.GOOGLE_CLOUD_SDK_HOME):
python_utils.PRINT(
'Downloading Google Cloud SDK (this may take a little while)...')
os.makedirs(common.GOOGLE_CLOUD_SDK_HOME)
try:
python_utils.url_retrieve(
'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/'
'google-cloud-sdk-304.0.0-linux-x86_64.tar.gz',
filename='gcloud-sdk.tar.gz')
except Exception:
python_utils.PRINT('Error downloading Google Cloud SDK. Exiting.')
raise Exception('Error downloading Google Cloud SDK.')
python_utils.PRINT('Download complete. Installing Google Cloud SDK...')
tar = tarfile.open(name='gcloud-sdk.tar.gz')
tar.extractall(path=os.path.join(common.OPPIA_TOOLS_DIR,
'google-cloud-sdk-304.0.0/'))
tar.close()
# This command installs specific google cloud components for the google
# cloud sdk to prevent the need for developers to install it themselves
# when the app engine development server starts up. The --quiet
# parameter specifically tells the gcloud program to autofill all
# prompts with default values. In this case, that means accepting all
# installations of gcloud packages.
subprocess.call([
common.GCLOUD_PATH, 'components', 'install',
'cloud-datastore-emulator', 'app-engine-python',
'app-engine-python-extras', '--quiet'
])
os.remove('gcloud-sdk.tar.gz')
def test_url_retrieve(self):
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.name = 'temp_file.txt'
python_utils.url_retrieve('http://www.google.com',
filename='temp_file.txt')
with python_utils.open_file('temp_file.txt', 'rb', encoding=None) as f:
content = f.read()
self.assertIn(b'<title>Google</title>', content)
tmp_file.close()
def install_prototool():
"""Installs prototool for Linux or Darwin, depending upon the platform."""
if os.path.exists(PROTOTOOL_BIN_PATH):
return
prototool_url = PROTOTOOL_LINUX_BIN_URL
if common.is_mac_os():
prototool_url = PROTOTOOL_DARWIN_BIN_URL
common.ensure_directory_exists(PROTOTOOL_DIR)
python_utils.url_retrieve(prototool_url, filename=PROTOTOOL_BIN_PATH)
common.recursive_chmod(PROTOTOOL_BIN_PATH, 0o744)
def download_and_install_package(url_to_retrieve, filename):
"""Downloads and installs package in Oppia tools directory.
Args:
url_to_retrieve: string. The url from which package is to be
downloaded.
filename: string. The name of the tar file.
"""
python_utils.url_retrieve(url_to_retrieve, filename=filename)
tar = tarfile.open(name=filename)
tar.extractall(path=common.OPPIA_TOOLS_DIR)
tar.close()
os.remove(filename)
def main(args=None):
"""Runs the script to setup GAE."""
unused_parsed_args = _PARSER.parse_args(args=args)
sys.path.append('.')
sys.path.append(common.GOOGLE_APP_ENGINE_SDK_HOME)
sys.path.append(os.path.join(common.OPPIA_TOOLS_DIR, 'webtest-2.0.35'))
# Delete old *.pyc files.
for directory, _, files in os.walk('.'):
for file_name in files:
if file_name.endswith('.pyc'):
filepath = os.path.join(directory, file_name)
os.remove(filepath)
python_utils.PRINT('Checking whether google-cloud-sdk is installed in %s' %
common.GOOGLE_CLOUD_SDK_HOME)
if not os.path.exists(common.GOOGLE_CLOUD_SDK_HOME):
python_utils.PRINT(
'Downloading Google Cloud SDK (this may take a little while)...')
os.makedirs(common.GOOGLE_CLOUD_SDK_HOME)
try:
# If the google cloud version is updated here, the corresponding
# lines (GAE_DIR and GCLOUD_PATH) in release_constants.json should
# also be updated.
python_utils.url_retrieve(
'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/'
'google-cloud-sdk-335.0.0-linux-x86_64.tar.gz',
filename='gcloud-sdk.tar.gz')
except Exception:
python_utils.PRINT('Error downloading Google Cloud SDK. Exiting.')
raise Exception('Error downloading Google Cloud SDK.')
python_utils.PRINT('Download complete. Installing Google Cloud SDK...')
tar = tarfile.open(name='gcloud-sdk.tar.gz')
tar.extractall(path=os.path.join(common.OPPIA_TOOLS_DIR,
'google-cloud-sdk-335.0.0/'))
tar.close()
os.remove('gcloud-sdk.tar.gz')
# This command installs specific google cloud components for the google
# cloud sdk to prevent the need for developers to install it themselves when
# the app engine development server starts up. The --quiet parameter
# specifically tells the gcloud program to autofill all prompts with default
# values. In this case, that means accepting all installations of gcloud
# packages.
subprocess.call([
common.GCLOUD_PATH, 'components', 'install', 'beta',
'cloud-datastore-emulator', 'app-engine-python',
'app-engine-python-extras', '--quiet'
])
def download_and_unzip_files(
source_url, target_parent_dir, zip_root_name, target_root_name):
"""Downloads a zip file, unzips it, and saves the result in a given dir.
The download occurs only if the target directory that the zip file unzips
to does not exist.
NB: This function assumes that the root level of the zip file has exactly
one folder.
Args:
source_url: str. The URL from which to download the zip file.
target_parent_dir: str. The directory to save the contents of the zip
file to.
zip_root_name: str. The name of the top-level folder in the zip
directory.
target_root_name: str. The name that the top-level folder should be
renamed to in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
python_utils.PRINT('Downloading and unzipping file %s to %s ...' % (
zip_root_name, target_parent_dir))
common.ensure_directory_exists(target_parent_dir)
python_utils.url_retrieve(source_url, filename=TMP_UNZIP_PATH)
try:
with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
os.remove(TMP_UNZIP_PATH)
except Exception:
if os.path.exists(TMP_UNZIP_PATH):
os.remove(TMP_UNZIP_PATH)
# Some downloads (like jqueryui-themes) may require a user-agent.
req = python_utils.url_request(source_url, None, {})
req.add_header('User-agent', 'python')
# This is needed to get a seekable filestream that can be used
# by zipfile.ZipFile.
file_stream = python_utils.string_io(
buffer_value=python_utils.url_open(req).read())
with zipfile.ZipFile(file_stream, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, zip_root_name),
os.path.join(target_parent_dir, target_root_name))
python_utils.PRINT('Download of %s succeeded.' % zip_root_name)
def download_and_install_package(url_to_retrieve, filename):
"""Downloads and installs package in Oppia tools directory.
Args:
url_to_retrieve: string. The url from which package is to be
downloaded.
filename: string. The name of the tar file.
"""
python_utils.url_retrieve(url_to_retrieve, filename=filename)
_, extension = os.path.splitext(filename)
if extension == '.gz' or extension == '.tgz':
tar = tarfile.open(name=filename)
tar.extractall(path=common.OPPIA_TOOLS_DIR)
tar.close()
elif common.OS_NAME == 'Windows' and extension == '.zip':
p = subprocess.Popen(
['powershell', 'Expand-Archive', filename, common.OPPIA_TOOLS_DIR],
stdout=sys.stdout)
p.communicate()
os.remove(filename)
def download_and_install_node():
"""Download and install node to Oppia tools directory."""
outfile_name = 'node-download'
if common.is_windows_os():
if common.is_x64_architecture():
architecture = 'x64'
else:
architecture = 'x86'
extension = '.zip'
node_file_name = 'node-v%s-win-%s' % (
common.NODE_VERSION, architecture)
url_to_retrieve = 'https://nodejs.org/dist/v%s/%s%s' % (
common.NODE_VERSION, node_file_name, extension)
python_utils.url_retrieve(url_to_retrieve, outfile_name)
subprocess.check_call(
['powershell.exe', '-c', 'expand-archive',
outfile_name, '-DestinationPath',
common.OPPIA_TOOLS_DIR])
else:
extension = '.tar.gz'
if common.is_x64_architecture():
if common.is_mac_os():
node_file_name = 'node-v%s-darwin-x64' % (common.NODE_VERSION)
elif common.is_linux_os():
node_file_name = 'node-v%s-linux-x64' % (common.NODE_VERSION)
else:
node_file_name = 'node-v%s' % common.NODE_VERSION
download_and_install_package(
'https://nodejs.org/dist/v%s/%s%s' % (
common.NODE_VERSION, node_file_name, extension),
outfile_name)
os.rename(
os.path.join(common.OPPIA_TOOLS_DIR, node_file_name),
common.NODE_PATH)
if node_file_name == 'node-v%s' % common.NODE_VERSION:
with common.CD(common.NODE_PATH):
subprocess.check_call(['./configure'])
subprocess.check_call(['make'])
def main(args=None):
"""Runs the script to setup GAE."""
unused_parsed_args = _PARSER.parse_args(args=args)
coverage_home = os.path.join(common.OPPIA_TOOLS_DIR, 'coverage-4.5.4')
# Note that if the following line is changed so that it uses webob_1_1_1,
# PUT requests from the frontend fail.
sys.path.append('.')
sys.path.append(coverage_home)
sys.path.append(common.GOOGLE_APP_ENGINE_HOME)
sys.path.append(
os.path.join(common.GOOGLE_APP_ENGINE_HOME, 'lib', 'webob_0_9'))
sys.path.append(os.path.join(common.OPPIA_TOOLS_DIR, 'webtest-2.0.33'))
# Delete old *.pyc files.
for directory, _, files in os.walk('.'):
for file_name in files:
if file_name.endswith('.pyc'):
filepath = os.path.join(directory, file_name)
os.remove(filepath)
python_utils.PRINT(
'Checking whether Google App Engine is installed in %s'
% common.GOOGLE_APP_ENGINE_HOME)
if not os.path.exists(common.GOOGLE_APP_ENGINE_HOME):
python_utils.PRINT(
'Downloading Google App Engine (this may take a little while)...')
os.makedirs(common.GOOGLE_APP_ENGINE_HOME)
try:
python_utils.url_retrieve(
'https://storage.googleapis.com/appengine-sdks/featured/'
'google_appengine_1.9.67.zip', filename='gae-download.zip')
except Exception:
python_utils.PRINT('Error downloading Google App Engine. Exiting.')
raise Exception
python_utils.PRINT('Download complete. Installing Google App Engine...')
with zipfile.ZipFile(GAE_DOWNLOAD_ZIP_PATH, 'r') as zip_ref:
zip_ref.extractall(
path=os.path.join(
common.OPPIA_TOOLS_DIR, 'google_appengine_1.9.67/'))
os.remove(GAE_DOWNLOAD_ZIP_PATH)
python_utils.PRINT(
'Checking whether google-cloud-sdk is installed in %s'
% common.GOOGLE_CLOUD_SDK_HOME)
if not os.path.exists(common.GOOGLE_CLOUD_SDK_HOME):
python_utils.PRINT(
'Downloading Google Cloud SDK (this may take a little while)...')
os.makedirs(common.GOOGLE_CLOUD_SDK_HOME)
try:
python_utils.url_retrieve(
'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/'
'google-cloud-sdk-251.0.0-linux-x86_64.tar.gz',
filename='gcloud-sdk.tar.gz')
except Exception:
python_utils.PRINT('Error downloading Google Cloud SDK. Exiting.')
raise Exception
python_utils.PRINT('Download complete. Installing Google Cloud SDK...')
tar = tarfile.open(name='gcloud-sdk.tar.gz')
tar.extractall(
path=os.path.join(
common.OPPIA_TOOLS_DIR, 'google-cloud-sdk-251.0.0/'))
tar.close()
os.remove('gcloud-sdk.tar.gz')
def main(args=None):
"""Runs the script to setup Oppia."""
unused_parsed_args = _PARSER.parse_args(args=args)
test_python_version()
# The second option allows this script to also be run from deployment
# folders.
if not os.getcwd().endswith('oppia') and not os.getcwd().endswith(
'deploy-'):
python_utils.PRINT('')
python_utils.PRINT(
'WARNING This script should be run from the oppia/ root folder.')
python_utils.PRINT('')
raise Exception
# Set COMMON_DIR to the absolute path of the directory above OPPIA_DIR. This
# is necessary becaue COMMON_DIR (or subsequent variables which refer to it)
# may use it in a situation where relative paths won't work as expected(such
# as $PYTHONPATH).
create_directory(common.OPPIA_TOOLS_DIR)
create_directory(common.THIRD_PARTY_DIR)
create_directory(common.NODE_MODULES_PATH)
os_info = os.uname()
if os_info[0] != 'Darwin' and os_info[0] != 'Linux':
# Node is a requirement for all installation scripts. Here, we check if
# the OS supports node.js installation; if not, we exit with an error.
common.print_each_string_after_two_new_lines([
'WARNING: Unsupported OS for installation of node.js.',
'If you are running this script on Windows, see the instructions',
'here regarding installation of node.js:',
'https://github.com/oppia/oppia/wiki/Installing-Oppia-%28Windows'
'%29',
'STATUS: Installation completed except for node.js. Exiting.'
])
raise Exception
# Download and install node.js.
python_utils.PRINT('Checking if node.js is installed in %s' %
common.OPPIA_TOOLS_DIR)
if not os.path.exists(common.NODE_PATH):
python_utils.PRINT('Installing Node.js')
if os_info[0] == 'Darwin':
if os_info[4] == 'x86_64':
node_file_name = 'node-v10.15.3-darwin-x64'
else:
node_file_name = 'node-v10.15.3-darwin-x86'
elif os_info[0] == 'Linux':
if os_info[4] == 'x86_64':
node_file_name = 'node-v10.15.3-linux-x64'
else:
node_file_name = 'node-v10.15.3-linux-x86'
python_utils.url_retrieve(
'https://nodejs.org/dist/v10.15.3/%s.tar.gz' % node_file_name,
filename='node-download.tgz')
tar = tarfile.open(name='node-download.tgz')
tar.extractall(path=common.OPPIA_TOOLS_DIR)
tar.close()
os.remove('node-download.tgz')
os.rename(os.path.join(common.OPPIA_TOOLS_DIR, node_file_name),
common.NODE_PATH)
# Change ownership of node_modules.
# Note: on some machines, these commands seem to take quite a long time.
common.recursive_chown(common.NODE_MODULES_PATH, os.getuid(), -1)
common.recursive_chmod(common.NODE_MODULES_PATH, 0o744)
# Download and install yarn.
python_utils.PRINT('Checking if yarn is installed in %s' %
common.OPPIA_TOOLS_DIR)
if not os.path.exists(common.YARN_PATH):
python_utils.PRINT('Removing package-lock.json')
clean.delete_file('package-lock.json')
common.print_each_string_after_two_new_lines([
'Installing yarn',
'WARNING: Please note that Oppia uses Yarn to manage node packages',
'do *NOT* use npm. For more information on how to use yarn,',
'visit https://yarnpkg.com/en/docs/usage.'
])
# NB: Update .yarnrc if the yarn version below is changed.
yarn_version = 'v1.17.3'
yarn_file_name = 'yarn-%s.tar.gz' % yarn_version
python_utils.url_retrieve(
'https://github.com/yarnpkg/yarn/releases/download/%s/%s' %
(yarn_version, yarn_file_name),
filename=yarn_file_name)
tar = tarfile.open(name=yarn_file_name)
tar.extractall(path=common.OPPIA_TOOLS_DIR)
tar.close()
os.remove(yarn_file_name)
# Adjust path to support the default Chrome locations for Unix, Windows and
# Mac OS.
if os.environ.get('TRAVIS'):
chrome_bin = '/usr/bin/chromium-browser'
elif os.path.isfile('/usr/bin/google-chrome'):
# Unix.
chrome_bin = '/usr/bin/google-chrome'
elif os.path.isfile('/usr/bin/chromium-browser'):
# Unix.
chrome_bin = '/usr/bin/chromium-browser'
elif os.path.isfile(
'/c/Program Files (x86)/Google/Chrome/Application/chrome.exe'):
# Windows.
chrome_bin = (
'/c/Program Files (x86)/Google/Chrome/Application/chrome.exe')
elif os.path.isfile(
'/mnt/c/Program Files (x86)/Google/Chrome/Application/chrome.exe'):
# WSL.
chrome_bin = (
'/mnt/c/Program Files (x86)/Google/Chrome/Application/chrome.exe')
elif os.path.isfile(
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'):
# Mac OS.
chrome_bin = (
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
else:
python_utils.PRINT('Chrome is not found, stopping ...')
raise Exception
os.environ['CHROME_BIN'] = chrome_bin
python_utils.PRINT('Environment setup completed.')