def install_buf_and_protoc():
"""Installs buf and protoc for Linux or Darwin, depending upon the
platform.
"""
buf_files = BUF_DARWIN_FILES if common.is_mac_os() else BUF_LINUX_FILES
protoc_file = (PROTOC_DARWIN_FILE
if common.is_mac_os() else PROTOC_LINUX_FILE)
buf_path = os.path.join(BUF_DIR, buf_files[0])
protoc_path = os.path.join(PROTOC_DIR, 'bin', 'protoc')
if os.path.isfile(buf_path) and os.path.isfile(protoc_path):
return
common.ensure_directory_exists(BUF_DIR)
for bin_file in buf_files:
python_utils.url_retrieve('%s/%s' % (BUF_BASE_URL, bin_file),
filename=os.path.join(BUF_DIR, bin_file))
python_utils.url_retrieve('%s/%s' % (PROTOC_URL, protoc_file),
filename=os.path.join(BUF_DIR, protoc_file))
try:
with zipfile.ZipFile(os.path.join(BUF_DIR, protoc_file), 'r') as zfile:
zfile.extractall(path=PROTOC_DIR)
os.remove(os.path.join(BUF_DIR, protoc_file))
except Exception:
raise Exception('Error installing protoc binary')
common.recursive_chmod(buf_path, 0o744)
common.recursive_chmod(protoc_path, 0o744)
def download_files(source_url_root, target_dir, source_filenames):
"""Downloads a group of files and saves them to a given directory.
Each file is downloaded only if it does not already exist.
Args:
source_url_root: str. The URL to prepend to all the filenames.
target_dir: str. The directory to save the files to.
source_filenames: list(str). Each filename is appended to the
end of the source_url_root in order to give the URL from which to
download the file. The downloaded file is then placed in target_dir,
and retains the same filename.
"""
assert isinstance(source_filenames,
list), ('Expected list of filenames, got \'%s\'' %
source_filenames)
common.ensure_directory_exists(target_dir)
for filename in source_filenames:
if not os.path.exists(os.path.join(target_dir, filename)):
python_utils.PRINT('Downloading file %s to %s ...' %
(filename, target_dir))
python_utils.url_retrieve('%s/%s' % (source_url_root, filename),
filename=os.path.join(
target_dir, filename))
python_utils.PRINT('Download of %s succeeded.' % filename)
def download_and_untar_files(source_url, target_parent_dir, tar_root_name,
target_root_name):
"""Downloads a tar file, untars it, and saves the result in a given dir.
The download occurs only if the target directory that the tar file untars
to does not exist.
NB: This function assumes that the root level of the tar file has exactly
one folder.
Args:
source_url: str. The URL from which to download the tar file.
target_parent_dir: str. The directory to save the contents of the tar
file to.
tar_root_name: str. The name of the top-level folder in the tar
directory.
target_root_name: str. The name that the top-level folder should be
renamed to in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
python_utils.PRINT('Downloading and untarring file %s to %s ...' %
(tar_root_name, target_parent_dir))
common.ensure_directory_exists(target_parent_dir)
python_utils.url_retrieve(source_url, filename=TMP_UNZIP_PATH)
with contextlib.closing(tarfile.open(name=TMP_UNZIP_PATH,
mode='r:gz')) as tfile:
tfile.extractall(target_parent_dir)
os.remove(TMP_UNZIP_PATH)
# Rename the target directory.
os.rename(os.path.join(target_parent_dir, tar_root_name),
os.path.join(target_parent_dir, target_root_name))
python_utils.PRINT('Download of %s succeeded.' % tar_root_name)
def test_url_retrieve(self):
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.name = 'temp_file.txt'
python_utils.url_retrieve('http://www.google.com',
filename='temp_file.txt')
with python_utils.open_file('temp_file.txt', 'rb', encoding=None) as f:
content = f.read()
self.assertIn(b'<title>Google</title>', content)
tmp_file.close()
def main(args=None):
"""Runs the script to setup GAE."""
unused_parsed_args = _PARSER.parse_args(args=args)
sys.path.append('.')
sys.path.append(common.GOOGLE_APP_ENGINE_SDK_HOME)
sys.path.append(os.path.join(common.OPPIA_TOOLS_DIR, 'webtest-2.0.35'))
# Delete old *.pyc files.
for directory, _, files in os.walk('.'):
for file_name in files:
if file_name.endswith('.pyc'):
filepath = os.path.join(directory, file_name)
os.remove(filepath)
python_utils.PRINT(
'Checking whether google-cloud-sdk is installed in %s'
% common.GOOGLE_CLOUD_SDK_HOME)
if not os.path.exists(common.GOOGLE_CLOUD_SDK_HOME):
python_utils.PRINT(
'Downloading Google Cloud SDK (this may take a little while)...')
os.makedirs(common.GOOGLE_CLOUD_SDK_HOME)
try:
# If the google cloud version is updated here, the corresponding
# lines (GAE_DIR and GCLOUD_PATH) in assets/release_constants.json
# should also be updated.
python_utils.url_retrieve(
'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/'
'google-cloud-sdk-335.0.0-linux-x86_64.tar.gz',
filename='gcloud-sdk.tar.gz')
except Exception:
python_utils.PRINT('Error downloading Google Cloud SDK. Exiting.')
raise Exception('Error downloading Google Cloud SDK.')
python_utils.PRINT('Download complete. Installing Google Cloud SDK...')
tar = tarfile.open(name='gcloud-sdk.tar.gz')
tar.extractall(
path=os.path.join(
common.OPPIA_TOOLS_DIR, 'google-cloud-sdk-335.0.0/'))
tar.close()
os.remove('gcloud-sdk.tar.gz')
# This command installs specific google cloud components for the google
# cloud sdk to prevent the need for developers to install it themselves when
# the app engine development server starts up. The --quiet parameter
# specifically tells the gcloud program to autofill all prompts with default
# values. In this case, that means accepting all installations of gcloud
# packages.
subprocess.call([
common.GCLOUD_PATH,
'components', 'install', 'beta', 'cloud-datastore-emulator',
'app-engine-python', 'app-engine-python-extras', '--quiet'])
def download_and_install_package(url_to_retrieve, filename):
"""Downloads and installs package in Oppia tools directory.
Args:
url_to_retrieve: string. The url from which package is to be
downloaded.
filename: string. The name of the tar file.
"""
python_utils.url_retrieve(url_to_retrieve, filename=filename)
tar = tarfile.open(name=filename)
tar.extractall(path=common.OPPIA_TOOLS_DIR)
tar.close()
rename_yarn_folder(filename, common.OPPIA_TOOLS_DIR)
os.remove(filename)
def download_and_unzip_files(source_url, target_parent_dir, zip_root_name,
target_root_name):
"""Downloads a zip file, unzips it, and saves the result in a given dir.
The download occurs only if the target directory that the zip file unzips
to does not exist.
NB: This function assumes that the root level of the zip file has exactly
one folder.
Args:
source_url: str. The URL from which to download the zip file.
target_parent_dir: str. The directory to save the contents of the zip
file to.
zip_root_name: str. The name of the top-level folder in the zip
directory.
target_root_name: str. The name that the top-level folder should be
renamed to in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
python_utils.PRINT('Downloading and unzipping file %s to %s ...' %
(zip_root_name, target_parent_dir))
common.ensure_directory_exists(target_parent_dir)
python_utils.url_retrieve(source_url, filename=TMP_UNZIP_PATH)
try:
with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
os.remove(TMP_UNZIP_PATH)
except Exception:
if os.path.exists(TMP_UNZIP_PATH):
os.remove(TMP_UNZIP_PATH)
# Some downloads (like jqueryui-themes) may require a user-agent.
req = python_utils.url_request(source_url, None, {})
req.add_header('User-agent', 'python')
# This is needed to get a seekable filestream that can be used
# by zipfile.ZipFile.
file_stream = python_utils.string_io(
buffer_value=python_utils.url_open(req).read())
with zipfile.ZipFile(file_stream, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
# Rename the target directory.
os.rename(os.path.join(target_parent_dir, zip_root_name),
os.path.join(target_parent_dir, target_root_name))
python_utils.PRINT('Download of %s succeeded.' % zip_root_name)
def download_and_install_node():
"""Download and install node to Oppia tools directory."""
outfile_name = 'node-download'
if common.is_windows_os():
if common.is_x64_architecture():
architecture = 'x64'
else:
architecture = 'x86'
extension = '.zip'
node_file_name = 'node-v%s-win-%s' % (common.NODE_VERSION,
architecture)
url_to_retrieve = 'https://nodejs.org/dist/v%s/%s%s' % (
common.NODE_VERSION, node_file_name, extension)
python_utils.url_retrieve(url_to_retrieve, filename=outfile_name)
subprocess.check_call([
'powershell.exe', '-c', 'expand-archive', outfile_name,
'-DestinationPath', common.OPPIA_TOOLS_DIR
])
else:
extension = '.tar.gz'
if common.is_x64_architecture():
if common.is_mac_os():
node_file_name = 'node-v%s-darwin-x64' % (common.NODE_VERSION)
elif common.is_linux_os():
node_file_name = 'node-v%s-linux-x64' % (common.NODE_VERSION)
else:
node_file_name = 'node-v%s' % common.NODE_VERSION
download_and_install_package(
'https://nodejs.org/dist/v%s/%s%s' %
(common.NODE_VERSION, node_file_name, extension), outfile_name)
os.rename(os.path.join(common.OPPIA_TOOLS_DIR, node_file_name),
common.NODE_PATH)
if node_file_name == 'node-v%s' % common.NODE_VERSION:
with common.CD(common.NODE_PATH):
subprocess.check_call(['./configure'])
subprocess.check_call(['make'])