def UpdateTarFile(tar_file, colabtools_src_dir, tmp_dir):
"""Updates the tar resources file.
Untar/zip's the resources file, adds colabtools to the site-packages,
and rezip/tar's the file.
args:
tar_file: the filename of the .tar.gz file
colabtools_src_dir: source directory of colabtools (should end in /colabtools)
tmp_dir: temporary directory to unzip tar file contents to. Cleanup
of this directory is the responsibility of the caller.
"""
with open(os.devnull, 'w') as devnull:
if subprocess.call(['tar', '-zxvf', tar_file, '-C', tmp_dir],
stdout=devnull,
stderr=devnull):
raise RuntimeError('Failed to extract tar file')
# Copy colabtools directory to site-pacakges directory in
# tar'ed resources.
colabtools_dest_dir = pjoin(tmp_dir, 'lib', 'python2.7', 'site-packages',
'colabtools')
RemoveFileOrDirectoryIfExist(colabtools_dest_dir)
MakeDirectoryIfNotExist(colabtools_dest_dir)
CopyTreeRecursively(colabtools_src_dir, colabtools_dest_dir)
# Overwrite original tar file
with open(os.devnull, 'w') as devnull:
if subprocess.call(['tar', '-zcvf', tar_file, '-C', tmp_dir, '.'],
stdout=devnull,
stderr=devnull):
raise RuntimeError('Failed to update tar file')
def UpdateTarFile(tar_file, colabtools_src_dir, tmp_dir):
"""Updates the tar resources file.
Untar/zip's the resources file, adds colabtools to the site-packages,
and rezip/tar's the file.
args:
tar_file: the filename of the .tar.gz file
colabtools_src_dir: source directory of colabtools (should end in /colabtools)
tmp_dir: temporary directory to unzip tar file contents to. Cleanup
of this directory is the responsibility of the caller.
"""
with open(os.devnull, 'w') as devnull:
if subprocess.call(['tar', '-zxvf', tar_file, '-C', tmp_dir],
stdout=devnull, stderr=devnull):
raise RuntimeError('Failed to extract tar file')
# Copy colabtools directory to site-pacakges directory in
# tar'ed resources.
colabtools_dest_dir = pjoin(tmp_dir, 'lib', 'python2.7', 'site-packages', 'colabtools')
RemoveFileOrDirectoryIfExist(colabtools_dest_dir)
MakeDirectoryIfNotExist(colabtools_dest_dir)
CopyTreeRecursively(colabtools_src_dir, colabtools_dest_dir)
# Overwrite original tar file
with open(os.devnull, 'w') as devnull:
if subprocess.call(['tar', '-zcvf', tar_file, '-C', tmp_dir, '.'],
stdout=devnull, stderr=devnull):
raise RuntimeError('Failed to update tar file')
def InstallChrome(release, colab_root, dest):
"""Installs the Chrome App.
args:
release: Whether to produce release version.
colab_root: root directory of colaboratory source
dest: destination directory for build
"""
# stage static files
app_id = DEBUG_APP_ID
if release:
app_id = RELEASE_APP_ID
extra_template_args = {
'app_mode': True,
'app_origin': 'chrome-extension://' + app_id
}
BundleStatic(colab_root, dest, extra_template_args=extra_template_args)
# copy chrome app files, putting pnacl files in root
# as the folder structure is hard coded into the pnacl kernel code
# right now
CopyTreeRecursively(pjoin(colab_root, 'chrome'), dest)
CopyTreeRecursively(pjoin(colab_root, 'chrome', 'pnacl'), dest)
# Download .pexe and .tar.gz files. Later these will be pulled
# from the naclports continuous builder
pexe_file = pjoin(dest, 'pnacl', 'kernel.pexe')
tar_file = pjoin(dest, 'pnacl_data.tar.gz')
url_opener = urllib.URLopener()
if not os.path.isfile(pexe_file):
print 'Downloading ' + NACL_PEXE_FILE_URL
url_opener.retrieve(NACL_PEXE_FILE_URL, pexe_file)
if not os.path.isfile(tar_file):
print 'Downloading ' + NACL_TAR_FILE_URL
url_opener.retrieve(NACL_TAR_FILE_URL, tar_file)
tmp_resources_dir = pjoin(dest, 'tmp_pnacl_resources')
colabtools_src_dir = pjoin(colab_root, 'colabtools')
MakeDirectoryIfNotExist(tmp_resources_dir)
UpdateTarFile(tar_file, colabtools_src_dir, tmp_resources_dir)
RemoveFileOrDirectoryIfExist(tmp_resources_dir)
if release:
# In release mode, we must change client IDS
manfiest_file = pjoin(dest, 'manifest.json')
manifest = {}
with open(manfiest_file, 'r') as f:
manifest = json.loads(f.read())
# Delete "key" entry, because published app's don't
# use private keys to determine to Chrome App ID.
del manifest['key']
# Change the Client ID to the release client ID
manifest['oauth2']['client_id'] = RELEASE_CLIENT_ID
# Pretty-print to same manifest file.
with open(manfiest_file, 'w') as f:
f.write(json.dumps(manifest, indent=2, separators=(',', ': ')))
def InstallChrome(release, colab_root, dest):
"""Installs the Chrome App.
args:
release: Whether to produce release version.
colab_root: root directory of colaboratory source
dest: destination directory for build
"""
# stage static files
app_id = DEBUG_APP_ID
if release:
app_id = RELEASE_APP_ID
extra_template_args={
'app_mode': True,
'app_origin': 'chrome-extension://' + app_id
}
BundleStatic(colab_root, dest, extra_template_args=extra_template_args)
# copy chrome app files, putting pnacl files in root
# as the folder structure is hard coded into the pnacl kernel code
# right now
CopyTreeRecursively(pjoin(colab_root, 'chrome'), dest)
CopyTreeRecursively(pjoin(colab_root, 'chrome', 'pnacl'), dest)
# Download .pexe and .tar.gz files. Later these will be pulled
# from the naclports continuous builder
pexe_file = pjoin(dest, 'pnacl', 'kernel.pexe')
tar_file = pjoin(dest, 'pnacl_data.tar.gz')
url_opener = urllib.URLopener()
if not os.path.isfile(pexe_file):
print 'Downloading ' + NACL_PEXE_FILE_URL
url_opener.retrieve(NACL_PEXE_FILE_URL, pexe_file)
if not os.path.isfile(tar_file):
print 'Downloading ' + NACL_TAR_FILE_URL
url_opener.retrieve(NACL_TAR_FILE_URL, tar_file)
tmp_resources_dir = pjoin(dest, 'tmp_pnacl_resources')
colabtools_src_dir = pjoin(colab_root, 'colabtools')
MakeDirectoryIfNotExist(tmp_resources_dir)
UpdateTarFile(tar_file, colabtools_src_dir, tmp_resources_dir)
RemoveFileOrDirectoryIfExist(tmp_resources_dir)
if release:
# In release mode, we must change client IDS
manfiest_file = pjoin(dest, 'manifest.json')
manifest = {}
with open(manfiest_file, 'r') as f:
manifest = json.loads(f.read())
# Delete "key" entry, because published app's don't
# use private keys to determine to Chrome App ID.
del manifest['key']
# Change the Client ID to the release client ID
manifest['oauth2']['client_id'] = RELEASE_CLIENT_ID
# Pretty-print to same manifest file.
with open(manfiest_file, 'w') as f:
f.write(json.dumps(manifest, indent=2, separators=(',', ': ')))
url_opener.retrieve(NACL_TAR_FILE_URL, tar_file)
tmp_resources_dir = pjoin(dest, 'tmp_pnacl_resources')
colabtools_src_dir = pjoin(colab_root, 'colabtools')
MakeDirectoryIfNotExist(tmp_resources_dir)
UpdateTarFile(tar_file, colabtools_src_dir, tmp_resources_dir)
RemoveFileOrDirectoryIfExist(tmp_resources_dir)
if release:
# In release mode, we must change client IDS
manfiest_file = pjoin(dest, 'manifest.json')
manifest = {}
with open(manfiest_file, 'r') as f:
manifest = json.loads(f.read())
# Delete "key" entry, because published app's don't
# use private keys to determine to Chrome App ID.
del manifest['key']
# Change the Client ID to the release client ID
manifest['oauth2']['client_id'] = RELEASE_CLIENT_ID
# Pretty-print to same manifest file.
with open(manfiest_file, 'w') as f:
f.write(json.dumps(manifest, indent=2, separators=(',', ': ')))
if __name__ == '__main__':
InstallChrome(args.release, COLAB_ROOT_PATH, pjoin(COLAB_ROOT_PATH, 'build_chrome'))
url_opener.retrieve(NACL_TAR_FILE_URL, tar_file)
tmp_resources_dir = pjoin(dest, 'tmp_pnacl_resources')
colabtools_src_dir = pjoin(colab_root, 'colabtools')
MakeDirectoryIfNotExist(tmp_resources_dir)
UpdateTarFile(tar_file, colabtools_src_dir, tmp_resources_dir)
RemoveFileOrDirectoryIfExist(tmp_resources_dir)
if release:
# In release mode, we must change client IDS
manfiest_file = pjoin(dest, 'manifest.json')
manifest = {}
with open(manfiest_file, 'r') as f:
manifest = json.loads(f.read())
# Delete "key" entry, because published app's don't
# use private keys to determine to Chrome App ID.
del manifest['key']
# Change the Client ID to the release client ID
manifest['oauth2']['client_id'] = RELEASE_CLIENT_ID
# Pretty-print to same manifest file.
with open(manfiest_file, 'w') as f:
f.write(json.dumps(manifest, indent=2, separators=(',', ': ')))
if __name__ == '__main__':
InstallChrome(args.release, COLAB_ROOT_PATH,
pjoin(COLAB_ROOT_PATH, 'build_chrome'))
import subprocess
import sys
from install_lib import COLAB_ROOT_PATH
from install_lib import pjoin
RESOURCES_DIR = pjoin(COLAB_ROOT_PATH, 'colaboratory', 'resources')
CLOSURE_DIR = pjoin(RESOURCES_DIR, 'closure-library')
CALC_DEPS_FILE = pjoin(CLOSURE_DIR, 'closure', 'bin', 'calcdeps.py')
GOOG_ROOT = CLOSURE_DIR
COLAB_ROOT = pjoin(RESOURCES_DIR, 'colab')
OUTPUT_FILE = pjoin(COLAB_ROOT, 'js', 'colab.dep')
subprocess.call([
'python', CALC_DEPS_FILE, '--dep', GOOG_ROOT, '--path', COLAB_ROOT,
'--output_mode', 'deps', '--output_file', OUTPUT_FILE
])
subprocess.call(['sed', '-i', '', 's/\.\.\/\.\.\/\.\./\.\./g', OUTPUT_FILE])
def BundleStatic(colab_root, dest, extra_template_args=None):
# Use the following default arguments for
template_args = {
'raw': '1',
'app_mode': False
}
if extra_template_args is not None:
template_args.update(extra_template_args)
# prepare the destination directory
MakeDirectoryIfNotExist(dest)
for d in ['colab', 'extern', 'ipython', 'closure', 'welcome', 'notebook']:
RemoveDirectoryIfExist(pjoin(dest, d))
ipython_static = IPython.html.DEFAULT_STATIC_FILES_PATH
colab_resources = pjoin(colab_root, 'colaboratory', 'resources')
colab_static = pjoin(colab_root, 'static')
closure = pjoin(colab_resources, 'closure-library', 'closure', 'goog')
# TODO: run git submodule init && git submodule update in COLAB_ROOT_PATH
# stage the basic colab and extern directories
CopyTreeRecursively(pjoin(colab_resources, 'colab'), pjoin(dest, 'colab'))
CopyTreeRecursively(pjoin(colab_resources, 'extern'), pjoin(dest, 'extern'))
# stage IPython's static files, then clobber them with patched versions
CopyTreeRecursively(ipython_static, pjoin(dest, 'ipython'))
RemoveFileOrDirectoryIfExist(pjoin(dest, 'ipython', 'components', '.git'))
CopyTreeRecursively(pjoin(colab_resources, 'ipython_patch'), pjoin(dest, 'ipython'))
# stage closure from the submodule
CopyTreeRecursively(closure, pjoin(dest, 'closure'))
# instantiate templates and stage the /, /welcome/, and /notebook/ URLs
template_path = os.path.join(colab_resources, "colab")
env = Environment(loader=FileSystemLoader(template_path))
CopyTreeRecursively(colab_static, pjoin(dest, 'static'))
for name in ['welcome', 'notebook']:
template = env.get_template(name + os.extsep + 'html');
for d in [pjoin(dest, name, 'index' + os.extsep + 'html'), pjoin(dest, 'colab', name + os.extsep + 'html')]:
path, filename = os.path.split(d)
MakeDirectoryIfNotExist(path)
with open(d, 'w') as f:
f.write(template.render(template_args))
# stage IPython's static files, then clobber them with patched versions
CopyTreeRecursively(ipython_static, pjoin(dest, 'ipython'))
RemoveFileOrDirectoryIfExist(pjoin(dest, 'ipython', 'components', '.git'))
CopyTreeRecursively(pjoin(colab_resources, 'ipython_patch'), pjoin(dest, 'ipython'))
# stage closure from the submodule
CopyTreeRecursively(closure, pjoin(dest, 'closure'))
# instantiate templates and stage the /, /welcome/, and /notebook/ URLs
template_path = os.path.join(colab_resources, "colab")
env = Environment(loader=FileSystemLoader(template_path))
CopyTreeRecursively(colab_static, pjoin(dest, 'static'))
for name in ['welcome', 'notebook']:
template = env.get_template(name + os.extsep + 'html');
for d in [pjoin(dest, name, 'index' + os.extsep + 'html'), pjoin(dest, 'colab', name + os.extsep + 'html')]:
path, filename = os.path.split(d)
MakeDirectoryIfNotExist(path)
with open(d, 'w') as f:
f.write(template.render(template_args))
if __name__ == '__main__':
import sys
if len(sys.argv) < 2:
dest = pjoin(COLAB_ROOT_PATH, 'build')
else:
dest = sys.argv[1]
BundleStatic(COLAB_ROOT_PATH, dest)
import subprocess import sys from install_lib import COLAB_ROOT_PATH from install_lib import pjoin RESOURCES_DIR = pjoin(COLAB_ROOT_PATH, 'colaboratory', 'resources') CLOSURE_DIR = pjoin(RESOURCES_DIR, 'closure-library') CALC_DEPS_FILE = pjoin(CLOSURE_DIR, 'closure', 'bin', 'calcdeps.py') GOOG_ROOT = CLOSURE_DIR COLAB_ROOT = pjoin(RESOURCES_DIR, 'colab') OUTPUT_FILE = pjoin(COLAB_ROOT, 'js', 'colab.dep') subprocess.call([ 'python', CALC_DEPS_FILE, '--dep', GOOG_ROOT, '--path', COLAB_ROOT, '--output_mode', 'deps', '--output_file', OUTPUT_FILE]) subprocess.call([ 'sed',
