def _gen_composer_lock(self):
gen_composer_lock_cmd = ['composer', 'update', '--lock']
ftl_util.run_command(
'composer_update_lock',
gen_composer_lock_cmd,
cmd_cwd=self._args.directory,
err_type=ftl_error.FTLErrors.USER())
def run_cached_tests(self):
logging.info('Beginning building {0} images'.format(self._runtime))
# For the binary
builder_path = 'ftl/{0}_builder.par'.format(self._runtime)
# For container builder
if not os.path.isfile(builder_path):
builder_path = 'bazel-bin/ftl/{0}_builder.par'.format(
self._runtime)
lyr_shas = []
for label, dir in zip(self._labels, self._dirs):
logging.info("label: %s" % label)
logging.info("dir: %s" % dir)
img_name = ''.join([self._name.split(":")[0], ":", label])
ftl_args = [
builder_path, '--base', self._base, '--name', img_name,
'--directory', dir
]
if label == "original":
ftl_args.extend(['--no-cache'])
try:
ftl_util.run_command("cached-ftl-build-%s" % img_name,
ftl_args)
lyr_shas.append(self._fetch_lyr_shas(img_name))
except ftl_util.FTLException as e:
logging.error(e)
exit(1)
finally:
self._cleanup(constants.VENV_DIR)
if len(lyr_shas) is not 2:
logging.error("Incorrect number of layers to compare")
exit(1)
self._compare_layers(lyr_shas[0], lyr_shas[1], self._offset)
def _pip_download_wheels(self, pkg_txt):
pip_cmd_args = list(self._pip_cmd)
pip_cmd_args.extend(
['wheel', '-w', self._wheel_dir, '-r', '/dev/stdin'])
pip_cmd_args.extend(['--no-deps'])
pip_cmd_args.extend(constants.PIP_OPTIONS)
ftl_util.run_command('pip_download_wheel', pip_cmd_args, None,
self._gen_pip_env(), pkg_txt)
def _setup_venv(self):
venv_cmd_args = list(self._venv_cmd)
venv_cmd_args.extend([
'--no-download',
self._venv_dir,
'-p',
])
venv_cmd_args.extend(list(self._python_cmd))
ftl_util.run_command('create_virtualenv', venv_cmd_args)
def _whl_to_fslayer(self, whl):
tmp_dir = tempfile.mkdtemp()
pkg_dir = os.path.join(tmp_dir, self._venv_dir.lstrip('/'))
os.makedirs(pkg_dir)
pip_cmd_args = list(self._pip_cmd)
pip_cmd_args.extend(['install', '--no-deps', '--prefix', pkg_dir, whl])
pip_cmd_args.extend(constants.PIP_OPTIONS)
ftl_util.run_command('pip_install_from_wheels', pip_cmd_args, None,
self._gen_pip_env())
return tmp_dir
def _pip_download_wheels(self, pkg_txt):
pip_cmd_args = list(self._pip_cmd)
pip_cmd_args.extend(
['wheel', '-w', self._wheel_dir, '-r', 'requirements.txt'])
pip_cmd_args.extend(constants.PIP_OPTIONS)
ftl_util.run_command('pip_download_wheels',
pip_cmd_args,
cmd_cwd=self._directory,
cmd_env=self._gen_pip_env(),
cmd_input=pkg_txt,
err_type=ftl_error.FTLErrors.USER())
def setup_venv(venv_dir, venv_cmd, python_cmd):
if os.path.isdir(venv_dir):
return
venv_cmd_args = list(venv_cmd)
venv_cmd_args.extend([
'--no-download',
venv_dir,
'-p',
])
venv_cmd_args.extend(list(python_cmd))
ftl_util.run_command('create_virtualenv', venv_cmd_args)
def _pip_download_wheels(self, pkg_txt):
pip_cmd_args = list(self._pip_cmd)
pip_cmd_args.extend(
['wheel', '-w', self._wheel_dir, '-r', '/dev/stdin'])
pip_cmd_args.extend(constants.PIP_OPTIONS)
ftl_util.run_command('pip_download_wheels',
pip_cmd_args,
None,
self._gen_pip_env(),
pkg_txt,
err_type=ftl_error.FTLErrors.USER())
def _gen_yarn_install_tar(self, app_dir):
yarn_install_cmd = ['yarn', 'install', '--production']
ftl_util.run_command('yarn_install',
yarn_install_cmd,
cmd_cwd=app_dir,
err_type=ftl_error.FTLErrors.USER())
module_destination = os.path.join(self._destination_path,
'node_modules')
modules_dir = os.path.join(self._directory, "node_modules")
return ftl_util.zip_dir_to_layer_sha(modules_dir, module_destination)
def run_cached_tests(self):
logging.info('Beginning building {0} images'.format(self._runtime))
# For the binary
builder_path = 'ftl/{0}_builder.par'.format(self._runtime)
# For container builder
if not os.path.isfile(builder_path):
builder_path = 'bazel-bin/ftl/{0}_builder.par'.format(
self._runtime)
lyr_shas = []
random_suffix = randomword(32)
WORKDIR = "/workspace-cached"
for label, dir in zip(self._labels, self._dirs):
logging.info("label: %s", label)
logging.info("dir: %s", dir)
img_name = "%s:%s" % (self._name.split(":")[0], label)
ftl_util.run_command("copy-%s-to-%s" % (dir, WORKDIR),
["cp", "-r", dir, WORKDIR])
ftl_args = [
builder_path, '--base', self._base, '--name', img_name,
'--directory', WORKDIR, '--destination', "/srv",
'--cache-repository',
'gcr.io/ftl-node-test/ftl-cache-repo-%s' % random_suffix
]
if label == "original":
ftl_args.extend(['--no-cache'])
out = ""
try:
out = ftl_util.run_command("cached-ftl-build-%s" % img_name,
ftl_args)
lyr_shas.append(self._fetch_lyr_shas(img_name))
except ftl_util.FTLException as e:
logging.error(e)
exit(1)
finally:
logging.info("cleaning up directories...")
self._cleanup(constants.VIRTUALENV_DIR)
ftl_util.run_command("cleanup-%s" % WORKDIR,
["rm", "-rf", WORKDIR])
if self._should_cache and label == "reupload":
if "[HIT]" not in out:
logging.error("Cache `[HIT]` not found on reupload: %s", out)
exit(1)
if len(lyr_shas) is not 2:
logging.error("Incorrect number of layers to compare")
exit(1)
self._compare_layers(lyr_shas[0], lyr_shas[1], self._offset)
def _gen_package_lock_if_required(self, ctx):
if not ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
return
if ctx.Contains(constants.PACKAGE_JSON) and \
not ctx.Contains(constants.YARN_LOCK) and \
not ctx.Contains(constants.PACKAGE_LOCK):
logging.info('Found neither yarn.lock or package-lock.json,'
'generating package-lock.json from package.json')
gen_package_lock_cmd = ['npm', 'install', '--package-lock-only']
ftl_util.run_command('gen_package_lock',
gen_package_lock_cmd,
cmd_cwd=self._args.directory,
err_type=ftl_error.FTLErrors.USER())
def _gen_composer_install_tar(self, app_dir, destination_path):
composer_install_cmd = [
'composer', 'install', '--no-dev', '--no-progress', '--no-suggest',
'--no-interaction'
]
ftl_util.run_command('composer_install',
composer_install_cmd,
cmd_cwd=app_dir,
cmd_env=php_util.gen_composer_env(),
err_type=ftl_error.FTLErrors.USER())
vendor_dir = os.path.join(self._directory, 'vendor')
vendor_destination = os.path.join(destination_path, 'vendor')
return ftl_util.zip_dir_to_layer_sha(vendor_dir, vendor_destination)
def _gcp_build(self, app_dir, install_bin, run_cmd):
env = os.environ.copy()
env["NODE_ENV"] = "development"
install_cmd = [install_bin, 'install']
ftl_util.run_command('%s_install' % install_bin,
install_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
npm_run_script_cmd = [install_bin, run_cmd, 'gcp-build']
ftl_util.run_command('%s_%s_gcp_build' % (install_bin, run_cmd),
npm_run_script_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
def _gen_composer_install_tar(self, destination_path, pkg_descriptor):
# Create temp directory to write package descriptor to
pkg_dir = tempfile.mkdtemp()
app_dir = os.path.join(pkg_dir, destination_path.strip("/"))
os.makedirs(app_dir)
rm_cmd = ['rm', '-rf', os.path.join(app_dir, 'vendor')]
ftl_util.run_command('rm_vendor_dir', rm_cmd)
pkg, version = pkg_descriptor
composer_install_cmd = ['composer', 'require', str(pkg), str(version)]
ftl_util.run_command('composer_require',
composer_install_cmd,
app_dir,
err_type=ftl_error.FTLErrors.USER())
return ftl_util.zip_dir_to_layer_sha(pkg_dir)
def _build_layer(self):
self._setup_venv()
tar_path = tempfile.mktemp(suffix='.tar')
tar_cmd = ['tar', '-cf', tar_path, self._venv_dir]
ftl_util.run_command('tar_venv_interpreter', tar_cmd)
u_blob = open(tar_path, 'r').read()
# We use gzip for performance instead of python's zip.
gzip_cmd = ['gzip', tar_path, '-1']
ftl_util.run_command('gzip_venv_interpreter', gzip_cmd)
blob = open(os.path.join(tar_path + '.gz'), 'rb').read()
overrides = ftl_util.generate_overrides(True, self._venv_dir)
self._img = tar_to_dockerimage.FromFSImage([blob], [u_blob], overrides)
def _gcp_build(self, app_dir):
env = os.environ.copy()
env["NODE_ENV"] = "development"
npm_install_cmd = ['npm', 'install']
ftl_util.run_command(
'npm_install',
npm_install_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
npm_run_script_cmd = ['npm', 'run-script', 'gcp-build']
ftl_util.run_command(
'npm_run_script_gcp_build',
npm_run_script_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
def _gen_yarn_install_tar(self, app_dir):
is_gcp_build = False
if self._ctx and self._ctx.Contains(constants.PACKAGE_JSON):
is_gcp_build = self._is_gcp_build(
json.loads(self._ctx.GetFile(constants.PACKAGE_JSON)))
if is_gcp_build:
self._gcp_build(app_dir, 'yarn', 'run')
else:
yarn_install_cmd = ['yarn', 'install', '--production']
ftl_util.run_command('yarn_install',
yarn_install_cmd,
cmd_cwd=app_dir,
err_type=ftl_error.FTLErrors.USER())
module_destination = os.path.join(self._destination_path,
'node_modules')
modules_dir = os.path.join(self._directory, "node_modules")
return ftl_util.zip_dir_to_layer_sha(modules_dir, module_destination)
def _gen_npm_install_tar(self, pkg_descriptor, app_dir):
try:
os.makedirs(app_dir)
except OSError:
logging.info("%s already exists, skipping creation", app_dir)
# Copy out the relevant package descriptors to a tempdir.
if self._descriptor_files and self._ctx:
ftl_util.descriptor_copy(self._ctx, self._descriptor_files,
app_dir)
if self._ctx:
self._check_gcp_build(
json.loads(self._ctx.GetFile(constants.PACKAGE_JSON)), app_dir)
rm_cmd = ['rm', '-rf', os.path.join(app_dir, 'node_modules')]
ftl_util.run_command('rm_node_modules', rm_cmd)
if not pkg_descriptor:
npm_install_cmd = ['npm', 'install', '--production']
ftl_util.run_command(
'npm_install',
npm_install_cmd,
app_dir,
err_type=ftl_error.FTLErrors.USER())
else:
npm_install_cmd = [
'npm', 'install', '--production', pkg_descriptor
]
ftl_util.run_command(
'npm_install',
npm_install_cmd,
app_dir,
err_type=ftl_error.FTLErrors.USER())
tar_path = tempfile.mktemp(suffix='.tar')
tar_cmd = ['tar', '-cf', tar_path, app_dir]
ftl_util.run_command('tar_node_dependencies', tar_cmd)
u_blob = open(tar_path, 'r').read()
# We use gzip for performance instead of python's zip.
gzip_cmd = ['gzip', tar_path, '-1']
ftl_util.run_command('gzip_node_dependencies', gzip_cmd)
return open(os.path.join(tar_path + '.gz'), 'rb').read(), u_blob
def _gen_composer_install_tar(self, destination_path):
# Create temp directory to write package descriptor to
pkg_dir = tempfile.mkdtemp()
app_dir = os.path.join(pkg_dir, destination_path.strip("/"))
os.makedirs(app_dir)
# Copy out the relevant package descriptors to a tempdir.
ftl_util.descriptor_copy(self._ctx, self._descriptor_files, app_dir)
rm_cmd = ['rm', '-rf', os.path.join(app_dir, 'vendor')]
ftl_util.run_command('rm_vendor_dir', rm_cmd)
composer_install_cmd = [
'composer', 'install', '--no-dev', '--no-scripts'
]
ftl_util.run_command('composer_install',
composer_install_cmd,
app_dir,
err_type=ftl_error.FTLErrors.USER())
return ftl_util.zip_dir_to_layer_sha(pkg_dir)
def _gen_npm_install_tar(self, pkg_descriptor, app_dir):
if self._ctx:
if self._ctx.Contains(constants.PACKAGE_JSON):
self._check_gcp_build(
json.loads(self._ctx.GetFile(constants.PACKAGE_JSON)),
app_dir)
rm_cmd = ['rm', '-rf', os.path.join(app_dir, 'node_modules')]
ftl_util.run_command('rm_node_modules', rm_cmd)
if not pkg_descriptor:
npm_install_cmd = ['npm', 'install', '--production']
ftl_util.run_command('npm_install',
npm_install_cmd,
cmd_cwd=app_dir,
err_type=ftl_error.FTLErrors.USER())
else:
npm_install_cmd = [
'npm', 'install', '--production', pkg_descriptor
]
ftl_util.run_command('npm_install',
npm_install_cmd,
cmd_cwd=app_dir,
err_type=ftl_error.FTLErrors.USER())
modules_dir = os.path.join(app_dir, "node_modules")
module_destination = os.path.join(self._destination_path,
'node_modules')
return ftl_util.zip_dir_to_layer_sha(modules_dir, module_destination)
def _check_gcp_build(self, package_json, app_dir):
scripts = package_json.get('scripts', {})
gcp_build = scripts.get('gcp-build')
if not gcp_build:
return
env = os.environ.copy()
env["NODE_ENV"] = "development"
npm_install_cmd = ['npm', 'install']
ftl_util.run_command('npm_install',
npm_install_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
npm_run_script_cmd = ['npm', 'run-script', 'gcp-build']
ftl_util.run_command('npm_run_script_gcp_build',
npm_run_script_cmd,
app_dir,
env,
err_type=ftl_error.FTLErrors.USER())
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
# delete any existing files in node_modules folder
if self._args.directory:
modules_dir = os.path.join(self._args.directory, "node_modules")
rm_cmd = ['rm', '-rf', modules_dir]
ftl_util.run_command('rm_node_modules', rm_cmd)
os.makedirs(os.path.join(modules_dir))
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
layer_builder = node_builder.LayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
destination_path=self._args.destination_path,
should_use_yarn=self._should_use_yarn,
cache_key_version=self._args.cache_key_version,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
if self._args.additional_directory:
additional_directory = base_builder.AppLayerBuilder(
directory=self._args.additional_directory,
destination_path=self._args.additional_directory,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
additional_directory.BuildLayer()
lyr_imgs.append(additional_directory.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
# delete any existing files in vendor folder
if self._args.directory:
vendor_dir = os.path.join(self._args.directory, 'vendor')
rm_cmd = ['rm', '-rf', vendor_dir]
ftl_util.run_command('rm_vendor_dir', rm_cmd)
os.makedirs(os.path.join(vendor_dir))
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
self._gen_composer_lock()
layer_builder = php_builder.PhaseOneLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
destination_path=self._args.destination_path,
cache_key_version=self._args.cache_key_version,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
if self._args.additional_directory:
additional_directory = base_builder.AppLayerBuilder(
directory=self._args.additional_directory,
destination_path=self._args.additional_directory,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
additional_directory.BuildLayer()
lyr_imgs.append(additional_directory.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def _gen_npm_install_tar(self, app_dir):
npm_install_cmd = ['npm', 'install', '--production']
npm_output = ftl_util.run_command('npm_install',
npm_install_cmd,
cmd_cwd=app_dir,
err_type=ftl_error.FTLErrors.USER())
module_destination = os.path.join(self._destination_path,
'node_modules')
modules_dir = os.path.join(self._directory, "node_modules")
if not os.path.isdir(modules_dir) or os.listdir(modules_dir) == []:
if "Invalid name" in npm_output:
raise ftl_error.UserError("%s\n%s" % (npm_output, "0"))
return ftl_util.zip_dir_to_layer_sha(modules_dir, module_destination)
def _cleanup_build_layer(self):
if self._directory:
vendor_dir = os.path.join(self._directory, 'vendor')
rm_cmd = ['rm', '-rf', vendor_dir]
ftl_util.run_command('rm_vendor_dir', rm_cmd)
def _cleanup_build_layer(self):
if self._directory:
modules_dir = os.path.join(self._directory, "node_modules")
rm_cmd = ['rm', '-rf', modules_dir]
ftl_util.run_command('rm_node_modules', rm_cmd)