def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
interpreter_builder = package_builder.InterpreterLayerBuilder(
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
venv_cmd=self._venv_cmd,
cache=self._cache)
interpreter_builder.BuildLayer()
lyr_imgs.append(interpreter_builder.GetImage())
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
# build interpreter layer
if self._is_phase2:
# do a phase 2 build of the package layers w/ Pipfile.lock
# iterate over package/version Pipfile.lock
python_util.setup_venv(self._venv_dir,
self._venv_cmd,
self._python_cmd)
pkgs = self._parse_pipfile_pkgs()
with ftl_util.Timing('uploading_all_package_layers'):
with concurrent.futures.ThreadPoolExecutor(
max_workers=constants.THREADS) as executor:
future_to_params = {executor.submit(
self._build_pkg, pkg,
interpreter_builder, lyr_imgs): pkg
for pkg in pkgs
}
for future in concurrent.futures.as_completed(
future_to_params):
future.result()
else:
# do a phase 1 build of the package layers w/ requirements.txt
req_txt_builder = package_builder.RequirementsLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
pkg_dir=None,
wheel_dir=self._wheel_dir,
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
req_txt_builder.BuildLayer()
if req_txt_builder.GetImage():
lyr_imgs.append(req_txt_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
# build interpreter layer
interpreter_builder = package_builder.InterpreterLayerBuilder(
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
venv_cmd=self._venv_cmd,
cache=self._cache)
interpreter_builder.BuildLayer()
lyr_imgs.append(interpreter_builder.GetImage())
if self._is_phase2:
# do a phase 2 build of the package layers w/ Pipfile.lock
# iterate over package/version Pipfile.lock
pkgs = self._parse_pipfile_pkgs()
for pkg in pkgs:
pipfile_builder = package_builder.PipfileLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_descriptor=pkg,
pkg_dir=None,
wheel_dir=ftl_util.gen_tmp_dir(constants.WHEEL_DIR),
venv_dir=self._venv_dir,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
pipfile_builder.BuildLayer()
lyr_imgs.append(pipfile_builder.GetImage())
else:
# do a phase 1 build of the package layers w/ requirements.txt
req_txt_builder = package_builder.RequirementsLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_dir=None,
wheel_dir=self._wheel_dir,
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
req_txt_builder.BuildLayer()
lyr_imgs.append(req_txt_builder.GetImage())
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def setUp(self, mock_from):
mock_from.return_value.__enter__.return_value = None
self.ctx = context.Memory()
self.ctx.AddFile("app.py", _APP)
args = mock.Mock()
args.name = 'gcr.io/test/test:latest'
args.base = 'gcr.io/google-appengine/python:latest'
args.python_version = 'python2.7'
args.tar_base_image_path = None
self.builder = builder.Python(self.ctx, args, "")
self.interpreter_builder = layer_builder.InterpreterLayerBuilder(
self.builder._venv_dir, self.builder._args.python_version)
self.builder._pip_install = mock.Mock()
def setUp(self, mock_from):
mock_from.return_value.__enter__.return_value = None
self.ctx = context.Memory()
self.ctx.AddFile("app.py", _APP)
args = mock.Mock()
args.name = 'gcr.io/test/test:latest'
args.base = 'gcr.io/google-appengine/python:latest'
args.python_cmd = 'python2.7'
args.pip_cmd = 'pip'
args.venv_cmd = 'virtualenv'
args.tar_base_image_path = None
self.builder = builder.Python(self.ctx, args)
# constants.VENV_DIR.replace('/', '') is used as the default path
# will give permissions errors in some build environments (eg: kokoro)
self.interpreter_builder = layer_builder.InterpreterLayerBuilder(
ftl_util.gen_tmp_dir(constants.VENV_DIR.replace('/', '')),
self.builder._python_cmd, self.builder._venv_cmd)
self.interpreter_builder._setup_venv = mock.Mock()
self.builder._pip_download_wheels = mock.Mock()
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
interpreter_builder = package_builder.InterpreterLayerBuilder(
self._venv_dir, self._args.python_version)
cached_int_img = None
if self._args.cache:
with ftl_util.Timing("checking cached int layer"):
key = interpreter_builder.GetCacheKey()
cached_int_img = self._cache.Get(key)
if cached_int_img is not None:
interpreter_builder.SetImage(cached_int_img)
else:
with ftl_util.Timing("building int layer"):
interpreter_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading int layer"):
self._cache.Set(interpreter_builder.GetCacheKey(),
interpreter_builder.GetImage())
lyr_imgs.append(interpreter_builder)
pkg_descriptor = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
with ftl_util.Timing("installing pip packages"):
self._pip_install(pkg_descriptor)
with ftl_util.Timing("resolving whl paths"):
whls = self._resolve_whls()
pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls]
for whl_pkg_dir in pkg_dirs:
layer_builder = package_builder.PackageLayerBuilder(
self._ctx, self._descriptor_files, whl_pkg_dir,
interpreter_builder)
cached_pkg_img = None
if self._args.cache:
with ftl_util.Timing("checking cached pkg layer"):
key = layer_builder.GetCacheKey()
cached_pkg_img = self._cache.Get(key)
if cached_pkg_img is not None:
layer_builder.SetImage(cached_pkg_img)
else:
with ftl_util.Timing("building pkg layer"):
layer_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading pkg layer"):
self._cache.Set(layer_builder.GetCacheKey(),
layer_builder.GetImage())
lyr_imgs.append(layer_builder)
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
with ftl_util.Timing("builder app layer"):
app.BuildLayer()
lyr_imgs.append(app)
with ftl_util.Timing("stitching lyrs into final image"):
ftl_image = self.AppendLayersIntoImage(lyr_imgs)
with ftl_util.Timing("uploading final image"):
self.StoreImage(ftl_image)