def squash_build(self, conf, context, stream, squash_commands):
"""Do a squash build"""
from harpoon.option_spec.image_objs import DockerFile
squashing = conf
output, status = command_output("which docker-squash")
if status != 0:
raise BadEnvironment(
"Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash"
)
if squash_commands:
squasher_conf = conf.clone()
squasher_conf.image_name = "{0}-for-squashing".format(conf.name)
if conf.image_name_prefix not in ("", None, NotSpecified):
squasher.conf.image_name = "{0}-{1}".format(
conf.image_name_prefix, squasher_conf.image_name)
with self.remove_replaced_images(squasher_conf) as info:
self.log_context_size(context, conf)
original_docker_file = conf.docker_file
new_docker_file = DockerFile(
["FROM {0}".format(conf.image_name)] + squash_commands,
original_docker_file.mtime)
with context.clone_with_new_dockerfile(
squasher_conf, new_docker_file) as squasher_context:
self.log_context_size(squasher_context, squasher_conf)
info['cached'] = self.do_build(squasher_conf,
squasher_context, stream)
squashing = squasher_conf
log.info("Saving image\timage=%s", squashing.image_name)
with hp.a_temp_file() as fle:
res = conf.harpoon.docker_context.get_image(squashing.image_name)
fle.write(res.read())
fle.close()
with hp.a_temp_file() as fle2:
output, status = command_output(
"sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(
fle.name, fle2.name, conf.image_name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to squash the image!")
output, status = command_output("docker load",
stdin=open(fle2.name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to load the squashed image")
if squashing is not conf:
log.info("Removing intermediate image %s", squashing.image_name)
conf.harpoon.docker_context.remove_image(squashing.image_name)
def make_context(self,
context,
docker_file,
silent_build=False,
extra_context=None):
"""
Context manager for creating the context of the image
Arguments:
context - ``harpoon.option_spec.image_objs.Context``
Knows all the context related options
docker_file - ``harpoon.option_spec.image_objs.Dockerfile``
Knows what is in the dockerfile and it's mtime
silent_build - boolean
If True, then suppress printing out information
extra_context - List of (string, string)
First string represents the content to put in a file and the second
string represents where in the context this extra file should go
"""
with a_temp_file() as tmpfile:
t = tarfile.open(mode='w:gz', fileobj=tmpfile)
for thing, mtime, arcname in self.find_mtimes(
context, silent_build):
if mtime:
os.utime(thing, (mtime, mtime))
t.add(thing, arcname=arcname)
mtime = docker_file.mtime
if extra_context:
for content, arcname in extra_context:
with a_temp_file() as fle:
fle.write(content.encode('utf-8'))
fle.seek(0)
if mtime:
os.utime(fle.name, (mtime, mtime))
t.add(fle.name, arcname=arcname)
# And add our docker file
with a_temp_file() as dockerfile:
dockerfile.write(docker_file.docker_lines.encode('utf-8'))
dockerfile.seek(0)
if mtime:
os.utime(dockerfile.name, (mtime, mtime))
t.add(dockerfile.name, arcname="./Dockerfile")
t.close()
tmpfile.seek(0)
yield tmpfile
def the_context(self, content, silent_build=False):
"""Return either a file with the content written to it, or a whole new context tar"""
if isinstance(content, six.string_types):
with a_temp_file() as fle:
fle.write(content.encode('utf-8'))
fle.seek(0)
yield fle
elif "context" in content:
with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper:
wrapper.close()
yield wrapper.tmpfile
elif "image" in content:
from harpoon.ship.runner import Runner
with a_temp_file() as fle:
content["conf"].command = "yes"
with Runner()._run_container(content["conf"], content["images"], detach=True, delete_anyway=True):
try:
strm, stat = content["docker_context"].get_archive(content["conf"].container_id, content["path"])
except docker.errors.NotFound:
raise BadOption("Trying to get something from an image that don't exist!", path=content["path"], image=content["conf"].image_name)
else:
log.debug(stat)
fo = BytesIO(strm.read())
tf = tarfile.TarFile(fileobj=fo)
if tf.firstmember.isdir():
tf2 = tarfile.TarFile(fileobj=fle, mode='w')
name = tf.firstmember.name
for member in tf.getmembers()[1:]:
member.name = member.name[len(name)+1:]
if member.issym():
with tempfile.NamedTemporaryFile() as symfle:
os.remove(symfle.name)
os.symlink(member.linkpath, symfle.name)
tf2.addfile(member, fileobj=symfle)
elif not member.isdir():
tf2.addfile(member, fileobj=tf.extractfile(member.name))
tf2.close()
else:
fle.write(tf.extractfile(tf.firstmember.name).read())
tf.close()
log.info("Got '{0}' from {1} for context".format(content["path"], content["conf"].container_id))
fle.seek(0)
yield fle
def add_docker_file_to_tarfile(self, docker_file, tar):
"""Add a Dockerfile to a tarfile"""
with hp.a_temp_file() as dockerfile:
log.debug("Context: ./Dockerfile")
dockerfile.write("\n".join(docker_file.docker_lines).encode('utf-8'))
dockerfile.seek(0)
os.utime(dockerfile.name, (docker_file.mtime, docker_file.mtime))
tar.add(dockerfile.name, arcname="./Dockerfile")
def add_docker_file_to_tarfile(self, docker_file, tar):
"""Add a Dockerfile to a tarfile"""
with hp.a_temp_file() as dockerfile:
log.debug("Context: ./Dockerfile")
dockerfile.write("\n".join(
docker_file.docker_lines).encode('utf-8'))
dockerfile.seek(0)
os.utime(dockerfile.name, (docker_file.mtime, docker_file.mtime))
tar.add(dockerfile.name, arcname="./Dockerfile")
def the_context(self, content, silent_build=False):
"""Return either a file with the content written to it, or a whole new context tar"""
if isinstance(content, six.string_types):
with a_temp_file() as fle:
fle.write(content.encode('utf-8'))
fle.seek(0)
yield fle
else:
with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper:
wrapper.close()
yield wrapper.tmpfile
def the_context(self, content, silent_build=False):
"""Return either a file with the content written to it, or a whole new context tar"""
if isinstance(content, six.string_types):
with a_temp_file() as fle:
fle.write(content.encode('utf-8'))
fle.seek(0)
yield fle
else:
with ContextBuilder().make_context(content["context"], silent_build=silent_build, use_gzip=False) as wrapper:
wrapper.close()
yield wrapper.tmpfile
def squash_build(self, conf, context, stream, squash_commands):
"""Do a squash build"""
from harpoon.option_spec.image_objs import DockerFile
squashing = conf
output, status = command_output("which docker-squash")
if status != 0:
raise BadEnvironment("Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash")
if squash_commands:
squasher_conf = conf.clone()
squasher_conf.image_name = "{0}-for-squashing".format(conf.name)
if conf.image_name_prefix not in ("", None, NotSpecified):
squasher.conf.image_name = "{0}-{1}".format(conf.image_name_prefix, squasher_conf.image_name)
with self.remove_replaced_images(squasher_conf) as info:
self.log_context_size(context, conf)
original_docker_file = conf.docker_file
new_docker_file = DockerFile(["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime)
with context.clone_with_new_dockerfile(squasher_conf, new_docker_file) as squasher_context:
self.log_context_size(squasher_context, squasher_conf)
info['cached'] = self.do_build(squasher_conf, squasher_context, stream)
squashing = squasher_conf
log.info("Saving image\timage=%s", squashing.image_name)
with hp.a_temp_file() as fle:
res = conf.harpoon.docker_context.get_image(squashing.image_name)
fle.write(res.read())
fle.close()
with hp.a_temp_file() as fle2:
output, status = command_output("sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(fle.name, fle2.name, conf.image_name), verbose=True, timeout=600)
if status != 0:
raise HarpoonError("Failed to squash the image!")
output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600)
if status != 0:
raise HarpoonError("Failed to load the squashed image")
if squashing is not conf:
log.info("Removing intermediate image %s", squashing.image_name)
conf.harpoon.docker_context.remove_image(squashing.image_name)
def clone_with_new_dockerfile(self, conf, docker_file):
"""Clone this tarfile and add in another filename before closing the new tar and returning"""
log.info("Copying context to add a different dockerfile")
self.close()
with a_temp_file() as tmpfile:
old_t = os.stat(self.tmpfile.name).st_size > 0
if old_t:
shutil.copy(self.tmpfile.name, tmpfile.name)
with tarfile.open(tmpfile.name, mode="a") as t:
conf.add_docker_file_to_tarfile(docker_file, t)
yield ContextWrapper(t, tmpfile)
def clone_with_new_dockerfile(self, conf, docker_file):
"""Clone this tarfile and add in another filename before closing the new tar and returning"""
log.info("Copying context to add a different dockerfile")
self.close()
with a_temp_file() as tmpfile:
old_t = os.stat(self.tmpfile.name).st_size > 0
if old_t:
shutil.copy(self.tmpfile.name, tmpfile.name)
with tarfile.open(tmpfile.name, mode="a") as t:
conf.add_docker_file_to_tarfile(docker_file, t)
yield ContextWrapper(t, tmpfile)
def clone_with_new_dockerfile(self, conf, docker_file):
"""Clone this tarfile and add in another filename before closing the new tar and returning"""
with open(self.tmpfile.name) as old_tmpfile:
old_t = None
if os.stat(old_tmpfile.name).st_size > 0:
old_t = tarfile.open(mode='r:gz', fileobj=open(old_tmpfile.name))
with a_temp_file() as tmpfile:
t = tarfile.open(mode='w:gz', fileobj=tmpfile)
if old_t:
for member in old_t:
t.addfile(member, old_t.extractfile(member.name))
conf.add_docker_file_to_tarfile(docker_file, t)
yield ContextWrapper(t, tmpfile)
def make_context(self, context, silent_build=False, extra_context=None):
"""
Context manager for creating the context of the image
Arguments:
context - ``harpoon.option_spec.image_objs.Context``
Knows all the context related options
docker_file - ``harpoon.option_spec.image_objs.Dockerfile``
Knows what is in the dockerfile and it's mtime
silent_build - boolean
If True, then suppress printing out information
extra_context - List of (content, string)
content is either a string repsenting the content to put in a file
or a dictionary representing what path to get from what docker image
The second string represents where in the context this extra file should go
"""
with a_temp_file() as tmpfile:
t = tarfile.open(mode='w', fileobj=tmpfile)
for thing, mtime, arcname in self.find_mtimes(context, silent_build):
if mtime:
os.utime(thing, (mtime, mtime))
log.debug("Context: {0}".format(arcname))
t.add(thing, arcname=arcname)
if extra_context:
extra = list(extra_context)
for content, arcname in extra:
mtime_match = re.search("mtime\((\d+)\)$", arcname)
specified_mtime = None if not mtime_match else int(mtime_match.groups()[0])
with self.the_context(content, silent_build=silent_build) as fle:
if specified_mtime:
os.utime(fle.name, (specified_mtime, specified_mtime))
log.debug("Context: {0}".format(arcname))
t.add(fle.name, arcname=arcname)
yield ContextWrapper(t, tmpfile)
def make_context(self, context, silent_build=False, use_gzip=True, extra_context=None):
"""
Context manager for creating the context of the image
Arguments:
context - ``harpoon.option_spec.image_objs.Context``
Knows all the context related options
docker_file - ``harpoon.option_spec.image_objs.Dockerfile``
Knows what is in the dockerfile and it's mtime
silent_build - boolean
If True, then suppress printing out information
extra_context - List of (string, string)
First string represents the content to put in a file and the second
string represents where in the context this extra file should go
"""
with a_temp_file() as tmpfile:
mode = "w:gz" if use_gzip else "w"
t = tarfile.open(mode=mode, fileobj=tmpfile)
for thing, mtime, arcname in self.find_mtimes(context, silent_build):
if mtime:
os.utime(thing, (mtime, mtime))
log.debug("Context: {0}".format(arcname))
t.add(thing, arcname=arcname)
if extra_context:
extra = list(extra_context)
for content, arcname in extra:
mtime_match = re.search("mtime\((\d+)\)$", arcname)
specified_mtime = None if not mtime_match else int(mtime_match.groups()[0])
with self.the_context(content, silent_build=silent_build) as fle:
if specified_mtime:
os.utime(fle.name, (specified_mtime, specified_mtime))
log.debug("Context: {0}".format(arcname))
t.add(fle.name, arcname=arcname)
yield ContextWrapper(t, tmpfile)
def make_context(self):
"""Context manager for creating the context of the image"""
class Nope(object): pass
host_context = not self.heira_formatted("no_host_context", default=False)
context_exclude = self.heira_formatted("context_exclude", default=None)
respect_gitignore = self.heira_formatted("respect_gitignore", default=Nope)
use_git_timestamps = self.heira_formatted("use_git_timestamps", default=Nope)
use_git = False
if respect_gitignore is not Nope and respect_gitignore:
use_git = True
if use_git_timestamps is not Nope and use_git_timestamps:
use_git = True
respect_gitignore = use_git if respect_gitignore is Nope else respect_gitignore
use_git_timestamps = use_git if use_git_timestamps is Nope else use_git_timestamps
git_files = set()
changed_files = set()
files = []
if host_context:
if use_git:
output, status = command_output("git diff --name-only", cwd=self.parent_dir)
if status != 0:
raise HarpoonError("Failed to determine what files have changed", directory=self.parent_dir, output=output)
changed_files = set(output)
if not self.silent_build: log.info("Determining context from git ls-files")
options = ""
if context_exclude:
for excluder in context_exclude:
options = "{0} --exclude={1}".format(options, excluder)
# Unfortunately --exclude doesn't work on committed/staged files, only on untracked things :(
output, status = command_output("git ls-files --exclude-standard", cwd=self.parent_dir)
if status != 0:
raise HarpoonError("Failed to do a git ls-files", directory=self.parent_dir, output=output)
others, status = command_output("git ls-files --exclude-standard --others {0}".format(options), cwd=self.parent_dir)
if status != 0:
raise HarpoonError("Failed to do a git ls-files to get untracked files", directory=self.parent_dir, output=others)
if not (output or others) or any(out and out[0].startswith("fatal: Not a git repository") for out in (output, others)):
raise HarpoonError("Told to use git features, but git ls-files says no", directory=self.parent_dir, output=output, others=others)
combined = set(output + others)
git_files = set(output)
else:
combined = set()
if context_exclude:
combined = set([os.path.relpath(location, self.parent_dir) for location in glob2.glob("{0}/**".format(self.parent_dir))])
else:
combined = set([self.parent_dir])
if context_exclude:
if not self.silent_build: log.info("Filtering %s items\texcluding=%s", len(combined), context_exclude)
excluded = set()
for filename in combined:
for excluder in context_exclude:
if fnmatch.fnmatch(filename, excluder):
excluded.add(filename)
break
combined = combined - excluded
files = sorted(os.path.join(self.parent_dir, filename) for filename in combined)
if context_exclude and not self.silent_build: log.info("Adding %s things from %s to the context", len(files), self.parent_dir)
mtime = self.mtime
docker_lines = '\n'.join(self.commands)
def matches_glob(string, globs):
"""Returns whether this string matches any of the globs"""
if isinstance(globs, bool):
return globs
return any(fnmatch.fnmatch(string, glob) for glob in globs)
with a_temp_file() as tmpfile:
t = tarfile.open(mode='w:gz', fileobj=tmpfile)
for thing in files:
if os.path.exists(thing):
relname = os.path.relpath(thing, self.parent_dir)
arcname = "./{0}".format(relname)
if use_git_timestamps and (relname in git_files and relname not in changed_files and matches_glob(relname, use_git_timestamps)):
# Set the modified date from git
date, status = command_output("git show -s --format=%at -n1 -- {0}".format(relname), cwd=self.parent_dir)
if status != 0 or not date or not date[0].isdigit():
log.error("Couldn't determine git date for a file\tdirectory=%s\trelname=%s", self.parent_dir, relname)
if date:
date = int(date[0])
os.utime(thing, (date, date))
t.add(thing, arcname=arcname)
for content, arcname in self.extra_context:
with a_temp_file() as fle:
fle.write(content)
fle.seek(0)
if mtime:
os.utime(fle.name, (mtime, mtime))
t.add(fle.name, arcname=arcname)
# And add our docker file
with a_temp_file() as dockerfile:
dockerfile.write(docker_lines)
dockerfile.seek(0)
if mtime:
os.utime(dockerfile.name, (mtime, mtime))
t.add(dockerfile.name, arcname="./Dockerfile")
t.close()
tmpfile.seek(0)
yield tmpfile
# coding: spec
from harpoon.helpers import a_temp_file, until, memoized_property
from tests.helpers import HarpoonCase
from noseOfYeti.tokeniser.support import noy_sup_setUp
from contextlib import contextmanager
import mock
import os
describe HarpoonCase, "a_temp_file":
it "yields the file object of a file that disappears after the context":
with a_temp_file() as fle:
assert os.path.exists(fle.name)
assert not os.path.exists(fle.name)
it "can write to the temporary file, close it and still read from it":
with a_temp_file() as fle:
fle.write("blah".encode("utf-8"))
fle.close()
with open(fle.name) as fread:
self.assertEqual(fread.read(), "blah")
assert not os.path.exists(fle.name)
describe HarpoonCase, "until":
@contextmanager
def mock_log_and_time(self):
"""Mock out the log object and time, yield (log, time)"""
fake_log = mock.Mock(name="log")
