def squash_build(self, conf, context, stream, squash_commands): """Do a squash build""" from harpoon.option_spec.image_objs import DockerFile squashing = conf output, status = command_output("which docker-squash") if status != 0: raise BadEnvironment( "Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash" ) if squash_commands: squasher_conf = conf.clone() squasher_conf.image_name = "{0}-for-squashing".format(conf.name) if conf.image_name_prefix not in ("", None, NotSpecified): squasher.conf.image_name = "{0}-{1}".format( conf.image_name_prefix, squasher_conf.image_name) with self.remove_replaced_images(squasher_conf) as info: self.log_context_size(context, conf) original_docker_file = conf.docker_file new_docker_file = DockerFile( ["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime) with context.clone_with_new_dockerfile( squasher_conf, new_docker_file) as squasher_context: self.log_context_size(squasher_context, squasher_conf) info['cached'] = self.do_build(squasher_conf, squasher_context, stream) squashing = squasher_conf log.info("Saving image\timage=%s", squashing.image_name) with hp.a_temp_file() as fle: res = conf.harpoon.docker_context.get_image(squashing.image_name) fle.write(res.read()) fle.close() with hp.a_temp_file() as fle2: output, status = command_output( "sudo docker-squash -i {0} -o {1} -t {2} -verbose".format( fle.name, fle2.name, conf.image_name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to squash the image!") output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to load the squashed image") if squashing is not conf: log.info("Removing intermediate image %s", squashing.image_name) conf.harpoon.docker_context.remove_image(squashing.image_name)
def make_context(self, context, docker_file, silent_build=False, extra_context=None): """ Context manager for creating the context of the image Arguments: context - ``harpoon.option_spec.image_objs.Context`` Knows all the context related options docker_file - ``harpoon.option_spec.image_objs.Dockerfile`` Knows what is in the dockerfile and it's mtime silent_build - boolean If True, then suppress printing out information extra_context - List of (string, string) First string represents the content to put in a file and the second string represents where in the context this extra file should go """ with a_temp_file() as tmpfile: t = tarfile.open(mode='w:gz', fileobj=tmpfile) for thing, mtime, arcname in self.find_mtimes( context, silent_build): if mtime: os.utime(thing, (mtime, mtime)) t.add(thing, arcname=arcname) mtime = docker_file.mtime if extra_context: for content, arcname in extra_context: with a_temp_file() as fle: fle.write(content.encode('utf-8')) fle.seek(0) if mtime: os.utime(fle.name, (mtime, mtime)) t.add(fle.name, arcname=arcname) # And add our docker file with a_temp_file() as dockerfile: dockerfile.write(docker_file.docker_lines.encode('utf-8')) dockerfile.seek(0) if mtime: os.utime(dockerfile.name, (mtime, mtime)) t.add(dockerfile.name, arcname="./Dockerfile") t.close() tmpfile.seek(0) yield tmpfile
def the_context(self, content, silent_build=False): """Return either a file with the content written to it, or a whole new context tar""" if isinstance(content, six.string_types): with a_temp_file() as fle: fle.write(content.encode('utf-8')) fle.seek(0) yield fle elif "context" in content: with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper: wrapper.close() yield wrapper.tmpfile elif "image" in content: from harpoon.ship.runner import Runner with a_temp_file() as fle: content["conf"].command = "yes" with Runner()._run_container(content["conf"], content["images"], detach=True, delete_anyway=True): try: strm, stat = content["docker_context"].get_archive(content["conf"].container_id, content["path"]) except docker.errors.NotFound: raise BadOption("Trying to get something from an image that don't exist!", path=content["path"], image=content["conf"].image_name) else: log.debug(stat) fo = BytesIO(strm.read()) tf = tarfile.TarFile(fileobj=fo) if tf.firstmember.isdir(): tf2 = tarfile.TarFile(fileobj=fle, mode='w') name = tf.firstmember.name for member in tf.getmembers()[1:]: member.name = member.name[len(name)+1:] if member.issym(): with tempfile.NamedTemporaryFile() as symfle: os.remove(symfle.name) os.symlink(member.linkpath, symfle.name) tf2.addfile(member, fileobj=symfle) elif not member.isdir(): tf2.addfile(member, fileobj=tf.extractfile(member.name)) tf2.close() else: fle.write(tf.extractfile(tf.firstmember.name).read()) tf.close() log.info("Got '{0}' from {1} for context".format(content["path"], content["conf"].container_id)) fle.seek(0) yield fle
def add_docker_file_to_tarfile(self, docker_file, tar): """Add a Dockerfile to a tarfile""" with hp.a_temp_file() as dockerfile: log.debug("Context: ./Dockerfile") dockerfile.write("\n".join(docker_file.docker_lines).encode('utf-8')) dockerfile.seek(0) os.utime(dockerfile.name, (docker_file.mtime, docker_file.mtime)) tar.add(dockerfile.name, arcname="./Dockerfile")
def add_docker_file_to_tarfile(self, docker_file, tar): """Add a Dockerfile to a tarfile""" with hp.a_temp_file() as dockerfile: log.debug("Context: ./Dockerfile") dockerfile.write("\n".join( docker_file.docker_lines).encode('utf-8')) dockerfile.seek(0) os.utime(dockerfile.name, (docker_file.mtime, docker_file.mtime)) tar.add(dockerfile.name, arcname="./Dockerfile")
def the_context(self, content, silent_build=False): """Return either a file with the content written to it, or a whole new context tar""" if isinstance(content, six.string_types): with a_temp_file() as fle: fle.write(content.encode('utf-8')) fle.seek(0) yield fle else: with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper: wrapper.close() yield wrapper.tmpfile
def the_context(self, content, silent_build=False): """Return either a file with the content written to it, or a whole new context tar""" if isinstance(content, six.string_types): with a_temp_file() as fle: fle.write(content.encode('utf-8')) fle.seek(0) yield fle else: with ContextBuilder().make_context(content["context"], silent_build=silent_build, use_gzip=False) as wrapper: wrapper.close() yield wrapper.tmpfile
def squash_build(self, conf, context, stream, squash_commands): """Do a squash build""" from harpoon.option_spec.image_objs import DockerFile squashing = conf output, status = command_output("which docker-squash") if status != 0: raise BadEnvironment("Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash") if squash_commands: squasher_conf = conf.clone() squasher_conf.image_name = "{0}-for-squashing".format(conf.name) if conf.image_name_prefix not in ("", None, NotSpecified): squasher.conf.image_name = "{0}-{1}".format(conf.image_name_prefix, squasher_conf.image_name) with self.remove_replaced_images(squasher_conf) as info: self.log_context_size(context, conf) original_docker_file = conf.docker_file new_docker_file = DockerFile(["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime) with context.clone_with_new_dockerfile(squasher_conf, new_docker_file) as squasher_context: self.log_context_size(squasher_context, squasher_conf) info['cached'] = self.do_build(squasher_conf, squasher_context, stream) squashing = squasher_conf log.info("Saving image\timage=%s", squashing.image_name) with hp.a_temp_file() as fle: res = conf.harpoon.docker_context.get_image(squashing.image_name) fle.write(res.read()) fle.close() with hp.a_temp_file() as fle2: output, status = command_output("sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(fle.name, fle2.name, conf.image_name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to squash the image!") output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to load the squashed image") if squashing is not conf: log.info("Removing intermediate image %s", squashing.image_name) conf.harpoon.docker_context.remove_image(squashing.image_name)
def clone_with_new_dockerfile(self, conf, docker_file): """Clone this tarfile and add in another filename before closing the new tar and returning""" log.info("Copying context to add a different dockerfile") self.close() with a_temp_file() as tmpfile: old_t = os.stat(self.tmpfile.name).st_size > 0 if old_t: shutil.copy(self.tmpfile.name, tmpfile.name) with tarfile.open(tmpfile.name, mode="a") as t: conf.add_docker_file_to_tarfile(docker_file, t) yield ContextWrapper(t, tmpfile)
def clone_with_new_dockerfile(self, conf, docker_file): """Clone this tarfile and add in another filename before closing the new tar and returning""" with open(self.tmpfile.name) as old_tmpfile: old_t = None if os.stat(old_tmpfile.name).st_size > 0: old_t = tarfile.open(mode='r:gz', fileobj=open(old_tmpfile.name)) with a_temp_file() as tmpfile: t = tarfile.open(mode='w:gz', fileobj=tmpfile) if old_t: for member in old_t: t.addfile(member, old_t.extractfile(member.name)) conf.add_docker_file_to_tarfile(docker_file, t) yield ContextWrapper(t, tmpfile)
def make_context(self, context, silent_build=False, extra_context=None): """ Context manager for creating the context of the image Arguments: context - ``harpoon.option_spec.image_objs.Context`` Knows all the context related options docker_file - ``harpoon.option_spec.image_objs.Dockerfile`` Knows what is in the dockerfile and it's mtime silent_build - boolean If True, then suppress printing out information extra_context - List of (content, string) content is either a string repsenting the content to put in a file or a dictionary representing what path to get from what docker image The second string represents where in the context this extra file should go """ with a_temp_file() as tmpfile: t = tarfile.open(mode='w', fileobj=tmpfile) for thing, mtime, arcname in self.find_mtimes(context, silent_build): if mtime: os.utime(thing, (mtime, mtime)) log.debug("Context: {0}".format(arcname)) t.add(thing, arcname=arcname) if extra_context: extra = list(extra_context) for content, arcname in extra: mtime_match = re.search("mtime\((\d+)\)$", arcname) specified_mtime = None if not mtime_match else int(mtime_match.groups()[0]) with self.the_context(content, silent_build=silent_build) as fle: if specified_mtime: os.utime(fle.name, (specified_mtime, specified_mtime)) log.debug("Context: {0}".format(arcname)) t.add(fle.name, arcname=arcname) yield ContextWrapper(t, tmpfile)
def make_context(self, context, silent_build=False, use_gzip=True, extra_context=None): """ Context manager for creating the context of the image Arguments: context - ``harpoon.option_spec.image_objs.Context`` Knows all the context related options docker_file - ``harpoon.option_spec.image_objs.Dockerfile`` Knows what is in the dockerfile and it's mtime silent_build - boolean If True, then suppress printing out information extra_context - List of (string, string) First string represents the content to put in a file and the second string represents where in the context this extra file should go """ with a_temp_file() as tmpfile: mode = "w:gz" if use_gzip else "w" t = tarfile.open(mode=mode, fileobj=tmpfile) for thing, mtime, arcname in self.find_mtimes(context, silent_build): if mtime: os.utime(thing, (mtime, mtime)) log.debug("Context: {0}".format(arcname)) t.add(thing, arcname=arcname) if extra_context: extra = list(extra_context) for content, arcname in extra: mtime_match = re.search("mtime\((\d+)\)$", arcname) specified_mtime = None if not mtime_match else int(mtime_match.groups()[0]) with self.the_context(content, silent_build=silent_build) as fle: if specified_mtime: os.utime(fle.name, (specified_mtime, specified_mtime)) log.debug("Context: {0}".format(arcname)) t.add(fle.name, arcname=arcname) yield ContextWrapper(t, tmpfile)
def make_context(self): """Context manager for creating the context of the image""" class Nope(object): pass host_context = not self.heira_formatted("no_host_context", default=False) context_exclude = self.heira_formatted("context_exclude", default=None) respect_gitignore = self.heira_formatted("respect_gitignore", default=Nope) use_git_timestamps = self.heira_formatted("use_git_timestamps", default=Nope) use_git = False if respect_gitignore is not Nope and respect_gitignore: use_git = True if use_git_timestamps is not Nope and use_git_timestamps: use_git = True respect_gitignore = use_git if respect_gitignore is Nope else respect_gitignore use_git_timestamps = use_git if use_git_timestamps is Nope else use_git_timestamps git_files = set() changed_files = set() files = [] if host_context: if use_git: output, status = command_output("git diff --name-only", cwd=self.parent_dir) if status != 0: raise HarpoonError("Failed to determine what files have changed", directory=self.parent_dir, output=output) changed_files = set(output) if not self.silent_build: log.info("Determining context from git ls-files") options = "" if context_exclude: for excluder in context_exclude: options = "{0} --exclude={1}".format(options, excluder) # Unfortunately --exclude doesn't work on committed/staged files, only on untracked things :( output, status = command_output("git ls-files --exclude-standard", cwd=self.parent_dir) if status != 0: raise HarpoonError("Failed to do a git ls-files", directory=self.parent_dir, output=output) others, status = command_output("git ls-files --exclude-standard --others {0}".format(options), cwd=self.parent_dir) if status != 0: raise HarpoonError("Failed to do a git ls-files to get untracked files", directory=self.parent_dir, output=others) if not (output or others) or any(out and out[0].startswith("fatal: Not a git repository") for out in (output, others)): raise HarpoonError("Told to use git features, but git ls-files says no", directory=self.parent_dir, output=output, others=others) combined = set(output + others) git_files = set(output) else: combined = set() if context_exclude: combined = set([os.path.relpath(location, self.parent_dir) for location in glob2.glob("{0}/**".format(self.parent_dir))]) else: combined = set([self.parent_dir]) if context_exclude: if not self.silent_build: log.info("Filtering %s items\texcluding=%s", len(combined), context_exclude) excluded = set() for filename in combined: for excluder in context_exclude: if fnmatch.fnmatch(filename, excluder): excluded.add(filename) break combined = combined - excluded files = sorted(os.path.join(self.parent_dir, filename) for filename in combined) if context_exclude and not self.silent_build: log.info("Adding %s things from %s to the context", len(files), self.parent_dir) mtime = self.mtime docker_lines = '\n'.join(self.commands) def matches_glob(string, globs): """Returns whether this string matches any of the globs""" if isinstance(globs, bool): return globs return any(fnmatch.fnmatch(string, glob) for glob in globs) with a_temp_file() as tmpfile: t = tarfile.open(mode='w:gz', fileobj=tmpfile) for thing in files: if os.path.exists(thing): relname = os.path.relpath(thing, self.parent_dir) arcname = "./{0}".format(relname) if use_git_timestamps and (relname in git_files and relname not in changed_files and matches_glob(relname, use_git_timestamps)): # Set the modified date from git date, status = command_output("git show -s --format=%at -n1 -- {0}".format(relname), cwd=self.parent_dir) if status != 0 or not date or not date[0].isdigit(): log.error("Couldn't determine git date for a file\tdirectory=%s\trelname=%s", self.parent_dir, relname) if date: date = int(date[0]) os.utime(thing, (date, date)) t.add(thing, arcname=arcname) for content, arcname in self.extra_context: with a_temp_file() as fle: fle.write(content) fle.seek(0) if mtime: os.utime(fle.name, (mtime, mtime)) t.add(fle.name, arcname=arcname) # And add our docker file with a_temp_file() as dockerfile: dockerfile.write(docker_lines) dockerfile.seek(0) if mtime: os.utime(dockerfile.name, (mtime, mtime)) t.add(dockerfile.name, arcname="./Dockerfile") t.close() tmpfile.seek(0) yield tmpfile
# coding: spec from harpoon.helpers import a_temp_file, until, memoized_property from tests.helpers import HarpoonCase from noseOfYeti.tokeniser.support import noy_sup_setUp from contextlib import contextmanager import mock import os describe HarpoonCase, "a_temp_file": it "yields the file object of a file that disappears after the context": with a_temp_file() as fle: assert os.path.exists(fle.name) assert not os.path.exists(fle.name) it "can write to the temporary file, close it and still read from it": with a_temp_file() as fle: fle.write("blah".encode("utf-8")) fle.close() with open(fle.name) as fread: self.assertEqual(fread.read(), "blah") assert not os.path.exists(fle.name) describe HarpoonCase, "until": @contextmanager def mock_log_and_time(self): """Mock out the log object and time, yield (log, time)""" fake_log = mock.Mock(name="log")