def squash_build(self, conf, context, stream, squash_commands):
"""Do a squash build"""
from harpoon.option_spec.image_objs import DockerFile
squashing = conf
output, status = command_output("which docker-squash")
if status != 0:
raise BadEnvironment(
"Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash"
)
if squash_commands:
squasher_conf = conf.clone()
squasher_conf.image_name = "{0}-for-squashing".format(conf.name)
if conf.image_name_prefix not in ("", None, NotSpecified):
squasher.conf.image_name = "{0}-{1}".format(
conf.image_name_prefix, squasher_conf.image_name)
with self.remove_replaced_images(squasher_conf) as info:
self.log_context_size(context, conf)
original_docker_file = conf.docker_file
new_docker_file = DockerFile(
["FROM {0}".format(conf.image_name)] + squash_commands,
original_docker_file.mtime)
with context.clone_with_new_dockerfile(
squasher_conf, new_docker_file) as squasher_context:
self.log_context_size(squasher_context, squasher_conf)
info['cached'] = self.do_build(squasher_conf,
squasher_context, stream)
squashing = squasher_conf
log.info("Saving image\timage=%s", squashing.image_name)
with hp.a_temp_file() as fle:
res = conf.harpoon.docker_context.get_image(squashing.image_name)
fle.write(res.read())
fle.close()
with hp.a_temp_file() as fle2:
output, status = command_output(
"sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(
fle.name, fle2.name, conf.image_name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to squash the image!")
output, status = command_output("docker load",
stdin=open(fle2.name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to load the squashed image")
if squashing is not conf:
log.info("Removing intermediate image %s", squashing.image_name)
conf.harpoon.docker_context.remove_image(squashing.image_name)
def git(args, error_message, cwd=context.parent_dir, **error_kwargs):
output, status = command_output("git {0}".format(args), cwd=cwd)
if status != 0:
error_kwargs['output'] = output
error_kwargs['directory'] = context.parent_dir
raise HarpoonError(error_message, **error_kwargs)
return output
def assertExampleRepoStatus(self, root_folder, expected):
output, status = command_output("git status -s", cwd=root_folder)
if status != 0:
raise HarpoonError("Failed to run git status",
output='\n'.join(output))
self.assertEqual(
'\n'.join(sorted(output)).strip(),
dedent('\n'.join(sorted(expected.split('\n')))).strip())
def cloned_submodule_example(self):
with self.a_temp_dir() as directory:
shutil.rmtree(directory)
os.makedirs(directory)
output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'two.bundle'), os.path.join(directory, "two"))
if status != 0:
raise HarpoonError("Failed to run git clone", output='\n'.join(output))
output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'one.bundle'), os.path.join(directory, "one"))
if status != 0:
raise HarpoonError("Failed to run git clone", output='\n'.join(output))
output, status = command_output("bash -c 'cd {0}/one && git submodule add {0}/two vendor/two'".format(directory))
if status != 0:
raise HarpoonError("Failed to run git submodule add", output='\n'.join(output))
yield os.path.join(directory, "one")
def find_files(self, context, silent_build):
"""
Find the set of files from our parent_dir that we care about
"""
first_layer = ["'{0}'".format(thing) for thing in os.listdir(context.parent_dir)]
output, status = command_output("find {0} -type l -or -type f {1} -follow -print".format(' '.join(first_layer), context.find_options), cwd=context.parent_dir)
if status != 0:
raise HarpoonError("Couldn't find the files we care about", output=output, cwd=context.parent_dir)
all_files = set(self.convert_nonascii(output))
total_files = set(all_files)
combined = set(all_files)
mtime_ignoreable = set()
if context.use_git:
if context.use_gitignore and context.parent_dir == context.git_root:
all_files = set([path for path in all_files if not path.startswith(".git")])
combined = set(all_files)
changed_files, untracked_files, valid_files = self.find_ignored_git_files(context, silent_build)
mtime_ignoreable = set(list(changed_files) + list(untracked_files))
removed = set()
if valid_files:
for fle in combined:
if fle not in valid_files:
removed.add(fle)
if removed and not silent_build: log.info("Ignoring %s/%s files", len(removed), len(combined))
combined -= removed
if context.exclude:
excluded = set()
for filename in combined:
for excluder in context.exclude:
if fnmatch.fnmatch(filename, excluder):
excluded.add(filename)
break
if not silent_build: log.info("Filtering %s/%s items\texcluding=%s", len(excluded), len(combined), context.exclude)
combined -= excluded
if context.include:
extra_included = []
for filename in total_files:
for includer in context.include:
if fnmatch.fnmatch(filename, includer):
extra_included.append(filename)
break
if not silent_build: log.info("Adding back %s items\tincluding=%s", len(extra_included), context.include)
combined = set(list(combined) + extra_included)
files = sorted(os.path.join(context.parent_dir, filename) for filename in combined)
if not silent_build: log.info("Adding %s things from %s to the context", len(files), context.parent_dir)
return files, mtime_ignoreable
def git_root(self):
"""
Find the root git folder
"""
if not getattr(self, "_git_folder", None):
root_folder = os.path.abspath(self.parent_dir)
while not os.path.exists(os.path.join(root_folder, '.git')):
if root_folder == '/':
raise HarpoonError("Couldn't find a .git folder",
start_at=self.parent_dir)
root_folder = os.path.dirname(root_folder)
self._git_folder = root_folder
return self._git_folder
def cloned_repo_example(self, shallow=False):
with self.a_temp_dir() as directory:
shutil.rmtree(directory)
output, status = command_output(
"git clone",
os.path.join(this_dir, '..', 'repo_example', 'example.bundle'),
directory)
if status != 0:
raise HarpoonError("Failed to run git clone",
output='\n'.join(output))
# For shallow clones, have to clone twice, seems --depth with bundles don't work
if shallow:
with self.a_temp_dir() as directory2:
shutil.rmtree(directory2)
output, status = command_output(
"git clone --depth 1", "file://{0}".format(directory),
directory2)
if status != 0:
raise HarpoonError("Failed to run git clone",
output='\n'.join(output))
yield directory2
else:
yield directory
def find_git_mtimes(self, context, silent_build):
"""
Use git to find the mtimes of the files we care about
"""
if not context.use_git_timestamps:
return {}
parent_dir = context.parent_dir
root_folder = context.git_root
# Can't use git timestamps if it's just a shallow clone
# Otherwise all the files get the timestamp of the latest commit
if context.use_git_timestamps and os.path.exists(os.path.join(root_folder, ".git", "shallow")):
raise HarpoonError("Can't get git timestamps from a shallow clone", directory=parent_dir)
options = {"include": context.include, "exclude": context.exclude, "timestamps_for": context.use_git_timestamps, "silent": silent_build}
return dict(GitTimes(root_folder, os.path.relpath(parent_dir, root_folder), **options).find())
def find_git_mtimes(self, context, silent_build):
"""
Use git to find the mtimes of the files we care about
"""
if not context.use_git_timestamps:
return {}
parent_dir = context.parent_dir
root_folder = context.git_root
# Can't use git timestamps if it's just a shallow clone
# Otherwise all the files get the timestamp of the latest commit
if context.use_git_timestamps and os.path.exists(
os.path.join(root_folder, ".git", "shallow")):
raise HarpoonError("Can't get git timestamps from a shallow clone",
directory=parent_dir)
git = Repo(root_folder)
mtimes = {}
all_files = set(git.open_index())
use_files = set()
for filename in all_files:
relpath = os.path.relpath(os.path.join(root_folder, filename),
context.parent_dir)
# Only include files under the parent_dir
if relpath.startswith("../"):
continue
# Ignore files that we don't want git_timestamps from
if context.use_git_timestamps and type(
context.use_git_timestamps) is not bool:
match = False
for line in context.use_git_timestamps:
if fnmatch.fnmatch(relpath, line):
match = True
break
if not match:
continue
# Matched is true by default if
# * Have context.exclude
# * No context.exclude and no context.include
matched = context.exclude or not any(
[context.exclude, context.include])
# Anything not matching exclude gets included
if context.exclude:
for line in context.exclude:
if fnmatch.fnmatch(relpath, line):
matched = False
# Anything matching include gets included
if context.include:
for line in context.include:
if fnmatch.fnmatch(relpath, line):
matched = True
break
# Either didn't match any exclude or matched an include
if matched:
use_files.add(filename)
if not silent_build:
log.info(
"Finding modified times for %s/%s git controlled files in %s",
len(use_files), len(all_files), root_folder)
for entry in git.get_walker(paths=use_files):
date = entry.commit.author_time
for changes in entry.changes():
if type(changes) is not list:
changes = [changes]
for change in changes:
path = change.new.path
if root_folder and change.new.path and context.parent_dir:
new_relpath = os.path.relpath(
os.path.join(root_folder, change.new.path),
context.parent_dir)
if path in use_files and mtimes.get(
new_relpath, 0
) < date and not new_relpath.startswith("../"):
mtimes[new_relpath] = date
if len(use_files - set(mtimes)) == 0:
break
return mtimes