def squash_build(self, conf, context, stream, squash_commands):
"""Do a squash build"""
from harpoon.option_spec.image_objs import DockerFile
squashing = conf
output, status = command_output("which docker-squash")
if status != 0:
raise BadEnvironment(
"Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash"
)
if squash_commands:
squasher_conf = conf.clone()
squasher_conf.image_name = "{0}-for-squashing".format(conf.name)
if conf.image_name_prefix not in ("", None, NotSpecified):
squasher.conf.image_name = "{0}-{1}".format(
conf.image_name_prefix, squasher_conf.image_name)
with self.remove_replaced_images(squasher_conf) as info:
self.log_context_size(context, conf)
original_docker_file = conf.docker_file
new_docker_file = DockerFile(
["FROM {0}".format(conf.image_name)] + squash_commands,
original_docker_file.mtime)
with context.clone_with_new_dockerfile(
squasher_conf, new_docker_file) as squasher_context:
self.log_context_size(squasher_context, squasher_conf)
info['cached'] = self.do_build(squasher_conf,
squasher_context, stream)
squashing = squasher_conf
log.info("Saving image\timage=%s", squashing.image_name)
with hp.a_temp_file() as fle:
res = conf.harpoon.docker_context.get_image(squashing.image_name)
fle.write(res.read())
fle.close()
with hp.a_temp_file() as fle2:
output, status = command_output(
"sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(
fle.name, fle2.name, conf.image_name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to squash the image!")
output, status = command_output("docker load",
stdin=open(fle2.name),
verbose=True,
timeout=600)
if status != 0:
raise HarpoonError("Failed to load the squashed image")
if squashing is not conf:
log.info("Removing intermediate image %s", squashing.image_name)
conf.harpoon.docker_context.remove_image(squashing.image_name)
def git(args, error_message, cwd=context.parent_dir, **error_kwargs):
output, status = command_output("git {0}".format(args), cwd=cwd)
if status != 0:
error_kwargs['output'] = output
error_kwargs['directory'] = context.parent_dir
raise HarpoonError(error_message, **error_kwargs)
return output
def git(args, error_message, **error_kwargs):
output, status = command_output("git {0}".format(args), cwd=root_folder)
if status != 0:
error_kwargs['output'] = output
error_kwargs['directory'] = context.parent_dir
raise HarpoonError(error_message, **error_kwargs)
return output
def assertExampleRepoStatus(self, root_folder, expected):
output, status = command_output("git status -s", cwd=root_folder)
if status != 0:
raise HarpoonError("Failed to run git status",
output='\n'.join(output))
self.assertEqual(
'\n'.join(sorted(output)).strip(),
dedent('\n'.join(sorted(expected.split('\n')))).strip())
def cloned_submodule_example(self):
with self.a_temp_dir() as directory:
shutil.rmtree(directory)
os.makedirs(directory)
output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'two.bundle'), os.path.join(directory, "two"))
if status != 0:
raise HarpoonError("Failed to run git clone", output='\n'.join(output))
output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'one.bundle'), os.path.join(directory, "one"))
if status != 0:
raise HarpoonError("Failed to run git clone", output='\n'.join(output))
output, status = command_output("bash -c 'cd {0}/one && git submodule add {0}/two vendor/two'".format(directory))
if status != 0:
raise HarpoonError("Failed to run git submodule add", output='\n'.join(output))
yield os.path.join(directory, "one")
def get_committime_or_mtime(self, context, location):
"""Get the commit time of some file or the modified time of of it if can't get from git"""
status, date = 0, None
if context.use_git:
date, status = command_output("git show -s --format=%at -n1 -- {0}".format(os.path.basename(location)), cwd=os.path.dirname(location))
if status == 0 and date:
return int(date[0])
else:
return os.path.getmtime(location)
def squash_build(self, conf, context, stream, squash_commands):
"""Do a squash build"""
from harpoon.option_spec.image_objs import DockerFile
squashing = conf
output, status = command_output("which docker-squash")
if status != 0:
raise BadEnvironment("Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash")
if squash_commands:
squasher_conf = conf.clone()
squasher_conf.image_name = "{0}-for-squashing".format(conf.name)
if conf.image_name_prefix not in ("", None, NotSpecified):
squasher.conf.image_name = "{0}-{1}".format(conf.image_name_prefix, squasher_conf.image_name)
with self.remove_replaced_images(squasher_conf) as info:
self.log_context_size(context, conf)
original_docker_file = conf.docker_file
new_docker_file = DockerFile(["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime)
with context.clone_with_new_dockerfile(squasher_conf, new_docker_file) as squasher_context:
self.log_context_size(squasher_context, squasher_conf)
info['cached'] = self.do_build(squasher_conf, squasher_context, stream)
squashing = squasher_conf
log.info("Saving image\timage=%s", squashing.image_name)
with hp.a_temp_file() as fle:
res = conf.harpoon.docker_context.get_image(squashing.image_name)
fle.write(res.read())
fle.close()
with hp.a_temp_file() as fle2:
output, status = command_output("sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(fle.name, fle2.name, conf.image_name), verbose=True, timeout=600)
if status != 0:
raise HarpoonError("Failed to squash the image!")
output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600)
if status != 0:
raise HarpoonError("Failed to load the squashed image")
if squashing is not conf:
log.info("Removing intermediate image %s", squashing.image_name)
conf.harpoon.docker_context.remove_image(squashing.image_name)
def get_committime_or_mtime(self, context, location):
"""Get the commit time of some file or the modified time of of it if can't get from git"""
status, date = 0, None
if context.use_git:
date, status = command_output(
"git show -s --format=%at -n1 -- {0}".format(
os.path.basename(location)),
cwd=os.path.dirname(location))
if status == 0 and date:
return int(date[0])
else:
return os.path.getmtime(location)
def find_files(self, context, silent_build):
"""
Find the set of files from our parent_dir that we care about
"""
first_layer = ["'{0}'".format(thing) for thing in os.listdir(context.parent_dir)]
output, status = command_output("find {0} -type l -or -type f {1} -follow -print".format(' '.join(first_layer), context.find_options), cwd=context.parent_dir)
if status != 0:
raise HarpoonError("Couldn't find the files we care about", output=output, cwd=context.parent_dir)
all_files = set(self.convert_nonascii(output))
total_files = set(all_files)
combined = set(all_files)
mtime_ignoreable = set()
if context.use_git:
if context.use_gitignore and context.parent_dir == context.git_root:
all_files = set([path for path in all_files if not path.startswith(".git")])
combined = set(all_files)
changed_files, untracked_files, valid_files = self.find_ignored_git_files(context, silent_build)
mtime_ignoreable = set(list(changed_files) + list(untracked_files))
removed = set()
if valid_files:
for fle in combined:
if fle not in valid_files:
removed.add(fle)
if removed and not silent_build: log.info("Ignoring %s/%s files", len(removed), len(combined))
combined -= removed
if context.exclude:
excluded = set()
for filename in combined:
for excluder in context.exclude:
if fnmatch.fnmatch(filename, excluder):
excluded.add(filename)
break
if not silent_build: log.info("Filtering %s/%s items\texcluding=%s", len(excluded), len(combined), context.exclude)
combined -= excluded
if context.include:
extra_included = []
for filename in total_files:
for includer in context.include:
if fnmatch.fnmatch(filename, includer):
extra_included.append(filename)
break
if not silent_build: log.info("Adding back %s items\tincluding=%s", len(extra_included), context.include)
combined = set(list(combined) + extra_included)
files = sorted(os.path.join(context.parent_dir, filename) for filename in combined)
if not silent_build: log.info("Adding %s things from %s to the context", len(files), context.parent_dir)
return files, mtime_ignoreable
def find_files(self, context, silent_build):
"""
Find the set of files from our parent_dir that we care about
"""
first_layer = ["'{0}'".format(thing) for thing in os.listdir(context.parent_dir)]
output, status = command_output("find {0} -type f -print".format(' '.join(first_layer)), cwd=context.parent_dir)
if status != 0:
raise HarpoonError("Couldn't find the files we care about", output=output, cwd=context.parent_dir)
all_files = set(self.convert_nonascii(output))
total_files = set(all_files)
combined = set(all_files)
mtime_ignoreable = set()
if context.use_git:
if context.use_gitignore and context.parent_dir == context.git_root:
all_files = set([path for path in all_files if not path.startswith(".git")])
combined = set(all_files)
changed_files, untracked_files, ignored_files = self.find_ignored_git_files(context, silent_build)
mtime_ignoreable = set(list(changed_files) + list(untracked_files) + list(ignored_files))
removed = set()
for fle in ignored_files:
if fle in combined:
removed.add(fle)
if removed and not silent_build: log.info("Ignoring %s/%s files", len(removed), len(combined))
combined -= removed
if context.exclude:
excluded = set()
for filename in combined:
for excluder in context.exclude:
if fnmatch.fnmatch(filename, excluder):
excluded.add(filename)
break
if not silent_build: log.info("Filtering %s/%s items\texcluding=%s", len(excluded), len(combined), context.exclude)
combined -= excluded
if context.include:
extra_included = []
for filename in total_files:
for includer in context.include:
if fnmatch.fnmatch(filename, includer):
extra_included.append(filename)
break
if not silent_build: log.info("Adding back %s items\tincluding=%s", len(extra_included), context.include)
combined = set(list(combined) + extra_included)
files = sorted(os.path.join(context.parent_dir, filename) for filename in combined)
if not silent_build: log.info("Adding %s things from %s to the context", len(files), context.parent_dir)
return files, mtime_ignoreable
def cloned_repo_example(self, shallow=False):
with self.a_temp_dir() as directory:
shutil.rmtree(directory)
output, status = command_output(
"git clone",
os.path.join(this_dir, '..', 'repo_example', 'example.bundle'),
directory)
if status != 0:
raise HarpoonError("Failed to run git clone",
output='\n'.join(output))
# For shallow clones, have to clone twice, seems --depth with bundles don't work
if shallow:
with self.a_temp_dir() as directory2:
shutil.rmtree(directory2)
output, status = command_output(
"git clone --depth 1", "file://{0}".format(directory),
directory2)
if status != 0:
raise HarpoonError("Failed to run git clone",
output='\n'.join(output))
yield directory2
else:
yield directory
def cmd(command):
output, status = command_output(command, cwd=directory)
print(output)
assert status is 0
return output
def cmd(command):
output, status = command_output(command, cwd=directory)
print(output)
assert status is 0
return output
filename = None
try:
filename = tempfile.NamedTemporaryFile(delete=False).name
yield filename
finally:
if filename and os.path.exists(filename):
os.remove(filename)
it "returns the output and exit code from running a command":
with self.a_temp_file() as filename:
with open(filename, 'w') as fle:
fle.write(dedent("""
print("hello")
print("there")
"""))
output, exit_code = command_output("{0} {1}".format(sys.executable, filename))
self.assertEqual(output, ["hello", "there"])
self.assertEqual(exit_code, 0)
it "can kill a command after a certain timeout":
with self.a_temp_file() as filename:
with open(filename, 'w') as fle:
fle.write(dedent("""
import time
print("hello")
import sys; sys.stdout.flush()
time.sleep(3)
print("there")
"""))
output, exit_code = command_output("{0} {1}".format(sys.executable, filename), timeout=0.05)
