def squash_build(self, conf, context, stream, squash_commands): """Do a squash build""" from harpoon.option_spec.image_objs import DockerFile squashing = conf output, status = command_output("which docker-squash") if status != 0: raise BadEnvironment( "Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash" ) if squash_commands: squasher_conf = conf.clone() squasher_conf.image_name = "{0}-for-squashing".format(conf.name) if conf.image_name_prefix not in ("", None, NotSpecified): squasher.conf.image_name = "{0}-{1}".format( conf.image_name_prefix, squasher_conf.image_name) with self.remove_replaced_images(squasher_conf) as info: self.log_context_size(context, conf) original_docker_file = conf.docker_file new_docker_file = DockerFile( ["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime) with context.clone_with_new_dockerfile( squasher_conf, new_docker_file) as squasher_context: self.log_context_size(squasher_context, squasher_conf) info['cached'] = self.do_build(squasher_conf, squasher_context, stream) squashing = squasher_conf log.info("Saving image\timage=%s", squashing.image_name) with hp.a_temp_file() as fle: res = conf.harpoon.docker_context.get_image(squashing.image_name) fle.write(res.read()) fle.close() with hp.a_temp_file() as fle2: output, status = command_output( "sudo docker-squash -i {0} -o {1} -t {2} -verbose".format( fle.name, fle2.name, conf.image_name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to squash the image!") output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to load the squashed image") if squashing is not conf: log.info("Removing intermediate image %s", squashing.image_name) conf.harpoon.docker_context.remove_image(squashing.image_name)
def git(args, error_message, cwd=context.parent_dir, **error_kwargs): output, status = command_output("git {0}".format(args), cwd=cwd) if status != 0: error_kwargs['output'] = output error_kwargs['directory'] = context.parent_dir raise HarpoonError(error_message, **error_kwargs) return output
def git(args, error_message, **error_kwargs): output, status = command_output("git {0}".format(args), cwd=root_folder) if status != 0: error_kwargs['output'] = output error_kwargs['directory'] = context.parent_dir raise HarpoonError(error_message, **error_kwargs) return output
def assertExampleRepoStatus(self, root_folder, expected): output, status = command_output("git status -s", cwd=root_folder) if status != 0: raise HarpoonError("Failed to run git status", output='\n'.join(output)) self.assertEqual( '\n'.join(sorted(output)).strip(), dedent('\n'.join(sorted(expected.split('\n')))).strip())
def cloned_submodule_example(self): with self.a_temp_dir() as directory: shutil.rmtree(directory) os.makedirs(directory) output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'two.bundle'), os.path.join(directory, "two")) if status != 0: raise HarpoonError("Failed to run git clone", output='\n'.join(output)) output, status = command_output("git clone", os.path.join(this_dir, '..', 'submodule_example', 'one.bundle'), os.path.join(directory, "one")) if status != 0: raise HarpoonError("Failed to run git clone", output='\n'.join(output)) output, status = command_output("bash -c 'cd {0}/one && git submodule add {0}/two vendor/two'".format(directory)) if status != 0: raise HarpoonError("Failed to run git submodule add", output='\n'.join(output)) yield os.path.join(directory, "one")
def get_committime_or_mtime(self, context, location): """Get the commit time of some file or the modified time of of it if can't get from git""" status, date = 0, None if context.use_git: date, status = command_output("git show -s --format=%at -n1 -- {0}".format(os.path.basename(location)), cwd=os.path.dirname(location)) if status == 0 and date: return int(date[0]) else: return os.path.getmtime(location)
def squash_build(self, conf, context, stream, squash_commands): """Do a squash build""" from harpoon.option_spec.image_objs import DockerFile squashing = conf output, status = command_output("which docker-squash") if status != 0: raise BadEnvironment("Please put docker-squash in your PATH first: https://github.com/jwilder/docker-squash") if squash_commands: squasher_conf = conf.clone() squasher_conf.image_name = "{0}-for-squashing".format(conf.name) if conf.image_name_prefix not in ("", None, NotSpecified): squasher.conf.image_name = "{0}-{1}".format(conf.image_name_prefix, squasher_conf.image_name) with self.remove_replaced_images(squasher_conf) as info: self.log_context_size(context, conf) original_docker_file = conf.docker_file new_docker_file = DockerFile(["FROM {0}".format(conf.image_name)] + squash_commands, original_docker_file.mtime) with context.clone_with_new_dockerfile(squasher_conf, new_docker_file) as squasher_context: self.log_context_size(squasher_context, squasher_conf) info['cached'] = self.do_build(squasher_conf, squasher_context, stream) squashing = squasher_conf log.info("Saving image\timage=%s", squashing.image_name) with hp.a_temp_file() as fle: res = conf.harpoon.docker_context.get_image(squashing.image_name) fle.write(res.read()) fle.close() with hp.a_temp_file() as fle2: output, status = command_output("sudo docker-squash -i {0} -o {1} -t {2} -verbose".format(fle.name, fle2.name, conf.image_name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to squash the image!") output, status = command_output("docker load", stdin=open(fle2.name), verbose=True, timeout=600) if status != 0: raise HarpoonError("Failed to load the squashed image") if squashing is not conf: log.info("Removing intermediate image %s", squashing.image_name) conf.harpoon.docker_context.remove_image(squashing.image_name)
def get_committime_or_mtime(self, context, location): """Get the commit time of some file or the modified time of of it if can't get from git""" status, date = 0, None if context.use_git: date, status = command_output( "git show -s --format=%at -n1 -- {0}".format( os.path.basename(location)), cwd=os.path.dirname(location)) if status == 0 and date: return int(date[0]) else: return os.path.getmtime(location)
def find_files(self, context, silent_build): """ Find the set of files from our parent_dir that we care about """ first_layer = ["'{0}'".format(thing) for thing in os.listdir(context.parent_dir)] output, status = command_output("find {0} -type l -or -type f {1} -follow -print".format(' '.join(first_layer), context.find_options), cwd=context.parent_dir) if status != 0: raise HarpoonError("Couldn't find the files we care about", output=output, cwd=context.parent_dir) all_files = set(self.convert_nonascii(output)) total_files = set(all_files) combined = set(all_files) mtime_ignoreable = set() if context.use_git: if context.use_gitignore and context.parent_dir == context.git_root: all_files = set([path for path in all_files if not path.startswith(".git")]) combined = set(all_files) changed_files, untracked_files, valid_files = self.find_ignored_git_files(context, silent_build) mtime_ignoreable = set(list(changed_files) + list(untracked_files)) removed = set() if valid_files: for fle in combined: if fle not in valid_files: removed.add(fle) if removed and not silent_build: log.info("Ignoring %s/%s files", len(removed), len(combined)) combined -= removed if context.exclude: excluded = set() for filename in combined: for excluder in context.exclude: if fnmatch.fnmatch(filename, excluder): excluded.add(filename) break if not silent_build: log.info("Filtering %s/%s items\texcluding=%s", len(excluded), len(combined), context.exclude) combined -= excluded if context.include: extra_included = [] for filename in total_files: for includer in context.include: if fnmatch.fnmatch(filename, includer): extra_included.append(filename) break if not silent_build: log.info("Adding back %s items\tincluding=%s", len(extra_included), context.include) combined = set(list(combined) + extra_included) files = sorted(os.path.join(context.parent_dir, filename) for filename in combined) if not silent_build: log.info("Adding %s things from %s to the context", len(files), context.parent_dir) return files, mtime_ignoreable
def find_files(self, context, silent_build): """ Find the set of files from our parent_dir that we care about """ first_layer = ["'{0}'".format(thing) for thing in os.listdir(context.parent_dir)] output, status = command_output("find {0} -type f -print".format(' '.join(first_layer)), cwd=context.parent_dir) if status != 0: raise HarpoonError("Couldn't find the files we care about", output=output, cwd=context.parent_dir) all_files = set(self.convert_nonascii(output)) total_files = set(all_files) combined = set(all_files) mtime_ignoreable = set() if context.use_git: if context.use_gitignore and context.parent_dir == context.git_root: all_files = set([path for path in all_files if not path.startswith(".git")]) combined = set(all_files) changed_files, untracked_files, ignored_files = self.find_ignored_git_files(context, silent_build) mtime_ignoreable = set(list(changed_files) + list(untracked_files) + list(ignored_files)) removed = set() for fle in ignored_files: if fle in combined: removed.add(fle) if removed and not silent_build: log.info("Ignoring %s/%s files", len(removed), len(combined)) combined -= removed if context.exclude: excluded = set() for filename in combined: for excluder in context.exclude: if fnmatch.fnmatch(filename, excluder): excluded.add(filename) break if not silent_build: log.info("Filtering %s/%s items\texcluding=%s", len(excluded), len(combined), context.exclude) combined -= excluded if context.include: extra_included = [] for filename in total_files: for includer in context.include: if fnmatch.fnmatch(filename, includer): extra_included.append(filename) break if not silent_build: log.info("Adding back %s items\tincluding=%s", len(extra_included), context.include) combined = set(list(combined) + extra_included) files = sorted(os.path.join(context.parent_dir, filename) for filename in combined) if not silent_build: log.info("Adding %s things from %s to the context", len(files), context.parent_dir) return files, mtime_ignoreable
def cloned_repo_example(self, shallow=False): with self.a_temp_dir() as directory: shutil.rmtree(directory) output, status = command_output( "git clone", os.path.join(this_dir, '..', 'repo_example', 'example.bundle'), directory) if status != 0: raise HarpoonError("Failed to run git clone", output='\n'.join(output)) # For shallow clones, have to clone twice, seems --depth with bundles don't work if shallow: with self.a_temp_dir() as directory2: shutil.rmtree(directory2) output, status = command_output( "git clone --depth 1", "file://{0}".format(directory), directory2) if status != 0: raise HarpoonError("Failed to run git clone", output='\n'.join(output)) yield directory2 else: yield directory
def cmd(command): output, status = command_output(command, cwd=directory) print(output) assert status is 0 return output
filename = None try: filename = tempfile.NamedTemporaryFile(delete=False).name yield filename finally: if filename and os.path.exists(filename): os.remove(filename) it "returns the output and exit code from running a command": with self.a_temp_file() as filename: with open(filename, 'w') as fle: fle.write(dedent(""" print("hello") print("there") """)) output, exit_code = command_output("{0} {1}".format(sys.executable, filename)) self.assertEqual(output, ["hello", "there"]) self.assertEqual(exit_code, 0) it "can kill a command after a certain timeout": with self.a_temp_file() as filename: with open(filename, 'w') as fle: fle.write(dedent(""" import time print("hello") import sys; sys.stdout.flush() time.sleep(3) print("there") """)) output, exit_code = command_output("{0} {1}".format(sys.executable, filename), timeout=0.05)