def run_with_altered_context(self,
name,
conf,
context,
stream,
dockerfile,
volumes_from=None,
tag=None,
detach=False,
command=None,
volumes=None):
"""Helper to build and run a new dockerfile"""
with self.build_with_altered_context(name,
conf,
context,
stream,
dockerfile,
volumes_from=volumes_from,
command=command,
volumes=volumes) as (new_conf, _):
# Hackity hack hakc hack hack hack
# I should just be able to use conf.configuration["images"]
# But it seems a bug in option_merge behaviour means the converters stop working :(
# Much strange
configuration = conf.configuration
class Images(object):
def __getitem__(self, key):
return configuration[["images", key]]
images = Images()
if detach:
Runner().run_container(new_conf,
images,
detach=True,
dependency=True)
else:
Runner().run_container(new_conf,
images,
detach=False,
dependency=False,
tag=True)
new_conf.image_name = new_conf.committed
return new_conf
def build_and_run(self, images):
"""Make this image and run it"""
Builder().make_image(self, images)
try:
Runner().run_container(self, images)
except DockerAPIError as error:
raise BadImage("Failed to start the container", error=error)
def build_image(self, conf, share_with_deps=None, pushing=False):
"""Build this image"""
if share_with_deps is None:
share_with_deps = []
with self.context(conf) as context:
try:
stream = BuildProgressStream(conf.harpoon.silent_build)
with self.remove_replaced_images(conf) as info:
if conf.recursive is NotSpecified:
cached = self.do_build(conf, context, stream)
else:
cached = self.do_recursive_build(
conf,
context,
stream,
needs_provider=conf.name in share_with_deps)
info['cached'] = cached
except (KeyboardInterrupt, Exception) as error:
exc_info = sys.exc_info()
if stream.current_container:
Runner().stage_build_intervention(conf,
stream.current_container)
if isinstance(error, KeyboardInterrupt):
raise UserQuit()
else:
six.reraise(*exc_info)
try:
for squash_options, condition in [(conf.squash_after, True),
(conf.squash_before_push,
pushing)]:
if squash_options is not NotSpecified and condition:
if type(squash_options) is command_objs.Commands:
squash_commands = squash_options.docker_lines_list
self.squash_build(conf, context, stream,
squash_commands)
cached = False
except (KeyboardInterrupt, Exception) as error:
exc_info = sys.exc_info()
if isinstance(error, KeyboardInterrupt):
raise UserQuit()
else:
six.reraise(*exc_info)
return cached
def the_context(self, content, silent_build=False):
"""Return either a file with the content written to it, or a whole new context tar"""
if isinstance(content, six.string_types):
with a_temp_file() as fle:
fle.write(content.encode('utf-8'))
fle.seek(0)
yield fle
elif "context" in content:
with ContextBuilder().make_context(content["context"], silent_build=silent_build) as wrapper:
wrapper.close()
yield wrapper.tmpfile
elif "image" in content:
from harpoon.ship.runner import Runner
with a_temp_file() as fle:
content["conf"].command = "yes"
with Runner()._run_container(content["conf"], content["images"], detach=True, delete_anyway=True):
try:
strm, stat = content["docker_context"].get_archive(content["conf"].container_id, content["path"])
except docker.errors.NotFound:
raise BadOption("Trying to get something from an image that don't exist!", path=content["path"], image=content["conf"].image_name)
else:
log.debug(stat)
fo = BytesIO(strm.read())
tf = tarfile.TarFile(fileobj=fo)
if tf.firstmember.isdir():
tf2 = tarfile.TarFile(fileobj=fle, mode='w')
name = tf.firstmember.name
for member in tf.getmembers()[1:]:
member.name = member.name[len(name)+1:]
if member.issym():
with tempfile.NamedTemporaryFile() as symfle:
os.remove(symfle.name)
os.symlink(member.linkpath, symfle.name)
tf2.addfile(member, fileobj=symfle)
elif not member.isdir():
tf2.addfile(member, fileobj=tf.extractfile(member.name))
tf2.close()
else:
fle.write(tf.extractfile(tf.firstmember.name).read())
tf.close()
log.info("Got '{0}' from {1} for context".format(content["path"], content["conf"].container_id))
fle.seek(0)
yield fle
def build_image(self, conf):
"""Build this image"""
with self.context(conf) as context:
try:
stream = BuildProgressStream(conf.harpoon.silent_build)
with self.remove_replaced_images(conf):
self.do_build(conf, context, stream)
except (KeyboardInterrupt, Exception) as error:
exc_info = sys.exc_info()
if stream.current_container:
Runner().stage_build_intervention(conf,
stream.current_container)
if isinstance(error, KeyboardInterrupt):
raise UserQuit()
else:
six.reraise(*exc_info)
def build_image(self, conf, pushing=False):
"""Build this image"""
with conf.make_context() as context:
try:
stream = BuildProgressStream(conf.harpoon.silent_build)
with self.remove_replaced_images(conf) as info:
if conf.persistence is NotSpecified:
cached = NormalBuilder().build(conf, context, stream)
else:
cached = PersistenceBuilder().build(
conf, context, stream)
info['cached'] = cached
except (KeyboardInterrupt, Exception) as error:
exc_info = sys.exc_info()
if stream.current_container:
Runner().stage_build_intervention(conf,
stream.current_container)
if isinstance(error, KeyboardInterrupt):
raise UserQuit()
else:
six.reraise(*exc_info)
try:
for squash_options, condition in [(conf.squash_after, True),
(conf.squash_before_push,
pushing)]:
if squash_options is not NotSpecified and condition:
if type(squash_options) is command_objs.Commands:
squash_commands = squash_options.docker_lines_list
SquashedBuilder(squash_commands).build(
conf, context, stream)
cached = False
except (KeyboardInterrupt, Exception) as error:
exc_info = sys.exc_info()
if isinstance(error, KeyboardInterrupt):
raise UserQuit()
else:
six.reraise(*exc_info)
return cached
self.assertEqual(tester_commands
, [ '/bin/sh -c echo sh'
, '/bin/sh -c echo /tmp'
, '/bin/sh -c echo \'cat /tmp/lines > /tmp/lines2; echo \'"\'"\'another_line\'"\'"\' >> /tmp/lines2; mv /tmp/lines2 /tmp/lines\''
]
)
@test
it "can be run after the first time":
conf.command = "/bin/sh -c 'cat /tmp/lines'"
fake_sys_stdout = self.make_temp_file()
fake_sys_stderr = self.make_temp_file()
conf.harpoon.tty_stdout = fake_sys_stdout
conf.harpoon.tty_stderr = fake_sys_stderr
try:
Runner().run_container(conf, {conf.name: conf})
except (docker.errors.APIError, BadImage) as error:
log.exception(error)
with open(fake_sys_stdout.name) as fle:
output = fle.read().strip().replace("\r", "")
with open(fake_sys_stderr.name) as fle:
self.assertEqual(fle.read().strip(), '')
self.assertEqual(output, "a_line\nanother_line")
assert_only_extra_tags("{0}:latest".format(conf.image_name), "{0}-tester:latest".format(conf.image_name))
@test
it "says image is cached if nothing has changed":
cached = Builder().make_image(conf, {conf.name: conf})
def commit_and_run(*args, **kwargs):
kwargs["command"] = "echo 'intervention_goes_here'"
called.append("commit_and_run")
return original_commit_and_run(Runner(), *args, **kwargs)
def do_recursive_build(self, conf, context, stream, needs_provider=False):
"""Do a recursive build!"""
from harpoon.option_spec.image_objs import Volumes
from harpoon.ship.runner import Runner
conf_image_name = conf.name
if conf.image_name_prefix not in (NotSpecified, "", None):
conf_image_name = "{0}-{1}".format(conf.image_name_prefix,
conf.name)
test_conf = conf.clone()
test_conf.image_name = "{0}-tester".format(conf_image_name)
log.info(
"Building test image for recursive image to see if the cache changed"
)
with self.remove_replaced_images(test_conf) as info:
cached = self.do_build(test_conf, context, stream)
info['cached'] = cached
have_final = "{0}:latest".format(
conf.image_name) in chain.from_iterable([
image["RepoTags"]
for image in conf.harpoon.docker_context.images()
])
provider_name = "{0}-provider".format(conf_image_name)
provider_conf = conf.clone()
provider_conf.name = "provider"
provider_conf.image_name = provider_name
provider_conf.container_id = None
provider_conf.container_name = "{0}-intermediate-{1}".format(
provider_name, str(uuid.uuid1())).replace("/", "__")
provider_conf.bash = NotSpecified
provider_conf.command = NotSpecified
if not have_final:
log.info("Building first image for recursive image")
with context.clone_with_new_dockerfile(
conf, conf.recursive.make_first_dockerfile(
conf.docker_file)) as new_context:
self.do_build(conf, new_context, stream)
if not needs_provider and cached:
return cached
with self.remove_replaced_images(provider_conf) as info:
if cached:
with conf.make_context(
docker_file=conf.recursive.make_provider_dockerfile(
conf.docker_file,
conf.image_name)) as provider_context:
self.log_context_size(provider_context, provider_conf)
info['cached'] = self.do_build(provider_conf,
provider_context,
stream,
image_name=provider_name)
conf.from_name = conf.image_name
conf.image_name = provider_name
conf.deleteable = True
return cached
else:
log.info("Building intermediate provider for recursive image")
with context.clone_with_new_dockerfile(
conf,
conf.recursive.make_changed_dockerfile(
conf.docker_file,
conf.image_name)) as provider_context:
self.log_context_size(provider_context, provider_conf)
self.do_build(provider_conf,
provider_context,
stream,
image_name=provider_name)
builder_name = "{0}-for-commit".format(conf_image_name)
builder_conf = conf.clone()
builder_conf.image_name = builder_name
builder_conf.container_id = None
builder_conf.container_name = "{0}-intermediate-{1}".format(
builder_name, str(uuid.uuid1())).replace("/", "__")
builder_conf.volumes = Volumes(mount=[], share_with=[provider_conf])
builder_conf.bash = NotSpecified
builder_conf.command = NotSpecified
log.info("Building intermediate builder for recursive image")
with self.remove_replaced_images(builder_conf) as info:
with context.clone_with_new_dockerfile(
conf,
conf.recursive.make_builder_dockerfile(
conf.docker_file)) as builder_context:
self.log_context_size(builder_context, builder_conf)
info['cached'] = self.do_build(builder_conf,
builder_context,
stream,
image_name=builder_name)
log.info(
"Running and committing builder container for recursive image")
with self.remove_replaced_images(conf):
Runner().run_container(builder_conf, {
provider_conf.name: provider_conf,
builder_conf.name: builder_conf
},
detach=False,
dependency=False,
tag=conf.image_name)
log.info("Removing intermediate image %s", builder_conf.image_name)
conf.harpoon.docker_context.remove_image(builder_conf.image_name)
if not needs_provider:
return cached
log.info("Building final provider of recursive image")
with self.remove_replaced_images(provider_conf) as info:
with conf.make_context(
docker_file=conf.recursive.make_provider_dockerfile(
conf.docker_file,
conf.image_name)) as provider_context:
self.log_context_size(provider_context, provider_conf)
info['cached'] = self.do_build(provider_conf,
provider_context,
stream,
image_name=provider_name)
conf.from_name = conf.image_name
conf.image_name = provider_name
conf.deleteable = True
return cached
def make_image(self, conf, context, stream, existing_image):
"""
If the image doesn't already exist, then we just run the normal docker_file
commands followed by the action and we are done.
Otherwise, we first create a container with a volume containing the folders from
the existing container we want to persist. We then make a new image with the normal
docker_file commands and run it in a container, copy over the folders from the VOLUME
and commit into an image.
Finally, we construct an image from that committed image and add a CMD command to
the one specified in the options, or sh
After all this we clean up everything, including that volume we created
"""
if not existing_image:
# Don't have an existing image to steal from
# Just have to make it, no volumes or trickery involved!
docker_file = conf.persistence.make_first_dockerfile(
conf.docker_file)
with self.build_with_altered_context(None, conf, context, stream,
docker_file):
pass
# Make the test image so the next time we run this, it's already cached
self.make_test_image(conf, context, stream)
return
# We have an existing image, let's steal from it!
first_conf = None
try:
docker_file = conf.persistence.make_rerunner_prep_dockerfile(
conf.docker_file, existing_image)
volumes = conf.volumes
if conf.persistence.no_volumes:
volumes = None
first_conf = self.run_with_altered_context(
"rerunner_prep",
conf,
context,
stream,
docker_file,
detach=True,
command="while true; do sleep 5; done",
volumes=volumes)
log.info("Built {0}".format(first_conf.image_name))
# Make the second image, which copies over from the VOLUME into the image
docker_file = conf.persistence.make_second_dockerfile(
conf.docker_file)
second_image_conf = self.run_with_altered_context(
"second",
conf,
context,
stream,
docker_file,
volumes_from=[first_conf.container_id],
volumes=volumes)
log.info("Built {0}".format(second_image_conf.image_name))
# Build the final image, which just appends the desired CMD to the end
docker_file = conf.persistence.make_final_dockerfile(
conf.docker_file, second_image_conf.image_name)
with self.build_with_altered_context(None,
conf,
None,
stream,
docker_file,
volumes=volumes):
pass
finally:
if first_conf:
Runner().stop_container(first_conf, remove_volumes=True)
try:
conf.harpoon.docker_context.remove_image(
first_conf.image_name)
except DockerAPIError as error:
log.error(error)
, "content":
{ "image": conf1
, "path": "/tmp/other"
}
, "mtime": 1463473251
}
]
, "CMD find /tmp/copied -type f | sort | xargs -t cat"
]
fake_sys_stdout = self.make_temp_file()
fake_sys_stderr = self.make_temp_file()
harpoon_options = {"no_intervention": True, "stdout": fake_sys_stdout, "tty_stdout": fake_sys_stdout, "tty_stderr": fake_sys_stderr}
with self.a_built_image({"context": False, "commands": commands2}, harpoon_options=harpoon_options, images={"one": conf1}, image_name="two") as (_, conf2):
Runner().run_container(conf2, {"one": conf1, "two": conf2})
with codecs.open(fake_sys_stdout.name) as fle:
output = fle.read().strip()
if isinstance(output, six.binary_type):
output = output.decode('utf-8')
output = '\n'.join([line for line in output.split('\n') if "lxc-start" not in line])
expected = """
Step 1 : .+
.+
Step 2 : .+
.+
Removing .+
Step 3 : .+