def wait_till_stopped(self, conf, container_id, timeout=10, message=None, waiting=True):
"""Wait till a container is stopped"""
stopped = False
inspection = None
for _ in until(timeout=timeout, action=message):
try:
inspection = conf.harpoon.docker_context.inspect_container(container_id)
if not isinstance(inspection, dict):
log.error("Weird response from inspecting the container\tresponse=%s", inspection)
else:
if not inspection["State"]["Running"]:
stopped = True
conf.container_id = None
break
else:
break
except (socket.timeout, ValueError):
log.warning("Failed to inspect the container\tcontainer_id=%s", container_id)
except DockerAPIError as error:
if error.response.status_code != 404:
raise
else:
break
if not inspection:
log.warning("Failed to inspect the container!")
stopped = True
exit_code = 1
else:
exit_code = inspection["State"]["ExitCode"]
return stopped, exit_code
def wait_till_stopped(self,
conf,
container_id,
timeout=10,
message=None,
waiting=True):
"""Wait till a container is stopped"""
stopped = False
for _ in until(timeout=timeout, action=message):
try:
inspection = conf.harpoon.docker_context.inspect_container(
container_id)
if not isinstance(inspection, dict):
log.error(
"Weird response from inspecting the container\tresponse=%s",
inspection)
else:
if not inspection["State"]["Running"]:
stopped = True
conf.container_id = None
break
else:
break
except (socket.timeout, ValueError):
log.warning("Failed to inspect the container\tcontainer_id=%s",
container_id)
except DockerAPIError as error:
if error.response.status_code != 404:
raise
else:
break
exit_code = inspection["State"]["ExitCode"]
return stopped, exit_code
def get_exit_code(self, conf):
"""Determine how a container exited"""
for _ in until(timeout=0.5, step=0.1, silent=True):
try:
inspection = conf.harpoon.docker_context.inspect_container(conf.container_id)
if not isinstance(inspection, dict) or "State" not in inspection:
raise BadResult("Expected inspect result to be a dictionary with 'State' in it", found=inspection)
elif not inspection["State"]["Running"]:
return inspection
except Exception as error:
log.error("Failed to see if container exited normally or not\thash=%s\terror=%s", conf.container_id, error)
def stop_container(self, conf, fail_on_bad_exit=False, fail_reason=None, tag=None):
"""Stop some container"""
stopped = False
container_id = conf.container_id
if not container_id:
return
container_name = conf.container_name
stopped, exit_code = self.is_stopped(conf, container_id)
if stopped:
if exit_code != 0 and fail_on_bad_exit:
if not conf.harpoon.interactive:
print_logs = True
else:
hp.write_to(conf.harpoon.stdout, "!!!!\n")
hp.write_to(conf.harpoon.stdout, "Container had already exited with a non zero exit code\tcontainer_name={0}\tcontainer_id={1}\texit_code={2}\n".format(container_name, container_id, exit_code))
hp.write_to(conf.harpoon.stdout, "Do you want to see the logs from this container?\n")
conf.harpoon.stdout.flush()
answer = input("[y]: ")
print_logs = not answer or answer.lower().startswith("y")
if print_logs:
hp.write_to(conf.harpoon.stdout, "=================== Logs for failed container {0} ({1})\n".format(container_id, container_name))
for line in conf.harpoon.docker_context.logs(container_id).split("\n"):
hp.write_to(conf.harpoon.stdout, "{0}\n".format(line))
hp.write_to(conf.harpoon.stdout, "------------------- End logs for failed container\n")
fail_reason = fail_reason or "Failed to run container"
raise BadImage(fail_reason, container_id=container_id, container_name=container_name)
else:
try:
log.info("Killing container %s:%s", container_name, container_id)
conf.harpoon.docker_context.kill(container_id, 9)
except DockerAPIError:
pass
self.wait_till_stopped(conf, container_id, timeout=10, message="waiting for container to die\tcontainer_name={0}\tcontainer_id={1}".format(container_name, container_id))
if tag:
log.info("Tagging a container\tcontainer_id=%s\ttag=%s", container_id, tag)
new_id = conf.harpoon.docker_context.commit(container_id)["Id"]
conf.harpoon.docker_context.tag(new_id, repository=tag, tag="latest", force=True)
if not conf.harpoon.no_cleanup:
log.info("Removing container %s:%s", container_name, container_id)
for _ in until(timeout=10, action="removing container\tcontainer_name={0}\tcontainer_id={1}".format(container_name, container_id)):
try:
conf.harpoon.docker_context.remove_container(container_id)
break
except socket.timeout:
break
except (ValueError, DockerAPIError) as error:
log.warning("Failed to remove container\tcontainer_id=%s\terror=%s", container_id, error)
def get_exit_code(self, conf):
"""Determine how a container exited"""
for _ in until(timeout=0.5, step=0.1, silent=True):
try:
inspection = conf.harpoon.docker_context.inspect_container(
conf.container_id)
if not isinstance(inspection,
dict) or "State" not in inspection:
raise BadResult(
"Expected inspect result to be a dictionary with 'State' in it",
found=inspection)
elif not inspection["State"]["Running"]:
return inspection
except Exception as error:
log.error(
"Failed to see if container exited normally or not\thash=%s\terror=%s",
conf.container_id, error)
def _stop_container(self, container_id, container_name, fail_on_bad_exit=False, fail_reason=None):
"""Stop some container"""
stopped = False
for _ in until(timeout=10):
try:
inspection = self.docker_context.inspect_container(container_id)
if not isinstance(inspection, dict):
log.error("Weird response from inspecting the container\tresponse=%s", inspection)
else:
if not inspection["State"]["Running"]:
stopped = True
break
else:
break
except (socket.timeout, ValueError):
log.warning("Failed to inspect the container\tcontainer_id=%s", container_id)
except DockerAPIError as error:
if error.response.status_code != 404:
raise
else:
break
if stopped:
exit_code = inspection["State"]["ExitCode"]
if exit_code != 0 and fail_on_bad_exit:
if not self.interactive:
print_logs = True
else:
print("!!!!")
print("Container had already exited with a non zero exit code\tcontainer_name={0}\tcontainer_id={1}\texit_code={2}".format(container_name, container_id, exit_code))
print("Do you want to see the logs from this container?")
answer = raw_input("[y]: ")
print_logs = not answer or answer.lower().startswith("y")
if print_logs:
print("=================== Logs for failed container {0} ({1})".format(container_id, container_name))
for line in self.docker_context.logs(container_id).split("\n"):
print(line)
print("------------------- End logs for failed container")
fail_reason = fail_reason or "Failed to run container"
raise BadImage(fail_reason, container_id=container_id, container_name=container_name)
else:
try:
log.info("Killing container %s:%s", container_name, container_id)
self.docker_context.kill(container_id, 9)
except DockerAPIError:
pass
for _ in until(timeout=10, action="waiting for container to die\tcontainer_name={0}\tcontainer_id={1}".format(container_name, container_id)):
try:
inspection = self.docker_context.inspect_container(container_id)
if not inspection["State"]["Running"]:
break
except socket.timeout:
pass
except ValueError:
log.warning("Failed to inspect the container\tcontainer_id=%s", container_id)
except DockerAPIError as error:
if error.response.status_code != 404:
raise
else:
break
log.info("Removing container %s:%s", container_name, container_id)
for _ in until(timeout=10, action="removing container\tcontainer_name={0}\tcontainer_id={1}".format(container_name, container_id)):
try:
self.docker_context.remove_container(container_id)
break
except socket.timeout:
break
except ValueError:
log.warning("Failed to remove container\tcontainer_id=%s", container_id)
except DockerAPIError as error:
if error.response.status_code != 404:
raise
else:
break
def _run_container(self, name, image_name, container_name
, detach=False, command=None, tty=True, volumes=None, volumes_from=None, links=None, delete_on_exit=False, env=None, ports=None, dependency=False, no_intervention=False
):
"""Run a single container"""
if not detach and dependency:
tty = True
log.info("Creating container from %s\timage=%s\tcontainer_name=%s\ttty=%s", image_name, name, container_name, tty)
binds = {}
volume_names = []
if volumes is None:
volumes = []
uncreated = []
for volume in volumes:
if ":" in volume:
name, bound = volume.split(":", 1)
permissions = "rw"
if ":" in bound:
bound, permissions = volume.split(":", 1)
binds[name] = {"bind": bound, permissions: True}
volume_names.append(bound)
if not os.path.exists(name):
log.info("Making volume for mounting\tvolume=%s", name)
try:
os.makedirs(name)
except OSError as error:
uncreated.append((name, error))
if uncreated:
raise BadOption("Failed to create some volumes on the host", uncreated=uncreated)
if volumes:
log.info("\tUsing volumes\tvolumes=%s", volumes)
if env:
log.info("\tUsing environment\tenv=%s", env)
if ports:
log.info("\tUsing ports\tports=%s", ports.keys())
container = self.docker_context.create_container(image_name
, name=container_name
, detach=detach
, command=command
, volumes=volumes
, environment=env
, tty = tty
, ports = (ports or {}).keys()
, stdin_open = tty
)
container_id = container
if isinstance(container_id, dict):
if "errorDetail" in container_id:
raise BadImage("Failed to create container", image=name, error=container_id["errorDetail"])
container_id = container_id["Id"]
self._container_id = container_id
self.already_running = True
try:
log.info("Starting container %s", container_name)
if links:
log.info("\tLinks: %s", links)
if volumes_from:
log.info("\tVolumes from: %s", volumes_from)
if ports:
log.info("\tPort Bindings: %s", ports)
self.docker_context.start(container_id
, links = links
, binds = binds
, volumes_from = volumes_from
, port_bindings = ports
)
if not detach and not dependency:
try:
dockerpty.start(self.docker_context, container_id)
except KeyboardInterrupt:
pass
inspection = None
if not detach and not dependency:
for _ in until(timeout=0.5, step=0.1, silent=True):
try:
inspection = self.docker_context.inspect_container(container_id)
if not isinstance(inspection, dict) or "State" not in inspection:
raise BadResult("Expected inspect result to be a dictionary with 'State' in it", found=inspection)
elif not inspection["State"]["Running"]:
break
except Exception as error:
log.error("Failed to see if container exited normally or not\thash=%s\terror=%s", container_id, error)
if inspection and not no_intervention:
if not inspection["State"]["Running"] and inspection["State"]["ExitCode"] != 0:
if self.interactive and not self.heira_formatted("harpoon.no_intervention", default=False):
print("!!!!")
print("Failed to run the container!")
print("Do you want commit the container in it's current state and /bin/bash into it to debug?")
answer = raw_input("[y]: ")
if not answer or answer.lower().startswith("y"):
with self.commit_and_run(container_id, command="/bin/bash"):
pass
raise BadImage("Failed to run container", container_id=container_id, container_name=container_name)
finally:
if delete_on_exit:
self._stop_container(container_id, container_name)
def stop_container(self,
conf,
fail_on_bad_exit=False,
fail_reason=None,
tag=None,
remove_volumes=False):
"""Stop some container"""
stopped = False
container_id = conf.container_id
if not container_id:
return
container_name = conf.container_name
stopped, exit_code = self.is_stopped(conf, container_id)
if stopped:
if exit_code != 0 and fail_on_bad_exit:
if not conf.harpoon.interactive:
print_logs = True
else:
hp.write_to(conf.harpoon.stdout, "!!!!\n")
hp.write_to(
conf.harpoon.stdout,
"Container had already exited with a non zero exit code\tcontainer_name={0}\tcontainer_id={1}\texit_code={2}\n"
.format(container_name, container_id, exit_code))
hp.write_to(
conf.harpoon.stdout,
"Do you want to see the logs from this container?\n")
conf.harpoon.stdout.flush()
answer = input("[y]: ")
print_logs = not answer or answer.lower().startswith("y")
if print_logs:
hp.write_to(
conf.harpoon.stdout,
"=================== Logs for failed container {0} ({1})\n"
.format(container_id, container_name))
for line in conf.harpoon.docker_context.logs(
container_id).split("\n"):
hp.write_to(conf.harpoon.stdout, "{0}\n".format(line))
hp.write_to(
conf.harpoon.stdout,
"------------------- End logs for failed container\n")
fail_reason = fail_reason or "Failed to run container"
raise BadImage(fail_reason,
container_id=container_id,
container_name=container_name)
else:
try:
log.info("Killing container %s:%s", container_name,
container_id)
conf.harpoon.docker_context.kill(container_id, 9)
except DockerAPIError:
pass
self.wait_till_stopped(
conf,
container_id,
timeout=10,
message=
"waiting for container to die\tcontainer_name={0}\tcontainer_id={1}"
.format(container_name, container_id))
if tag:
log.info("Tagging a container\tcontainer_id=%s\ttag=%s",
container_id, tag)
new_id = conf.harpoon.docker_context.commit(container_id)["Id"]
conf["committed"] = new_id
if tag is not True:
the_tag = "latest" if conf.tag is NotSpecified else conf.tag
conf.harpoon.docker_context.tag(new_id,
repository=tag,
tag=the_tag,
force=True)
mounts = []
if remove_volumes:
inspection = conf.harpoon.docker_context.inspect_container(
container_id)
if "Mounts" in inspection:
for m in inspection["Mounts"]:
if "Name" in m:
mounts.append(m['Name'])
else:
log.warning("Your docker can't inspect and delete volumes :(")
if not conf.harpoon.no_cleanup:
log.info("Removing container %s:%s", container_name, container_id)
for _ in until(
timeout=10,
action=
"removing container\tcontainer_name={0}\tcontainer_id={1}".
format(container_name, container_id)):
try:
conf.harpoon.docker_context.remove_container(container_id)
break
except socket.timeout:
break
except (ValueError, DockerAPIError) as error:
log.warning(
"Failed to remove container\tcontainer_id=%s\terror=%s",
container_id, error)
for mount in mounts:
try:
log.info("Cleaning up volume {0}".format(mount))
conf.harpoon.docker_context.remove_volume(mount)
except DockerAPIError as error:
log.warning("Failed to cleanup volume\tvolume=%s\terror=%s",
mount, error)
describe HarpoonCase, "until":
@contextmanager
def mock_log_and_time(self):
"""Mock out the log object and time, yield (log, time)"""
fake_log = mock.Mock(name="log")
fake_time = mock.Mock(name="time")
with mock.patch("harpoon.helpers.log", fake_log):
with mock.patch("harpoon.helpers.time", fake_time):
yield (fake_log, fake_time)
it "yields before doing anything else":
done = []
with self.mock_log_and_time() as (fake_log, fake_time):
for _ in until():
done.append(1)
break
self.assertEqual(len(fake_time.time.mock_calls), 0)
self.assertEqual(len(fake_log.info.mock_calls), 0)
self.assertEqual(done, [1])
it "logs the action each time":
done = []
action = mock.Mock(name="action")
with self.mock_log_and_time() as (fake_log, fake_time):
def timer():
if not done:
return 10
else:
