def run(self, *args, output_wrangler=None):
"""
Run a singularity container instance
"""
if self.volumes:
volumes = " --bind " + " --bind ".join(self.volumes)
else:
volumes = ""
if not os.path.exists(self.image):
self.logger.error(f"The image, {self.image}, required to run this cab does not exist."\
" Please run 'stimela pull --help' for help on how to download the image")
raise SystemExit from None
self.status = "running"
self._print("Starting container [{0:s}]. Timeout set to {1:d}. The container ID is printed below.".format(
self.name, self.time_out))
utils.xrun(f"cd {self.execdir} && singularity", ["run", "--workdir", self.execdir, "--containall"] \
+ list(args) + [volumes, self.image, self.RUNSCRIPT],
log=self.logger, timeout=self.time_out, output_wrangler=output_wrangler,
env=self._env, logfile=self.logfile)
self.status = "exited"
return 0
def create(self, *args):
if self.volumes:
volumes = " -v " + " -v ".join(self.volumes)
else:
volumes = ""
if self.environs:
environs = environs = " -e "+" -e ".join(self.environs)
else:
environs = ""
# non-root podman users cannot allocate resources on all linux kernel setups. So minisise stress I don't
# allow it
args = list(args)
for rsrc in ["--memory", "--cpus", "--user"]:
for arg in args:
if arg.startswith(rsrc):
args.remove(arg)
self._print(
"Instantiating container [{}]. The container ID is printed below.".format(self.name))
utils.xrun("podman create", args + [volumes, environs, "--rm",
"-w %s" % (self.WORKDIR) if self.WORKDIR else "",
"--name", self.name, "--shm-size", self.shared_memory,
self.image,
self.COMMAND or ""])
self.status = "created"
def build(image, build_path, tag=None, build_args=None, fromline=None, args=[]):
""" build a podman image"""
if tag:
image = ":".join([image, tag])
bdir = tempfile.mkdtemp()
os.system('cp -r {0:s}/* {1:s}'.format(build_path, bdir))
if build_args:
stdw = tempfile.NamedTemporaryFile(dir=bdir, mode='w')
with open("{}/Dockerfile".format(bdir)) as std:
dfile = std.readlines()
for line in dfile:
if fromline and line.lower().startswith('from'):
stdw.write('FROM {:s}\n'.format(fromline))
elif line.lower().startswith("cmd"):
for arg in build_args:
stdw.write(arg+"\n")
stdw.write(line)
else:
stdw.write(line)
stdw.flush()
utils.xrun("podman build", args+["--force-rm", "--no-cache", "-f", stdw.name,
"-t", image,
bdir])
stdw.close()
else:
utils.xrun("podman build", args+["--force-rm", "--no-cache", "-t", image,
bdir])
os.system('rm -rf {:s}'.format(bdir))
def start(self, *args):
"""
Create a singularity container instance
"""
if self.volumes:
volumes = " --bind " + " --bind ".join(self.volumes)
else:
volumes = ""
self._print(
"Instantiating container [{0:s}]. Timeout set to {1:d}. The container ID is printed below."
.format(self.name, self.time_out))
utils.xrun(
"singularity instance.start",
list(args) + [
volumes,
# "-c",
self.image,
self.name
])
self.status = "created"
return 0
def run(self, *args):
if self.volumes:
volumes = " --volume=" + " --volume=".join(self.volumes)
else:
volumes = ""
if self.environs:
environs = environs = " --env="+" --env=".join(self.environs)
else:
environs = ""
self._print("Running container [{}]".format(self.name))
tstart = time.time()
utils.xrun("udocker run", ["=".join(args)] + [volumes, environs,
"--workdir=%s" % (
self.WORKDIR) if self.WORKDIR else "",
"--rm",
"--dri" if self.use_graphics else "",
self.name, self.COMMAND or ""],
logfile=self.logfile,
log=self.logger,
timeout=self.time_out)
self.status = "running"
uptime = seconds_hms(time.time() - tstart)
self.uptime = uptime
self._print("Runtime was {0}.".format(uptime))
def create(self):
running = True
self.status = "created"
# self.cont_logger.log_container(self.name)
# self.cont_logger.write()
self._print("Creating container [{0:s}]. Timeout set to {1:d}. The container ID is printed below.".format(
self.name, self.time_out))
utils.xrun(
"udocker", ["create", "--name={0:s}".format(self.name), self.image])
def pull(image, store_path, docker=True):
"""
pull an image
"""
if docker:
fp = "docker://{0:s}".format(image)
else:
fp = image
utils.xrun("singularity", ["pull", "--force", "--name", store_path, fp])
return 0
def stop(self):
dinfo = self.info()
status = dinfo["State"]["Status"]
killed = False
if status in ["running", "paused"]:
try:
utils.xrun("docker stop", [self.name])
except KeyboardInterrupt("Received terminate signal. Will stop and remove container first"):
killed = True
self.status = 'exited'
self._print("Container {} has been stopped.".format(self.name))
if killed:
self.remove()
raise KeyboardInterrupt
def remove(self):
dinfo = self.info()
status = dinfo["State"]["Status"]
killed = False
if status == "exited":
try:
utils.xrun("docker rm", [self.name])
except KeyboardInterrupt:
killed = True
if killed:
raise KeyboardInterrupt
else:
raise DockerError(
"Container [{}] has not been stopped, cannot remove".format(self.name))
def pull(image, store_path, docker=True, directory=".", force=False):
"""
pull an image
"""
if docker:
fp = "docker://{0:s}".format(image)
else:
fp = image
if not os.path.exists(directory):
os.mkdir(directory)
utils.xrun("singularity", ["build",
"--force" if force else "",
os.path.join(directory,store_path), fp])
return 0
def generate_report_notebooks(notebooks, output_dir, prefix, container_tech):
opts = ["--non-interactive"]
if container_tech == "docker":
opts.append("--docker")
elif container_tech == "singularity":
opts.append("--singularity")
else:
log.warning(
"Container technology '{}' not supported by radiopadre, skipping report rendering"
)
return
if caracal.DEBUG:
opts += ['-v', '2', '--container-debug']
## disabling as per https://github.com/caracal-pipeline/caracal/issues/1161
# # first time run with -u
# global _radiopadre_updated
# if not _radiopadre_updated:
# opts.append('--update')
# _radiopadre_updated = True
start_time = time.time()
log.info("Rendering report(s)")
for notebook in notebooks:
if prefix:
notebook = "{}-{}".format(prefix, notebook)
nbdest = os.path.join(output_dir, notebook + ".ipynb")
nbhtml = os.path.join(output_dir, notebook + ".html")
if os.path.exists(nbdest):
try:
xrun("run-radiopadre", opts + ["--nbconvert", nbdest], log=log)
except StimelaCabRuntimeError as exc:
log.warning(
"Report {} failed to render ({}). HTML report will not be available."
.format(nbhtml, exc))
# check that HTML file actually showed up (sometimes the container doesn't report an error)
if os.path.exists(
nbhtml) and os.path.getmtime(nbhtml) >= start_time:
log.info("Rendered report {}".format(nbhtml))
else:
log.warning("Report {} failed to render".format(nbhtml))
else:
log.warning(
"Report notebook {} not found, skipping report rendering".
format(nbdest))
def stop(self, *args):
"""
Stop a singularity container instance
"""
if self.volumes:
volumes = " --bind " + " --bind ".join(self.volumes)
else:
volumes = ""
self._print(
"Stopping container [{}]. The container ID is printed below.".
format(self.name))
utils.xrun("singularity", ["instance.stop {0:s}".format(self.name)])
self.status = "exited"
return 0
def start(self, output_wrangler=None):
running = True
tstart = time.time()
self.status = "running"
self._print("Starting container [{0:s}]. Timeout set to {1:d}. The container ID is printed below.".format(
self.name, self.time_out))
utils.xrun("docker", ["start", "-a", self.name],
timeout=self.time_out,
logfile=self.logfile,
log=self.logger, output_wrangler=output_wrangler,
kill_callback=lambda: utils.xrun("docker", ["kill", self.name]))
uptime = seconds_hms(time.time() - tstart)
self.uptime = uptime
self._print(
"Container [{0}] has executed successfully".format(self.name))
self._print("Runtime was {0}.".format(uptime))
self.status = "exited"
def pull(image, name, docker=True, directory=".", force=False):
"""
pull an image
"""
if docker:
fp = "docker://{0:s}".format(image)
else:
fp = image
if not os.path.exists(directory):
os.mkdir(directory)
image_path = os.path.abspath(os.path.join(directory, name))
if os.path.exists(image_path) and not force:
stimela.logger().info(f"Singularity image already exists at '{image_path}'. To replace it, please re-run with the 'force' option")
else:
utils.xrun(f"cd {directory} && singularity", ["pull",
"--force" if force else "", "--name",
name, fp])
return 0
def create(self, *args):
if self.volumes:
volumes = " -v " + " -v ".join(self.volumes)
else:
volumes = ""
if self.environs:
environs = environs = " -e "+" -e ".join(self.environs)
else:
environs = ""
self._print(
"Instantiating container [{}]. The container ID is printed below.".format(self.name))
utils.xrun("docker create", list(args) + [volumes, environs, "--rm",
"-w %s" % (self.WORKDIR),
"--name", self.name,
self.image,
self.RUNSCRIPT or ""], log=self.logger)
self.status = "created"
def run(self, *args):
"""
Run a singularity container instance
"""
if self.volumes:
volumes = " --bind " + " --bind ".join(self.volumes)
else:
volumes = ""
self._print(
"Starting container [{0:s}]. Timeout set to {1:d}. The container ID is printed below."
.format(self.name, self.time_out))
utils.xrun(
"singularity run",
["instance://{0:s} {1:s}".format(self.name, self.RUNSCRIPT)],
timeout=self.time_out,
kill_callback=self.stop)
self.status = "running"
return 0
def start(self):
running = True
tstart = time.time()
self.status = "running"
self.cont_logger.log_container(self.name)
self.cont_logger.write()
self._print("Starting container [{0:s}]. Timeout set to {1:d}. The container ID is printed below.".format(
self.name, self.time_out))
utils.xrun("podman", ["start", "-a", self.name],
timeout=self.time_out,
kill_callback=lambda: utils.xrun("podman", ["kill", self.name]),
logfile=list(filter(lambda x: "log" in x, self.volumes))[0].split(":")[0])
uptime = seconds_hms(time.time() - tstart)
self.uptime = uptime
self._print(
"Container [{0}] has executed successfully".format(self.name))
self._print("Runtime was {0}.".format(uptime))
self.status = "exited"
def create(self, *args):
if self.volumes:
volumes = " -v " + " -v ".join(self.volumes)
else:
volumes = ""
if self.environs:
environs = environs = " -e " + " -e ".join(self.environs)
else:
environs = ""
self._print(
"Instantiating container [{}]. The container ID is printed below.".
format(self.name))
utils.xrun(
"docker create",
list(args) + [
volumes, environs,
"-w %s" % (self.WORKDIR) if self.WORKDIR else "", "--name",
self.name, "--shm-size", self.shared_memory, self.image,
self.COMMAND or ""
])
self.status = "created"
def pull(image, tag=None, force=False):
""" pull a docker image """
if tag:
image = ":".join([image, tag])
utils.xrun("docker", ["pull", image])
def clean(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(
description='Convience tools for cleaning up after stimela')
add = parser.add_argument
add("-ai", "--all-images", action="store_true",
help="Remove all images pulled/built by stimela. This include CAB images")
add("-ab", "--all-base", action="store_true",
help="Remove all base images")
add("-ac", "--all-cabs", action="store_true",
help="Remove all CAB images")
add("-aC", "--all-containers", action="store_true",
help="Stop and/or Remove all stimela containers")
add("-bl", "--build-label", default=USER.lower(),
help="Label for cab images. All cab images will be named <CAB_LABEL>_<cab name>. The default is $USER")
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
log_cabs = logger.StimelaLogger('{0:s}/{1:s}_stimela_logfile.json'.format(LOG_HOME,
args.build_label))
if args.all_images:
images = log.info['images'].keys()
images = log_cabs.info['images'].keys()
for image in images:
utils.xrun('docker', ['rmi', image])
log.remove('images', image)
log.write()
images = log_cabs.info['images'].keys()
for image in images:
if log_cabs.info['images'][image]['CAB']:
utils.xrun('docker', ['rmi', image])
log_cabs.remove('images', image)
log_cabs.write()
if args.all_base:
images = log.info['images'].keys()
for image in images:
if log.info['images'][image]['CAB'] is False:
utils.xrun('docker', ['rmi', image])
log.remove('images', image)
log.write()
if args.all_cabs:
images = log_cabs.info['images'].keys()
for image in images:
if log_cabs.info['images'][image]['CAB']:
utils.xrun('docker', ['rmi', image])
log_cabs.remove('images', image)
log_cabs.write()
if args.all_containers:
containers = log.info['containers'].keys()
for container in containers:
cont = docker.Container(
log.info['containers'][container]['IMAGE'], container)
try:
status = cont.info()['State']['Status'].lower()
except:
print('Could not inspect container {}. It probably doesn\'t exist, will remove it from log'.format(
container))
status = "no there"
if status == 'running':
# Kill the container instead of stopping it, so that effect can be felt py parent process
utils.xrun('docker', ['kill', container])
cont.remove()
elif status in ['exited', 'dead']:
cont.remove()
log.remove('containers', container)
log.write()
def pull(image, tag=None, force=False):
""" pull a podman image """
if tag:
image = ":".join([image, tag])
utils.xrun("podman", ["pull", "docker.io/"+image])
def pull(image, tag=None):
""" pull a docker image """
if tag:
image = ":".join([image, tag])
utils.xrun("docker pull", [image])