def kill(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='Gracefully kill stimela process(s).')
add = parser.add_argument
add("pid", nargs="*",
help="Process ID")
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
for pid in args.pid:
found = pid in log.info['processes'].keys()
if not found:
print("Could not find process {0}".format(pid))
continue
try:
os.kill(int(pid), signal.SIGINT)
except OSError:
raise OSError(
'Process with PID {} could not be killed'.format(pid))
log.remove('processes', pid)
log.write()
def pull(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg
parser = ArgumentParser(description='Pull docker stimela base images')
add = parser.add_argument
add("-im",
"--image",
action="append",
metavar="IMAGE[:TAG]",
help=
"Pull base image along with its tag (or version). Can be called multiple times"
)
add("-t", "--tag", help="Tag")
add("-s",
"--singularity",
action="store_true",
help="Use singularity instead of docker."
"Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default"
)
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
images = log.read()['images']
if args.image:
for image in args.image:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
if args.singularity:
singularity.pull(image, simage)
else:
docker.pull(image)
log.log_image(image, 'pulled')
else:
base = []
for cab in CAB:
image = "{:s}/{:s}".format(cargo.CAB_PATH, cab)
base.append(utils.get_Dockerfile_base_image(image).split()[-1])
base = set(base)
for image in base:
if image not in ["stimela/ddfacet", "radioastro/ddfacet"]:
if args.singularity:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
singularity.pull(image, simage)
else:
docker.pull(image)
log.log_image(image, 'pulled')
log.write()
def get_cabs(logfile):
log = logger.StimelaLogger(logfile)
cabs_ = log.read()['images']
# Remove images that are not cabs
for key in cabs_.keys():
if not cabs_[key]['CAB']:
del cabs_[key]
return cabs_
def ps(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='List all running stimela processes')
add = parser.add_argument
add("-c", "--clear", action="store_true",
help="Clear logfile that keeps track of stimela processes. This doesn't do anything ot the processes themselves.")
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
log.display('processes')
if args.clear:
log.clear('processes')
log.write()
def images(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='List all stimela related images.')
add = parser.add_argument
add("-c", "--clear", action="store_true",
help="Clear the logfile that keeps track of stimela images. This does not do anythig to the images.")
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
log.display('images')
if args.clear:
log.clear('images')
log.write()
def clean(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(
description='Convience tools for cleaning up after stimela')
add = parser.add_argument
add("-ai", "--all-images", action="store_true",
help="Remove all images pulled/built by stimela. This include CAB images")
add("-ab", "--all-base", action="store_true",
help="Remove all base images")
add("-ac", "--all-cabs", action="store_true",
help="Remove all CAB images")
add("-aC", "--all-containers", action="store_true",
help="Stop and/or Remove all stimela containers")
add("-bl", "--build-label", default=USER.lower(),
help="Label for cab images. All cab images will be named <CAB_LABEL>_<cab name>. The default is $USER")
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
log_cabs = logger.StimelaLogger('{0:s}/{1:s}_stimela_logfile.json'.format(LOG_HOME,
args.build_label))
if args.all_images:
images = log.info['images'].keys()
images = log_cabs.info['images'].keys()
for image in images:
utils.xrun('docker', ['rmi', image])
log.remove('images', image)
log.write()
images = log_cabs.info['images'].keys()
for image in images:
if log_cabs.info['images'][image]['CAB']:
utils.xrun('docker', ['rmi', image])
log_cabs.remove('images', image)
log_cabs.write()
if args.all_base:
images = log.info['images'].keys()
for image in images:
if log.info['images'][image]['CAB'] is False:
utils.xrun('docker', ['rmi', image])
log.remove('images', image)
log.write()
if args.all_cabs:
images = log_cabs.info['images'].keys()
for image in images:
if log_cabs.info['images'][image]['CAB']:
utils.xrun('docker', ['rmi', image])
log_cabs.remove('images', image)
log_cabs.write()
if args.all_containers:
containers = log.info['containers'].keys()
for container in containers:
cont = docker.Container(
log.info['containers'][container]['IMAGE'], container)
try:
status = cont.info()['State']['Status'].lower()
except:
print('Could not inspect container {}. It probably doesn\'t exist, will remove it from log'.format(
container))
status = "no there"
if status == 'running':
# Kill the container instead of stopping it, so that effect can be felt py parent process
utils.xrun('docker', ['kill', container])
cont.remove()
elif status in ['exited', 'dead']:
cont.remove()
log.remove('containers', container)
log.write()
def build(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv = ' ' + arg
parser = ArgumentParser(description='Build executor (a.k.a cab) images')
parser.add_argument("-b", "--base", action="store_true",
help="Build base images")
parser.add_argument("-c", "--cab", metavar="CAB,CAB_DIR",
help="Executor image (name) name, location of executor image files")
parser.add_argument("-uo", "--us-only",
help="Only build these cabs. Comma separated cab names")
parser.add_argument("-i", "--ignore-cabs", default="",
help="Comma separated cabs (executor images) to ignore.")
parser.add_argument("-nc", "--no-cache", action="store_true",
help="Do not use cache when building the image")
parser.add_argument("-bl", "--build-label", default=USER.lower(),
help="Label for cab images. All cab images will be named <CAB_LABEL>_<cab name>. The default is $USER")
args = parser.parse_args(argv)
log = logger.StimelaLogger(
'{0:s}/{1:s}_stimela_logfile.json'.format(LOG_HOME, args.build_label), jtype="docker")
no_cache = ["--no-cache"] if args.no_cache else []
if args.base:
# Build base and meqtrees images first
BASE.remove("base")
BASE.remove("meqtrees")
BASE.remove("casa")
BASE.remove("astropy")
for image in ["base", "meqtrees", "casa", "astropy"] + BASE:
dockerfile = "{:s}/{:s}".format(stimela.BASE_PATH, image)
image = "stimela/{0}:{1}".format(image, stimela.__version__)
docker.build(image,
dockerfile, args=no_cache)
log.log_image(image, dockerfile, replace=True)
log.write()
return 0
workdir = "/home/{}/output/".format(USER)
build_args = [
"WORKDIR {:s}".format(workdir),
"RUN useradd -r -u {0:d} -U {1:s}".format(UID, USER),
"USER {0:s}".format(USER),
]
if args.cab:
cab_args = args.cab.split(",")
if len(cab_args) == 2:
cab, path = cab_args
else:
raise ValueError("Not enough arguments for build command.")
image = "{:s}_cab/{:s}".format(args.build_label, cab)
docker.build(image,
path,
build_args=build_args, args=no_cache)
log.log_image(image, path, replace=True, cab=True)
log.write()
return
# Cabs and their locations
cabs = []
dockerfiles = []
if args.us_only:
CABS = args.us_only.split(',')
else:
# Images that have been logged
# This is crucial for making custom cabs
logged_images = log.read().get('images', {})
for key, val in logged_images.items():
if val['CAB']:
if key.endswith("custom"):
continue
else:
cabs.append(key)
dockerfiles.append(val['DIR'])
# If user wants to ignore some cabs
IGNORE = args.ignore_cabs.split(",")
CABS = set(CAB).difference(set(IGNORE))
# Prioritise package images over logged images
cabs = ["{:s}_cab/{:s}".format(args.build_label, cab)
for cab in CABS] + cabs
dockerfiles = ["{:s}/{:s}".format(stimela.CAB_PATH, cab)
for cab in CABS] + dockerfiles
built = []
for image, dockerfile in zip(cabs, dockerfiles):
if image not in built:
docker.build(image,
dockerfile,
build_args=build_args, args=no_cache)
log.log_image(image, dockerfile, replace=True, cab=True)
log.write()
built.append(image)
def pull(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='Pull docker stimela base images')
add = parser.add_argument
add("-im", "--image", nargs="+", metavar="IMAGE[:TAG]",
help="Pull base image along with its tag (or version). Can be called multiple times")
add("-f", "--force", action="store_true",
help="force pull if image already exists")
add("-s", "--singularity", action="store_true",
help="Pull base images using singularity."
"Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default")
add("-d", "--docker", action="store_true",
help="Pull base images using docker.")
add("-p", "--podman", action="store_true",
help="Pull base images using podman.")
add("-cb", "--cab-base", nargs="+",
help="Pull base image for specified cab")
add("-pf", "--pull-folder",
help="Images will be placed in this folder. Else, if the environmnental variable 'SINGULARITY_PULLFOLDER' is set, then images will be placed there. "
"Else, images will be placed in the current directory")
args = parser.parse_args(argv)
if args.pull_folder:
pull_folder = args.pull_folder
else:
try:
pull_folder = os.environ["SINGULARITY_PULLFOLDER"]
except KeyError:
pull_folder = "."
if args.docker:
jtype = "docker"
elif args.podman:
jtype = "podman"
elif args.singularity:
jtype = "singularity"
else:
jtype = "udocker"
log = logger.StimelaLogger(LOG_FILE, jtype=jtype)
images = log.read()['images']
images_ = []
for cab in args.cab_base or []:
if cab in CAB:
filename = "/".join([stimela.CAB_PATH, cab, "parameters.json"])
param = utils.readJson(filename)
images_.append(":".join([param["base"], param["tag"]]))
args.image = images_ or args.image
if args.image:
for image in args.image:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
if args.singularity:
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image)
log.log_image(image, 'pulled')
elif args.podman:
podman.pull(image)
log.log_image(image, 'pulled')
else:
udocker.pull(image)
log.log_image(image, 'pulled')
else:
base = []
for cab in CAB:
image = "{:s}/{:s}".format(stimela.CAB_PATH, cab)
base.append(utils.get_Dockerfile_base_image(image).split()[-1])
base = set(base)
for image in base:
if args.singularity:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image, force=args.force)
log.log_image(image, 'pulled')
elif args.podman:
podman.pull(image, force=args.force)
log.log_image(image, 'pulled')
else:
udocker.pull(image, force=args.force)
log.log_image(image, 'pulled')
log.write()
