def pull(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg
parser = ArgumentParser(description='Pull docker stimela base images')
add = parser.add_argument
add("-im",
"--image",
action="append",
metavar="IMAGE[:TAG]",
help=
"Pull base image along with its tag (or version). Can be called multiple times"
)
add("-t", "--tag", help="Tag")
add("-s",
"--singularity",
action="store_true",
help="Use singularity instead of docker."
"Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default"
)
args = parser.parse_args(argv)
log = logger.StimelaLogger(LOG_FILE)
images = log.read()['images']
if args.image:
for image in args.image:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
if args.singularity:
singularity.pull(image, simage)
else:
docker.pull(image)
log.log_image(image, 'pulled')
else:
base = []
for cab in CAB:
image = "{:s}/{:s}".format(cargo.CAB_PATH, cab)
base.append(utils.get_Dockerfile_base_image(image).split()[-1])
base = set(base)
for image in base:
if image not in ["stimela/ddfacet", "radioastro/ddfacet"]:
if args.singularity:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
singularity.pull(image, simage)
else:
docker.pull(image)
log.log_image(image, 'pulled')
log.write()
def pull(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='Pull docker stimela base images')
add = parser.add_argument
add("-im", "--image", nargs="+", metavar="IMAGE[:TAG]",
help="Pull base image along with its tag (or version). Can be called multiple times")
add("-f", "--force", action="store_true",
help="force pull if image already exists")
add("-s", "--singularity", action="store_true",
help="Pull base images using singularity."
"Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default")
add("-d", "--docker", action="store_true",
help="Pull base images using docker.")
add("-p", "--podman", action="store_true",
help="Pull base images using podman.")
add("-cb", "--cab-base", nargs="+",
help="Pull base image for specified cab")
add("-pf", "--pull-folder",
help="Images will be placed in this folder. Else, if the environmnental variable 'SINGULARITY_PULLFOLDER' is set, then images will be placed there. "
"Else, images will be placed in the current directory")
args = parser.parse_args(argv)
if args.pull_folder:
pull_folder = args.pull_folder
else:
try:
pull_folder = os.environ["SINGULARITY_PULLFOLDER"]
except KeyError:
pull_folder = "."
if args.docker:
jtype = "docker"
elif args.podman:
jtype = "podman"
elif args.singularity:
jtype = "singularity"
else:
jtype = "udocker"
log = logger.StimelaLogger(LOG_FILE, jtype=jtype)
images = log.read()['images']
images_ = []
for cab in args.cab_base or []:
if cab in CAB:
filename = "/".join([stimela.CAB_PATH, cab, "parameters.json"])
param = utils.readJson(filename)
images_.append(":".join([param["base"], param["tag"]]))
args.image = images_ or args.image
if args.image:
for image in args.image:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
if args.singularity:
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image)
log.log_image(image, 'pulled')
elif args.podman:
podman.pull(image)
log.log_image(image, 'pulled')
else:
udocker.pull(image)
log.log_image(image, 'pulled')
else:
base = []
for cab in CAB:
image = "{:s}/{:s}".format(stimela.CAB_PATH, cab)
base.append(utils.get_Dockerfile_base_image(image).split()[-1])
base = set(base)
for image in base:
if args.singularity:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + ".img"
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image, force=args.force)
log.log_image(image, 'pulled')
elif args.podman:
podman.pull(image, force=args.force)
log.log_image(image, 'pulled')
else:
udocker.pull(image, force=args.force)
log.log_image(image, 'pulled')
log.write()
def setup_job(self, image, config,
indir=None, outdir=None, msdir=None,
singularity_image_dir=None):
"""
Setup job
image : stimela cab name, e.g. 'cab/simms'
name : This name will be part of the name of the contaier that will
execute the task (now optional)
config : Dictionary of options to parse to the task. This will modify
the parameters in the default parameter file which
can be viewd by running 'stimela cabs -i <cab name>', e.g 'stimela cabs -i simms'
indir : input dirctory for cab
outdir : output directory for cab
msdir : MS directory for cab. Only specify if different from recipe ms_dir
function : Python callable to execute
name : Name of function (if not given, will used function.__name__)
parameters : Parameters to parse to function
label : Function label; for logging purposes
"""
if self.jtype == "python":
self.image = image.__name__
if not callable(image):
raise utils.StimelaCabRuntimeError(
'Object given as function is not callable')
if self.name is None:
self.name = image.__name__
self.job = {
'function': image,
'parameters': config,
}
self.setup_job_log()
return 0
# check if name has any offending characters
offenders = re.findall('[^\w .-]', self.name)
if offenders:
raise StimelaCabParameterError('The cab name \'{:s}\' contains invalid characters.'
' Allowed charcaters are alphanumeric, plus [-_. ].'.format(self.name))
self.setup_job_log()
# make name palatable as container name
pausterized_name = re.sub("[\W]", "_", self.name)
name = '{0}-{1}{2}'.format(pausterized_name, id(image),
str(time.time()).replace('.', ''))
cont = getattr(CONT_MOD[self.jtype], "Container")(image, name,
logger=self.log,
workdir=CONT_IO["output"],
time_out=self.time_out)
cabpath = os.path.join(CAB_PATH, image.split("/")[1])
# In case the user specified a custom cab
cabpath = os.path.join(self.cabpath, image.split("/")[1]) if self.cabpath else cabpath
parameter_file = os.path.join(cabpath, 'parameters.json')
_cab = cab.CabDefinition(indir=indir, outdir=outdir,
msdir=msdir, parameter_file=parameter_file)
self.setup_output_wranglers(_cab.wranglers)
cont.IODEST = CONT_IO
cont.cabname = _cab.task
#
#Example
# ----------------
# casa_listobs:
# tag: <tag> ## optional
# version: <version> ## optional. If version is a dict, then ignore tag and priority and use <tag>:<version> pairs in dict
# force: true ## Continue even if tag is specified in the parameters.json file
no_tag_version = False
if self.tag or self.version:
tvi = None
if self.tag:
try:
tvi = _cab.tag.index(self.tag)
except ValueError:
pass
elif self.version:
try:
tvi = _cab.version.index(self.version)
except ValueError:
self.log.error(f"The version, {self.version}, specified for cab '{_cab.task}' is unknown. Available versions are {_cab.version}")
raise ValueError
if tvi is None:
tvi = -1
self.tag = _cab.tag[tvi]
self.version = _cab.version[tvi]
else:
self.tag = _cab.tag[-1]
self.version = _cab.version[-1]
cabspecs = self.recipe.cabspecs.get(cont.cabname, None)
if cabspecs:
_tag = cabspecs.get("tag", None)
_version = cabspecs.get("version", None)
_force_tag = cabspecs.get("force", False)
if isinstance(_version, dict):
if self.version in _version:
self.tag = _version[self.version]
elif _version:
self.version = _version
else:
self.tag = _tag
if self.version and self.version not in _cab.version:
self.log.error(f"The version, {self.version}, specified for cab '{_cab.task}' is unknown. Available versions are {_cab.version}")
raise ValueError
if not _tag:
idx = _cab.version.index(self.version)
self.tag = _cab.tag[idx]
self.force_tag = _force_tag
if self.tag not in _cab.tag:
if self.force_tag:
self.log.warn(f"You have chosen to use an unverified base image '{_cab.base}:{self.tag}'. May the force be with you.")
else:
raise StimelaBaseImageError(f"The base image '{_cab.base}' with tag '{self.tag}' has not been verified. If you wish to continue with it, please add the 'force_tag' when adding it to your recipe")
if self.jtype == "singularity":
simage = _cab.base.replace("/", "_")
cont.image = '{0:s}/{1:s}_{2:s}{3:s}'.format(singularity_image_dir,
simage, self.tag, singularity.suffix)
cont.image = os.path.abspath(cont.image)
if not os.path.exists(cont.image):
singularity.pull(":".join([_cab.base, self.tag]),
os.path.basename(cont.image), directory=singularity_image_dir)
else:
cont.image = ":".join([_cab.base, self.tag])
# Container parameter file will be updated and validated before the container is executed
cont._cab = _cab
cont.parameter_file_name = '{0}/{1}.json'.format(
self.recipe.parameter_file_dir, name)
self.image = str(cont.image)
# Remove dismissable kw arguments:
ops_to_pop = []
for op in config:
if isinstance(config[op], dismissable):
ops_to_pop.append(op)
for op in ops_to_pop:
arg = config.pop(op)()
if arg is not None:
config[op] = arg
cont.config = config
# These are standard volumes and
# environmental variables. These will be
# always exist in a cab container
cont.add_volume(cont.parameter_file_name,
f'{cab.MOUNT}/configfile', perm='ro', noverify=True)
cont.add_volume(os.path.join(cabpath, "src"), f"{cab.MOUNT}/code", "ro")
cont.add_volume(os.path.join(self.workdir, "passwd"), "/etc/passwd")
cont.add_volume(os.path.join(self.workdir, "group"), "/etc/group")
cont.RUNSCRIPT = f"/{self.jtype}_run"
if self.jtype == "singularity":
cont.RUNSCRIPT = f"/{self.jtype}"
if _cab.base.startswith("stimela/casa") or _cab.base.startswith("stimela/simms"):
cont.add_environ("LANGUAGE", "en_US.UTF-8")
cont.add_environ("LANG", "en_US.UTF-8")
cont.add_environ("LC_ALL", "en_US.UTF-8")
cont.execdir = self.workdir
else:
cont.RUNSCRIPT = f"/{self.jtype}_run"
runscript = shutil.which("stimela_runscript")
if runscript:
cont.add_volume(runscript,
cont.RUNSCRIPT, perm="ro")
else:
self.log.error("Stimela container runscript could not found.\
This may due to conflicting python or stimela installations in your $PATH.")
raise OSError
cont.add_environ('CONFIG', f'{cab.MOUNT}/configfile')
cont.add_environ('HOME', cont.IODEST["output"])
cont.add_environ('STIMELA_MOUNT', cab.MOUNT)
if msdir:
md = cont.IODEST["msfile"]
os.makedirs(msdir, exist_ok=True)
cont.add_volume(msdir, md)
cont.add_environ("MSDIR", md)
# Keep a record of the content of the
# volume
dirname, dirs, files = [a for a in next(os.walk(msdir))]
cont.msdir_content = {
"volume": dirname,
"dirs": dirs,
"files": files,
}
self.log.debug(
'Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format(msdir, md))
if indir:
cont.add_volume(indir, cont.IODEST["input"], perm='ro')
cont.add_environ("INPUT", cont.IODEST["input"])
# Keep a record of the content of the
# volume
dirname, dirs, files = [a for a in next(os.walk(indir))]
cont.input_content = {
"volume": dirname,
"dirs": dirs,
"files": files,
}
self.log.debug('Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format(
indir, cont.IODEST["input"]))
os.makedirs(outdir, exist_ok=True)
od = cont.IODEST["output"]
cont.logfile = self.logfile
cont.add_volume(outdir, od, "rw")
cont.add_environ("OUTPUT", od)
# temp files go into output
tmpfol = os.path.join(outdir, "tmp")
if not os.path.exists(tmpfol):
os.mkdir(tmpfol)
cont.add_volume(tmpfol, cont.IODEST["tmp"], "rw")
cont.add_environ("TMPDIR", cont.IODEST["tmp"])
self.log.debug(
'Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format(outdir, od))
# Added and ready for execution
self.job = cont
return 0
def pull(argv):
for i, arg in enumerate(argv):
if (arg[0] == '-') and arg[1].isdigit():
argv[i] = ' ' + arg
parser = ArgumentParser(description='Pull docker stimela base images')
add = parser.add_argument
add("-im", "--image", nargs="+", metavar="IMAGE[:TAG]",
help="Pull base image along with its tag (or version). Can be called multiple times")
add("-f", "--force", action="store_true",
help="force pull if image already exists")
add("-s", "--singularity", action="store_true",
help="Pull base images using singularity."
"Images will be pulled into the directory specified by the enviroment varaible, STIMELA_PULLFOLDER. $PWD by default")
add("-d", "--docker", action="store_true",
help="Pull base images using docker.")
add("-p", "--podman", action="store_true",
help="Pull base images using podman.")
add("-cb", "--cab-base", nargs="+",
help="Pull base image for specified cab")
add("-pf", "--pull-folder",
help="Images will be placed in this folder. Else, if the environmnental variable 'STIMELA_PULLFOLDER' is set, then images will be placed there. "
"Else, images will be placed in the current directory")
args = parser.parse_args(argv)
if args.pull_folder:
pull_folder = args.pull_folder
else:
try:
pull_folder = os.environ["STIMELA_PULLFOLDER"]
except KeyError:
pull_folder = "."
if args.podman:
jtype = "podman"
elif args.singularity:
jtype = "singularity"
elif args.docker:
jtype = "docker"
else:
jtype = "docker"
images_ = []
for cab in args.cab_base or []:
if cab in CAB:
filename = "/".join([stimela.CAB_PATH, cab, "parameters.json"])
param = utils.readJson(filename)
tags = param["tag"]
if not isinstance(tags, list):
tags = [tags]
for tag in tags:
images_.append(":".join([param["base"], tag]))
args.image = images_ or args.image
if args.image:
for image in args.image:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + singularity.suffix
if args.singularity:
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image)
elif args.podman:
podman.pull(image)
else:
docker.pull(image)
else:
base = []
for cab_ in CAB:
cabdir = "{:s}/{:s}".format(stimela.CAB_PATH, cab_)
_cab = info(cabdir, display=False)
tags = _cab.tag
if not isinstance(tags, list):
tags = [tags]
for tag in tags:
base.append(f"{_cab.base}:{tag}")
base = set(base)
for image in base:
if args.singularity:
simage = image.replace("/", "_")
simage = simage.replace(":", "_") + singularity.suffix
singularity.pull(
image, simage, directory=pull_folder, force=args.force)
elif args.docker:
docker.pull(image, force=args.force)
elif args.podman:
podman.pull(image, force=args.force)
else:
docker.pull(image, force=args.force)