def pull(argv): for i, arg in enumerate(argv): if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg parser = ArgumentParser(description='Pull docker stimela base images') add = parser.add_argument add("-im", "--image", action="append", metavar="IMAGE[:TAG]", help= "Pull base image along with its tag (or version). Can be called multiple times" ) add("-t", "--tag", help="Tag") add("-s", "--singularity", action="store_true", help="Use singularity instead of docker." "Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default" ) args = parser.parse_args(argv) log = logger.StimelaLogger(LOG_FILE) images = log.read()['images'] if args.image: for image in args.image: simage = image.replace("/", "_") simage = simage.replace(":", "_") + ".img" if args.singularity: singularity.pull(image, simage) else: docker.pull(image) log.log_image(image, 'pulled') else: base = [] for cab in CAB: image = "{:s}/{:s}".format(cargo.CAB_PATH, cab) base.append(utils.get_Dockerfile_base_image(image).split()[-1]) base = set(base) for image in base: if image not in ["stimela/ddfacet", "radioastro/ddfacet"]: if args.singularity: simage = image.replace("/", "_") simage = simage.replace(":", "_") + ".img" singularity.pull(image, simage) else: docker.pull(image) log.log_image(image, 'pulled') log.write()
def pull(argv): for i, arg in enumerate(argv): if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg parser = ArgumentParser(description='Pull docker stimela base images') add = parser.add_argument add("-im", "--image", nargs="+", metavar="IMAGE[:TAG]", help="Pull base image along with its tag (or version). Can be called multiple times") add("-f", "--force", action="store_true", help="force pull if image already exists") add("-s", "--singularity", action="store_true", help="Pull base images using singularity." "Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default") add("-d", "--docker", action="store_true", help="Pull base images using docker.") add("-p", "--podman", action="store_true", help="Pull base images using podman.") add("-cb", "--cab-base", nargs="+", help="Pull base image for specified cab") add("-pf", "--pull-folder", help="Images will be placed in this folder. Else, if the environmnental variable 'SINGULARITY_PULLFOLDER' is set, then images will be placed there. " "Else, images will be placed in the current directory") args = parser.parse_args(argv) if args.pull_folder: pull_folder = args.pull_folder else: try: pull_folder = os.environ["SINGULARITY_PULLFOLDER"] except KeyError: pull_folder = "." if args.docker: jtype = "docker" elif args.podman: jtype = "podman" elif args.singularity: jtype = "singularity" else: jtype = "udocker" log = logger.StimelaLogger(LOG_FILE, jtype=jtype) images = log.read()['images'] images_ = [] for cab in args.cab_base or []: if cab in CAB: filename = "/".join([stimela.CAB_PATH, cab, "parameters.json"]) param = utils.readJson(filename) images_.append(":".join([param["base"], param["tag"]])) args.image = images_ or args.image if args.image: for image in args.image: simage = image.replace("/", "_") simage = simage.replace(":", "_") + ".img" if args.singularity: singularity.pull( image, simage, directory=pull_folder, force=args.force) elif args.docker: docker.pull(image) log.log_image(image, 'pulled') elif args.podman: podman.pull(image) log.log_image(image, 'pulled') else: udocker.pull(image) log.log_image(image, 'pulled') else: base = [] for cab in CAB: image = "{:s}/{:s}".format(stimela.CAB_PATH, cab) base.append(utils.get_Dockerfile_base_image(image).split()[-1]) base = set(base) for image in base: if args.singularity: simage = image.replace("/", "_") simage = simage.replace(":", "_") + ".img" singularity.pull( image, simage, directory=pull_folder, force=args.force) elif args.docker: docker.pull(image, force=args.force) log.log_image(image, 'pulled') elif args.podman: podman.pull(image, force=args.force) log.log_image(image, 'pulled') else: udocker.pull(image, force=args.force) log.log_image(image, 'pulled') log.write()
def setup_job(self, image, config, indir=None, outdir=None, msdir=None, singularity_image_dir=None): """ Setup job image : stimela cab name, e.g. 'cab/simms' name : This name will be part of the name of the contaier that will execute the task (now optional) config : Dictionary of options to parse to the task. This will modify the parameters in the default parameter file which can be viewd by running 'stimela cabs -i <cab name>', e.g 'stimela cabs -i simms' indir : input dirctory for cab outdir : output directory for cab msdir : MS directory for cab. Only specify if different from recipe ms_dir function : Python callable to execute name : Name of function (if not given, will used function.__name__) parameters : Parameters to parse to function label : Function label; for logging purposes """ if self.jtype == "python": self.image = image.__name__ if not callable(image): raise utils.StimelaCabRuntimeError( 'Object given as function is not callable') if self.name is None: self.name = image.__name__ self.job = { 'function': image, 'parameters': config, } self.setup_job_log() return 0 # check if name has any offending characters offenders = re.findall('[^\w .-]', self.name) if offenders: raise StimelaCabParameterError('The cab name \'{:s}\' contains invalid characters.' ' Allowed charcaters are alphanumeric, plus [-_. ].'.format(self.name)) self.setup_job_log() # make name palatable as container name pausterized_name = re.sub("[\W]", "_", self.name) name = '{0}-{1}{2}'.format(pausterized_name, id(image), str(time.time()).replace('.', '')) cont = getattr(CONT_MOD[self.jtype], "Container")(image, name, logger=self.log, workdir=CONT_IO["output"], time_out=self.time_out) cabpath = os.path.join(CAB_PATH, image.split("/")[1]) # In case the user specified a custom cab cabpath = os.path.join(self.cabpath, image.split("/")[1]) if self.cabpath else cabpath parameter_file = os.path.join(cabpath, 'parameters.json') _cab = cab.CabDefinition(indir=indir, outdir=outdir, msdir=msdir, parameter_file=parameter_file) self.setup_output_wranglers(_cab.wranglers) cont.IODEST = CONT_IO cont.cabname = _cab.task # #Example # ---------------- # casa_listobs: # tag: <tag> ## optional # version: <version> ## optional. If version is a dict, then ignore tag and priority and use <tag>:<version> pairs in dict # force: true ## Continue even if tag is specified in the parameters.json file no_tag_version = False if self.tag or self.version: tvi = None if self.tag: try: tvi = _cab.tag.index(self.tag) except ValueError: pass elif self.version: try: tvi = _cab.version.index(self.version) except ValueError: self.log.error(f"The version, {self.version}, specified for cab '{_cab.task}' is unknown. Available versions are {_cab.version}") raise ValueError if tvi is None: tvi = -1 self.tag = _cab.tag[tvi] self.version = _cab.version[tvi] else: self.tag = _cab.tag[-1] self.version = _cab.version[-1] cabspecs = self.recipe.cabspecs.get(cont.cabname, None) if cabspecs: _tag = cabspecs.get("tag", None) _version = cabspecs.get("version", None) _force_tag = cabspecs.get("force", False) if isinstance(_version, dict): if self.version in _version: self.tag = _version[self.version] elif _version: self.version = _version else: self.tag = _tag if self.version and self.version not in _cab.version: self.log.error(f"The version, {self.version}, specified for cab '{_cab.task}' is unknown. Available versions are {_cab.version}") raise ValueError if not _tag: idx = _cab.version.index(self.version) self.tag = _cab.tag[idx] self.force_tag = _force_tag if self.tag not in _cab.tag: if self.force_tag: self.log.warn(f"You have chosen to use an unverified base image '{_cab.base}:{self.tag}'. May the force be with you.") else: raise StimelaBaseImageError(f"The base image '{_cab.base}' with tag '{self.tag}' has not been verified. If you wish to continue with it, please add the 'force_tag' when adding it to your recipe") if self.jtype == "singularity": simage = _cab.base.replace("/", "_") cont.image = '{0:s}/{1:s}_{2:s}{3:s}'.format(singularity_image_dir, simage, self.tag, singularity.suffix) cont.image = os.path.abspath(cont.image) if not os.path.exists(cont.image): singularity.pull(":".join([_cab.base, self.tag]), os.path.basename(cont.image), directory=singularity_image_dir) else: cont.image = ":".join([_cab.base, self.tag]) # Container parameter file will be updated and validated before the container is executed cont._cab = _cab cont.parameter_file_name = '{0}/{1}.json'.format( self.recipe.parameter_file_dir, name) self.image = str(cont.image) # Remove dismissable kw arguments: ops_to_pop = [] for op in config: if isinstance(config[op], dismissable): ops_to_pop.append(op) for op in ops_to_pop: arg = config.pop(op)() if arg is not None: config[op] = arg cont.config = config # These are standard volumes and # environmental variables. These will be # always exist in a cab container cont.add_volume(cont.parameter_file_name, f'{cab.MOUNT}/configfile', perm='ro', noverify=True) cont.add_volume(os.path.join(cabpath, "src"), f"{cab.MOUNT}/code", "ro") cont.add_volume(os.path.join(self.workdir, "passwd"), "/etc/passwd") cont.add_volume(os.path.join(self.workdir, "group"), "/etc/group") cont.RUNSCRIPT = f"/{self.jtype}_run" if self.jtype == "singularity": cont.RUNSCRIPT = f"/{self.jtype}" if _cab.base.startswith("stimela/casa") or _cab.base.startswith("stimela/simms"): cont.add_environ("LANGUAGE", "en_US.UTF-8") cont.add_environ("LANG", "en_US.UTF-8") cont.add_environ("LC_ALL", "en_US.UTF-8") cont.execdir = self.workdir else: cont.RUNSCRIPT = f"/{self.jtype}_run" runscript = shutil.which("stimela_runscript") if runscript: cont.add_volume(runscript, cont.RUNSCRIPT, perm="ro") else: self.log.error("Stimela container runscript could not found.\ This may due to conflicting python or stimela installations in your $PATH.") raise OSError cont.add_environ('CONFIG', f'{cab.MOUNT}/configfile') cont.add_environ('HOME', cont.IODEST["output"]) cont.add_environ('STIMELA_MOUNT', cab.MOUNT) if msdir: md = cont.IODEST["msfile"] os.makedirs(msdir, exist_ok=True) cont.add_volume(msdir, md) cont.add_environ("MSDIR", md) # Keep a record of the content of the # volume dirname, dirs, files = [a for a in next(os.walk(msdir))] cont.msdir_content = { "volume": dirname, "dirs": dirs, "files": files, } self.log.debug( 'Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format(msdir, md)) if indir: cont.add_volume(indir, cont.IODEST["input"], perm='ro') cont.add_environ("INPUT", cont.IODEST["input"]) # Keep a record of the content of the # volume dirname, dirs, files = [a for a in next(os.walk(indir))] cont.input_content = { "volume": dirname, "dirs": dirs, "files": files, } self.log.debug('Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format( indir, cont.IODEST["input"])) os.makedirs(outdir, exist_ok=True) od = cont.IODEST["output"] cont.logfile = self.logfile cont.add_volume(outdir, od, "rw") cont.add_environ("OUTPUT", od) # temp files go into output tmpfol = os.path.join(outdir, "tmp") if not os.path.exists(tmpfol): os.mkdir(tmpfol) cont.add_volume(tmpfol, cont.IODEST["tmp"], "rw") cont.add_environ("TMPDIR", cont.IODEST["tmp"]) self.log.debug( 'Mounting volume \'{0}\' from local file system to \'{1}\' in the container'.format(outdir, od)) # Added and ready for execution self.job = cont return 0
def pull(argv): for i, arg in enumerate(argv): if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg parser = ArgumentParser(description='Pull docker stimela base images') add = parser.add_argument add("-im", "--image", nargs="+", metavar="IMAGE[:TAG]", help="Pull base image along with its tag (or version). Can be called multiple times") add("-f", "--force", action="store_true", help="force pull if image already exists") add("-s", "--singularity", action="store_true", help="Pull base images using singularity." "Images will be pulled into the directory specified by the enviroment varaible, STIMELA_PULLFOLDER. $PWD by default") add("-d", "--docker", action="store_true", help="Pull base images using docker.") add("-p", "--podman", action="store_true", help="Pull base images using podman.") add("-cb", "--cab-base", nargs="+", help="Pull base image for specified cab") add("-pf", "--pull-folder", help="Images will be placed in this folder. Else, if the environmnental variable 'STIMELA_PULLFOLDER' is set, then images will be placed there. " "Else, images will be placed in the current directory") args = parser.parse_args(argv) if args.pull_folder: pull_folder = args.pull_folder else: try: pull_folder = os.environ["STIMELA_PULLFOLDER"] except KeyError: pull_folder = "." if args.podman: jtype = "podman" elif args.singularity: jtype = "singularity" elif args.docker: jtype = "docker" else: jtype = "docker" images_ = [] for cab in args.cab_base or []: if cab in CAB: filename = "/".join([stimela.CAB_PATH, cab, "parameters.json"]) param = utils.readJson(filename) tags = param["tag"] if not isinstance(tags, list): tags = [tags] for tag in tags: images_.append(":".join([param["base"], tag])) args.image = images_ or args.image if args.image: for image in args.image: simage = image.replace("/", "_") simage = simage.replace(":", "_") + singularity.suffix if args.singularity: singularity.pull( image, simage, directory=pull_folder, force=args.force) elif args.docker: docker.pull(image) elif args.podman: podman.pull(image) else: docker.pull(image) else: base = [] for cab_ in CAB: cabdir = "{:s}/{:s}".format(stimela.CAB_PATH, cab_) _cab = info(cabdir, display=False) tags = _cab.tag if not isinstance(tags, list): tags = [tags] for tag in tags: base.append(f"{_cab.base}:{tag}") base = set(base) for image in base: if args.singularity: simage = image.replace("/", "_") simage = simage.replace(":", "_") + singularity.suffix singularity.pull( image, simage, directory=pull_folder, force=args.force) elif args.docker: docker.pull(image, force=args.force) elif args.podman: podman.pull(image, force=args.force) else: docker.pull(image, force=args.force)