def _image_file(self, image): lib_path = path(os.environ['SOS_SINGULARITY_LIBRARY'] ) if 'SOS_SINGULARITY_LIBRARY' in os.environ else path( '~/.sos/singularity/library') if not os.path.isdir(lib_path): try: os.makedirs(lib_path, exist_ok=True) except: raise RuntimeError( f'Failed to create singularity library directory {lib_path}' ) if '://' in image: ctx, cname = image.split('://', 1) if ctx == 'file': return image else: return os.path.join( lib_path, cname.replace('/', '-').replace(':', '-') + '.sif') elif os.path.isfile(image): # if image is a filename, ok return image else: # otherwise assuming it is an image in SoS Singulariry Library return os.path.join(lib_path, image)
def pull(self, image): self._ensure_singularity() if image in self.pulled_images: return if image.startswith('instance://'): return image image_file = self._image_file(image) if os.path.exists(image_file): env.logger.debug(f'Using existing singularity image {image_file}') return if '://' not in image: raise ValueError( f'Cannot locate or pull singularity image {image}') # if image is specified, check if it is available locally. If not, pull it try: print(f'HINT: Pulling image {image} to {image_file}') subprocess.check_output('singularity pull --name {} {}'.format( image_file, image), stderr=subprocess.STDOUT, shell=True, universal_newlines=True) self.pulled_images.add(image) except subprocess.CalledProcessError as exc: env.logger.warning(f'Failed to pull {image}: {exc.output}') if not path(image_file).exists(): raise ValueError( f'Image {image_file} does not exist after pulling {image}.')
def get_step_option(self): if not self.prepare: if self.conf is None or (self.step.name in self.conf and self.conf[self.step.name]['queue'] is None) \ or (self.step.name not in self.conf and self.conf['default']['queue'] is None): return self.step_option += f"task: {', '.join([str(k) + ' = ' + (repr(v) if isinstance(v, str) and k != 'trunk_workers' else str(v)) for k, v in self.conf[self.step.name if self.step.name in self.conf else 'default'].items()])}, tags = f'{self.step.name}_{{_output:bn}}'" self.step_option += '\n' if path(self.step.workdir).absolute( ) == path.cwd() else f', workdir = {repr(self.step.workdir)}\n'
def build(self, script, **kwargs): if not self.client: raise RuntimeError( 'Cannot connect to the Docker daemon. Is the docker daemon running on this host?') with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: if script: with open(os.path.join(tempdir, 'Dockerfile'), 'w') as df: df.write(script) file_opt = [tempdir] else: if 'file' not in kwargs: raise RuntimeError( 'Docker file must be specified with option file if not directly included.') file_opt = ['--file', kwargs['file']] other_opts = [] for arg, value in kwargs.items(): # boolean args if arg in ('compress', 'disable_content_trust', 'force_rm', 'memory_swap', 'no_cache', 'pull', 'quiet', 'rm', 'squash', 'stream'): if value is True: other_opts.append(f'--{arg.replace("_", "-")}') else: env.logger.warning( f'Boolean {arg} is ignored (True should be provided)') elif arg in ('add_host', 'build_arg', 'cache_from', 'cgroup_parent', 'cpu_period', 'cpu_quota', 'cpu-shares', 'cpuset_cpus', 'cpuset_mems', 'label', 'memory', 'network', 'platform', 'security_opt', 'shm_size', 'tag', 'target', 'ulimit'): other_opts.extend([f'--{arg.replace("_", "-")}', value]) cmd = subprocess.list2cmdline( ['docker', 'build'] + file_opt + other_opts) env.logger.debug(cmd) if env.config['run_mode'] == 'dryrun': print(f'HINT: {cmd}') print(script) return 0 ret = self._run_cmd(cmd, **kwargs) if ret != 0: if script: debug_script_dir = os.path.join(env.exec_dir, '.sos') msg = 'The Dockerfile has been saved to {}/Dockerfile. To reproduce the error please run:\n``{}``'.format( debug_script_dir, cmd.replace(tempdir, path(debug_script_dir))) shutil.copy(os.path.join( tempdir, 'Dockerfile'), debug_script_dir) else: msg = f'To reproduce this error please run {cmd}' raise subprocess.CalledProcessError( returncode=ret, cmd=cmd, stderr=msg) # if a tag is given, check if the image is built if 'tag' in kwargs and not self._is_image_avail(kwargs['tag']): raise RuntimeError( 'Image with tag {} is not created.'.format(kwargs['tag']))
def pull(self, image): self._ensure_singularity() if image in self.pulled_images: return if image.startswith('instance://'): return image image_file = self._image_file(image) if os.path.exists(image_file): env.logger.debug( f'Using existing singularity image {image_file.replace(os.path.expanduser("~"), "~")}' ) return if '://' not in image: raise ValueError( f'Cannot locate or pull singularity image {image}') # ask controller while True: res = request_answer_from_controller( ['resource', 'singularity_image', 'request', image]) if res == 'pending': time.sleep(0.5) elif res == 'available': return elif res == 'unavailable': raise RuntimeError(f'Docker image {image} is unavailable') elif res == 'help yourself': break else: raise ValueError(f'Unrecognized request from controller {res}') # if image is specified, check if it is available locally. If not, pull it try: print( f'HINT: Pulling singularity image {image} to {image_file.replace(os.path.expanduser("~"), "~")}' ) subprocess.check_output('singularity pull {} {}'.format( image_file, image), stderr=subprocess.STDOUT, shell=True, universal_newlines=True) self.pulled_images.add(image) except subprocess.CalledProcessError as exc: send_message_to_controller( ['resource', 'singularity_image', 'unavailable', image]) env.logger.warning(f'Failed to pull {image}: {exc.output}') if not path(image_file).exists(): raise ValueError( f'Image {image_file} does not exist after pulling {image}.') else: print(f'HINT: Singularity image {image} is now up to date') send_message_to_controller( ['resource', 'singularity_image', 'available', image])
def test_zap(self): '''Test zap''' with open('testzap.txt', 'w') as sf: sf.write('some text') path('testzap.txt').zap() self.assertTrue(os.path.isfile('testzap.txt.zapped')) self.assertFalse(os.path.isfile('testzap.txt')) # re-zap is ok file_target('testzap.txt').zap() self.assertTrue(os.path.isfile('testzap.txt.zapped')) self.assertFalse(os.path.isfile('testzap.txt')) # non-existent file os.remove('testzap.txt.zapped') self.assertRaises(FileNotFoundError, path('testzap.txt').zap) # with open('testzap.txt', 'w') as sf: sf.write('some text') with open('testzap1.txt', 'w') as sf: sf.write('some text') paths('testzap.txt', 'testzap1.txt').zap() self.assertTrue(os.path.isfile('testzap.txt.zapped')) self.assertFalse(os.path.isfile('testzap.txt')) self.assertTrue(os.path.isfile('testzap1.txt.zapped')) self.assertFalse(os.path.isfile('testzap1.txt')) # os.remove('testzap.txt.zapped') os.remove('testzap1.txt.zapped') with open('testzap.txt', 'w') as sf: sf.write('some text') with open('testzap1.txt', 'w') as sf: sf.write('some text') sos_targets(['testzap.txt', 'testzap1.txt']).zap() self.assertTrue(os.path.isfile('testzap.txt.zapped')) self.assertFalse(os.path.isfile('testzap.txt')) self.assertTrue(os.path.isfile('testzap1.txt.zapped')) self.assertFalse(os.path.isfile('testzap1.txt'))
def test_zap(self): """Test zap""" with open("testzap.txt", "w") as sf: sf.write("some text") path("testzap.txt").zap() self.assertTrue(os.path.isfile("testzap.txt.zapped")) self.assertFalse(os.path.isfile("testzap.txt")) # re-zap is ok file_target("testzap.txt").zap() self.assertTrue(os.path.isfile("testzap.txt.zapped")) self.assertFalse(os.path.isfile("testzap.txt")) # non-existent file os.remove("testzap.txt.zapped") self.assertRaises(FileNotFoundError, path("testzap.txt").zap) # with open("testzap.txt", "w") as sf: sf.write("some text") with open("testzap1.txt", "w") as sf: sf.write("some text") paths("testzap.txt", "testzap1.txt").zap() self.assertTrue(os.path.isfile("testzap.txt.zapped")) self.assertFalse(os.path.isfile("testzap.txt")) self.assertTrue(os.path.isfile("testzap1.txt.zapped")) self.assertFalse(os.path.isfile("testzap1.txt")) # os.remove("testzap.txt.zapped") os.remove("testzap1.txt.zapped") with open("testzap.txt", "w") as sf: sf.write("some text") with open("testzap1.txt", "w") as sf: sf.write("some text") sos_targets(["testzap.txt", "testzap1.txt"]).zap() self.assertTrue(os.path.isfile("testzap.txt.zapped")) self.assertFalse(os.path.isfile("testzap.txt")) self.assertTrue(os.path.isfile("testzap1.txt.zapped")) self.assertFalse(os.path.isfile("testzap1.txt"))
def run(self, image, script='', interpreter='', args='', suffix='.sh', **kwargs): self._ensure_singularity() # env.logger.debug('singularity_run with keyword args {}'.format(kwargs)) # # now, write a temporary file to a tempoary directory under the current directory, this is because # we need to share the directory to ... with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: # keep the temporary script for debugging purposes # tempdir = tempfile.mkdtemp(dir=os.getcwd()) tempscript = 'singularity_run_{}{}'.format(os.getpid(), suffix) if script: with open(os.path.join(tempdir, tempscript), 'w') as script_file: # the input script might have windows new line but the container # will need linux new line for proper execution #1023 script_file.write('\n'.join(script.splitlines())) # # if there is an interpreter and with args if not args: args = '{filename:pq}' # # under mac, we by default share /Users within Singularity if 'bind' in kwargs: binds = [kwargs['bind']] if isinstance(kwargs['bind'], str) else kwargs['bind'] bind_opt = ' '.join('-B {}'.format(x) for x in binds) else: bind_opt = '' cmd_opt = interpolate( f'{interpreter if isinstance(interpreter, str) else interpreter[0]} {args}', { 'filename': path(tempdir) / tempscript, 'script': script }) cmd = 'singularity exec {} {} {}'.format( bind_opt, # volumes self._image_file(image), cmd_opt) env.logger.debug(cmd) if env.config['run_mode'] == 'dryrun': print(f'HINT: {cmd}') print(script) return 0 ret = self._run_cmd(cmd, **kwargs) if ret != 0: debug_script_dir = os.path.join(os.path.expanduser('~'), '.sos') msg = 'The script has been saved to {}/{}. To reproduce the error please run:\n``{}``'.format( debug_script_dir, tempscript, cmd.replace(f'{path(tempdir):p}', f'{path(debug_script_dir):p}')) shutil.copy(os.path.join(tempdir, tempscript), debug_script_dir) out = f", stdout={kwargs['stdout']}" if 'stdout' in kwargs and os.path.isfile( kwargs['stdout']) and os.path.getsize( kwargs['stdout']) > 0 else '' err = f", stderr={kwargs['stderr']}" if 'stderr' in kwargs and os.path.isfile( kwargs['stderr']) and os.path.getsize( kwargs['stderr']) > 0 else '' msg = f"Executing script in Singularity returns an error (exitcode={ret}{err}{out}).\n{msg}" raise subprocess.CalledProcessError( returncode=ret, cmd=cmd.replace(tempdir, debug_script_dir), stderr=msg) return 0
def get_action(self): if self.prepare: combined_params = '[([{0}], {1}) {2}]'.\ format(', '.join([f"('{x}', _{x})" for x in reversed(self.params)]), None if self.loop_string[1] == '' else ("f\"{' '.join(__i__)}\"" if len(self.current_depends) > 1 else "f'{__i__}'"), ' '.join(self.loop_string) + self.filter_string) input_str = '[]' if self.input_vars is None else '{0} if {0} is not None else []'.format( self.input_vars) output_str = f"__{n2a(int(self.step_map[self.step.name][1])).lower()}_{self.step.name}_output__" # FIXME: multiple output to be implemented ext_str = self.step.plugin.output_ext if ( len(self.step.exe['path']) == 0 and len(self.step.rv) > 0) else 'yml' if len(self.current_depends): self.action += f"__io_db__['{self.step.name}:' + str(__pipeline_id__)] = dict([(' '.join((y, x[1])), dict([('__pipeline_id__', __pipeline_id__), ('__pipeline_name__', __pipeline_name__), ('__module__', '{self.step.name}'), ('__out_vars__', __out_vars__)] + x[0])) for x, y in zip({combined_params}, {output_str})] + [('__input_output___', ({input_str}, {output_str})), ('__ext__', '{ext_str}')])\n" else: self.action += f"__io_db__['{self.step.name}:' + str(__pipeline_id__)] = dict([(y, dict([('__pipeline_id__', __pipeline_id__), ('__pipeline_name__', __pipeline_name__), ('__module__', '{self.step.name}'), ('__out_vars__', __out_vars__)] + x[0])) for x, y in zip({combined_params}, {output_str})] + [('__input_output___', ({input_str}, {output_str})), ('__ext__', '{ext_str}')])\n" else: # FIXME: have not considered multi-action module (or compound module) yet # Create fake loop for now with idx going around for idx, (plugin, cmd) in enumerate( zip([self.step.plugin], [self.step.exe])): sigil = '$[ ]' if plugin.name == 'bash' else '${ }' if self.conf is None: self.action += f'{"python3" if plugin.name == "python" else plugin.name}: expand = "{sigil}"' if path(self.step.workdir).absolute() != path.cwd(): self.action += f", workdir = {repr(self.step.workdir)}" self.action += f', stderr = f"{{_output:n}}.stderr", stdout = f"{{_output:n}}.stdout"' else: self.action += f'{"python3" if plugin.name == "python" else plugin.name}: expand = "{sigil}"' if self.step.container: self.action += f", container={repr(self.step.container)}" if self.step.container_engine: self.action += f", engine={repr(self.step.container_engine)}" if len(self.step.path): self.action += ", env={'PATH': '%s:' + os.environ['PATH']}" % ":".join(self.step.path) self.action += plugin.get_cmd_args(cmd['args'], self.params) # Add action if len(cmd['path']) == 0: if self.debug: script = plugin.get_return(None) else: script_begin = plugin.load_env( self.step.depends, idx > 0 and len(self.step.rv)) script_begin += '\n' + plugin.get_input( self.params, self.step.libpath if self.step.libpath else []) if len(self.step.rf): script_begin += '\n' + plugin.get_output( self.step.rf) script_begin = '\n'.join( [x for x in script_begin.split('\n') if x]) script_begin = f"{cmd['header']}\n{script_begin.strip()}\n\n## BEGIN DSC CORE" script_end = plugin.get_return( self.step.rv) if len(self.step.rv) else '' script_end = f'## END DSC CORE\n\n{script_end.strip()}'.strip( ) script = '\n'.join( [script_begin, cmd['content'], script_end]) if self.try_catch: script = plugin.add_try( script, len([self.step.rf.values()])) script = f"""## {str(plugin)} script UUID: ${{DSC_STEP_ID_}}\n{script}\n""" script = '\n'.join( [f' {x}' for x in script.split('\n')]) self.action += script self.exe_signature.append(cmd['signature']) else: self.exe_check.append( f"executable({repr(cmd['path'])})") self.action += f"\t{cmd['path']} {'$*' if cmd['args'] else ''}\n"