def _image_file(self, image):
lib_path = path(os.environ['SOS_SINGULARITY_LIBRARY']
) if 'SOS_SINGULARITY_LIBRARY' in os.environ else path(
'~/.sos/singularity/library')
if not os.path.isdir(lib_path):
try:
os.makedirs(lib_path, exist_ok=True)
except:
raise RuntimeError(
f'Failed to create singularity library directory {lib_path}'
)
if '://' in image:
ctx, cname = image.split('://', 1)
if ctx == 'file':
return image
else:
return os.path.join(
lib_path,
cname.replace('/', '-').replace(':', '-') + '.sif')
elif os.path.isfile(image):
# if image is a filename, ok
return image
else:
# otherwise assuming it is an image in SoS Singulariry Library
return os.path.join(lib_path, image)
def pull(self, image):
self._ensure_singularity()
if image in self.pulled_images:
return
if image.startswith('instance://'):
return image
image_file = self._image_file(image)
if os.path.exists(image_file):
env.logger.debug(f'Using existing singularity image {image_file}')
return
if '://' not in image:
raise ValueError(
f'Cannot locate or pull singularity image {image}')
# if image is specified, check if it is available locally. If not, pull it
try:
print(f'HINT: Pulling image {image} to {image_file}')
subprocess.check_output('singularity pull --name {} {}'.format(
image_file, image),
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
self.pulled_images.add(image)
except subprocess.CalledProcessError as exc:
env.logger.warning(f'Failed to pull {image}: {exc.output}')
if not path(image_file).exists():
raise ValueError(
f'Image {image_file} does not exist after pulling {image}.')
def get_step_option(self):
if not self.prepare:
if self.conf is None or (self.step.name in self.conf and self.conf[self.step.name]['queue'] is None) \
or (self.step.name not in self.conf and self.conf['default']['queue'] is None):
return
self.step_option += f"task: {', '.join([str(k) + ' = ' + (repr(v) if isinstance(v, str) and k != 'trunk_workers' else str(v)) for k, v in self.conf[self.step.name if self.step.name in self.conf else 'default'].items()])}, tags = f'{self.step.name}_{{_output:bn}}'"
self.step_option += '\n' if path(self.step.workdir).absolute(
) == path.cwd() else f', workdir = {repr(self.step.workdir)}\n'
def build(self, script, **kwargs):
if not self.client:
raise RuntimeError(
'Cannot connect to the Docker daemon. Is the docker daemon running on this host?')
with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir:
if script:
with open(os.path.join(tempdir, 'Dockerfile'), 'w') as df:
df.write(script)
file_opt = [tempdir]
else:
if 'file' not in kwargs:
raise RuntimeError(
'Docker file must be specified with option file if not directly included.')
file_opt = ['--file', kwargs['file']]
other_opts = []
for arg, value in kwargs.items():
# boolean args
if arg in ('compress', 'disable_content_trust', 'force_rm', 'memory_swap',
'no_cache', 'pull', 'quiet', 'rm', 'squash', 'stream'):
if value is True:
other_opts.append(f'--{arg.replace("_", "-")}')
else:
env.logger.warning(
f'Boolean {arg} is ignored (True should be provided)')
elif arg in ('add_host', 'build_arg', 'cache_from', 'cgroup_parent',
'cpu_period', 'cpu_quota', 'cpu-shares', 'cpuset_cpus', 'cpuset_mems',
'label', 'memory', 'network', 'platform', 'security_opt', 'shm_size',
'tag', 'target', 'ulimit'):
other_opts.extend([f'--{arg.replace("_", "-")}', value])
cmd = subprocess.list2cmdline(
['docker', 'build'] + file_opt + other_opts)
env.logger.debug(cmd)
if env.config['run_mode'] == 'dryrun':
print(f'HINT: {cmd}')
print(script)
return 0
ret = self._run_cmd(cmd, **kwargs)
if ret != 0:
if script:
debug_script_dir = os.path.join(env.exec_dir, '.sos')
msg = 'The Dockerfile has been saved to {}/Dockerfile. To reproduce the error please run:\n``{}``'.format(
debug_script_dir, cmd.replace(tempdir, path(debug_script_dir)))
shutil.copy(os.path.join(
tempdir, 'Dockerfile'), debug_script_dir)
else:
msg = f'To reproduce this error please run {cmd}'
raise subprocess.CalledProcessError(
returncode=ret, cmd=cmd, stderr=msg)
# if a tag is given, check if the image is built
if 'tag' in kwargs and not self._is_image_avail(kwargs['tag']):
raise RuntimeError(
'Image with tag {} is not created.'.format(kwargs['tag']))
def pull(self, image):
self._ensure_singularity()
if image in self.pulled_images:
return
if image.startswith('instance://'):
return image
image_file = self._image_file(image)
if os.path.exists(image_file):
env.logger.debug(
f'Using existing singularity image {image_file.replace(os.path.expanduser("~"), "~")}'
)
return
if '://' not in image:
raise ValueError(
f'Cannot locate or pull singularity image {image}')
# ask controller
while True:
res = request_answer_from_controller(
['resource', 'singularity_image', 'request', image])
if res == 'pending':
time.sleep(0.5)
elif res == 'available':
return
elif res == 'unavailable':
raise RuntimeError(f'Docker image {image} is unavailable')
elif res == 'help yourself':
break
else:
raise ValueError(f'Unrecognized request from controller {res}')
# if image is specified, check if it is available locally. If not, pull it
try:
print(
f'HINT: Pulling singularity image {image} to {image_file.replace(os.path.expanduser("~"), "~")}'
)
subprocess.check_output('singularity pull {} {}'.format(
image_file, image),
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
self.pulled_images.add(image)
except subprocess.CalledProcessError as exc:
send_message_to_controller(
['resource', 'singularity_image', 'unavailable', image])
env.logger.warning(f'Failed to pull {image}: {exc.output}')
if not path(image_file).exists():
raise ValueError(
f'Image {image_file} does not exist after pulling {image}.')
else:
print(f'HINT: Singularity image {image} is now up to date')
send_message_to_controller(
['resource', 'singularity_image', 'available', image])
def test_zap(self):
'''Test zap'''
with open('testzap.txt', 'w') as sf:
sf.write('some text')
path('testzap.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
# re-zap is ok
file_target('testzap.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
# non-existent file
os.remove('testzap.txt.zapped')
self.assertRaises(FileNotFoundError, path('testzap.txt').zap)
#
with open('testzap.txt', 'w') as sf:
sf.write('some text')
with open('testzap1.txt', 'w') as sf:
sf.write('some text')
paths('testzap.txt', 'testzap1.txt').zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
self.assertTrue(os.path.isfile('testzap1.txt.zapped'))
self.assertFalse(os.path.isfile('testzap1.txt'))
#
os.remove('testzap.txt.zapped')
os.remove('testzap1.txt.zapped')
with open('testzap.txt', 'w') as sf:
sf.write('some text')
with open('testzap1.txt', 'w') as sf:
sf.write('some text')
sos_targets(['testzap.txt', 'testzap1.txt']).zap()
self.assertTrue(os.path.isfile('testzap.txt.zapped'))
self.assertFalse(os.path.isfile('testzap.txt'))
self.assertTrue(os.path.isfile('testzap1.txt.zapped'))
self.assertFalse(os.path.isfile('testzap1.txt'))
def test_zap(self):
"""Test zap"""
with open("testzap.txt", "w") as sf:
sf.write("some text")
path("testzap.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
# re-zap is ok
file_target("testzap.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
# non-existent file
os.remove("testzap.txt.zapped")
self.assertRaises(FileNotFoundError, path("testzap.txt").zap)
#
with open("testzap.txt", "w") as sf:
sf.write("some text")
with open("testzap1.txt", "w") as sf:
sf.write("some text")
paths("testzap.txt", "testzap1.txt").zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
self.assertTrue(os.path.isfile("testzap1.txt.zapped"))
self.assertFalse(os.path.isfile("testzap1.txt"))
#
os.remove("testzap.txt.zapped")
os.remove("testzap1.txt.zapped")
with open("testzap.txt", "w") as sf:
sf.write("some text")
with open("testzap1.txt", "w") as sf:
sf.write("some text")
sos_targets(["testzap.txt", "testzap1.txt"]).zap()
self.assertTrue(os.path.isfile("testzap.txt.zapped"))
self.assertFalse(os.path.isfile("testzap.txt"))
self.assertTrue(os.path.isfile("testzap1.txt.zapped"))
self.assertFalse(os.path.isfile("testzap1.txt"))
def run(self,
image,
script='',
interpreter='',
args='',
suffix='.sh',
**kwargs):
self._ensure_singularity()
#
env.logger.debug('singularity_run with keyword args {}'.format(kwargs))
#
# now, write a temporary file to a tempoary directory under the current directory, this is because
# we need to share the directory to ...
with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir:
# keep the temporary script for debugging purposes
# tempdir = tempfile.mkdtemp(dir=os.getcwd())
tempscript = 'singularity_run_{}{}'.format(os.getpid(), suffix)
if script:
with open(os.path.join(tempdir, tempscript),
'w') as script_file:
# the input script might have windows new line but the container
# will need linux new line for proper execution #1023
script_file.write('\n'.join(script.splitlines()))
#
# if there is an interpreter and with args
if not args:
args = '{filename:pq}'
#
# under mac, we by default share /Users within Singularity
if 'bind' in kwargs:
binds = [kwargs['bind']] if isinstance(kwargs['bind'],
str) else kwargs['bind']
bind_opt = ' '.join('-B {}'.format(x) for x in binds)
else:
bind_opt = ''
cmd_opt = interpolate(
f'{interpreter if isinstance(interpreter, str) else interpreter[0]} {args}',
{
'filename': path(tempdir) / tempscript,
'script': script
})
cmd = 'singularity exec {} {} {}'.format(
bind_opt, # volumes
self._image_file(image),
cmd_opt)
env.logger.debug(cmd)
if env.config['run_mode'] == 'dryrun':
print(f'HINT: {cmd}')
print(script)
return 0
ret = self._run_cmd(cmd, **kwargs)
if ret != 0:
debug_script_dir = os.path.join(os.path.expanduser('~'), '.sos')
msg = 'The script has been saved to {}/{}. To reproduce the error please run:\n``{}``'.format(
debug_script_dir, tempscript,
cmd.replace(f'{path(tempdir):p}',
f'{path(debug_script_dir):p}'))
shutil.copy(os.path.join(tempdir, tempscript), debug_script_dir)
out = f", stdout={kwargs['stdout']}" if 'stdout' in kwargs and os.path.isfile(
kwargs['stdout']) and os.path.getsize(
kwargs['stdout']) > 0 else ''
err = f", stderr={kwargs['stderr']}" if 'stderr' in kwargs and os.path.isfile(
kwargs['stderr']) and os.path.getsize(
kwargs['stderr']) > 0 else ''
msg = f"Executing script in Singularity returns an error (exitcode={ret}{err}{out}).\n{msg}"
raise subprocess.CalledProcessError(
returncode=ret,
cmd=cmd.replace(tempdir, debug_script_dir),
stderr=msg)
return 0
def get_action(self):
if self.prepare:
combined_params = '[([{0}], {1}) {2}]'.\
format(', '.join([f"('{x}', _{x})" for x in reversed(self.params)]),
None if self.loop_string[1] == '' else ("f\"{' '.join(__i__)}\"" if len(self.current_depends) > 1 else "f'{__i__}'"),
' '.join(self.loop_string) + self.filter_string)
input_str = '[]' if self.input_vars is None else '{0} if {0} is not None else []'.format(
self.input_vars)
output_str = f"__{n2a(int(self.step_map[self.step.name][1])).lower()}_{self.step.name}_output__"
# FIXME: multiple output to be implemented
ext_str = self.step.plugin.output_ext if (
len(self.step.exe['path']) == 0
and len(self.step.rv) > 0) else 'yml'
if len(self.current_depends):
self.action += f"__io_db__['{self.step.name}:' + str(__pipeline_id__)] = dict([(' '.join((y, x[1])), dict([('__pipeline_id__', __pipeline_id__), ('__pipeline_name__', __pipeline_name__), ('__module__', '{self.step.name}'), ('__out_vars__', __out_vars__)] + x[0])) for x, y in zip({combined_params}, {output_str})] + [('__input_output___', ({input_str}, {output_str})), ('__ext__', '{ext_str}')])\n"
else:
self.action += f"__io_db__['{self.step.name}:' + str(__pipeline_id__)] = dict([(y, dict([('__pipeline_id__', __pipeline_id__), ('__pipeline_name__', __pipeline_name__), ('__module__', '{self.step.name}'), ('__out_vars__', __out_vars__)] + x[0])) for x, y in zip({combined_params}, {output_str})] + [('__input_output___', ({input_str}, {output_str})), ('__ext__', '{ext_str}')])\n"
else:
# FIXME: have not considered multi-action module (or compound module) yet
# Create fake loop for now with idx going around
for idx, (plugin, cmd) in enumerate(
zip([self.step.plugin], [self.step.exe])):
sigil = '$[ ]' if plugin.name == 'bash' else '${ }'
if self.conf is None:
self.action += f'{"python3" if plugin.name == "python" else plugin.name}: expand = "{sigil}"'
if path(self.step.workdir).absolute() != path.cwd():
self.action += f", workdir = {repr(self.step.workdir)}"
self.action += f', stderr = f"{{_output:n}}.stderr", stdout = f"{{_output:n}}.stdout"'
else:
self.action += f'{"python3" if plugin.name == "python" else plugin.name}: expand = "{sigil}"'
if self.step.container:
self.action += f", container={repr(self.step.container)}"
if self.step.container_engine:
self.action += f", engine={repr(self.step.container_engine)}"
if len(self.step.path):
self.action += ", env={'PATH': '%s:' + os.environ['PATH']}" % ":".join(self.step.path)
self.action += plugin.get_cmd_args(cmd['args'],
self.params)
# Add action
if len(cmd['path']) == 0:
if self.debug:
script = plugin.get_return(None)
else:
script_begin = plugin.load_env(
self.step.depends, idx > 0
and len(self.step.rv))
script_begin += '\n' + plugin.get_input(
self.params,
self.step.libpath if self.step.libpath else [])
if len(self.step.rf):
script_begin += '\n' + plugin.get_output(
self.step.rf)
script_begin = '\n'.join(
[x for x in script_begin.split('\n') if x])
script_begin = f"{cmd['header']}\n{script_begin.strip()}\n\n## BEGIN DSC CORE"
script_end = plugin.get_return(
self.step.rv) if len(self.step.rv) else ''
script_end = f'## END DSC CORE\n\n{script_end.strip()}'.strip(
)
script = '\n'.join(
[script_begin, cmd['content'], script_end])
if self.try_catch:
script = plugin.add_try(
script, len([self.step.rf.values()]))
script = f"""## {str(plugin)} script UUID: ${{DSC_STEP_ID_}}\n{script}\n"""
script = '\n'.join(
[f' {x}' for x in script.split('\n')])
self.action += script
self.exe_signature.append(cmd['signature'])
else:
self.exe_check.append(
f"executable({repr(cmd['path'])})")
self.action += f"\t{cmd['path']} {'$*' if cmd['args'] else ''}\n"
