def run(self,
image,
script='',
interpreter='',
args='',
suffix='.sh',
**kwargs):
if self.client is None:
raise RuntimeError(
'Cannot connect to the Docker daemon. Is the docker daemon running on this host?'
)
#
env.logger.debug('docker_run with keyword args {}'.format(kwargs))
#
# now, write a temporary file to a tempoary directory under the current directory, this is because
# we need to share the directory to ...
with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir:
# keep the temporary script for debugging purposes
# tempdir = tempfile.mkdtemp(dir=os.getcwd())
if script:
tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix)
with open(os.path.join(tempdir, tempscript),
'w') as script_file:
script_file.write(script)
#
# if there is an interpreter and with args
if not args:
args = '{filename:q}'
if not interpreter:
interpreter = '/bin/bash'
# if there is a shebang line, we ...
if script.startswith('#!'):
# make the script executable
env.logger.warning(
'Shebang line in a docker-run script is ignored')
elif not isinstance(interpreter, str):
interpreter = interpreter[0]
#
binds = []
if 'volumes' in kwargs:
volumes = [kwargs['volumes']] if isinstance(
kwargs['volumes'], str) else kwargs['volumes']
for vol in volumes:
if not vol:
continue
if vol.count(':') != 1:
raise RuntimeError(
'Please specify columes in the format of host_dir:mnt_dir'
)
host_dir, mnt_dir = vol.split(':')
if platform.system() == 'Darwin':
# under Darwin, host_dir must be under /Users
if not os.path.abspath(host_dir).startswith(
'/Users') and not (
self.has_volumes and os.path.abspath(
host_dir).startswith('/Volumes')):
raise RuntimeError(
'hostdir ({}) under MacOSX must be under /Users or /Volumes (if properly configured, see https://github.com/vatlab/SOS/wiki/SoS-Docker-guide for details) to be usable in docker container'
.format(host_dir))
binds.append('{}:{}'.format(os.path.abspath(host_dir),
mnt_dir))
#
volumes_opt = ' '.join('-v {}'.format(x) for x in binds)
# under mac, we by default share /Users within docker
if platform.system() == 'Darwin':
if not any(x.startswith('/Users:') for x in binds):
volumes_opt += ' -v /Users:/Users'
if self.has_volumes:
volumes_opt += ' -v /Volumes:/Volumes'
elif platform.system() == 'Linux':
if not any(x.startswith('/home:') for x in binds):
volumes_opt += ' -v /home:/home'
if not any(x.startswith('/tmp:') for x in binds):
volumes_opt += ' -v /tmp:/tmp'
#
mem_limit_opt = ''
if 'mem_limit' in kwargs:
mem_limit_opt = '--memory={}'.format(kwargs['mem_limit'])
#
volumes_from_opt = ''
if 'volumes_from' in kwargs:
if isinstance(kwargs['volumes_from'], str):
volumes_from_opt = '--volumes_from={}'.format(
kwargs['volumes_from'])
elif isinstance(kwargs['volumes_from'], list):
volumes_from_opt = ' '.join(
'--volumes_from={}'.format(x)
for x in kwargs['volumes_from'])
else:
raise RuntimeError(
'Option volumes_from only accept a string or list of string: {} specified'
.format(kwargs['volumes_from']))
# we also need to mount the script
cmd_opt = ''
if script and interpreter:
volumes_opt += ' -v {}:{}'.format(
shlex.quote(os.path.join(tempdir, tempscript)),
'/var/lib/sos/{}'.format(tempscript))
cmd_opt = interpolate(
'{} {}'.format(interpreter, args),
{'filename': sos_targets(f'/var/lib/sos/{tempscript}')})
#
working_dir_opt = '-w={}'.format(
shlex.quote(os.path.abspath(os.getcwd())))
if 'working_dir' in kwargs:
if not os.path.isabs(kwargs['working_dir']):
env.logger.warning(
'An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.'
.format(
kwargs['working_dir'],
os.path.abspath(
os.path.expanduser(kwargs['working_dir']))))
working_dir_opt = '-w={}'.format(
os.path.abspath(
os.path.expanduser(kwargs['working_dir'])))
else:
working_dir_opt = '-w={}'.format(kwargs['working_dir'])
#
env_opt = ''
if 'environment' in kwargs:
if isinstance(kwargs['environment'], dict):
env_opt = ' '.join(
'-e {}={}'.format(x, y)
for x, y in kwargs['environment'].items())
elif isinstance(kwargs['environment'], list):
env_opt = ' '.join('-e {}'.format(x)
for x in kwargs['environment'])
elif isinstance(kwargs['environment'], str):
env_opt = '-e {}'.format(kwargs['environment'])
else:
raise RuntimeError(
'Invalid value for option environment (str, list, or dict is allowd, {} provided)'
.format(kwargs['environment']))
#
port_opt = '-P'
if 'port' in kwargs:
if isinstance(kwargs['port'], (str, int)):
port_opt = '-p {}'.format(kwargs['port'])
elif isinstance(kwargs['port'], list):
port_opt = ' '.join('-p {}'.format(x)
for x in kwargs['port'])
else:
raise RuntimeError(
'Invalid value for option port (a list of intergers), {} provided'
.format(kwargs['port']))
#
name_opt = ''
if 'name' in kwargs:
name_opt = '--name={}'.format(kwargs['name'])
#
stdin_opt = ''
if 'stdin_open' in kwargs and kwargs['stdin_optn']:
stdin_opt = '-i'
#
tty_opt = '-t'
if 'tty' in kwargs and not kwargs['tty']:
tty_opt = ''
#
user_opt = ''
if 'user' in kwargs:
user_opt = '-u {}'.format(kwargs['user'])
#
extra_opt = ''
if 'extra_args' in kwargs:
extra_opt = kwargs['extra_args']
#
security_opt = ''
if platform.system() == 'Linux':
# this is for a selinux problem when /var/sos/script cannot be executed
security_opt = '--security-opt label:disable'
command = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(
security_opt, # security option
volumes_opt, # volumes
volumes_from_opt, # volumes_from
name_opt, # name
stdin_opt, # stdin_optn
tty_opt, # tty
port_opt, # port
working_dir_opt, # working dir
user_opt, # user
env_opt, # environment
mem_limit_opt, # memory limit
extra_opt, # any extra parameters
image, # image
cmd_opt)
env.logger.info(command)
ret = subprocess.call(command, shell=True)
if ret != 0:
msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format(
tempscript,
command.replace(tempdir, os.path.abspath('./.sos')))
shutil.copy(os.path.join(tempdir, tempscript), '.sos')
if ret == 125:
raise RuntimeError(
'Docker daemon failed (exitcode=125). ' + msg)
elif ret == 126:
raise RuntimeError(
'Failed to invoke specified command (exitcode=126). ' +
msg)
elif ret == 127:
raise RuntimeError(
'Failed to locate specified command (exitcode=127). ' +
msg)
elif ret == 137:
if not hasattr(self, 'tot_mem'):
self.tot_mem = self.total_memory(image)
if self.tot_mem is None:
raise RuntimeError('Script killed by docker. ' + msg)
else:
raise RuntimeError(
'Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). '
.format(self.tot_mem / 1024 / 1024) + msg)
else:
raise RuntimeError(
'Executing script in docker returns an error (exitcode={}). '
.format(ret) + msg)
return 0
def run(self,
image,
script='',
interpreter='',
args='',
suffix='.sh',
**kwargs):
if self.client is None:
raise RuntimeError(
'Cannot connect to the Docker daemon. Is the docker daemon running on this host?'
)
#
env.logger.debug('docker_run with keyword args {}'.format(kwargs))
#
# now, write a temporary file to a tempoary directory under the current directory, this is because
# we need to share the directory to ...
with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir:
# keep the temporary script for debugging purposes
# tempdir = tempfile.mkdtemp(dir=os.getcwd())
tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix)
if script:
with open(os.path.join(tempdir, tempscript),
'w') as script_file:
# the input script might have windows new line but the container
# will need linux new line for proper execution #1023
script_file.write('\n'.join(script.splitlines()))
#
# if there is an interpreter and with args
if not args:
args = '{filename:pq}'
#
# under mac, we by default share /Users within docker
wdir = os.path.abspath(os.getcwd())
binds = []
if 'volumes' in kwargs:
volumes = [kwargs['volumes']] if isinstance(
kwargs['volumes'], str) else kwargs['volumes']
has_wdir = False
for vol in volumes:
if not vol:
continue
if isinstance(vol, (str, path)):
vol = str(vol)
else:
raise ValueError(
f'Unacceptable value {vol} for parameter volumes')
if vol.count(':') == 0:
host_dir, mnt_dir = vol, vol
elif vol.count(':') in (1, 2):
host_dir, mnt_dir = vol.split(':', 1)
else:
raise ValueError(
f'Invalid format for volume specification: {vol}')
binds.append(
f'{path(host_dir).resolve():p}:{path(mnt_dir):p}')
if wdir.startswith(
os.path.abspath(os.path.expanduser(host_dir))):
has_wdir = True
volumes_opt = ' '.join('-v {}'.format(x) for x in binds)
if not has_wdir:
volumes_opt += f' -v {path(wdir):p}:{path(wdir):p}'
else:
volumes_opt = f' -v {path(wdir):p}:{path(wdir):p}'
#
mem_limit_opt = ''
if 'mem_limit' in kwargs:
mem_limit_opt = '--memory={}'.format(kwargs['mem_limit'])
#
volumes_from_opt = ''
if 'volumes_from' in kwargs:
if isinstance(kwargs['volumes_from'], str):
volumes_from_opt = f'--volumes_from={kwargs["volumes_from"]}'
elif isinstance(kwargs['volumes_from'], list):
volumes_from_opt = ' '.join(
f'--volumes_from={x}' for x in kwargs['volumes_from'])
else:
raise RuntimeError(
'Option volumes_from only accept a string or list of string: {} specified'
.format(kwargs['volumes_from']))
# we also need to mount the script
if script:
volumes_opt += f' -v {path(tempdir)/tempscript:p}:/var/lib/sos/{tempscript}'
cmd_opt = interpolate(
f'{interpreter if isinstance(interpreter, str) else interpreter[0]} {args}',
{
'filename': sos_targets(f'/var/lib/sos/{tempscript}'),
'script': script
})
#
workdir_opt = ''
if 'docker_workdir' in kwargs and kwargs[
'docker_workdir'] is not None:
if not os.path.isabs(kwargs['docker_workdir']):
env.logger.warning(
'An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.'
.format(
kwargs['docker_workdir'],
os.path.abspath(
os.path.expanduser(kwargs['docker_workdir']))))
workdir_opt = f'-w={path(kwargs["docker_workdir"]).resolve():p}'
else:
workdir_opt = f'-w={path(kwargs["docker_workdir"]):p}'
elif 'docker_workdir' not in kwargs:
# by default, map current working directoryself.
workdir_opt = f'-w={path(wdir):p}'
env_opt = ''
if 'environment' in kwargs:
if isinstance(kwargs['environment'], dict):
env_opt = ' '.join(
f'-e {x}={y}'
for x, y in kwargs['environment'].items())
elif isinstance(kwargs['environment'], list):
env_opt = ' '.join(f'-e {x}'
for x in kwargs['environment'])
elif isinstance(kwargs['environment'], str):
env_opt = f'-e {kwargs["environment"]}'
else:
raise RuntimeError(
'Invalid value for option environment (str, list, or dict is allowd, {} provided)'
.format(kwargs['environment']))
#
port_opt = ''
if 'port' in kwargs:
if kwargs['port'] is True:
port_opt = '-P'
elif isinstance(kwargs['port'], (str, int)):
port_opt = '-p {}'.format(kwargs['port'])
elif isinstance(kwargs['port'], list):
port_opt = ' '.join('-p {}'.format(x)
for x in kwargs['port'])
else:
raise RuntimeError(
'Invalid value for option port (a list of intergers or True), {} provided'
.format(kwargs['port']))
#
name_opt = ''
if 'name' in kwargs:
name_opt = f'--name={kwargs["name"]}'
#
stdin_opt = ''
if 'stdin_open' in kwargs and kwargs['stdin_optn']:
stdin_opt = '-i'
#
tty_opt = '-t'
if 'tty' in kwargs and not kwargs['tty']:
tty_opt = ''
#
user_opt = ''
if 'user' in kwargs:
if kwargs['user'] is not None:
user_opt = f'-u {kwargs["user"]}'
elif platform.system() != 'Windows':
# Tocket #922
user_opt = f'-u {os.getuid()}:{os.getgid()}'
#
extra_opt = ''
if 'extra_args' in kwargs:
extra_opt = kwargs['extra_args']
#
security_opt = ''
if platform.system() == 'Linux':
# this is for a selinux problem when /var/sos/script cannot be executed
security_opt = '--security-opt label:disable'
cmd = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(
security_opt, # security option
volumes_opt, # volumes
volumes_from_opt, # volumes_from
name_opt, # name
stdin_opt, # stdin_optn
tty_opt, # tty
port_opt, # port
workdir_opt, # working dir
user_opt, # user
env_opt, # environment
mem_limit_opt, # memory limit
extra_opt, # any extra parameters
image, # image
cmd_opt)
env.logger.debug(cmd)
if env.config['run_mode'] == 'dryrun':
print(f'HINT: {cmd}')
print(script)
return 0
ret = self._run_cmd(cmd, **kwargs)
if ret != 0:
debug_script_dir = os.path.join(env.exec_dir, '.sos')
msg = 'The script has been saved to {}/{}. To reproduce the error please run:\n``{}``'.format(
debug_script_dir, tempscript,
cmd.replace(f'{path(tempdir):p}',
f'{path(debug_script_dir):p}'))
shutil.copy(os.path.join(tempdir, tempscript),
debug_script_dir)
if ret == 125:
msg = 'Docker daemon failed (exitcode=125). ' + msg
elif ret == 126:
msg = 'Failed to invoke specified command (exitcode=126). ' + msg
elif ret == 127:
msg = 'Failed to locate specified command (exitcode=127). ' + msg
elif ret == 137:
if not hasattr(self, 'tot_mem'):
self.tot_mem = self.total_memory(image)
if self.tot_mem is None:
msg = 'Script killed by docker. ' + msg
else:
msg = 'Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). '.format(
self.tot_mem / 1024 / 1024) + msg
else:
out = f", stdout={kwargs['stdout']}" if 'stdout' in kwargs and os.path.isfile(
kwargs['stdout']) and os.path.getsize(
kwargs['stdout']) > 0 else ''
err = f", stderr={kwargs['stderr']}" if 'stderr' in kwargs and os.path.isfile(
kwargs['stderr']) and os.path.getsize(
kwargs['stderr']) > 0 else ''
msg = f"Executing script in docker returns an error (exitcode={ret}{err}{out}).\n{msg}"
raise subprocess.CalledProcessError(returncode=ret,
cmd=cmd.replace(
tempdir,
debug_script_dir),
stderr=msg)
return 0
def Rmarkdown(script=None,
input=None,
output=None,
args='{input:r}, output_file={output:ar}',
**kwargs):
'''Convert input file to output using Rmarkdown
The input can be specified in three ways:
1. instant script, which is assumed to be in md format
Rmarkdown: output='report.html'
script
2. one or more input files. The format is determined by extension of input file
Rmarkdown(input, output='report.html')
3. input file specified by command line option `-r` .
Rmarkdown(output='report.html')
If no output is specified, it is assumed to be in html format
and is written to standard output.
You can specify more options using the args parameter of the action. The default value
of args is `${input!r} --output ${output!ar}'
'''
if not R_library('rmarkdown').target_exists():
raise UnknownTarget(R_library('rmarkdown'))
input = sos_targets(collect_input(script, input))
output = sos_targets(output)
if len(output) == 0:
write_to_stdout = True
output = sos_targets(
tempfile.NamedTemporaryFile(mode='w+t',
suffix='.html',
delete=False).name)
else:
write_to_stdout = False
#
ret = 1
try:
# render(input, output_format = NULL, output_file = NULL, output_dir = NULL,
# output_options = NULL, intermediates_dir = NULL,
# runtime = c("auto", "static", "shiny"),
# clean = TRUE, params = NULL, knit_meta = NULL, envir = parent.frame(),
# run_Rmarkdown = TRUE, quiet = FALSE, encoding = getOption("encoding"))
cmd = interpolate(f'Rscript -e "rmarkdown::render({args})"', {
'input': input,
'output': output
})
env.logger.trace(f'Running command "{cmd}"')
if env.config['run_mode'] == 'interactive':
# need to catch output and send to python output, which will in trun be hijacked by SoS notebook
p = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
#pid = p.pid
out, err = p.communicate()
sys.stdout.write(out.decode())
sys.stderr.write(err.decode())
ret = p.returncode
else:
p = subprocess.Popen(cmd, shell=True)
#pid = p.pid
ret = p.wait()
except Exception as e:
env.logger.error(e)
if ret != 0:
temp_file = os.path.join('.sos', f'{"Rmarkdown"}_{os.getpid()}.md')
shutil.copyfile(str(input), temp_file)
cmd = interpolate(f'Rscript -e "rmarkdown::render({args})"', {
'input': input,
'output': sos_targets(temp_file)
})
raise RuntimeError(
f'Failed to execute script. Please use command \n"{cmd}"\nunder {os.getcwd()} to test it.'
)
if write_to_stdout:
with open(str(output[0])) as out:
sys.stdout.write(out.read())
else:
env.logger.info(f'Report saved to {output}')