def testInterpolation(self):
'''Test string interpolation'''
env.sos_dict = WorkflowDict({
'os': os,
'a': 100,
'b': 20,
'c': ['file1', 'file2', 'file3'],
'd': {'a': 'file1', 'b':'file2'},
'e': set([1, 'a']),
'var1': 1/2.,
'var2': [1, 2, 3.1],
'file': 'a/b.txt',
'files2': ['a/b.txt', 'c.d.txt'],
'file_ws': ['d i r/f .txt']
})
for sigil in ('${ }', '{ }', '[ ]', '%( )', '[[ ]]', '%( )s', '# #', '` `'):
l, r = sigil.split(' ')
for expr, result, nested, exclude in [
('{0}1{1}', '1', False, []),
('{0}a{1}', '100', False, []),
('{0}a+b{1}', '120', False, []),
('{0}a+b*5{1}', '200', False, []),
('{0}a+b*5{1} is 200', '200 is 200', False, []),
('{0}a+b*5{1} and {0}a{1}', '200 and 100', False, []),
('Pre {0}a+b*5{1} and {0}a{1} after', 'Pre 200 and 100 after', False, []),
('Nested {0}a+b*{0}b//2{1}{1}', 'Nested 300', True, []),
('Format {0}a:.5f{1}', 'Format 100.00000', False, []),
('{0}var2[:2]{1}', '1 2', False, []),
('{0}var2[1:]{1}', '2 3.1', False, []),
# nested
('Nested {0}a:.{0}4+1{1}f{1}', 'Nested 100.00000', True, []),
# deep nested
('Triple Nested {0}a:.{0}4+{0}5//5{1}{1}f{1}', 'Triple Nested 100.00000', True, []),
# nested invalid
('Nested invalid {0}"{0}a-{1}"{1}', 'Nested invalid {}a-{}'.format(l, r), True, []),
('Nested valid {0}"{0}a{1}-"{1}', 'Nested valid 100-', True, []),
#
('Dict {0}d{1}', ['Dict a b', 'Dict b a'], False, []),
('set {0}e{1}', ['set 1 a', 'set a 1'], False, []),
('Fmt {0}var1:.2f{1}', 'Fmt 0.50', False, []),
('Fmt {0}var2:.2f{1}', 'Fmt 1.00 2.00 3.10', False, []),
('LC {0}[x*2 for x in var2]{1}', 'LC 2 4 6.2', False, []),
('LC {0}[x*2 for x in var2]:.2f{1}', 'LC 2.00 4.00 6.20', False, []),
#
# [['a':'b', 'c':'d']['a']] works because
# ['a':'b', 'c':'d']a
# is invalid so SoS does not consider ['a'] as nested expression
#
('Newline {0}{{"a": "b", \n"c": "d"}}["a"]{1}', 'Newline b', False, []),
#
# string literal within interpolated expression
('Literal {0}"{0} {1}"{1}', 'Literal ' + sigil, True, []),
# this case does not work because {} would become '' with sigil {}
("{0}' {{}} '.format(a){1}", ' 100 ', False, ['{ }']),
#
("{0}os.path.basename(file){1}", 'b.txt', False, []),
('{0}os.path.basename(file_ws[0]){1}', 'f .txt', False, []),
#
# ! conversion
('{0}file!r{1}', "'a/b.txt'", False, []),
('{0}file!s{1}', "a/b.txt", False, []),
('''{0}"a'b"!r{1}''', '"a\'b"', False, []),
('''{0}'a"b'!r{1}''', "'a\"b'", False, []),
#
# !q conversion (added by SoS)
('{0}file_ws[0]!q{1}', "'d i r/f .txt'", False, []),
#
# !, joined by ,
('{0}var2!r,{1}', "1, 2, 3.1", False, []),
('{0}c!r,{1}', "'file1', 'file2', 'file3'", False, []),
('{0}c!,{1}', "file1, file2, file3", False, []),
#
('{0}10000:,{1}', "10,000", False, []),
#
# full name by 'a'
('{0}"test_utils.py"!a{1}', os.path.abspath('test_utils.py'), False, []),
('{0}"a/b/c/test_utils.py"!b{1}', 'test_utils.py', False, []),
('{0}"a/b/c/test_utils.py"!d{1}', 'a/b/c', False, []),
('{0}"a/b/c/test_utils.py"!dd{1}', 'a/b', False, []),
('{0}"a/b/c/test_utils.py"!ddb{1}', 'b', False, []),
('{0}"a/b/c/test_utils.py"!n{1}', 'a/b/c/test_utils', False, []),
('{0}"a/b/c/test_utils.py"!bn{1}', 'test_utils', False, []),
('{0}"~/test_utils.py"!a{1}', os.path.expanduser('~/test_utils.py'), False, []),
('{0}"~/test_utils.py"!e{1}', os.path.expanduser('~/test_utils.py'), False, []),
('{0}"test/test_utils.py"!b{1}', "test_utils.py", False, []),
# preceded by \
(r'\{0}100{1}', '{0}100{1}'.format(l, r), True, []),
(r'\{0}100{1} {0}100+4{1}', '{0}100{1} 104'.format(l, r), True, []),
(r'''{0}{1}''', '', False, []),
]:
if l == r and nested:
continue
if sigil in exclude:
continue
if isinstance(result, str):
self.assertEqual(interpolate(expr.format(l, r), sigil=sigil), result, 'Failed to interpolate {} with sigil {}'.format(expr, sigil))
else:
# for cases when the order of output is not guaranteed
self.assertTrue(interpolate(expr.format(l, r), sigil=sigil) in result, 'Failed to interpolate {} with sigil {}'.format(expr, sigil))
#
# locals should be the one passed to the expression
self.assertTrue('file_ws' in interpolate('${globals().keys()}', '${ }'))
# 5:.5.0f does not work.
self.assertRaises(InterpolationError, interpolate, '${5:.${4/2.}f}', '${ }')
def Rmarkdown(script=None, input=None, output=None, args="${input!r}, output_file=${output!ar}", **kwargs):
"""Convert input file to output using Rmarkdown
The input can be specified in three ways:
1. instant script, which is assumed to be in md format
Rmarkdown: output='report.html'
script
2. one or more input files. The format is determined by extension of input file
Rmarkdown(input, output='report.html')
3. input file specified by command line option `-r` .
Rmarkdown(output='report.html')
If no output is specified, it is assumed to be in html format
and is written to standard output.
You can specify more options using the args parameter of the action. The default value
of args is `${input!r} --output ${output!ar}'
"""
if not R_library("rmarkdown").exists():
raise UnknownTarget(R_library("rmarkdown"))
input_file = collect_input(script, input)
write_to_stdout = False
if output is None:
write_to_stdout = True
output_file = tempfile.NamedTemporaryFile(mode="w+t", suffix=".html", delete=False).name
elif isinstance(output, str):
output_file = output
else:
raise RuntimeError("A filename is expected, {} provided".format(output))
#
ret = 1
try:
# render(input, output_format = NULL, output_file = NULL, output_dir = NULL,
# output_options = NULL, intermediates_dir = NULL,
# runtime = c("auto", "static", "shiny"),
# clean = TRUE, params = NULL, knit_meta = NULL, envir = parent.frame(),
# run_Rmarkdown = TRUE, quiet = FALSE, encoding = getOption("encoding"))
cmd = interpolate(
'Rscript -e "rmarkdown::render({})"'.format(args), "${ }", {"input": input_file, "output": output_file}
)
env.logger.trace('Running command "{}"'.format(cmd))
if env.run_mode == "interactive":
# need to catch output and send to python output, which will in trun be hijacked by SoS notebook
p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
pid = p.pid
env.register_process(p.pid, "Runing {}".format(input_file))
out, err = p.communicate()
sys.stdout.write(out.decode())
sys.stderr.write(err.decode())
ret = p.returncode
else:
p = subprocess.Popen(cmd, shell=True)
pid = p.pid
env.register_process(pid, "Runing {}".format(input_file))
ret = p.wait()
except Exception as e:
env.logger.error(e)
finally:
env.deregister_process(p.pid)
if ret != 0:
temp_file = os.path.join(".sos", "{}_{}.md".format("Rmarkdown", os.getpid()))
shutil.copyfile(input_file, temp_file)
cmd = interpolate(
'Rscript -e "rmarkdown::render({})"'.format(args), "${ }", {"input": input_file, "output": output_file}
)
raise RuntimeError(
'Failed to execute script. Please use command \n"{}"\nunder {} to test it.'.format(cmd, os.getcwd())
)
if write_to_stdout:
with open(output_file) as out:
sys.stdout.write(out.read())
else:
env.logger.info("Report saved to {}".format(output_file))
def _install(self, name, version, repos):
'''Check existence and version match of R library.
cran and bioc packages are unique yet might overlap with github.
Therefore if the input name is {repo}/{pkg} the package will be
installed from github if not available, else from cran or bioc
'''
from sos.pattern import glob_wildcards
from sos.sos_eval import interpolate
import tempfile
import shlex
import subprocess
output_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.txt', delete=False).name
script_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.R', delete=False).name
if len(glob_wildcards('{repo}/{pkg}', [name])['repo']):
# package is from github
self._intall('devtools', version, repos)
install_script = interpolate('''
options(warn=-1)
package_repo <- ${name!r}
package <- basename(package_repo)
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}")
} else {
devtools::install_github(package_repo)
# if it still does not exist, write the package name to output
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}")
} else {
write(paste(package, "NA", "MISSING"), file="${output_file}")
quit("no")
}
}
cur_version <- packageVersion(package)
''', '${ }', locals())
else:
# package is from cran or bioc
install_script = interpolate('''
options(warn=-1)
package <- ${name!r}
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}")
} else {
install.packages(package, repos="${repos}",
quiet=FALSE)
# if the package still does not exist
if (!require(package, character.only=TRUE, quietly=TRUE)) {
source("http://bioconductor.org/biocLite.R")
biocLite(package, suppressUpdates=TRUE, suppressAutoUpdate=TRUE, ask=FALSE)
}
# if it still does not exist, write the package name to output
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}")
} else {
write(paste(package, "NA", "MISSING"), file="${output_file}")
quit("no")
}
}
cur_version <- packageVersion(package)
''', '${ }', locals())
version_script = ''
if version is not None:
version = [version] if isinstance(version, str) else version
operators = []
for idx, value in enumerate(version):
value = str(value)
if value.endswith('+'):
operators.append('>=')
version[idx] = value[:-1]
elif value.endswith('-'):
operators.append('<')
version[idx] = value[:-1]
else:
operators.append('==')
# check version and mark version mismatch
# if current version satisfies any of the
# requirement the check program quits
for x, y in zip(version, operators):
version_script += '''
if (cur_version {1} {0}) {{
quit("no")
}}
'''.format(repr(x), y)
version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\
format(repr(output_file))
# temporarily change the run mode to run to execute script
try:
with open(script_file, 'w') as sfile:
sfile.write(install_script + version_script)
cmd = 'Rscript --default-packages=utils ' + shlex.quote(script_file)
#
p = subprocess.Popen(cmd, shell=True)
ret = p.wait()
if ret != 0:
env.logger.warning('Failed to detect or install R library')
return False
except Exception as e:
env.logger.error('Failed to execute script: {}'.format(e))
return False
finally:
os.remove(script_file)
ret_val = False
with open(output_file) as tmp:
for line in tmp:
lib, version, status = line.split()
if status.strip() == "MISSING":
env.logger.warning('R Library {} is not available and cannot be installed.'.format(lib))
elif status.strip() == 'AVAILABLE':
env.logger.debug('R library {} ({}) is available'.format(lib, version))
ret_val = True
elif status.strip() == 'INSTALLED':
env.logger.debug('R library {} ({}) has been installed'.format(lib, version))
ret_val = True
elif status.strip() == 'VERSION_MISMATCH':
env.logger.warning('R library {} ({}) does not satisfy version requirement!'.format(lib, version))
else:
raise RuntimeError('This should not happen: {}'.format(line))
try:
os.remove(output_file)
except:
pass
return ret_val
def testInterpolation(self):
"""Test string interpolation"""
env.sos_dict = WorkflowDict(
{
"os": os,
"a": 100,
"b": 20,
"c": ["file1", "file2", "file3"],
"d": {"a": "file1", "b": "file2"},
"e": set([1, "a"]),
"var1": 1 / 2.0,
"var2": [1, 2, 3.1],
"file": "a/b.txt",
"files2": ["a/b.txt", "c.d.txt"],
"file_ws": ["d i r/f .txt"],
}
)
for sigil in ("${ }", "{ }", "[ ]", "%( )", "[[ ]]", "%( )s", "# #", "` `"):
l, r = sigil.split(" ")
for expr, result, nested, exclude in [
("{0}1{1}", "1", False, []),
("{0}a{1}", "100", False, []),
("{0}a+b{1}", "120", False, []),
("{0}a+b*5{1}", "200", False, []),
("{0}a+b*5{1} is 200", "200 is 200", False, []),
("{0}a+b*5{1} and {0}a{1}", "200 and 100", False, []),
("Pre {0}a+b*5{1} and {0}a{1} after", "Pre 200 and 100 after", False, []),
("Nested {0}a+b*{0}b//2{1}{1}", "Nested 300", True, []),
("Format {0}a:.5f{1}", "Format 100.00000", False, []),
("{0}var2[:2]{1}", "1 2", False, []),
("{0}var2[1:]{1}", "2 3.1", False, []),
# nested
("Nested {0}a:.{0}4+1{1}f{1}", "Nested 100.00000", True, []),
# deep nested
("Triple Nested {0}a:.{0}4+{0}5//5{1}{1}f{1}", "Triple Nested 100.00000", True, []),
# nested invalid
('Nested invalid {0}"{0}a-{1}"{1}', "Nested invalid {}a-{}".format(l, r), True, []),
('Nested valid {0}"{0}a{1}-"{1}', "Nested valid 100-", True, []),
#
("Dict {0}d{1}", ["Dict a b", "Dict b a"], False, []),
("set {0}e{1}", ["set 1 a", "set a 1"], False, []),
("Fmt {0}var1:.2f{1}", "Fmt 0.50", False, []),
("Fmt {0}var2:.2f{1}", "Fmt 1.00 2.00 3.10", False, []),
("LC {0}[x*2 for x in var2]{1}", "LC 2 4 6.2", False, []),
("LC {0}[x*2 for x in var2]:.2f{1}", "LC 2.00 4.00 6.20", False, []),
#
# [['a':'b', 'c':'d']['a']] works because
# ['a':'b', 'c':'d']a
# is invalid so SoS does not consider ['a'] as nested expression
#
('Newline {0}{{"a": "b", \n"c": "d"}}["a"]{1}', "Newline b", False, []),
#
# string literal within interpolated expression
('Literal {0}"{0} {1}"{1}', "Literal " + sigil, True, []),
# this case does not work because {} would become '' with sigil {}
("{0}' {{}} '.format(a){1}", " 100 ", False, ["{ }"]),
#
("{0}os.path.basename(file){1}", "b.txt", False, []),
("{0}os.path.basename(file_ws[0]){1}", "f .txt", False, []),
#
# ! conversion
("{0}file!r{1}", "'a/b.txt'", False, []),
("{0}file!s{1}", "a/b.txt", False, []),
("""{0}"a'b"!r{1}""", '"a\'b"', False, []),
("""{0}'a"b'!r{1}""", "'a\"b'", False, []),
#
# !q conversion (added by SoS)
("{0}file_ws[0]!q{1}", "'d i r/f .txt'", False, []),
#
# !, joined by ,
("{0}var2!r,{1}", "1, 2, 3.1", False, []),
("{0}c!r,{1}", "'file1', 'file2', 'file3'", False, []),
("{0}c!,{1}", "file1, file2, file3", False, []),
#
("{0}10000:,{1}", "10,000", False, []),
#
# full name by 'a'
('{0}"test_utils.py"!a{1}', os.path.abspath("test_utils.py"), False, []),
('{0}"a/b/c/test_utils.py"!b{1}', "test_utils.py", False, []),
('{0}"a/b/c/test_utils.py"!d{1}', "a/b/c", False, []),
('{0}"a/b/c/test_utils.py"!dd{1}', "a/b", False, []),
('{0}"a/b/c/test_utils.py"!ddb{1}', "b", False, []),
('{0}"a/b/c/test_utils.py"!n{1}', "a/b/c/test_utils", False, []),
('{0}"a/b/c/test_utils.py"!bn{1}', "test_utils", False, []),
('{0}"~/test_utils.py"!a{1}', os.path.expanduser("~/test_utils.py"), False, []),
('{0}"~/test_utils.py"!e{1}', os.path.expanduser("~/test_utils.py"), False, []),
('{0}"test/test_utils.py"!b{1}', "test_utils.py", False, []),
# preceded by \
(r"\{0}100{1}", "{0}100{1}".format(l, r), True, []),
(r"\{0}100{1} {0}100+4{1}", "{0}100{1} 104".format(l, r), True, []),
(r"""{0}{1}""", "", False, []),
]:
if l == r and nested:
continue
if sigil in exclude:
continue
if isinstance(result, str):
self.assertEqual(
interpolate(expr.format(l, r), sigil=sigil),
result,
"Failed to interpolate {} with sigil {}".format(expr, sigil),
)
else:
# for cases when the order of output is not guaranteed
self.assertTrue(
interpolate(expr.format(l, r), sigil=sigil) in result,
"Failed to interpolate {} with sigil {}".format(expr, sigil),
)
#
# locals should be the one passed to the expression
self.assertTrue("file_ws" in interpolate("${globals().keys()}", "${ }"))
# 5:.5.0f does not work.
self.assertRaises(InterpolationError, interpolate, "${5:.${4/2.}f}", "${ }")
def _install(self, name, version, repos):
'''Check existence and version match of R library.
cran and bioc packages are unique yet might overlap with github.
Therefore if the input name is {repo}/{pkg} the package will be
installed from github if not available, else from cran or bioc
'''
from sos.pattern import glob_wildcards
from sos.sos_eval import interpolate
import tempfile
import shlex
import subprocess
output_file = tempfile.NamedTemporaryFile(mode='w+t',
suffix='.txt',
delete=False).name
script_file = tempfile.NamedTemporaryFile(mode='w+t',
suffix='.R',
delete=False).name
if len(glob_wildcards('{repo}/{pkg}', [name])['repo']):
# package is from github
self._intall('devtools', version, repos)
install_script = interpolate(
'''
options(warn=-1)
package_repo <- ${name!r}
package <- basename(package_repo)
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}")
} else {
devtools::install_github(package_repo)
# if it still does not exist, write the package name to output
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}")
} else {
write(paste(package, "NA", "MISSING"), file="${output_file}")
quit("no")
}
}
cur_version <- packageVersion(package)
''', '${ }', locals())
else:
# package is from cran or bioc
install_script = interpolate(
'''
options(warn=-1)
package <- ${name!r}
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}")
} else {
install.packages(package, repos="${repos}",
quiet=FALSE)
# if the package still does not exist
if (!require(package, character.only=TRUE, quietly=TRUE)) {
source("http://bioconductor.org/biocLite.R")
biocLite(package, suppressUpdates=TRUE, suppressAutoUpdate=TRUE, ask=FALSE)
}
# if it still does not exist, write the package name to output
if (require(package, character.only=TRUE, quietly=TRUE)) {
write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}")
} else {
write(paste(package, "NA", "MISSING"), file="${output_file}")
quit("no")
}
}
cur_version <- packageVersion(package)
''', '${ }', locals())
version_script = ''
if version is not None:
version = [version] if isinstance(version, str) else version
operators = []
for idx, value in enumerate(version):
value = str(value)
if value.endswith('+'):
operators.append('>=')
version[idx] = value[:-1]
elif value.endswith('-'):
operators.append('<')
version[idx] = value[:-1]
else:
operators.append('==')
# check version and mark version mismatch
# if current version satisfies any of the
# requirement the check program quits
for x, y in zip(version, operators):
version_script += '''
if (cur_version {1} {0}) {{
quit("no")
}}
'''.format(repr(x), y)
version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\
format(repr(output_file))
# temporarily change the run mode to run to execute script
try:
with open(script_file, 'w') as sfile:
sfile.write(install_script + version_script)
cmd = 'Rscript --default-packages=utils ' + shlex.quote(
script_file)
#
p = subprocess.Popen(cmd, shell=True)
ret = p.wait()
if ret != 0:
env.logger.warning('Failed to detect or install R library')
return False
except Exception as e:
env.logger.error('Failed to execute script: {}'.format(e))
return False
finally:
os.remove(script_file)
ret_val = False
with open(output_file) as tmp:
for line in tmp:
lib, version, status = line.split()
if status.strip() == "MISSING":
env.logger.warning(
'R Library {} is not available and cannot be installed.'
.format(lib))
elif status.strip() == 'AVAILABLE':
env.logger.debug('R library {} ({}) is available'.format(
lib, version))
ret_val = True
elif status.strip() == 'INSTALLED':
env.logger.debug(
'R library {} ({}) has been installed'.format(
lib, version))
ret_val = True
elif status.strip() == 'VERSION_MISMATCH':
env.logger.warning(
'R library {} ({}) does not satisfy version requirement!'
.format(lib, version))
else:
raise RuntimeError(
'This should not happen: {}'.format(line))
try:
os.remove(output_file)
except:
pass
return ret_val
def run(self, image, script='', interpreter='', args='', suffix='.sh', **kwargs):
if self.client is None:
raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?')
#
env.logger.debug('docker_run with keyword args {}'.format(kwargs))
#
# now, write a temporary file to a tempoary directory under the current directory, this is because
# we need to share the directory to ...
with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir:
# keep the temporary script for debugging purposes
# tempdir = tempfile.mkdtemp(dir=os.getcwd())
if script:
tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix)
with open(os.path.join(tempdir, tempscript), 'w') as script_file:
script_file.write(script)
#
# if there is an interpreter and with args
if not args:
args = '${filename!q}'
if not interpreter:
interpreter = '/bin/bash'
# if there is a shebang line, we ...
if script.startswith('#!'):
# make the script executable
env.logger.warning('Shebang line in a docker-run script is ignored')
#
binds = []
if 'volumes' in kwargs:
volumes = [kwargs['volumes']] if isinstance(kwargs['volumes'], str) else kwargs['volumes']
for vol in volumes:
if not vol:
continue
if vol.count(':') != 1:
raise RuntimeError('Please specify columes in the format of host_dir:mnt_dir')
host_dir, mnt_dir = vol.split(':')
if platform.system() == 'Darwin':
# under Darwin, host_dir must be under /Users
if not os.path.abspath(host_dir).startswith('/Users') and not (self.has_volumes and os.path.abspath(host_dir).startswith('/Volumes')):
raise RuntimeError('hostdir ({}) under MacOSX must be under /Users or /Volumes (if properly configured, see https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide for details) to be usable in docker container'.format(host_dir))
binds.append('{}:{}'.format(os.path.abspath(host_dir), mnt_dir))
#
volumes_opt = ' '.join('-v {}'.format(x) for x in binds)
# under mac, we by default share /Users within docker
if platform.system() == 'Darwin':
if not any(x.startswith('/Users:') for x in binds):
volumes_opt += ' -v /Users:/Users'
if self.has_volumes:
volumes_opt += ' -v /Volumes:/Volumes'
if not any(x.startswith('/tmp:') for x in binds):
volumes_opt += ' -v /tmp:/tmp'
#
mem_limit_opt = ''
if 'mem_limit' in kwargs:
mem_limit_opt = '--memory={}'.format(kwargs['mem_limit'])
#
volumes_from_opt = ''
if 'volumes_from' in kwargs:
if isinstance(kwargs['volumes_from'], str):
volumes_from_opt = '--volumes_from={}'.format(kwargs['volumes_from'])
elif isinstance(kwargs['volumes_from'], list):
volumes_from_opt = ' '.join('--volumes_from={}'.format(x) for x in kwargs['volumes_from'])
else:
raise RuntimeError('Option volumes_from only accept a string or list of string'.format(kwargs['volumes_from']))
# we also need to mount the script
cmd_opt = ''
if script and interpreter:
volumes_opt += ' -v {}:{}'.format(os.path.join(tempdir, tempscript), '/var/lib/sos/{}'.format(tempscript))
cmd_opt = interpolate('{} {}'.format(interpreter, args), '${ }',
{'filename': '/var/lib/sos/{}'.format(tempscript)})
#
working_dir_opt = '-w={}'.format(os.path.abspath(os.getcwd()))
if 'working_dir' in kwargs:
if not os.path.isabs(kwargs['working_dir']):
env.logger.warning('An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.'
.format(kwargs['working_dir'], os.path.abspath(os.path.expanduser(kwargs['working_dir']))))
working_dir_opt = '-w={}'.format(os.path.abspath(os.path.expanduser(kwargs['working_dir'])))
else:
working_dir_opt = '-w={}'.format(kwargs['working_dir'])
#
env_opt = ''
if 'environment' in kwargs:
if isinstance(kwargs['environment'], dict):
env_opt = ' '.join('-e {}={}'.format(x,y) for x,y in kwargs['environment'].items())
elif isinstance(kwargs['environment'], list):
env_opt = ' '.join('-e {}'.format(x) for x in kwargs['environment'])
elif isinstance(kwargs['environment'], str):
env_opt = '-e {}'.format(kwargs['environment'])
else:
raise RuntimeError('Invalid value for option environment (str, list, or dict is allowd, {} provided)'.format(kwargs['environment']))
#
port_opt = '-P'
if 'port' in kwargs:
if isinstance(kwargs['port'], (str, int)):
port_opt = '-p {}'.format(kwargs['port'])
elif isinstance(kwargs['port'], list):
port_opt = ' '.join('-p {}'.format(x) for x in kwargs['port'])
else:
raise RuntimeError('Invalid value for option port (a list of intergers), {} provided'.format(kwargs['port']))
#
name_opt = ''
if 'name' in kwargs:
name_opt = '--name={}'.format(kwargs['name'])
#
stdin_opt = ''
if 'stdin_open' in kwargs and kwargs['stdin_optn']:
stdin_opt = '-i'
#
tty_opt = '-t'
if 'tty' in kwargs and not kwargs['tty']:
tty_opt = ''
#
user_opt = ''
if 'user' in kwargs:
user_opt = '-u {}'.format(kwargs['user'])
#
extra_opt = ''
if 'extra_args' in kwargs:
extra_opt = kwargs['extra_args']
#
security_opt = ''
if platform.system() == 'Linux':
# this is for a selinux problem when /var/sos/script cannot be executed
security_opt = '--security-opt label:disable'
command = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(
security_opt, # security option
volumes_opt, # volumes
volumes_from_opt, # volumes_from
name_opt, # name
stdin_opt, # stdin_optn
tty_opt, # tty
port_opt, # port
working_dir_opt, # working dir
user_opt, # user
env_opt, # environment
mem_limit_opt, # memory limit
extra_opt, # any extra parameters
image, # image
cmd_opt
)
env.logger.info(command)
ret = subprocess.call(command, shell=True)
if ret != 0:
msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format(
tempscript, command.replace(tempdir, os.path.abspath('./.sos')))
shutil.copy(os.path.join(tempdir, tempscript), '.sos')
if ret == 125:
raise RuntimeError('Docker daemon failed (exitcode=125). ' + msg)
elif ret == 126:
raise RuntimeError('Failed to invoke specified command (exitcode=126). ' + msg)
elif ret == 127:
raise RuntimeError('Failed to locate specified command (exitcode=127). ' + msg)
elif ret == 137:
if not hasattr(self, 'tot_mem'):
self.tot_mem = self.total_memory(image)
if self.tot_mem is None:
raise RuntimeError('Script killed by docker. ' + msg)
else:
raise RuntimeError('Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). '.format(self.tot_mem/1024/1024) + msg)
else:
raise RuntimeError('Executing script in docker returns an error (exitcode={}). '.format(ret) + msg)
return 0
