def testInterpolation(self): '''Test string interpolation''' env.sos_dict = WorkflowDict({ 'os': os, 'a': 100, 'b': 20, 'c': ['file1', 'file2', 'file3'], 'd': {'a': 'file1', 'b':'file2'}, 'e': set([1, 'a']), 'var1': 1/2., 'var2': [1, 2, 3.1], 'file': 'a/b.txt', 'files2': ['a/b.txt', 'c.d.txt'], 'file_ws': ['d i r/f .txt'] }) for sigil in ('${ }', '{ }', '[ ]', '%( )', '[[ ]]', '%( )s', '# #', '` `'): l, r = sigil.split(' ') for expr, result, nested, exclude in [ ('{0}1{1}', '1', False, []), ('{0}a{1}', '100', False, []), ('{0}a+b{1}', '120', False, []), ('{0}a+b*5{1}', '200', False, []), ('{0}a+b*5{1} is 200', '200 is 200', False, []), ('{0}a+b*5{1} and {0}a{1}', '200 and 100', False, []), ('Pre {0}a+b*5{1} and {0}a{1} after', 'Pre 200 and 100 after', False, []), ('Nested {0}a+b*{0}b//2{1}{1}', 'Nested 300', True, []), ('Format {0}a:.5f{1}', 'Format 100.00000', False, []), ('{0}var2[:2]{1}', '1 2', False, []), ('{0}var2[1:]{1}', '2 3.1', False, []), # nested ('Nested {0}a:.{0}4+1{1}f{1}', 'Nested 100.00000', True, []), # deep nested ('Triple Nested {0}a:.{0}4+{0}5//5{1}{1}f{1}', 'Triple Nested 100.00000', True, []), # nested invalid ('Nested invalid {0}"{0}a-{1}"{1}', 'Nested invalid {}a-{}'.format(l, r), True, []), ('Nested valid {0}"{0}a{1}-"{1}', 'Nested valid 100-', True, []), # ('Dict {0}d{1}', ['Dict a b', 'Dict b a'], False, []), ('set {0}e{1}', ['set 1 a', 'set a 1'], False, []), ('Fmt {0}var1:.2f{1}', 'Fmt 0.50', False, []), ('Fmt {0}var2:.2f{1}', 'Fmt 1.00 2.00 3.10', False, []), ('LC {0}[x*2 for x in var2]{1}', 'LC 2 4 6.2', False, []), ('LC {0}[x*2 for x in var2]:.2f{1}', 'LC 2.00 4.00 6.20', False, []), # # [['a':'b', 'c':'d']['a']] works because # ['a':'b', 'c':'d']a # is invalid so SoS does not consider ['a'] as nested expression # ('Newline {0}{{"a": "b", \n"c": "d"}}["a"]{1}', 'Newline b', False, []), # # string literal within interpolated expression ('Literal {0}"{0} {1}"{1}', 'Literal ' + sigil, True, []), # this case does not work because {} would become '' with sigil {} ("{0}' {{}} '.format(a){1}", ' 100 ', False, ['{ }']), # ("{0}os.path.basename(file){1}", 'b.txt', False, []), ('{0}os.path.basename(file_ws[0]){1}', 'f .txt', False, []), # # ! conversion ('{0}file!r{1}', "'a/b.txt'", False, []), ('{0}file!s{1}', "a/b.txt", False, []), ('''{0}"a'b"!r{1}''', '"a\'b"', False, []), ('''{0}'a"b'!r{1}''', "'a\"b'", False, []), # # !q conversion (added by SoS) ('{0}file_ws[0]!q{1}', "'d i r/f .txt'", False, []), # # !, joined by , ('{0}var2!r,{1}', "1, 2, 3.1", False, []), ('{0}c!r,{1}', "'file1', 'file2', 'file3'", False, []), ('{0}c!,{1}', "file1, file2, file3", False, []), # ('{0}10000:,{1}', "10,000", False, []), # # full name by 'a' ('{0}"test_utils.py"!a{1}', os.path.abspath('test_utils.py'), False, []), ('{0}"a/b/c/test_utils.py"!b{1}', 'test_utils.py', False, []), ('{0}"a/b/c/test_utils.py"!d{1}', 'a/b/c', False, []), ('{0}"a/b/c/test_utils.py"!dd{1}', 'a/b', False, []), ('{0}"a/b/c/test_utils.py"!ddb{1}', 'b', False, []), ('{0}"a/b/c/test_utils.py"!n{1}', 'a/b/c/test_utils', False, []), ('{0}"a/b/c/test_utils.py"!bn{1}', 'test_utils', False, []), ('{0}"~/test_utils.py"!a{1}', os.path.expanduser('~/test_utils.py'), False, []), ('{0}"~/test_utils.py"!e{1}', os.path.expanduser('~/test_utils.py'), False, []), ('{0}"test/test_utils.py"!b{1}', "test_utils.py", False, []), # preceded by \ (r'\{0}100{1}', '{0}100{1}'.format(l, r), True, []), (r'\{0}100{1} {0}100+4{1}', '{0}100{1} 104'.format(l, r), True, []), (r'''{0}{1}''', '', False, []), ]: if l == r and nested: continue if sigil in exclude: continue if isinstance(result, str): self.assertEqual(interpolate(expr.format(l, r), sigil=sigil), result, 'Failed to interpolate {} with sigil {}'.format(expr, sigil)) else: # for cases when the order of output is not guaranteed self.assertTrue(interpolate(expr.format(l, r), sigil=sigil) in result, 'Failed to interpolate {} with sigil {}'.format(expr, sigil)) # # locals should be the one passed to the expression self.assertTrue('file_ws' in interpolate('${globals().keys()}', '${ }')) # 5:.5.0f does not work. self.assertRaises(InterpolationError, interpolate, '${5:.${4/2.}f}', '${ }')
def Rmarkdown(script=None, input=None, output=None, args="${input!r}, output_file=${output!ar}", **kwargs): """Convert input file to output using Rmarkdown The input can be specified in three ways: 1. instant script, which is assumed to be in md format Rmarkdown: output='report.html' script 2. one or more input files. The format is determined by extension of input file Rmarkdown(input, output='report.html') 3. input file specified by command line option `-r` . Rmarkdown(output='report.html') If no output is specified, it is assumed to be in html format and is written to standard output. You can specify more options using the args parameter of the action. The default value of args is `${input!r} --output ${output!ar}' """ if not R_library("rmarkdown").exists(): raise UnknownTarget(R_library("rmarkdown")) input_file = collect_input(script, input) write_to_stdout = False if output is None: write_to_stdout = True output_file = tempfile.NamedTemporaryFile(mode="w+t", suffix=".html", delete=False).name elif isinstance(output, str): output_file = output else: raise RuntimeError("A filename is expected, {} provided".format(output)) # ret = 1 try: # render(input, output_format = NULL, output_file = NULL, output_dir = NULL, # output_options = NULL, intermediates_dir = NULL, # runtime = c("auto", "static", "shiny"), # clean = TRUE, params = NULL, knit_meta = NULL, envir = parent.frame(), # run_Rmarkdown = TRUE, quiet = FALSE, encoding = getOption("encoding")) cmd = interpolate( 'Rscript -e "rmarkdown::render({})"'.format(args), "${ }", {"input": input_file, "output": output_file} ) env.logger.trace('Running command "{}"'.format(cmd)) if env.run_mode == "interactive": # need to catch output and send to python output, which will in trun be hijacked by SoS notebook p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) pid = p.pid env.register_process(p.pid, "Runing {}".format(input_file)) out, err = p.communicate() sys.stdout.write(out.decode()) sys.stderr.write(err.decode()) ret = p.returncode else: p = subprocess.Popen(cmd, shell=True) pid = p.pid env.register_process(pid, "Runing {}".format(input_file)) ret = p.wait() except Exception as e: env.logger.error(e) finally: env.deregister_process(p.pid) if ret != 0: temp_file = os.path.join(".sos", "{}_{}.md".format("Rmarkdown", os.getpid())) shutil.copyfile(input_file, temp_file) cmd = interpolate( 'Rscript -e "rmarkdown::render({})"'.format(args), "${ }", {"input": input_file, "output": output_file} ) raise RuntimeError( 'Failed to execute script. Please use command \n"{}"\nunder {} to test it.'.format(cmd, os.getcwd()) ) if write_to_stdout: with open(output_file) as out: sys.stdout.write(out.read()) else: env.logger.info("Report saved to {}".format(output_file))
def _install(self, name, version, repos): '''Check existence and version match of R library. cran and bioc packages are unique yet might overlap with github. Therefore if the input name is {repo}/{pkg} the package will be installed from github if not available, else from cran or bioc ''' from sos.pattern import glob_wildcards from sos.sos_eval import interpolate import tempfile import shlex import subprocess output_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.txt', delete=False).name script_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.R', delete=False).name if len(glob_wildcards('{repo}/{pkg}', [name])['repo']): # package is from github self._intall('devtools', version, repos) install_script = interpolate(''' options(warn=-1) package_repo <- ${name!r} package <- basename(package_repo) if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}") } else { devtools::install_github(package_repo) # if it still does not exist, write the package name to output if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}") } else { write(paste(package, "NA", "MISSING"), file="${output_file}") quit("no") } } cur_version <- packageVersion(package) ''', '${ }', locals()) else: # package is from cran or bioc install_script = interpolate(''' options(warn=-1) package <- ${name!r} if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}") } else { install.packages(package, repos="${repos}", quiet=FALSE) # if the package still does not exist if (!require(package, character.only=TRUE, quietly=TRUE)) { source("http://bioconductor.org/biocLite.R") biocLite(package, suppressUpdates=TRUE, suppressAutoUpdate=TRUE, ask=FALSE) } # if it still does not exist, write the package name to output if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}") } else { write(paste(package, "NA", "MISSING"), file="${output_file}") quit("no") } } cur_version <- packageVersion(package) ''', '${ }', locals()) version_script = '' if version is not None: version = [version] if isinstance(version, str) else version operators = [] for idx, value in enumerate(version): value = str(value) if value.endswith('+'): operators.append('>=') version[idx] = value[:-1] elif value.endswith('-'): operators.append('<') version[idx] = value[:-1] else: operators.append('==') # check version and mark version mismatch # if current version satisfies any of the # requirement the check program quits for x, y in zip(version, operators): version_script += ''' if (cur_version {1} {0}) {{ quit("no") }} '''.format(repr(x), y) version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\ format(repr(output_file)) # temporarily change the run mode to run to execute script try: with open(script_file, 'w') as sfile: sfile.write(install_script + version_script) cmd = 'Rscript --default-packages=utils ' + shlex.quote(script_file) # p = subprocess.Popen(cmd, shell=True) ret = p.wait() if ret != 0: env.logger.warning('Failed to detect or install R library') return False except Exception as e: env.logger.error('Failed to execute script: {}'.format(e)) return False finally: os.remove(script_file) ret_val = False with open(output_file) as tmp: for line in tmp: lib, version, status = line.split() if status.strip() == "MISSING": env.logger.warning('R Library {} is not available and cannot be installed.'.format(lib)) elif status.strip() == 'AVAILABLE': env.logger.debug('R library {} ({}) is available'.format(lib, version)) ret_val = True elif status.strip() == 'INSTALLED': env.logger.debug('R library {} ({}) has been installed'.format(lib, version)) ret_val = True elif status.strip() == 'VERSION_MISMATCH': env.logger.warning('R library {} ({}) does not satisfy version requirement!'.format(lib, version)) else: raise RuntimeError('This should not happen: {}'.format(line)) try: os.remove(output_file) except: pass return ret_val
def testInterpolation(self): """Test string interpolation""" env.sos_dict = WorkflowDict( { "os": os, "a": 100, "b": 20, "c": ["file1", "file2", "file3"], "d": {"a": "file1", "b": "file2"}, "e": set([1, "a"]), "var1": 1 / 2.0, "var2": [1, 2, 3.1], "file": "a/b.txt", "files2": ["a/b.txt", "c.d.txt"], "file_ws": ["d i r/f .txt"], } ) for sigil in ("${ }", "{ }", "[ ]", "%( )", "[[ ]]", "%( )s", "# #", "` `"): l, r = sigil.split(" ") for expr, result, nested, exclude in [ ("{0}1{1}", "1", False, []), ("{0}a{1}", "100", False, []), ("{0}a+b{1}", "120", False, []), ("{0}a+b*5{1}", "200", False, []), ("{0}a+b*5{1} is 200", "200 is 200", False, []), ("{0}a+b*5{1} and {0}a{1}", "200 and 100", False, []), ("Pre {0}a+b*5{1} and {0}a{1} after", "Pre 200 and 100 after", False, []), ("Nested {0}a+b*{0}b//2{1}{1}", "Nested 300", True, []), ("Format {0}a:.5f{1}", "Format 100.00000", False, []), ("{0}var2[:2]{1}", "1 2", False, []), ("{0}var2[1:]{1}", "2 3.1", False, []), # nested ("Nested {0}a:.{0}4+1{1}f{1}", "Nested 100.00000", True, []), # deep nested ("Triple Nested {0}a:.{0}4+{0}5//5{1}{1}f{1}", "Triple Nested 100.00000", True, []), # nested invalid ('Nested invalid {0}"{0}a-{1}"{1}', "Nested invalid {}a-{}".format(l, r), True, []), ('Nested valid {0}"{0}a{1}-"{1}', "Nested valid 100-", True, []), # ("Dict {0}d{1}", ["Dict a b", "Dict b a"], False, []), ("set {0}e{1}", ["set 1 a", "set a 1"], False, []), ("Fmt {0}var1:.2f{1}", "Fmt 0.50", False, []), ("Fmt {0}var2:.2f{1}", "Fmt 1.00 2.00 3.10", False, []), ("LC {0}[x*2 for x in var2]{1}", "LC 2 4 6.2", False, []), ("LC {0}[x*2 for x in var2]:.2f{1}", "LC 2.00 4.00 6.20", False, []), # # [['a':'b', 'c':'d']['a']] works because # ['a':'b', 'c':'d']a # is invalid so SoS does not consider ['a'] as nested expression # ('Newline {0}{{"a": "b", \n"c": "d"}}["a"]{1}', "Newline b", False, []), # # string literal within interpolated expression ('Literal {0}"{0} {1}"{1}', "Literal " + sigil, True, []), # this case does not work because {} would become '' with sigil {} ("{0}' {{}} '.format(a){1}", " 100 ", False, ["{ }"]), # ("{0}os.path.basename(file){1}", "b.txt", False, []), ("{0}os.path.basename(file_ws[0]){1}", "f .txt", False, []), # # ! conversion ("{0}file!r{1}", "'a/b.txt'", False, []), ("{0}file!s{1}", "a/b.txt", False, []), ("""{0}"a'b"!r{1}""", '"a\'b"', False, []), ("""{0}'a"b'!r{1}""", "'a\"b'", False, []), # # !q conversion (added by SoS) ("{0}file_ws[0]!q{1}", "'d i r/f .txt'", False, []), # # !, joined by , ("{0}var2!r,{1}", "1, 2, 3.1", False, []), ("{0}c!r,{1}", "'file1', 'file2', 'file3'", False, []), ("{0}c!,{1}", "file1, file2, file3", False, []), # ("{0}10000:,{1}", "10,000", False, []), # # full name by 'a' ('{0}"test_utils.py"!a{1}', os.path.abspath("test_utils.py"), False, []), ('{0}"a/b/c/test_utils.py"!b{1}', "test_utils.py", False, []), ('{0}"a/b/c/test_utils.py"!d{1}', "a/b/c", False, []), ('{0}"a/b/c/test_utils.py"!dd{1}', "a/b", False, []), ('{0}"a/b/c/test_utils.py"!ddb{1}', "b", False, []), ('{0}"a/b/c/test_utils.py"!n{1}', "a/b/c/test_utils", False, []), ('{0}"a/b/c/test_utils.py"!bn{1}', "test_utils", False, []), ('{0}"~/test_utils.py"!a{1}', os.path.expanduser("~/test_utils.py"), False, []), ('{0}"~/test_utils.py"!e{1}', os.path.expanduser("~/test_utils.py"), False, []), ('{0}"test/test_utils.py"!b{1}', "test_utils.py", False, []), # preceded by \ (r"\{0}100{1}", "{0}100{1}".format(l, r), True, []), (r"\{0}100{1} {0}100+4{1}", "{0}100{1} 104".format(l, r), True, []), (r"""{0}{1}""", "", False, []), ]: if l == r and nested: continue if sigil in exclude: continue if isinstance(result, str): self.assertEqual( interpolate(expr.format(l, r), sigil=sigil), result, "Failed to interpolate {} with sigil {}".format(expr, sigil), ) else: # for cases when the order of output is not guaranteed self.assertTrue( interpolate(expr.format(l, r), sigil=sigil) in result, "Failed to interpolate {} with sigil {}".format(expr, sigil), ) # # locals should be the one passed to the expression self.assertTrue("file_ws" in interpolate("${globals().keys()}", "${ }")) # 5:.5.0f does not work. self.assertRaises(InterpolationError, interpolate, "${5:.${4/2.}f}", "${ }")
def _install(self, name, version, repos): '''Check existence and version match of R library. cran and bioc packages are unique yet might overlap with github. Therefore if the input name is {repo}/{pkg} the package will be installed from github if not available, else from cran or bioc ''' from sos.pattern import glob_wildcards from sos.sos_eval import interpolate import tempfile import shlex import subprocess output_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.txt', delete=False).name script_file = tempfile.NamedTemporaryFile(mode='w+t', suffix='.R', delete=False).name if len(glob_wildcards('{repo}/{pkg}', [name])['repo']): # package is from github self._intall('devtools', version, repos) install_script = interpolate( ''' options(warn=-1) package_repo <- ${name!r} package <- basename(package_repo) if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}") } else { devtools::install_github(package_repo) # if it still does not exist, write the package name to output if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}") } else { write(paste(package, "NA", "MISSING"), file="${output_file}") quit("no") } } cur_version <- packageVersion(package) ''', '${ }', locals()) else: # package is from cran or bioc install_script = interpolate( ''' options(warn=-1) package <- ${name!r} if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "AVAILABLE"), file="${output_file}") } else { install.packages(package, repos="${repos}", quiet=FALSE) # if the package still does not exist if (!require(package, character.only=TRUE, quietly=TRUE)) { source("http://bioconductor.org/biocLite.R") biocLite(package, suppressUpdates=TRUE, suppressAutoUpdate=TRUE, ask=FALSE) } # if it still does not exist, write the package name to output if (require(package, character.only=TRUE, quietly=TRUE)) { write(paste(package, packageVersion(package), "INSTALLED"), file="${output_file}") } else { write(paste(package, "NA", "MISSING"), file="${output_file}") quit("no") } } cur_version <- packageVersion(package) ''', '${ }', locals()) version_script = '' if version is not None: version = [version] if isinstance(version, str) else version operators = [] for idx, value in enumerate(version): value = str(value) if value.endswith('+'): operators.append('>=') version[idx] = value[:-1] elif value.endswith('-'): operators.append('<') version[idx] = value[:-1] else: operators.append('==') # check version and mark version mismatch # if current version satisfies any of the # requirement the check program quits for x, y in zip(version, operators): version_script += ''' if (cur_version {1} {0}) {{ quit("no") }} '''.format(repr(x), y) version_script += 'write(paste(package, cur_version, "VERSION_MISMATCH"), file = {})'.\ format(repr(output_file)) # temporarily change the run mode to run to execute script try: with open(script_file, 'w') as sfile: sfile.write(install_script + version_script) cmd = 'Rscript --default-packages=utils ' + shlex.quote( script_file) # p = subprocess.Popen(cmd, shell=True) ret = p.wait() if ret != 0: env.logger.warning('Failed to detect or install R library') return False except Exception as e: env.logger.error('Failed to execute script: {}'.format(e)) return False finally: os.remove(script_file) ret_val = False with open(output_file) as tmp: for line in tmp: lib, version, status = line.split() if status.strip() == "MISSING": env.logger.warning( 'R Library {} is not available and cannot be installed.' .format(lib)) elif status.strip() == 'AVAILABLE': env.logger.debug('R library {} ({}) is available'.format( lib, version)) ret_val = True elif status.strip() == 'INSTALLED': env.logger.debug( 'R library {} ({}) has been installed'.format( lib, version)) ret_val = True elif status.strip() == 'VERSION_MISMATCH': env.logger.warning( 'R library {} ({}) does not satisfy version requirement!' .format(lib, version)) else: raise RuntimeError( 'This should not happen: {}'.format(line)) try: os.remove(output_file) except: pass return ret_val
def run(self, image, script='', interpreter='', args='', suffix='.sh', **kwargs): if self.client is None: raise RuntimeError('Cannot connect to the Docker daemon. Is the docker daemon running on this host?') # env.logger.debug('docker_run with keyword args {}'.format(kwargs)) # # now, write a temporary file to a tempoary directory under the current directory, this is because # we need to share the directory to ... with tempfile.TemporaryDirectory(dir=os.getcwd()) as tempdir: # keep the temporary script for debugging purposes # tempdir = tempfile.mkdtemp(dir=os.getcwd()) if script: tempscript = 'docker_run_{}{}'.format(os.getpid(), suffix) with open(os.path.join(tempdir, tempscript), 'w') as script_file: script_file.write(script) # # if there is an interpreter and with args if not args: args = '${filename!q}' if not interpreter: interpreter = '/bin/bash' # if there is a shebang line, we ... if script.startswith('#!'): # make the script executable env.logger.warning('Shebang line in a docker-run script is ignored') # binds = [] if 'volumes' in kwargs: volumes = [kwargs['volumes']] if isinstance(kwargs['volumes'], str) else kwargs['volumes'] for vol in volumes: if not vol: continue if vol.count(':') != 1: raise RuntimeError('Please specify columes in the format of host_dir:mnt_dir') host_dir, mnt_dir = vol.split(':') if platform.system() == 'Darwin': # under Darwin, host_dir must be under /Users if not os.path.abspath(host_dir).startswith('/Users') and not (self.has_volumes and os.path.abspath(host_dir).startswith('/Volumes')): raise RuntimeError('hostdir ({}) under MacOSX must be under /Users or /Volumes (if properly configured, see https://github.com/bpeng2000/SOS/wiki/SoS-Docker-guide for details) to be usable in docker container'.format(host_dir)) binds.append('{}:{}'.format(os.path.abspath(host_dir), mnt_dir)) # volumes_opt = ' '.join('-v {}'.format(x) for x in binds) # under mac, we by default share /Users within docker if platform.system() == 'Darwin': if not any(x.startswith('/Users:') for x in binds): volumes_opt += ' -v /Users:/Users' if self.has_volumes: volumes_opt += ' -v /Volumes:/Volumes' if not any(x.startswith('/tmp:') for x in binds): volumes_opt += ' -v /tmp:/tmp' # mem_limit_opt = '' if 'mem_limit' in kwargs: mem_limit_opt = '--memory={}'.format(kwargs['mem_limit']) # volumes_from_opt = '' if 'volumes_from' in kwargs: if isinstance(kwargs['volumes_from'], str): volumes_from_opt = '--volumes_from={}'.format(kwargs['volumes_from']) elif isinstance(kwargs['volumes_from'], list): volumes_from_opt = ' '.join('--volumes_from={}'.format(x) for x in kwargs['volumes_from']) else: raise RuntimeError('Option volumes_from only accept a string or list of string'.format(kwargs['volumes_from'])) # we also need to mount the script cmd_opt = '' if script and interpreter: volumes_opt += ' -v {}:{}'.format(os.path.join(tempdir, tempscript), '/var/lib/sos/{}'.format(tempscript)) cmd_opt = interpolate('{} {}'.format(interpreter, args), '${ }', {'filename': '/var/lib/sos/{}'.format(tempscript)}) # working_dir_opt = '-w={}'.format(os.path.abspath(os.getcwd())) if 'working_dir' in kwargs: if not os.path.isabs(kwargs['working_dir']): env.logger.warning('An absolute path is needed for -w option of docker run command. "{}" provided, "{}" used.' .format(kwargs['working_dir'], os.path.abspath(os.path.expanduser(kwargs['working_dir'])))) working_dir_opt = '-w={}'.format(os.path.abspath(os.path.expanduser(kwargs['working_dir']))) else: working_dir_opt = '-w={}'.format(kwargs['working_dir']) # env_opt = '' if 'environment' in kwargs: if isinstance(kwargs['environment'], dict): env_opt = ' '.join('-e {}={}'.format(x,y) for x,y in kwargs['environment'].items()) elif isinstance(kwargs['environment'], list): env_opt = ' '.join('-e {}'.format(x) for x in kwargs['environment']) elif isinstance(kwargs['environment'], str): env_opt = '-e {}'.format(kwargs['environment']) else: raise RuntimeError('Invalid value for option environment (str, list, or dict is allowd, {} provided)'.format(kwargs['environment'])) # port_opt = '-P' if 'port' in kwargs: if isinstance(kwargs['port'], (str, int)): port_opt = '-p {}'.format(kwargs['port']) elif isinstance(kwargs['port'], list): port_opt = ' '.join('-p {}'.format(x) for x in kwargs['port']) else: raise RuntimeError('Invalid value for option port (a list of intergers), {} provided'.format(kwargs['port'])) # name_opt = '' if 'name' in kwargs: name_opt = '--name={}'.format(kwargs['name']) # stdin_opt = '' if 'stdin_open' in kwargs and kwargs['stdin_optn']: stdin_opt = '-i' # tty_opt = '-t' if 'tty' in kwargs and not kwargs['tty']: tty_opt = '' # user_opt = '' if 'user' in kwargs: user_opt = '-u {}'.format(kwargs['user']) # extra_opt = '' if 'extra_args' in kwargs: extra_opt = kwargs['extra_args'] # security_opt = '' if platform.system() == 'Linux': # this is for a selinux problem when /var/sos/script cannot be executed security_opt = '--security-opt label:disable' command = 'docker run --rm {} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format( security_opt, # security option volumes_opt, # volumes volumes_from_opt, # volumes_from name_opt, # name stdin_opt, # stdin_optn tty_opt, # tty port_opt, # port working_dir_opt, # working dir user_opt, # user env_opt, # environment mem_limit_opt, # memory limit extra_opt, # any extra parameters image, # image cmd_opt ) env.logger.info(command) ret = subprocess.call(command, shell=True) if ret != 0: msg = 'The script has been saved to .sos/{} so that you can execute it using the following command:\n{}'.format( tempscript, command.replace(tempdir, os.path.abspath('./.sos'))) shutil.copy(os.path.join(tempdir, tempscript), '.sos') if ret == 125: raise RuntimeError('Docker daemon failed (exitcode=125). ' + msg) elif ret == 126: raise RuntimeError('Failed to invoke specified command (exitcode=126). ' + msg) elif ret == 127: raise RuntimeError('Failed to locate specified command (exitcode=127). ' + msg) elif ret == 137: if not hasattr(self, 'tot_mem'): self.tot_mem = self.total_memory(image) if self.tot_mem is None: raise RuntimeError('Script killed by docker. ' + msg) else: raise RuntimeError('Script killed by docker, probably because of lack of RAM (available RAM={:.1f}GB, exitcode=137). '.format(self.tot_mem/1024/1024) + msg) else: raise RuntimeError('Executing script in docker returns an error (exitcode={}). '.format(ret) + msg) return 0