def test_multi_named_output(clear_now_and_after):
'''Test DAG built from multiple named_output #1166'''
clear_now_and_after('a.txt', 'b.txt', 'test_named_output.dot')
execute_workflow('''
[A]
output: A='a.txt', B='b.txt'
_output.touch()
[default]
input: named_output('A'), named_output('B')
''',
options={'output_dag': 'test_named_output.dot'})
# note that A2 is no longer mentioned
assertDAG('test_named_output.dot', [
'''
strict digraph "" {
default;
"A (B)";
"A (B)" -> default;
}
''', '''
strict digraph "" {
default;
"A (A)";
"A (A)" -> default;
}
'''
])
def testRun():
'''Test action run'''
execute_workflow(r'''
[0]
run:
echo 'Echo'
''')
def test_remote_exec_named_path(clear_now_and_after):
clear_now_and_after("result_named_path.txt")
root_dir = "/root/build" if "TRAVIS" in os.environ else "/root"
execute_workflow(
"""
output: '#home/result_named_path.txt'
task:
sh: expand=True
echo Output: {_output} > {_output}
echo PWD: `pwd`. >> {_output}
""",
options={
"config_file": "~/docker.yml",
"default_queue": "docker",
"sig_mode": "force",
},
)
assert file_target("#home/result_named_path.txt").target_exists()
with open(file_target("#home/result_named_path.txt")) as res:
result = res.read()
print(result)
assert "Output: /root/result_named_path.txt" in result
assert f"PWD: {root_dir}/vatlab/sos/test." in result
def testAcceptableArgs():
'''test acceptable args of options'''
with pytest.raises(Exception):
execute_workflow(r"""
run: unrecog=1
echo 'a'
""")
def testRunWithShebang():
execute_workflow(r'''
[0]
run:
#!/usr/bin/env python
print('Echo')
''')
def test_to_host_option_with_named_path(clear_now_and_after):
"""Test from_remote option"""
clear_now_and_after(
os.path.expanduser("~/to_host_named_testfile.txt"),
"to_host_named_linecount.txt",
)
with open(os.path.expanduser("~/to_host_named_testfile.txt"), "w") as tf:
for i in range(200):
tf.write(f"line {i+1}\n")
execute_workflow(
"""
[10]
output: 'to_host_named_linecount.txt'
task: to_host='#home/to_host_named_testfile.txt'
sh:
wc -l ~/to_host_named_testfile.txt > to_host_named_linecount.txt
""",
options={
"config_file": "~/docker.yml",
"default_queue": "docker",
"sig_mode": "force",
},
)
assert os.path.isfile("to_host_named_linecount.txt")
with open("to_host_named_linecount.txt") as lc:
assert lc.read().strip().startswith("200")
def test_bash_in_docker():
'''Test action bash in docker environment'''
execute_workflow(r'''
[0]
run: container='docker://ubuntu'
echo 'Echo'
''')
def test_python2():
execute_workflow(r'''
[0]
python2: expand='${ }'
a = {'1', '2'}
print a
''')
def test_rmarkdown(clear_now_and_after):
"""Test action Rmarkdown"""
if not R_library('rmarkdown1').target_exists():
pytest.xfail('rmarkdown is not properly installed.')
clear_now_and_after("myreport.html")
execute_workflow(
r"""
[10]
report:
## Some random figure
Generated by matplotlib
[100]
# generate report
output: 'myreport.html'
Rmarkdown(output=_output[0])
""",
options={"report_output": "report.md"},
)
assert os.path.isfile("myreport.html")
def test_delayed_interpolation(clear_now_and_after):
"""Test delayed interpolation with expression involving remote objects"""
# purge all previous tasks
clear_now_and_after('test.py', 'test.py.bak')
execute_workflow(
"""
[10]
output: remote('test.py')
task:
run:
touch test.py
[20]
output: remote(f"{_input:R}.bak")
task:
run: expand=True
cp {_input} {_output}
""", options={
"config_file": "~/docker.yml",
# do not wait for jobs
"wait_for_task": True,
"default_queue": "docker",
"sig_mode": "force",
},
)
# this file is remote only
assert not os.path.isfile("test.py")
assert not os.path.isfile("test.py.bak")
def testRegenerateReport(clear_now_and_after):
'''Testing the regeneration of report once is needed. The problem
here is the 'input' parameter of report.'''
clear_now_and_after('a1.md', 'a2.md', 'out.md')
script = r'''
[A_1]
output: 'a1.txt', 'a1.md'
run:
echo 'a1' >> a1.txt
report: output='a1.md'
a1
[A_2]
output: 'a2.txt', 'a2.md'
run:
echo 'a2' >> a2.txt
report: output='a2.md'
a2
[A_3]
input: 'a1.md', 'a2.md'
output: 'out.md'
report: input=['a1.md', 'a2.md'], output='out.md'
'''
execute_workflow(script)
with open('a1.md') as a:
assert a.read() == 'a1\n\n'
with open('a2.md') as a:
assert a.read() == 'a2\n\n'
with open('out.md') as a:
assert a.read() == 'a1\n\na2\n\n'
clear_now_and_after('a1.md', 'a2.md', 'out.md')
execute_workflow(script)
def testGetOutput():
'''Test utility function get_output'''
execute_workflow(r"""
[0: shared='ret']
ret = get_output('echo blah')
""")
# use strip because there would be \r\n under windows
assert env.sos_dict['ret'].strip() == 'blah'
execute_workflow(r"""
[0: shared='ret']
ret = get_output('echo blah', show_command=True)
""")
assert [x.strip() for x in env.sos_dict['ret'].splitlines()
] == ['$ echo blah', 'blah']
execute_workflow(r"""
[0: shared='ret']
ret = get_output('echo blah', show_command=True, prompt='% ')
""")
assert [x.strip() for x in env.sos_dict['ret'].splitlines()
] == ['% echo blah', 'blah']
with pytest.raises(Exception):
execute_workflow(r"""
[0]
get_output('catmouse')
""")
with pytest.raises(Exception):
execute_workflow(r"""
[0]
ret = get_output('cat -h')
""")
def test_r_library():
"""Test target R_Library"""
execute_workflow("""
[default]
depends: R_library("dplyr", autoinstall=True)
R:
library('dplyr')
""")
def test_action_option_template():
if os.path.isfile('template_output.txt'):
os.remove('template_output.txt')
execute_workflow('''
run: template='cat {filename}'
echo 'whatever' > template_output.txt
''')
assert not os.path.isfile('template_output.txt')
def test_docker_script_action():
'''Test action sh in docker environment'''
# test docker
execute_workflow(r'''
[0]
script: container='docker://ubuntu', args='{script}'
echo 'Echo'
''')
def test_node():
'''Test action node'''
execute_workflow(r'''
[0]
node:
var args = process.argv.slice(2);
console.log('Hello ' + args.join(' ') + '!');
''')
def test_run_1():
with pytest.raises(Exception):
execute_workflow(r'''
[0]
run:
echo 'Echo
''')
def test_action_option_template(clear_now_and_after):
clear_now_and_after('template_output.txt')
execute_workflow('''
run: template='cat {filename}'
echo 'whatever' > template_output.txt
''')
assert not os.path.isfile('template_output.txt')
def test_auto_output_mount():
'''Test use of option port in action'''
execute_workflow(r'''
output: '../data/1.txt'
run: container='ubuntu'
sh: expand=True
touch {_output}
''')
assert os.path.isfile('../data/1.txt')
def test_to_named_path_path():
execute_workflow("""
[10: shared="a"]
a = path('/root/xxx/whatever').to_named_path(host='docker')
""",
options={
"config_file":
os.path.join(os.path.expanduser("~"), "docker.yml"),
})
assert env.sos_dict['a'] == '#home/xxx/whatever'
def testProvidesSoSVariable(self):
'''Test provides non-filename targets #1341'''
execute_workflow('''
[count: provides=sos_variable('numNotebooks')]
numNotebooks = 1
[default]
depends: sos_variable('numNotebooks')
print(f"There are {numNotebooks} notebooks in this directory")
''')
def pull_images(self, containers):
for container, engine in containers:
script = f'[container]\nrun: stderr=False, stdout=False, container={repr(container)}'
if not engine is None:
script += f', engine={repr(engine)}'
script += '\n dsc -h'
try:
execute_workflow(script, workflow='container', options=dict(verbosity=1))
except Exception as e:
raise ModuleNotFoundError(f'Please make sure ``{container}`` is available (locally or online) and has DSC software (including R package dscrutils) installed for use in DSC environment.')
def test_report_5(clear_now_and_after):
'''Test action report'''
clear_now_and_after('a.txt')
execute_workflow(r'''
[A_1]
output: 'a.txt'
report: output=_output[0]
something
''')
def test_report_4(clear_now_and_after):
'''Test action report'''
clear_now_and_after('a.txt')
# test report to other types of output: path
execute_workflow(r'''
[A_1]
report: output=path('a.txt')
something
''')
def test_group_with_with_no_output(self):
execute_workflow(r'''
[10]
input: for_each=dict(i=range(3))
output: group_with=dict(var=['A', 'B', 'C'][_index])
print(i)
[20]
print(f'Input is {_input} {var}')
''')
def testPerl():
'''Test action perl'''
execute_workflow(r'''
[0]
perl:
use strict;
use warnings;
print "hi NAME\n";
''')
def test_rmarkdown_to_stdout():
if not R_library('rmarkdown1').target_exists():
pytest.xfail('rmarkdown is not properly installed.')
execute_workflow(
r"""
# generate report
Rmarkdown:
# this is title
"""
)
def test_report_6(clear_now_and_after):
'''Test action report'''
clear_now_and_after('a.txt')
# test report to other types of output: sos_targets
execute_workflow(r'''
[A_1]
output: 'a.txt'
report: output=_output
something
''')
def test_report_7(clear_now_and_after):
'''Test action report'''
clear_now_and_after('a.txt')
with pytest.raises(Exception):
execute_workflow(r'''
[A_1]
output: 'a.txt', 'b.txt'
report: output=_output
something
''')
def test_global_vars(config_factory):
'''Test SoS defined variables'''
execute_workflow("[0]", options={'mode': 'dryrun'})
assert env.sos_dict['SOS_VERSION'] == __version__
assert isinstance(env.sos_dict['CONFIG'], dict)
cfg = config_factory({'my_config': 5})
execute_workflow("[0]", options={'config_file': cfg})
assert env.sos_dict['CONFIG']['my_config'] == 5
