def test_magics_magic_command(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
commands = [f'\t- {cmd}' for cmd in kernel._magic_commands.keys()]
commands.sort()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% magics""")
)
self.assertEqual(
'List of Available Magic commands\n' + os.linesep.join(commands),
responses[-1][0][2]['text']
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% magics data""")
)
from cwlkernel.kernel_magics import data as data_magic_command
import inspect
self.assertEqual(
inspect.getdoc(data_magic_command),
responses[-1][0][2]['text']
)
self.assertIn(' '.join('Display all the data which are registered in the kernel session.'.split()),
' '.join(responses[-1][0][2]['text'].split()))
def test_all_magic_commands_have_methods(self):
kernel = CWLKernel()
for magic in kernel._magic_commands:
try:
kernel.__getattribute__(f'_execute_magic_{magic}')
except AttributeError as e:
self.fail(f'Missing function for magic command: {magic}. \nAttribute error raises: {e}')
def test_send_workflow_without_id(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'without_id.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str)
)
self.assertRegex(
responses[-1][0][2]['text'],
r"^tool '[a-zA-Z0-9-]+' registered"
)
with open(os.sep.join([self.cwl_directory, 'echo.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str)
)
self.assertTupleEqual(
((None, 'stream', {'name': 'stdout', 'text': "tool 'echo' registered"}),
{}),
responses[-1]
)
def test_snippet_builder(self):
import yaml
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'echo.cwl'])) as f:
echo_workflow = f.read().splitlines()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"% snippet add\n{os.linesep.join(echo_workflow[:6])}")
)
self.assertDictEqual(
yaml.load(StringIO(os.linesep.join(echo_workflow[:6])), yaml.Loader),
dict(responses[-1][0][2]['data']['application/json'])
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"% snippet build\n{os.linesep.join(echo_workflow[6:])}")
)
self.assertDictEqual(
yaml.load(StringIO(os.linesep.join(echo_workflow)), yaml.Loader),
dict(responses[-1][0][2]['data']['application/json'])
)
self.assertDictEqual(
yaml.load(StringIO(kernel._workflow_repository.__repo__.get_by_id("echo").to_yaml()), yaml.Loader),
yaml.load(StringIO('\n'.join(echo_workflow)), yaml.Loader),
)
def test_display_json_output_after_execution(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
# prepare data
with open(os.sep.join([self.data_directory, 'tar_job.yml'])) as f:
data = f.read()
tar_directory = kernel._cwl_executor.file_manager.ROOT_DIRECTORY
with open(os.path.join(tar_directory, 'hello.txt'), 'w') as temp_hello_world_file:
temp_hello_world_file.write("hello world")
tar_full_name = os.path.join(tar_directory, 'tarfile.tar')
print('create tar file:', tar_full_name)
with tarfile.open(tar_full_name, 'w') as tar:
tar.add(temp_hello_world_file.name)
data = f"% execute extract-tar\n{data.format(tar_directory=tar_directory)}"
# set workflow
with open(os.sep.join([self.cwl_directory, 'extract_tar.cwl'])) as f:
workflow_str = f.read().format(example_out=temp_hello_world_file.name[1:])
result = kernel.do_execute(workflow_str, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
result = kernel.do_execute(data, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
self.assertDictEqual(
{
'example_out': {
'location': f'file://{tar_directory}/hello.txt', 'basename': 'hello.txt',
'nameroot': 'hello', 'nameext': '.txt', 'class': 'File',
'checksum': 'sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'size': 11,
'http://commonwl.org/cwltool#generation': 0,
'id': 'example_out',
"result_counter": 0,
'_produced_by': 'extract-tar',
}
},
responses[-1][0][2]['data']['application/json']
)
self.assertDictEqual(
{
'example_out': {
'location': f'file://{tar_directory}/hello.txt', 'basename': 'hello.txt',
'nameroot': 'hello', 'nameext': '.txt', 'class': 'File',
'checksum': 'sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'size': 11,
'http://commonwl.org/cwltool#generation': 0,
'id': 'example_out',
"result_counter": 0,
'_produced_by': 'extract-tar',
}
},
json.loads(responses[-1][0][2]['data']['text/plain'])
)
def test_display_data_magic_command(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# monitor responses
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.data_directory, 'echo-job.yml'])) as f:
data = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data, False)
)
with open(os.sep.join([self.cwl_directory, 'echo_stdout.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str, False)
)
kernel.do_execute('% display_data')
self.assertEqual(
'ERROR: you must select an output to display. Correct format:\n % display_data [output name]',
responses[-1][0][2]['text']
)
kernel.do_execute('% display_data echo_output')
self.assertEqual(
'Hello world!\n',
responses[-1][0][2]['text']
)
def test_get_past_results_without_input(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.cwl_directory, 'touched.cwl'])) as f:
workflow_str = f.read()
result = kernel.do_execute(workflow_str, False)
full_path, basename = [(f, os.path.basename(f)) for f in kernel.get_past_results()][0]
self.assertTrue(full_path.startswith(kernel._results_manager.ROOT_DIRECTORY), 'output is in a wrong directory')
self.assertTrue(basename, 'touchedfile.txt')
def test_display_json_output_after_execution(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.data_directory, 'tar_job.yml'])) as f:
data = f.read()
tar_directory = kernel._cwl_executor.file_manager.ROOT_DIRECTORY
with open(os.path.join(tar_directory, 'hello.txt'), 'w') as temp_hello_world_file:
temp_hello_world_file.write("hello world")
tar_full_name = os.path.join(tar_directory, 'tarfile.tar')
print('create tar file:', tar_full_name)
with tarfile.open(tar_full_name, 'w') as tar:
tar.add(temp_hello_world_file.name)
data = data.format(tar_directory=tar_directory)
result = kernel.do_execute(data, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
with open(os.sep.join([self.cwl_directory, 'extract_tar.cwl'])) as f:
workflow_str = f.read().format(example_out=temp_hello_world_file.name[1:])
result = kernel.do_execute(workflow_str, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
self.assertTupleEqual(
(None, 'display_data',
{
'data': {
'text/plain': '<IPython.core.display.JSON object>',
'application/json': {
'example_out': {
'location': f'file://{tar_directory}/hello.txt', 'basename': 'hello.txt',
'nameroot': 'hello', 'nameext': '.txt', 'class': 'File',
'checksum': 'sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'size': 11,
'http://commonwl.org/cwltool#generation': 0,
'id': 'example_out'
}
}
},
'metadata': {
'application/json': {
'expanded': False,
'root': 'root'
}
}
}
),
responses[1][0]
)
def test_array_output(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.cwl_directory, 'array-outputs.cwl'])) as f:
workflow_str = f.read()
with open(os.sep.join([self.data_directory, 'array-outputs-job.yml'])) as f:
data = f"% execute touch\n{f.read()}"
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data)
)
def test_execute_multiple_steps(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'head.cwl'])) as f:
head_cwl = f.read()
with open(os.sep.join([self.cwl_directory, 'tail.cwl'])) as f:
tail = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(head_cwl)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(tail)
)
execute_tail = os.linesep.join([
f"% execute tail",
f"tailinput:",
f" class: File",
f" location: {os.sep.join([self.data_directory, 'data.csv'])}",
f"number_of_lines: 15"
])
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(execute_tail)
)
execute_head = os.linesep.join([
'% execute head',
'headinput:',
' class: File',
' $data: tailoutput',
'number_of_lines: 5',
])
self.assertDictEqual(
{'status': 'error', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(execute_head)
)
# execute by referencing id/output
execute_head = os.linesep.join([
'% execute head',
'headinput:',
' class: File',
' $data: tail/tailoutput',
'number_of_lines: 5',
])
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(execute_head)
)
def test_data_magic_command(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# monitor responses
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'echo_stdout.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str, False)
)
with open(os.sep.join([self.data_directory, 'echo-job.yml'])) as f:
data = f"% execute echo\n{f.read()}"
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data, False)
)
from lxml import etree
kernel.do_execute('% data')
self.assertEqual(
1,
len(etree.HTML(responses[-1][0][2]['data']['text/html']).xpath('//a'))
)
def test_execute_echo_cwl(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.data_directory, 'echo-job.yml'])) as f:
data = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data, False)
)
with open(os.sep.join([self.cwl_directory, 'echo.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str, False)
)
def test_send_invalid_yaml(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
invalid_yaml = """
this is an invalid yaml: fp
?: 1
?: 2
?
"""
exec_result = kernel.do_execute(invalid_yaml, False)
self.assertDictEqual({
'status': 'error',
# The base class increments the execution count
'execution_count': 0,
'payload': [],
'user_expressions': {},
}, exec_result)
def test_handle_input_data_files(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.data_directory, 'input_with_file.yml'])) as f:
data = yaml.load(f, Loader=yaml.Loader)
tmp_dir = tempfile.mkdtemp()
data['example_file']['location'] = os.path.join(tmp_dir, 'file.txt')
with open(data['example_file']['location'], 'w') as f:
f.write('')
data_stream = StringIO()
yaml.dump(data, data_stream)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data_stream.getvalue(), False)
)
with open(os.sep.join([self.cwl_directory, 'workflow_with_input_file.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str, False)
)
import uuid
input_with_missing_file = StringIO()
yaml.dump({"missing_file": {"class": "File", "location": f"/{uuid.uuid4()}"}}, input_with_missing_file)
response = kernel.do_execute(input_with_missing_file.getvalue())
self.assertDictEqual(
{'status': 'error', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
response
)
def test_githubImport_without_id(self):
when(requests) \
.get(
"https://api.github.com/repos/giannisdoukas/CWLJNIKernel/contents/tests/cwl/without_id.cwl?ref=dev") \
.thenReturn(mock({
'status_code': 200,
'json': lambda: {
"name": "without_id.cwl",
"path": "tests/cwl/without_id.cwl",
"sha": "8dd9d522c666d469c75bac566957e78acdb4a5f6",
"size": 129,
"url": "https://api.github.com/repos/giannisdoukas/CWLJNIKernel/contents/tests/cwl/without_id.cwl?ref=dev",
"html_url": "https://github.com/giannisdoukas/CWLJNIKernel/blob/dev/tests/cwl/without_id.cwl",
"git_url": "https://api.github.com/repos/giannisdoukas/CWLJNIKernel/git/blobs/8dd9d522c666d469c75bac566957e78acdb4a5f6",
"download_url": "https://raw.githubusercontent.com/giannisdoukas/CWLJNIKernel/dev/tests/cwl/without_id.cwl",
"type": "file",
"content": "Y3dsVmVyc2lvbjogdjEuMApjbGFzczogQ29tbWFuZExpbmVUb29sCmJhc2VD\nb21tYW5kOiBbZWNobywgImhlbGxvIHdvcmxkIl0KaW5wdXRzOiBbXQpvdXRw\ndXRzOgogIGV4YW1wbGVfb3V0cHV0OgogICAgdHlwZTogc3Rkb3V0\n",
"encoding": "base64",
"_links": {
"self": "https://api.github.com/repos/giannisdoukas/CWLJNIKernel/contents/tests/cwl/without_id.cwl?ref=dev",
"git": "https://api.github.com/repos/giannisdoukas/CWLJNIKernel/git/blobs/8dd9d522c666d469c75bac566957e78acdb4a5f6",
"html": "https://github.com/giannisdoukas/CWLJNIKernel/blob/dev/tests/cwl/without_id.cwl"
}
}}))
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(
"% githubImport https://github.com/giannisdoukas/CWLJNIKernel/blob/dev/tests/cwl/without_id.cwl")
)
self.assertRegex(responses[-1][0][2]['text'], r"^tool 'without_id' registered")
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% execute {responses[-1][0][2]['text'].split("'")[1]}""")
)
def test_get_past_results_from_kernel(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.data_directory, 'tar_job.yml'])) as f:
data = f.read()
tar_directory = kernel._cwl_executor.file_manager.ROOT_DIRECTORY
with open(os.path.join(tar_directory, 'hello.txt'), 'w') as temp_hello_world_file:
temp_hello_world_file.write("hello world")
tar_full_name = os.path.join(tar_directory, 'tarfile.tar')
print('create tar file:', tar_full_name)
with tarfile.open(tar_full_name, 'w') as tar:
tar.add(temp_hello_world_file.name)
data = data.format(tar_directory=tar_directory)
result = kernel.do_execute(data, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
with open(os.sep.join([self.cwl_directory, 'extract_tar.cwl'])) as f:
workflow_str = f.read().format(example_out=temp_hello_world_file.name[1:])
result = kernel.do_execute(workflow_str, False)
self.assertEqual('ok', result['status'], f'execution returned an error')
full_path, basename = [(f, os.path.basename(f)) for f in kernel.get_past_results()][0]
self.assertTrue(full_path.startswith(kernel._results_manager.ROOT_DIRECTORY), 'output is in a wrong directory')
self.assertTrue(basename, 'hello.txt')
def test_logs_magic_command(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.extend(args[2]['data']['application/json'])
exec_response = kernel.do_execute('% logs')
self.assertDictEqual(
{"status": "ok", "execution_count": 0, 'payload': [], 'user_expressions': {}},
exec_response
)
exec_response = kernel.do_execute('% logs')
self.assertDictEqual(
{"status": "ok", "execution_count": 0, 'payload': [], 'user_expressions': {}},
exec_response
)
number_of_responses = len(responses)
self.assertEqual(number_of_responses, 2)
exec_response = kernel.do_execute('% logs 1')
self.assertDictEqual(
{"status": "ok", "execution_count": 0, 'payload': [], 'user_expressions': {}},
exec_response
)
self.assertEqual(len(responses), number_of_responses + 1)
self.assertEqual(
responses[0]['process_id']['process_id'],
os.getpid()
)
def test_viewTool(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'echo.cwl'])) as f:
cwl_string = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""{cwl_string}""")
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% viewTool x""")
)
self.assertEqual(
"Tool 'x' is not registered",
responses[-1][0][2]['text']
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% viewTool echo""")
)
self.assertDictEqual(
yaml.load(StringIO(cwl_string), yaml.Loader),
responses[-1][0][2]['data']['application/json']
)
def test_system_magic_command(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute("% system echo 'Hello World'")
)
self.assertDictEqual(
{'name': 'stdout', 'text': 'Hello World\n'},
responses[-1][0][2],
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% system ls ERROR')
)
self.assertEqual(
'stderr',
responses[-1][0][2]['name'],
)
def test_get_input_data(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.data_directory, 'data1.yml'])) as f:
data = f.read()
exec_response = kernel.do_execute(data, False)
self.assertDictEqual(
{"status": "ok", "execution_count": 0, 'payload': [], 'user_expressions': {}},
exec_response
)
self.assertListEqual([data], kernel._yaml_input_data)
exec_response = kernel.do_execute(data, False)
# The base class increments the execution count. So, exec_count remains 0
self.assertDictEqual(
{"status": "ok", "execution_count": 0, 'payload': [], 'user_expressions': {}},
exec_response
)
self.assertListEqual([data, data], kernel._yaml_input_data)
def test_execute_with_provenance(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
yaml = YAML(typ='safe')
with open(os.sep.join([self.cwl_directory, 'echo.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str, False)
)
self.assertIsNotNone(kernel._workflow_repository.get_by_id(yaml.load(workflow_str)['id']))
with open(os.sep.join([self.data_directory, 'echo-job.yml'])) as f:
data = '\n'.join(["% executeWithProvenance echo", f.read()])
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(data)
)
provenance_directory = list(filter(
lambda r: 'text' in r[0][2] and "Provenance stored in directory" in r[0][2]['text'],
responses
))[0][0][2]['text'].split()[-1]
print(provenance_directory)
self.assertTrue(os.path.isdir(provenance_directory))
self.assertTrue(os.path.isdir(os.path.join(provenance_directory, 'data')))
self.assertTrue(os.path.isdir(os.path.join(provenance_directory, 'metadata')))
self.assertTrue(os.path.isdir(os.path.join(provenance_directory, 'snapshot')))
self.assertTrue(os.path.isfile(os.path.join(provenance_directory, 'snapshot', 'echo.cwl')))
self.assertTrue(os.path.isdir(os.path.join(provenance_directory, 'workflow')))
shutil.rmtree(provenance_directory)
def test_sample_csv(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% displayDataCSV no-existing""")
)
self.assertEqual(
'Result not found',
responses[-1][0][2]['text']
)
with open(os.sep.join([self.cwl_directory, 'head.cwl'])) as f:
head = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(head)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% execute head
headinput:
class: File
location: {os.sep.join([self.data_directory, 'data.csv'])}
number_of_lines: 15""")
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute("% sampleCSV headoutput 0.2")
)
shape = pd.read_html(responses[-1][0][2]['data']['text/html'], header=None)[0].shape
print('shape:', shape)
self.assertAlmostEqual(shape[0], 4, delta=4)
self.assertEqual(shape[1], 20)
def test_display_data_csv(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% displayDataCSV no-existing""")
)
self.assertEqual(
'Result not found',
responses[-1][0][2]['text']
)
with open(os.sep.join([self.cwl_directory, 'head.cwl'])) as f:
head = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(head)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"""% execute head
headinput:
class: File
location: {os.sep.join([self.data_directory, 'data.csv'])}
number_of_lines: 15""")
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute("% displayDataCSV headoutput")
)
self.assertListEqual(
[-54, -85, -5, 47, 39, 20, -58, 24, 12, 13, 4, -22, -1, -70, 44, -30, 91, -6, 40, -24],
list(pd.read_html(responses[-1][0][2]['data']['text/html'], header=None)[0].values[0])
)
def test_get_past_results_without_input(self):
from cwlkernel.CWLKernel import CWLKernel
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
with open(os.sep.join([self.cwl_directory, 'touched.cwl'])) as f:
workflow_str = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(workflow_str)
)
self.assertEqual(0, len(kernel.get_past_results()))
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% execute touch')
)
full_path, basename = [(f, os.path.basename(f)) for f in kernel.get_past_results()][0]
self.assertTrue(full_path.startswith(kernel._results_manager.ROOT_DIRECTORY), 'output is in a wrong directory')
self.assertTrue(basename, 'touchedfile.txt')
def setUp(self) -> None:
import tempfile
CWLKernel.clear_instance()
WorkflowRepository(Path(tempfile.mkdtemp()))
WorkflowRepository.get_instance().delete()
def get_kernel(self) -> CWLKernel:
kernel = CWLKernel()
# cancel send_response
kernel.send_response = lambda *args, **kwargs: None
return kernel
def test_scatter_tool(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.path.join(self.cwl_directory, 'echo_stdout.cwl')) as f:
yaml_tool = f.read()
expected_tool = {
'cwlVersion': 'v1.0',
'id': 'scattered_echo',
'class': 'Workflow',
'inputs': [{'id': 'message_scatter_array', 'type': 'string[]'}],
'outputs': [{'type': 'File[]', 'id': 'echo_output_scatter_array',
'outputSource': 'echo/echo_output'}],
'steps': {'echo': {'run': 'echo.cwl',
'scatter': 'message',
'in': {'message': 'message_scatter_array'},
'out': ['echo_output']}},
'requirements': {'ScatterFeatureRequirement': {}}
}
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(yaml_tool)
)
self.assertDictEqual(
{'status': 'error', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% scatter echo')
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% scatter NOT-EXISTING-TOOL-ID message')
)
self.assertEqual(
responses[-1][0][2]['name'], 'stderr'
)
self.assertEqual(
responses[-1][0][2]['text'], "Tool 'NOT-EXISTING-TOOL-ID' not found"
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% scatter echo NOT-EXISTING-INPUT')
)
self.assertEqual(
responses[-1][0][2]['name'], 'stderr'
)
self.assertEqual(
responses[-1][0][2]['text'], "There is no input 'NOT-EXISTING-INPUT' in tool 'echo'"
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% scatter echo message')
)
scattered_tool = dict(responses[-1][0][2]['data']['application/json'])
self.assertListEqual(
expected_tool['inputs'],
scattered_tool['inputs']
)
self.assertListEqual(
expected_tool['outputs'],
scattered_tool['outputs']
)
expected_tool['steps']['echo']['run'] = os.path.join(
os.path.dirname(scattered_tool['steps']['echo']['run']), expected_tool['steps']['echo']['run']
)
self.assertDictEqual(
expected_tool['steps'],
scattered_tool['steps']
)
self.assertDictEqual(
expected_tool,
scattered_tool
)
self.assertIsNotNone(kernel.workflow_repository.get_instance().get_by_id(scattered_tool['id']))
def setUp(self) -> None:
CWLKernel.clear_instance()
def test_edit_magic(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.path.join(self.cwl_directory, 'echo_stdout.cwl')) as f:
yaml_tool = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(yaml_tool)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute("% edit ")
)
self.assertEqual(
responses[-1][0][2]['name'], 'stderr'
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute("% edit NOT-EXISTING")
)
self.assertEqual(
responses[-1][0][2]['name'], 'stderr'
)
execute_response = kernel.do_execute("% edit echo")
self.assertEqual('ok', execute_response['status'])
self.assertListEqual(
execute_response['payload'],
[
{
'source': "set_next_input",
'replace': True,
'text': f'% edit echo\n{kernel.workflow_repository.get_instance().get_by_id("echo").to_yaml()}',
}
]
)
new_tool = yaml.safe_load(StringIO(yaml_tool))
new_tool['outputs'] = {
'new_echo_output': {'type': 'stdout'}
}
output = StringIO()
yaml.safe_dump(new_tool, output)
new_tool_yaml = output.getvalue()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(f"% edit echo\n{new_tool_yaml}")
)
new_tool = kernel.workflow_repository.get_instance().get_by_id('echo')
self.assertListEqual(
new_tool.outputs,
[{'id': 'new_echo_output', 'type': 'stdout'}]
)
def test_compile_executed_steps(self):
kernel = CWLKernel()
# cancel send_response
responses = []
kernel.send_response = lambda *args, **kwargs: responses.append((args, kwargs))
with open(os.sep.join([self.cwl_directory, 'head-no-optional.cwl'])) as f:
head_cwl = f.read()
with open(os.sep.join([self.cwl_directory, 'tail-no-optional.cwl'])) as f:
tail = f.read()
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(head_cwl)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(tail)
)
execute_tail = os.linesep.join([
f"% execute tail",
f"tailinput:",
f" class: File",
f" location: {os.sep.join([self.data_directory, 'data.csv'])}",
])
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(execute_tail)
)
execute_head = os.linesep.join([
'% execute head',
'headinput:',
' class: File',
' $data: tail/tailoutput',
])
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute(execute_head)
)
self.assertDictEqual(
{'status': 'ok', 'execution_count': 0, 'payload': [], 'user_expressions': {}},
kernel.do_execute('% compile main')
)
expected_workflow = {
'class': 'Workflow',
'cwlVersion': "v1.0",
'id': 'main',
'inputs': [
{
'id': 'tailinput',
'type': 'File'
},
],
'outputs': [
{
'id': 'headoutput',
'type': 'File',
'outputSource': "head/headoutput"
}
],
"steps": {
"head":
{
"run": "head.cwl",
"in": {
"headinput": "tail/tailoutput"
},
"out": ['headoutput']
},
"tail":
{
"run": "tail.cwl",
"in": {"tailinput": "tailinput"},
"out": ['tailoutput']
},
},
'requirements': {}
}
self.assertDictEqual(
expected_workflow,
responses[-1][0][2]['data']['application/json']
)
self.assertDictEqual(
expected_workflow,
kernel.workflow_repository.get_instance().get_by_id('main').to_dict()
)