def test_nested_parameters(self):
"""Test proper nesting of parameters for DT_LIST and DT_RECORD."""
# Create a new TemplateSpec with an empty workflow specification and
# a list of six parameters (one record and one list)
template = TemplateSpec(
workflow_spec={},
parameters=[
pd.parameter_declaration('A'),
pd.parameter_declaration('B', data_type=pd.DT_RECORD),
pd.parameter_declaration('C', parent='B'),
pd.parameter_declaration('D', parent='B'),
pd.parameter_declaration('E', data_type=pd.DT_LIST),
pd.parameter_declaration('F', parent='E'),
],
validate=True)
# Parameters 'A', 'C', 'D', and 'F' have no children
for key in ['A', 'C', 'D', 'F']:
self.assertFalse(template.get_parameter(key).has_children())
# Parameter 'B' has two children 'C' and 'D'
b = template.get_parameter('B')
self.assertTrue(b.has_children())
self.assertEqual(len(b.children), 2)
self.assertTrue('C' in [p.identifier for p in b.children])
self.assertTrue('D' in [p.identifier for p in b.children])
# Parameter 'E' has one childr 'F'
e = template.get_parameter('E')
self.assertTrue(e.has_children())
self.assertEqual(len(e.children), 1)
self.assertTrue('F' in [p.identifier for p in e.children])
def test_load_simple_template(self):
"""Test loading a simple REANA workflow template that does not contain
any template parameter.
"""
template = TemplateSpec.load('tests/files/simple-template.yaml')
self.assertTrue(isinstance(template.workflow_spec, dict))
self.assertEqual(len(template.parameters), 0)
def test_load_invalid_file(self):
"""Test loading files that are not valid reana templates or that do not
follow valid JSON or YAML syntax.
"""
# Missing workflow specification
with self.assertRaises(ValueError):
TemplateSpec.load(os.path.abspath('tests/files/invalid-template1.yaml'))
# Additional elements
with self.assertRaises(ValueError):
TemplateSpec.load(os.path.abspath('tests/files/invalid-template2.yaml'))
# Value error when loading invalid JSON file
with self.assertRaises(ValueError):
read_object(INVALID_JSON_FILE, format=FORMAT_JSON)
# Value error when loading invalid yaml file
with self.assertRaises(ValueError):
read_object(INVALID_YAML_FILE, format=FORMAT_YAML)
def test_sort(self):
"""Test the sort functionality of the template list_parameters method.
"""
# Create a new TemplateSpec with an empty workflow specification and
# a list of five parameters
template = TemplateSpec(workflow_spec={},
parameters=[
pd.parameter_declaration('A', index=1),
pd.parameter_declaration('B'),
pd.parameter_declaration('C'),
pd.parameter_declaration('D', index=2),
pd.parameter_declaration('E', index=1)
],
validate=True)
# Get list of sorted parameter identifier from listing
keys = [p.identifier for p in template.list_parameters()]
self.assertEqual(keys, ['B', 'C', 'A', 'E', 'D'])
def test_file_argument(self):
"""Test template specifications having file parameters with and without
a constant replacement value.
"""
template = TemplateSpec(
workflow_spec={'inputs': {
'files': ['$[[fileA]]', '$[[fileB]]']
}},
parameters=[
pd.parameter_declaration('fileA', data_type=pd.DT_FILE),
pd.parameter_declaration('fileB',
data_type=pd.DT_FILE,
as_const='names.txt')
],
validate=True)
arguments = {
'fileA': FileHandle('code/Hello.py'),
'fileB': FileHandle('data/inputs.txt')
}
spec = template.get_workflow_spec(arguments)
self.assertEqual(spec['inputs']['files'][0], 'Hello.py')
self.assertEqual(spec['inputs']['files'][1], 'names.txt')
# Get list of upload files
upload_files = template.get_upload_files(arguments)
self.assertEqual(len(upload_files), 2)
target_files = [fh.target_file for fh in upload_files]
self.assertTrue('Hello.py' in target_files)
self.assertTrue('names.txt' in target_files)
# Value error if we ensure that upload files exist
with self.assertRaises(ValueError):
template.get_upload_files(arguments, ensure_file_exists=True)
def test_read_with_record(self):
"""Read argument for a template that contains a parameter of data type
DT_RECORD.
"""
template = TemplateSpec.load('tests/files/template_with_record.yaml')
sc = Scanner(
reader=ListReader(['ABC.txt', 3, True, 'XYZ.txt', 6, 0.123]))
arguments = template.read(scanner=sc)
self.assertEqual(arguments['codeFile'].name, 'ABC.txt')
self.assertEqual(arguments['sleeptime'], 3)
self.assertEqual(arguments['verbose'], True)
self.assertEqual(arguments['outputTarget'].name, 'XYZ.txt')
self.assertEqual(arguments['outputType'], 6)
self.assertEqual(arguments['frac'], 0.123)
def test_simple_replace(self):
"""Replace parameter references in simple template with argument values.
"""
template = TemplateSpec.load('tests/files/template.yaml')
arguments = {'codeFile': FileHandle('code/Hello.py'), 'sleeptime': 10}
spec = template.get_workflow_spec(arguments)
self.assertEqual(spec['inputs']['files'][0], 'helloworld.py')
self.assertEqual(spec['inputs']['parameters']['helloworld'],
'helloworld.py')
self.assertEqual(spec['inputs']['parameters']['sleeptime'], 10)
self.assertEqual(spec['inputs']['parameters']['waittime'], 5)
# Error when argument for mandatory parameter is missing
with self.assertRaises(ValueError):
template.get_workflow_spec(dict())
def test_duplicate_id(self):
"""Ensure that exception is raised if parameter identifier are not
unique.
"""
with self.assertRaises(ValueError):
TemplateSpec(workflow_spec={},
parameters=[
pd.parameter_declaration('A', index=1),
pd.parameter_declaration('B'),
pd.parameter_declaration('C'),
pd.parameter_declaration('A', index=2),
pd.parameter_declaration('E', index=1)
],
validate=True)
def test_read_object_with_parameters(self):
"""Test loading a REANA workflow template that does contain template
parameter declarations.
"""
template = TemplateSpec.load('tests/files/template.yaml')
self.assertTrue(isinstance(template.workflow_spec, dict))
self.assertEqual(len(template.parameters), 3)
# Code file parameter
p_code = template.get_parameter('codeFile')
self.assertEqual(p_code.name, 'Code File')
self.assertEqual(p_code.data_type, pd.DT_FILE)
self.assertEqual(p_code.as_constant, 'helloworld.py')
# Sleep time parameter
p_sleep = template.get_parameter('sleeptime')
self.assertEqual(p_sleep.name, 'sleeptime')
self.assertEqual(p_sleep.data_type, pd.DT_INTEGER)
def main(input_file, output_file=None):
"""Prompt user to input values for all template parameters. Write workflow
specification with replaced template parameter arguments either to standard
output or to the given output file.
Parameters
----------
input_file: string
Path to REANA workflow template specification file
output_file: string, optional
Optional path to output file
"""
# Read the workflow template specification file
template = TemplateSpec.load(filename=input_file)
# Get the workflow specification. Will prompt the user to input parameter
# values via standard input.
workflow_spec = template.get_workflow_spec(template.read())
# Depending on whether an output file is given or not the workflow
# specification is either written to file or to standar output.
if not output_file is None:
write_object(output_file, workflow_spec)
else:
print(yaml.dump(workflow_spec))
def test_template_engine(self):
"""Test run method of template engine."""
# Create a workflow template
template = TemplateSpec(
workflow_spec={'inputs': {'files': ['$[[fileA]]', '$[[fileB]]']}},
parameters=[
pd.parameter_declaration('fileA', data_type=pd.DT_FILE),
pd.parameter_declaration('fileB', data_type=pd.DT_FILE, as_const='names.txt')
],
validate=True
)
arguments = {
'fileA': FileHandle('tests/files/template/code/helloworld.py'),
'fileB': FileHandle('tests/files/template/inputs/names.txt')
}
engine = FakeTemplateEngine()
engine.run(template, arguments, 'NAME')
self.assertTrue(engine.started)
self.assertEqual(len(engine.files), 2)
self.assertTrue('helloworld.py' in engine.files)
self.assertTrue('names.txt' in engine.files)
# Value error when running a workflow with a missing upload file
arguments = {
'fileA': FileHandle('Hello.py'),
'fileB': FileHandle('names.txt')
}
with self.assertRaises(ValueError):
engine.run(template, arguments, 'NAME')
engine = TemplateEngine()
# For completeness test calling the abstract methods
with self.assertRaises(NotImplementedError):
engine.create_workflow(dict())
with self.assertRaises(NotImplementedError):
engine.upload_file('ID', '/dev/null', 'dev/null')
with self.assertRaises(NotImplementedError):
engine.start_workflow('ID')