def testTaskParameterLabel(self):
# verify that "parameters" cannot be used as a task label
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
parameters: modA
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
def testReadTaskConfig(self):
# Verify that a task with a config is read in correctly
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA:
class: test.moduleA
config:
propertyA: 6
propertyB: 7
file: testfile.py
python: "config.testDict['a'] = 9"
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
# Verify that multiple files are read fine
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA:
class: test.moduleA
config:
file:
- testfile.py
- otherFile.py
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
# Test reading multiple Config entries
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA:
class: test.moduleA
config:
- propertyA: 6
propertyB: 7
dataId: {"visit": 6}
- propertyA: 8
propertyB: 9
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
def testSerialization(self):
# Test creating a pipeline, writing it to a file, reading the file
pipeline_str = textwrap.dedent("""
description: Test Pipeline
instrument: dummyCam
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
instrument: None
tasks:
modC:
class: test.moduleC
config:
- propertyA: 6
propertyB: 7
dataId: {"visit": 6}
- propertyA: 8
propertyB: 9
modD: test.moduleD
contracts:
- modA.foo == modB.bar
subsets:
subA:
- modA
- modC
""")
pipeline = PipelineIR.from_string(pipeline_str)
# Create the temp file, write and read
with tempfile.NamedTemporaryFile() as tf:
pipeline.to_file(tf.name)
loaded_pipeline = PipelineIR.from_file(tf.name)
self.assertEqual(pipeline, loaded_pipeline)
def testImportingInstrument(self):
# verify an instrument is imported, or ignored, (Or otherwise modified
# for potential future use)
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $TESTDIR/testPipeline1.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, "test.instrument")
# verify that an imported pipeline can have its instrument set to None
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $TESTDIR/testPipeline1.yaml
instrument: None
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, None)
# verify that an imported pipeline can have its instrument modified
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $TESTDIR/testPipeline1.yaml
instrument: new.instrument
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, "new.instrument")
# Test that multiple instruments can't be defined,
# and that the error message tells you what instruments were found.
pipeline_str = textwrap.dedent("""
description: Test Pipeline
instrument: new.instrument
imports:
- location: $TESTDIR/testPipeline1.yaml
""")
with self.assertRaisesRegex(ValueError,
"new.instrument .* test.instrument."):
PipelineIR.from_string(pipeline_str)
def testInstrument(self):
# Verify that if instrument is defined it is parsed out
pipeline_str = textwrap.dedent("""
description: Test Pipeline
instrument: dummyCam
tasks:
modA: test.moduleA
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, "dummyCam")
def testReadParameters(self):
# verify that parameters section are read in from a pipeline
pipeline_str = textwrap.dedent("""
description: Test Pipeline
parameters:
value1: A
value2: B
tasks:
modA: ModuleA
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"})
def testImportingInstrument(self):
# verify an instrument is imported, or ignored, (Or otherwise modified
# for potential future use)
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, "test.instrument")
# verify that an imported pipeline can have its instrument set to None
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
instrument: None
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, None)
# verify that an imported pipeline can have its instrument modified
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
instrument: new.instrument
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.instrument, "new.instrument")
# Test that multiple instruments can't be defined
pipeline_str = textwrap.dedent("""
description: Test Pipeline
instrument: new.instrument
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
def testPipelineIRInitChecks(self):
# Missing description
pipeline_str = """
tasks:
a: module.A
"""
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# Missing tasks
pipeline_str = """
description: Test Pipeline
"""
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# This should raise a FileNotFoundError, as there are imported defined
# so the __init__ method should pass but the imported file does not
# exist
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports: /dummy_pipeline.yaml
""")
with self.assertRaises(FileNotFoundError):
PipelineIR.from_string(pipeline_str)
def testTaskParsing(self):
# Should be able to parse a task defined both ways
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
def testParameterConfigFormatting(self):
# verify that a config properly is formatted with parameters
pipeline_str = textwrap.dedent("""
description: Test Pipeline
parameters:
value1: A
tasks:
modA:
class: ModuleA
config:
testKey: parameters.value1
""")
pipeline = PipelineIR.from_string(pipeline_str)
newConfig = pipeline.tasks['modA'].config[0].formatted(pipeline.parameters)
self.assertEqual(newConfig.rest['testKey'], "A")
def testSorting(self):
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
""")
pipeline = PipelineIR.from_string(pipeline_str)
newKeyOrder = ["modB", "modA"]
pipeline.reorder_tasks(newKeyOrder)
self.assertEqual(list(pipeline.tasks.keys()), newKeyOrder)
with self.assertRaises(KeyError):
pipeline.reorder_tasks(["modB"])
with self.assertRaises(KeyError):
pipeline.reorder_tasks(["modD"])
def testParameterImporting(self):
# verify that importing parameters happens correctly
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
- location: $PIPE_BASE_DIR/tests/testPipeline2.yaml
exclude:
- modA
parameters:
value4: valued
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.parameters.mapping, {"value4": "valued", "value1": "valueNew",
"value2": "valueB",
"value3": "valueC"})
def testSortingPrimitives(self):
pipeline_str = textwrap.dedent("""
description: Test Pipeline
parameters:
value2: A
value1: B
tasks:
modB: ModuleB
modA: ModuleA
contracts:
- contract: modB.foo == modA.Bar
msg: "Test message"
- contract: modA.foo == modB.Bar
msg: "Test message"
subsets:
subset2:
- modA
- modB
subset1:
subset:
- modA
- modB
description: "A test named subset"
""")
pipeline = PipelineIR.from_string(pipeline_str)
primitives = pipeline.to_primitives()
# verify subsets
self.assertEqual(list(pipeline.labeled_subsets.keys()),
["subset2", "subset1"])
self.assertEqual(list(primitives["subsets"].keys()),
["subset1", "subset2"])
# verify parameters
self.assertEqual(list(pipeline.parameters.mapping.keys()),
["value2", "value1"])
self.assertEqual(list(primitives["parameters"].keys()),
["value1", "value2"])
# verify contracts
self.assertEqual([c.contract for c in pipeline.contracts],
["modB.foo == modA.Bar", "modA.foo == modB.Bar"])
self.assertEqual([c["contract"] for c in primitives["contracts"]],
["modA.foo == modB.Bar", "modB.foo == modA.Bar"])
def testReadContracts(self):
# Verify that contracts are read in from a pipeline
location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
pipeline = PipelineIR.from_file(location)
self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
# Verify that a contract message is loaded
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
contracts:
- contract: modA.foo == modB.Bar
msg: "Test message"
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.contracts[0].msg, "Test message")
def testInheritParsing(self):
# This should raise, as the two pipelines, both define the same label
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# This should pass, as the conflicting task is excluded
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
exclude: modA
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
# This should pass, as the conflicting task is no in includes
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
include: modB
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
# Test that you cant include and exclude a task
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
exclude: modA
include: modB
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# Test that contracts are inherited
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
# Test that contracts are not inherited
pipeline_str = textwrap.dedent("""
description: Test Pipeline
inherits:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
importContracts: False
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.contracts, [])
def testImportParsing(self):
# This should raise, as the two pipelines, both define the same label
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# This should pass, as the conflicting task is excluded
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
exclude: modA
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
# This should pass, as the conflicting task is no in includes
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
include: modB
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
# Test that you cant include and exclude a task
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
exclude: modA
include: modB
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# Test that contracts are imported
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline1.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
# Test that contracts are not imported
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
importContracts: False
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.contracts, [])
# Test that configs are imported when defining the same task again
# with the same label
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
tasks:
modA:
class: "test.moduleA"
config:
value2: 2
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2})
# Test that configs are not imported when redefining the task
# associated with a label
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
tasks:
modA:
class: "test.moduleAReplace"
config:
value2: 2
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2})
# Test that named subsets are imported
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"})
# Test that imported and redeclaring a named subset works
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
tasks:
modE: "test.moduleE"
subsets:
modSubset:
- modE
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"})
# Test that imported from two pipelines that both declare a named
# subset with the same name fails
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
- $PIPE_BASE_DIR/tests/testPipeline3.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# Test that imported a named subset that duplicates a label declared
# in this pipeline fails
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
tasks:
modSubset: "test.moduleE"
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# Test that imported fails if a named subset and task label conflict
pipeline_str = textwrap.dedent("""
description: Test Pipeline
imports:
- $PIPE_BASE_DIR/tests/testPipeline2.yaml
- $PIPE_BASE_DIR/tests/testPipeline4.yaml
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
def testReadNamedSubsets(self):
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
modC: test.modC
modD: test.modD
subsets:
subset1:
- modA
- modB
subset2:
subset:
- modC
- modD
description: "A test named subset"
""")
pipeline = PipelineIR.from_string(pipeline_str)
self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"})
self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"})
self.assertEqual(pipeline.labeled_subsets["subset1"].description, None)
self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"})
self.assertEqual(pipeline.labeled_subsets["subset2"].description,
"A test named subset")
# verify that forgetting a subset key is an error
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
modC: test.modC
modD: test.modD
subsets:
subset2:
sub:
- modC
- modD
description: "A test named subset"
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
# verify putting a label in a named subset that is not in the task is
# an error
pipeline_str = textwrap.dedent("""
description: Test Pipeline
tasks:
modA: test.modA
modB:
class: test.modB
modC: test.modC
modD: test.modD
subsets:
subset2:
- modC
- modD
- modE
""")
with self.assertRaises(ValueError):
PipelineIR.from_string(pipeline_str)
