def test_parse_custom_output(self):
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "podpac.core.pipeline.output",
"output": "ImageOutput"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output, ImageOutput)
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "podpac",
"output": "core.pipeline.output.ImageOutput"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output, ImageOutput)
def test_empty(self):
s = '{ }'
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="Pipeline definition requires 'nodes' property"):
parse_pipeline_definition(d)
def test_parse_output_invalid_mode(self):
# invalid mode
s = '''
{
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {"mode": "nonexistent_mode"}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError, match='output has unexpected mode'):
parse_pipeline_definition(d)
def test_invalid_node(self):
# module does not exist
s = '{"nodes": {"a": {"node": "nonexistent.Arbitrary"} } }'
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError, match='No module found'):
parse_pipeline_definition(d)
# node does not exist in module
s = '{"nodes": {"a": {"node": "core.Nonexistent"} } }'
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match="Node 'Nonexistent' not found"):
parse_pipeline_definition(d)
def test_datasource_interpolation(self):
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"source": "my_data_string",
"interpolation": "nearest"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.interpolation == "nearest"
# not required
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
# incorrect
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"attrs": {
"interpolation": "nearest"
}
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="The 'interpolation' property cannot be in attrs"):
parse_pipeline_definition(d)
def test_algorithm_inputs(self):
# basic
s = '''
{
"nodes": {
"source1": {"node": "algorithm.Arange"},
"source2": {"node": "algorithm.CoordData"},
"result": {
"node": "algorithm.Arithmetic",
"inputs": {
"A": "source1",
"B": "source2"
},
"attrs": {
"eqn": "A + B"
}
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output.node.A, podpac.algorithm.Arange)
assert isinstance(output.node.B, podpac.algorithm.CoordData)
# sub-node
s = '''
{
"nodes": {
"mysource": {"node": "algorithm.Arange"},
"double": {
"node": "algorithm.Arithmetic",
"inputs": { "A": "mysource" },
"attrs": { "eqn": "2 * A" }
},
"quadruple": {
"node": "algorithm.Arithmetic",
"inputs": { "A": "double.A" },
"attrs": { "eqn": "2 * A" }
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output.node.A, podpac.algorithm.Arange)
def test_parse_custom_output_invalid(self):
# no module
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "nonexistent_module",
"output": "arbitrary"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError, match="No module found"):
parse_pipeline_definition(d)
# module okay, but no such class
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "podpac.core.pipeline.output",
"output": "Nonexistent"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match="Output 'Nonexistent' not found"):
parse_pipeline_definition(d)
# module okay, class found, could not create
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "numpy",
"output": "ndarray"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match='Could not create custom output'):
parse_pipeline_definition(d)
# module okay, class found, incorrect type
s = ''' {
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"plugin": "numpy",
"output": "array"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
m = "Custom output '.*' must subclass 'podpac.core.pipeline.output.Output'"
with pytest.raises(PipelineError, match=m):
parse_pipeline_definition(d)
def test_parse_output_nonexistent_node(self):
s = '''
{
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"node": "b",
"mode": "file",
"format": "pickle",
"outdir": "my_directory"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match="output' references nonexistent node"):
parse_pipeline_definition(d)
def test_parse_output_implicit_node(self):
s = '''
{
"nodes": {
"source1": {"node": "algorithm.Arange"},
"source2": {"node": "algorithm.Arange"},
"result": {
"node": "algorithm.Arithmetic",
"inputs": {
"A": "source1",
"B": "source2"
},
"attrs": {
"eqn": "A + B"
}
}
},
"output": {
"mode": "none"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output.node, podpac.algorithm.Arithmetic)
def test_invalid_property(self):
s = '''
{
"nodes": {
"a": {
"node": "algorithm.Arange",
"invalid_property": "value"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match="node 'a' has unexpected property"):
parse_pipeline_definition(d)
def test_compositor_sources(self):
# basic
s = '''
{
"nodes": {
"a": {"node": "algorithm.Arange"},
"b": {"node": "algorithm.CoordData"},
"c": {
"node": "compositor.OrderedCompositor",
"sources": ["a", "b"]
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output.node.sources[0], podpac.algorithm.Arange)
assert isinstance(output.node.sources[1], podpac.algorithm.CoordData)
# sub-node
s = '''
{
"nodes": {
"source1": {"node": "algorithm.Arange"},
"source2": {"node": "algorithm.CoordData"},
"double": {
"node": "algorithm.Arithmetic",
"inputs": { "A": "source1" },
"attrs": { "eqn": "2 * A" }
},
"c": {
"node": "compositor.OrderedCompositor",
"sources": ["double.A", "source2"]
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output.node.sources[0], podpac.algorithm.Arange)
assert isinstance(output.node.sources[1], podpac.algorithm.CoordData)
def test_parse_output_implicit(self):
s = '''
{
"nodes": {"a": {"node": "algorithm.Arange"} }
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output, NoOutput)
assert isinstance(output.node, podpac.algorithm.Arange)
assert output.name == 'a'
def test_lookup_attrs(self):
# attrs doesn't work
s = '''
{
"nodes": {
"a": {
"node": "algorithm.CoordData",
"attrs": { "coord_name": "lat" }
},
"b": {
"node": "algorithm.CoordData",
"attrs": { "coord_name": "a.coord_name" }
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
with pytest.raises(AssertionError):
assert output.node.coord_name == 'lat'
# but lookup_attrs does
s = '''
{
"nodes": {
"a": {
"node": "algorithm.CoordData",
"attrs": { "coord_name": "lat" }
},
"b": {
"node": "algorithm.CoordData",
"lookup_attrs": { "coord_name": "a.coord_name" }
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.coord_name == 'lat'
def test_array_lookup_source(self):
# source doesn't work
s = '''
{
"nodes": {
"a": {
"node": "data.Array",
"source": [0, 1, 2]
},
"b": {
"node": "data.Array",
"source": "a.source"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(ValueError):
parse_pipeline_definition(d)
# lookup_source does work
s = '''
{
"nodes": {
"a": {
"node": "data.Array",
"source": [0, 1, 2]
},
"b": {
"node": "data.Array",
"lookup_source": "a.source"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
np.testing.assert_array_equal(output.node.source, [0, 1, 2])
def test_array_source(self):
s = '''
{
"nodes": {
"mysource": {
"node": "data.Array",
"source": [0, 1, 2]
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
np.testing.assert_array_equal(output.node.source, [0, 1, 2])
def test_attrs(self):
s = '''
{
"nodes": {
"sm": {
"node": "datalib.smap.SMAP",
"attrs": {
"product": "SPL4SMGP"
}
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.product == "SPL4SMGP"
def test_parse_output_ftp(self):
s = '''
{
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"node": "a",
"mode": "ftp",
"url": "my_url",
"user": "******"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output, FTPOutput)
assert isinstance(output.node, podpac.algorithm.Arange)
assert output.name == 'a'
assert output.user == 'my_user'
assert output.url == 'my_url'
def test_parse_output_file(self):
s = '''
{
"nodes": {"a": {"node": "algorithm.Arange"} },
"output": {
"node": "a",
"mode": "file",
"format": "pickle",
"outdir": "my_directory"
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert isinstance(output, FileOutput)
assert isinstance(output.node, podpac.algorithm.Arange)
assert output.name == 'a'
assert output.format == 'pickle'
assert output.outdir == 'my_directory'
def test_compositor_interpolation(self):
s = '''
{
"nodes": {
"a": {
"node": "algorithm.Arange"
},
"b": {
"node": "algorithm.Arange"
},
"c": {
"node": "compositor.OrderedCompositor",
"sources": ["a", "b"],
"interpolation": "nearest"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.interpolation == "nearest"
def test_datasource_lookup_source(self):
# sub-node
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"source": "my_data_string"
},
"double": {
"node": "algorithm.Arithmetic",
"inputs": {"A": "mydata"},
"attrs": { "eqn": "2 * A" }
},
"mydata2": {
"node": "data.DataSource",
"lookup_source": "double.A.source"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.source == 'my_data_string'
# nonexistent node
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"source": "my_data_string"
},
"double": {
"node": "algorithm.Arithmetic",
"inputs": {"A": "mydata"},
"attrs": { "eqn": "2 * A" }
},
"mydata2": {
"node": "data.DataSource",
"lookup_source": "nonexistent.source"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="'mydata2' references nonexistent node/attribute"):
parse_pipeline_definition(d)
# nonexistent subattr
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"source": "my_data_string"
},
"double": {
"node": "algorithm.Arithmetic",
"inputs": {"A": "mydata"},
"attrs": { "eqn": "2 * A" }
},
"mydata2": {
"node": "data.DataSource",
"lookup_source": "double.nonexistent.source"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="'mydata2' references nonexistent node/attribute"):
parse_pipeline_definition(d)
# nonexistent subsubattr
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"source": "my_data_string"
},
"double": {
"node": "algorithm.Arithmetic",
"inputs": {"A": "mydata"},
"attrs": { "eqn": "2 * A" }
},
"mydata2": {
"node": "data.DataSource",
"lookup_source": "double.A.nonexistent"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="'mydata2' references nonexistent node/attribute"):
parse_pipeline_definition(d)
# in attrs (incorrect)
s = '''
{
"nodes": {
"mydata": {
"node": "data.DataSource",
"attrs": {
"lookup_source": "my_data_string"
}
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(
PipelineError,
match="The 'lookup_source' property cannot be in attrs"):
parse_pipeline_definition(d)
def test_reprojected_source_lookup_source(self):
# source doesn't work
s = '''
{
"nodes": {
"mysource": {
"node": "data.DataSource",
"source": "my_data_string"
},
"reprojected": {
"node": "data.ReprojectedSource",
"source": "mysource"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(tl.TraitError):
parse_pipeline_definition(d)
# lookup_source
s = '''
{
"nodes": {
"mysource": {
"node": "data.DataSource",
"source": "my_data_string"
},
"reprojected": {
"node": "data.ReprojectedSource",
"lookup_source": "mysource"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.source
assert output.node.source.source == 'my_data_string'
# lookup_source subattr
s = '''
{
"nodes": {
"mysource": {
"node": "data.DataSource",
"source": "my_data_string"
},
"double": {
"node": "algorithm.Arithmetic",
"inputs": {"A": "mysource"},
"attrs": { "eqn": "2 * A" }
},
"reprojected": {
"node": "data.ReprojectedSource",
"lookup_source": "double.A"
}
}
}
'''
d = json.loads(s, object_pairs_hook=OrderedDict)
output = parse_pipeline_definition(d)
assert output.node.source
assert output.node.source.source == "my_data_string"
def test_no_nodes(self):
s = '{"nodes": { } }'
d = json.loads(s, object_pairs_hook=OrderedDict)
with pytest.raises(PipelineError,
match="'nodes' property cannot be empty"):
parse_pipeline_definition(d)
