def test_plugin(self):
global MyPluginNode
class MyPluginNode(Node):
pass
s = """
{
"mynode": {
"plugin": "test_node",
"node": "MyPluginNode"
}
}
"""
node = Node.from_json(s)
assert isinstance(node, MyPluginNode)
# missing plugin
s = """
{
"mynode": {
"plugin": "missing",
"node": "MyPluginNode"
}
}
"""
with pytest.raises(ValueError, match="no module found"):
Node.from_json(s)
def test_debuggable(self):
s = """
{
"a": {
"node": "algorithm.Arange"
},
"mean": {
"node": "algorithm.Convolution",
"lookup_attrs": {"source": "a"},
"attrs": {"kernel_type": "mean,3", "kernel_dims": ["lat", "lon"]}
},
"c": {
"node": "algorithm.Arithmetic",
"lookup_attrs": {"A": "a", "B": "mean"},
"attrs": {"eqn": "a-b"}
}
}
"""
with warnings.catch_warnings(), podpac.settings:
warnings.filterwarnings("ignore", "Insecure evaluation.*")
# normally node objects can and should be re-used
podpac.settings["DEBUG"] = False
node = Node.from_json(s)
assert node.inputs["A"] is node.inputs["B"].source
# when debugging is on, node objects should be unique
podpac.settings["DEBUG"] = True
node = Node.from_json(s)
assert node.inputs["A"] is not node.inputs["B"].source
def test_cache_ctrl(self):
# settings
with podpac.settings:
podpac.settings["DEFAULT_CACHE"] = ["ram"]
node = Node()
assert node.cache_ctrl is not None
assert len(node.cache_ctrl._cache_stores) == 1
assert isinstance(node.cache_ctrl._cache_stores[0], RamCacheStore)
podpac.settings["DEFAULT_CACHE"] = ["ram", "disk"]
node = Node()
assert node.cache_ctrl is not None
assert len(node.cache_ctrl._cache_stores) == 2
assert isinstance(node.cache_ctrl._cache_stores[0], RamCacheStore)
assert isinstance(node.cache_ctrl._cache_stores[1], DiskCacheStore)
# specify
node = Node(cache_ctrl=["ram"])
assert node.cache_ctrl is not None
assert len(node.cache_ctrl._cache_stores) == 1
assert isinstance(node.cache_ctrl._cache_stores[0], RamCacheStore)
node = Node(cache_ctrl=["ram", "disk"])
assert node.cache_ctrl is not None
assert len(node.cache_ctrl._cache_stores) == 2
assert isinstance(node.cache_ctrl._cache_stores[0], RamCacheStore)
assert isinstance(node.cache_ctrl._cache_stores[1], DiskCacheStore)
def test_inputs(self):
# invalid type
s = """
{
"a": {
"node": "algorithm.Min",
"inputs": { "source": 10 }
}
}
"""
with pytest.raises(ValueError, match="Invalid definition for node"):
Node.from_json(s)
# nonexistent node
s = """
{
"a": {
"node": "algorithm.Min",
"inputs": { "source": "nonexistent" }
}
}
"""
with pytest.raises(ValueError, match="Invalid definition for node"):
Node.from_json(s)
def test_create_output_array_dtype(self):
node = Node(dtype=bool)
output = node.create_output_array(self.c1, data=0)
assert isinstance(output, UnitsDataArray)
assert output.shape == self.c1.shape
assert output.dtype == node.dtype
assert np.all(~output)
def test_definition_duplicate_base_ref(self):
n1 = Node(units="m")
n2 = Node(units="ft")
n3 = Node(units="in")
node = podpac.compositor.OrderedCompositor(sources=[n1, n2, n3])
d = node.definition
assert n1.base_ref == n2.base_ref == n3.base_ref
assert len(d) == 5
def test_cache_output(self):
with podpac.settings:
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = False
node = Node()
assert not node.cache_output
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = True
node = Node()
assert node.cache_output
def test_create_output_array_default(self):
node = Node()
for crd in self.crds:
output = node.create_output_array(crd)
assert isinstance(output, UnitsDataArray)
assert output.shape == crd.shape
assert output.dtype == node.dtype
assert np.all(np.isnan(output))
def test_create_output_array_crs(self):
crs = "+proj=merc +lat_ts=56.5 +ellps=GRS80"
c = podpac.Coordinates(
[podpac.clinspace((0, 0), (1, 1), 10), [0, 1, 2]],
dims=["lat_lon", "time"],
crs=crs)
node = Node()
output = node.create_output_array(c)
assert output.crs == crs
def test_base_definition_inputs_array(self):
class MyNode(Node):
my_attr = ArrayTrait().tag(attr=True)
a = Node()
b = Node()
node = MyNode(my_attr=[a, b])
d = node._base_definition
assert d["inputs"]["my_attr"][0] == a
assert d["inputs"]["my_attr"][1] == b
def test_base_definition_inputs_dict(self):
class MyNode(Node):
my_attr = tl.Dict().tag(attr=True)
a = Node()
b = Node()
node = MyNode(my_attr={"a": a, "b": b})
d = node._base_definition
assert d["inputs"]["my_attr"]["a"] == a
assert d["inputs"]["my_attr"]["b"] == b
def test_eval_output_crs(self):
coords = podpac.Coordinates([[0, 1, 2, 3], [0, 1]],
dims=["lat", "lon"])
node = Node()
with pytest.raises(
ValueError,
match="Output coordinate reference system .* does not match"):
node.eval(coords,
output=node.create_output_array(
coords.transform("EPSG:2193")))
def test_invalid_node(self):
# module does not exist
s = '{"a": {"node": "nonexistent.Arbitrary"} }'
with pytest.raises(ValueError, match="no module found"):
Node.from_json(s)
# node does not exist in module
s = '{"a": {"node": "core.Nonexistent"} }'
with pytest.raises(ValueError,
match="class 'Nonexistent' not found in module"):
Node.from_json(s)
def test_create_output_array_dtype(self):
c = podpac.Coordinates(
[podpac.clinspace((0, 0), (1, 1), 10), [0, 1, 2]],
dims=["lat_lon", "time"])
node = Node(dtype=bool)
output = node.create_output_array(c, data=0)
assert isinstance(output, UnitsDataArray)
assert output.shape == c.shape
assert output.dtype == node.dtype
assert output.crs == c.crs
assert np.all(~output)
def test_invalid_property(self):
s = """
{
"a": {
"node": "algorithm.Arange",
"invalid_property": "value"
}
}
"""
with pytest.raises(ValueError, match="unexpected property"):
Node.from_json(s)
def test_create_output_array_units(self):
c = podpac.Coordinates(
[podpac.clinspace((0, 0), (1, 1), 10), [0, 1, 2]],
dims=["lat_lon", "time"])
node = Node(units="meters")
output = node.create_output_array(c)
assert isinstance(output, UnitsDataArray)
from podpac.core.units import ureg as _ureg
assert output.units == _ureg.meters
def test_repr(self):
n = Node()
repr(n)
n = Node(outputs=["a", "b"])
repr(n)
assert "outputs=" in repr(n)
assert "output=" not in repr(n)
n = Node(outputs=["a", "b"], output="a")
repr(n)
assert "outputs=" not in repr(n)
assert "output=" in repr(n)
def test_str(self):
n = Node()
str(n)
n = Node(outputs=["a", "b"])
str(n)
assert "outputs=" in str(n)
assert "output=" not in str(n)
n = Node(outputs=["a", "b"], output="a")
str(n)
assert "outputs=" not in str(n)
assert "output=" in str(n)
def test_base_definition_remove_unnecessary_attrs(self):
node = Node(outputs=["a", "b"], output="a", units="m")
d = node._base_definition
assert "outputs" in d["attrs"]
assert "output" in d["attrs"]
assert "units" in d["attrs"]
node = Node()
d = node._base_definition
if "attrs" in d:
assert "outputs" not in d["attrs"]
assert "output" not in d["attrs"]
assert "units" not in d["attrs"]
def test_from_url_with_plugin_style_params(self):
url0 = (
r"https://mobility-devel.crearecomputing.com/geowatch?&SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&"
r"LAYERS=Arange&STYLES=&FORMAT=image%2Fpng&TRANSPARENT=true&HEIGHT=256&WIDTH=256"
r"&CRS=EPSG%3A3857&BBOX=-20037508.342789244,10018754.171394618,-10018754.171394622,20037508.34278071&"
r'PARAMS={"plugin": "podpac.algorithm"}')
url1 = (
r"https://mobility-devel.crearecomputing.com/geowatch?&SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&"
r"LAYERS=datalib.terraintiles.TerrainTiles&STYLES=&FORMAT=image%2Fpng&TRANSPARENT=true&HEIGHT=256&WIDTH=256&"
r"TIME=2021-03-01T12%3A00%3A00.000Z&CRS=EPSG%3A3857&BBOX=-10018754.171394622,5009377.08569731,-9392582.035682458,5635549.221409475"
r'&PARAMS={"style": {"name": "Aspect (Composited 30-90 m)","units": "radians","colormap": "hsv","clim": [0,6.283185307179586]}}'
)
node = Node.from_url(url0)
node = Node.from_url(url1)
def test_deserialize_reprojected_coordinates(self):
node1 = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
node2 = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates.definition)
node3 = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates.json)
assert node1.reprojected_coordinates == self.reprojected_coordinates
assert node2.reprojected_coordinates == self.reprojected_coordinates
assert node3.reprojected_coordinates == self.reprojected_coordinates
def test_node(self):
"""test node attribute and defaults"""
parent_node = Node()
node = S3(source=self.source, node=parent_node)
assert node.node_class
def test_traits(self):
""" check each of the s3 traits """
S3(source=self.source, s3_bucket=self.bucket)
with pytest.raises(TraitError):
S3(source=self.source, s3_bucket=5)
S3(source=self.source, node=Node())
with pytest.raises(TraitError):
S3(source=self.source, node='not a node')
S3(source=self.source, node_kwargs={})
with pytest.raises(TraitError):
S3(source=self.source, node_kwargs=5)
S3(source=self.source, node_class=DataSource)
with pytest.raises(TraitError):
S3(source=self.source, node_class=5)
S3(source=self.source, s3_bucket='testbucket')
with pytest.raises(TraitError):
S3(source=self.source, s3_bucket=5)
S3(source=self.source, return_type='path')
with pytest.raises(TraitError):
S3(source=self.source, return_type='notpath')
def test_hash_preserves_definition(self):
n = Node()
d_before = deepcopy(n.definition)
h = n.hash
d_after = deepcopy(n.definition)
assert d_before == d_after
def test_from_name_params(self):
# Normal
name = "algorithm.Arange"
node = Node.from_name_params(name)
# Normal with params
name = "algorithm.CoordData"
params = {"coord_name": "alt"}
node = Node.from_name_params(name, params)
assert node.coord_name == "alt"
# Plugin style
name = "CoordData"
params = {"plugin": "podpac.algorithm", "attrs": {"coord_name": "alt"}}
node = Node.from_name_params(name, params)
assert node.coord_name == "alt"
def eval_source(self, coordinates, coordinates_index, out, i, source=None):
if source is None:
source = self.source
# Make a copy to prevent any possibility of memory corruption
source = Node.from_definition(source.definition)
success = False
o = None
while not success:
if self.check_worker_available():
try:
o = source.eval(coordinates, output=out)
success = True
except self.async_exception:
# This exception is fine and constitutes a success
o = None
success = True
except self.no_worker_exception as e:
response = e.response
if not (response and response.get("Error", {}).get("Code")
== "TooManyRequestsException"):
raise e # Raise error again, not the right error
_log.debug("Worker {} exception {}".format(i, e))
success = False
time.sleep(self.sleep_time)
else:
_log.debug("Worker unavailable for {}".format(i, e))
time.sleep(self.sleep_time)
_log.info("Submitting source {}".format(i))
return (o, coordinates_index)
def test_base_ref(self):
"""test base ref"""
node = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
assert "_reprojected" in node.base_ref
def test_init(self):
"""test basic init of class"""
node = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
assert isinstance(node, ReprojectedSource)
def test_init(self):
"""test basic init of class"""
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
assert isinstance(node, ReprojectedSource)
def test_output(self):
node = Node()
assert node.output is None
node = Node(outputs=["a", "b"])
assert node.output is None
node = Node(outputs=["a", "b"], output="b")
assert node.output == "b"
# must be one of the outputs
with pytest.raises(ValueError, match="Invalid output"):
node = Node(outputs=["a", "b"], output="other")
# only valid for multiple-output nodes
with pytest.raises(TypeError, match="Invalid output"):
node = Node(output="other")
