def test_constructor(self):
# component id not string nor HeronComponentSpec
with self.assertRaises(TypeError):
GlobalStreamId(componentId=123, streamId="default")
# stream id not string
with self.assertRaises(TypeError):
GlobalStreamId(componentId="component", streamId=12345)
def test_get_item(self):
# HeronComponentSpec name set
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec['another_stream']
self.assertEqual(ret, GlobalStreamId("spout", "another_stream"))
# HeronComponentSpec name not set
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec['default']
self.assertEqual(ret, GlobalStreamId(spec, "default"))
# stream id not registered
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
with self.assertRaises(ValueError):
spec['non_existent_stream']
def test_component_id_property(self):
# component id is string
gsi = GlobalStreamId(componentId="component", streamId="stream")
self.assertEqual(gsi.component_id, "component")
# component id is HeronComponentSpec with name
spec = HeronComponentSpec("spout", "class", True, 1)
gsi = GlobalStreamId(spec, "stream")
self.assertEqual(gsi.component_id, "spout")
# component id is HeronComponentSpec without name
spec = HeronComponentSpec(None, "class", True, 1)
gsi = GlobalStreamId(spec, "stream")
# expecting "<No name available for HeronComponentSpec yet, uuid: %s>"
self.assertIn(spec.uuid, gsi.component_id)
def _calculate_inputs(self):
return {
GlobalStreamId(self._parents[0]._stage_name, self._parents[0]._output):
Grouping.SHUFFLE
}
def _calculate_inputs(self):
return {GlobalStreamId(self._parents[0]._stage_name, self._parents[0]._output) :
Grouping.custom("heron.dsl.src.python.reducebykeyandwindowbolt.ReduceGrouping")}
def _calculate_inputs(self):
inputs = {}
for parent in self._parents:
inputs[GlobalStreamId(parent._stage_name, parent._output)] = \
Grouping.custom("heron.dsl.src.python.joinbolt.JoinGrouping")
return inputs
def test_sanitize_inputs(self):
# Note that _sanitize_inputs() should only be called after HeronComponentSpec's
# name attribute is set
# invalid inputs given as argument (valid ones are either dict, list, tuple or None)
invalid_spec = HeronComponentSpec("name",
"classpath",
True,
1,
inputs="string")
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
invalid_spec = HeronComponentSpec("name",
"classpath",
True,
1,
inputs=100)
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
# dict <HeronComponentSpec -> Grouping>
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs={from_spec: Grouping.SHUFFLE})
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {GlobalStreamId("spout", "default"): Grouping.SHUFFLE})
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
from_spec.outputs = [Stream(name='another_stream')]
to_spec = HeronComponentSpec(
"bolt",
"bl_clspath",
False,
1,
inputs={from_spec['another_stream']: Grouping.ALL})
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {GlobalStreamId("spout", "another_stream"): Grouping.ALL})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs={from_spec: Grouping.ALL})
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# dict <GlobalStreamId -> Grouping>
inputs_dict = {
GlobalStreamId("some_spout", "some_stream"):
Grouping.NONE,
GlobalStreamId("another_spout", "default"):
Grouping.fields(['word', 'count'])
}
spec = HeronComponentSpec("bolt",
"classpath",
False,
1,
inputs=inputs_dict)
ret = spec._sanitize_inputs()
self.assertEqual(ret, inputs_dict)
# list of HeronComponentSpec
from_spec1 = HeronComponentSpec("spout1", "sp1_cls", True, 1)
from_spec2 = HeronComponentSpec("spout2", "sp2_cls", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=[from_spec1, from_spec2])
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {
GlobalStreamId("spout1", "default"): Grouping.SHUFFLE,
GlobalStreamId("spout2", "default"): Grouping.SHUFFLE
})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs=[from_spec])
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# list of GlobalStreamId
inputs_list = [
GlobalStreamId("spout1", "default"),
GlobalStreamId("spout2", "some_stream")
]
spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=inputs_list)
ret = spec._sanitize_inputs()
self.assertEqual(ret, dict(zip(inputs_list, [Grouping.SHUFFLE] * 2)))
# list of neither GlobalStreamId nor HeronComponentSpec
inputs_list = [None, 123, "string", [GlobalStreamId("sp", "default")]]
spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=inputs_list)
with self.assertRaises(ValueError):
spec._sanitize_inputs()