def test_sanitize_outputs(self):
# outputs is None (no argument to outputs)
spec = HeronComponentSpec("spout", "class", True, 1)
ret = spec._sanitize_outputs()
self.assertIsNone(ret)
# outputs neither list nor tuple
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = "string"
with self.assertRaises(TypeError):
spec._sanitize_outputs()
# output list contains a non-string and non-Stream object
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", False, 123]
with self.assertRaises(TypeError):
spec._sanitize_outputs()
# output list is all string
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", "hello", "heron"]
ret = spec._sanitize_outputs()
self.assertEqual(ret, {"default": ["string", "hello", "heron"]})
# output list has mixed stream
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", "hello", Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")]
ret = spec._sanitize_outputs()
self.assertEqual(ret, {"default": ["string", "hello", "another", "default"],
"another_stream": ["abc", "def"]})
def test_get_item(self):
# HeronComponentSpec name set
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec['another_stream']
self.assertEqual(ret, GlobalStreamId("spout", "another_stream"))
# HeronComponentSpec name not set
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec['default']
self.assertEqual(ret, GlobalStreamId(spec, "default"))
# stream id not registered
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
with self.assertRaises(ValueError):
spec['non_existent_stream']
def test_get_out_streamids(self):
# outputs is none
spec = HeronComponentSpec("spout", "class", True, 1)
ret = spec.get_out_streamids()
self.assertEqual(ret, set())
# outputs neither list nor tuple
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = "string"
with self.assertRaises(TypeError):
spec.get_out_streamids()
# outputs sane
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", "hello", Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")]
ret = spec.get_out_streamids()
self.assertEqual(ret, {"default", "another_stream"})
def test_get_item(self):
# HeronComponentSpec name set
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", "hello", Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")]
ret = spec['another_stream']
self.assertEqual(ret, GlobalStreamId("spout", "another_stream"))
# HeronComponentSpec name not set
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = ["string", "hello", Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")]
ret = spec['default']
self.assertEqual(ret, GlobalStreamId(spec, "default"))
# stream id not registered
spec = HeronComponentSpec(None, "class", True, 1)
spec.outputs = ["string", "hello", Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")]
with self.assertRaises(ValueError):
spec['non_existent_stream']
def test_get_out_streamids(self):
# outputs is none
spec = HeronComponentSpec("spout", "class", True, 1)
ret = spec.get_out_streamids()
self.assertEqual(ret, set())
# outputs neither list nor tuple
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = "string"
with self.assertRaises(TypeError):
spec.get_out_streamids()
# outputs sane
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec.get_out_streamids()
self.assertEqual(ret, {"default", "another_stream"})
def test_sanitize_outputs(self):
# outputs is None (no argument to outputs)
spec = HeronComponentSpec("spout", "class", True, 1)
ret = spec._sanitize_outputs()
self.assertIsNone(ret)
# outputs neither list nor tuple
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = "string"
with self.assertRaises(TypeError):
spec._sanitize_outputs()
# output list contains a non-string and non-Stream object
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", False, 123]
with self.assertRaises(TypeError):
spec._sanitize_outputs()
# output list is all string
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = ["string", "hello", "heron"]
ret = spec._sanitize_outputs()
self.assertEqual(ret, {"default": ["string", "hello", "heron"]})
# output list has mixed stream
spec = HeronComponentSpec("spout", "class", True, 1)
spec.outputs = [
"string", "hello",
Stream(fields=["abc", "def"], name="another_stream"),
Stream(fields=["another", "default"], name="default")
]
ret = spec._sanitize_outputs()
self.assertEqual(
ret, {
"default": ["string", "hello", "another", "default"],
"another_stream": ["abc", "def"]
})
def test_sanitize_inputs(self):
# Note that _sanitize_inputs() should only be called after HeronComponentSpec's
# name attribute is set
# invalid inputs given as argument (valid ones are either dict, list, tuple or None)
invalid_spec = HeronComponentSpec("name", "classpath", True, 1, inputs="string")
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
invalid_spec = HeronComponentSpec("name", "classpath", True, 1, inputs=100)
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
# dict <HeronComponentSpec -> Grouping>
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt", "bl_clspath", False, 1,
inputs={from_spec: Grouping.SHUFFLE})
ret = to_spec._sanitize_inputs()
self.assertEqual(ret, {GlobalStreamId("spout", "default"): Grouping.SHUFFLE})
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
from_spec.outputs = [Stream(name='another_stream')]
to_spec = HeronComponentSpec("bolt", "bl_clspath", False, 1,
inputs={from_spec['another_stream']: Grouping.ALL})
ret = to_spec._sanitize_inputs()
self.assertEqual(ret, {GlobalStreamId("spout", "another_stream"): Grouping.ALL})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt", "bl_clspath", False, 1,
inputs={from_spec: Grouping.ALL})
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# dict <GlobalStreamId -> Grouping>
inputs_dict = {GlobalStreamId("some_spout", "some_stream"): Grouping.NONE,
GlobalStreamId("another_spout", "default"): Grouping.fields(['word', 'count'])}
spec = HeronComponentSpec("bolt", "classpath", False, 1, inputs=inputs_dict)
ret = spec._sanitize_inputs()
self.assertEqual(ret, inputs_dict)
# list of HeronComponentSpec
from_spec1 = HeronComponentSpec("spout1", "sp1_cls", True, 1)
from_spec2 = HeronComponentSpec("spout2", "sp2_cls", True, 1)
to_spec = HeronComponentSpec("bolt", "bl_cls", False, 1, inputs=[from_spec1, from_spec2])
ret = to_spec._sanitize_inputs()
self.assertEqual(ret, {GlobalStreamId("spout1", "default"): Grouping.SHUFFLE,
GlobalStreamId("spout2", "default"): Grouping.SHUFFLE})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt", "bl_clspath", False, 1, inputs=[from_spec])
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# list of GlobalStreamId
inputs_list = [GlobalStreamId("spout1", "default"), GlobalStreamId("spout2", "some_stream")]
spec = HeronComponentSpec("bolt", "bl_cls", False, 1, inputs=inputs_list)
ret = spec._sanitize_inputs()
self.assertEqual(ret, dict(zip(inputs_list, [Grouping.SHUFFLE] * 2)))
# list of neither GlobalStreamId nor HeronComponentSpec
inputs_list = [None, 123, "string", [GlobalStreamId("sp", "default")]]
spec = HeronComponentSpec("bolt", "bl_cls", False, 1, inputs=inputs_list)
with self.assertRaises(ValueError):
spec._sanitize_inputs()
def test_sanitize_inputs(self):
# Note that _sanitize_inputs() should only be called after HeronComponentSpec's
# name attribute is set
# invalid inputs given as argument (valid ones are either dict, list, tuple or None)
invalid_spec = HeronComponentSpec("name",
"classpath",
True,
1,
inputs="string")
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
invalid_spec = HeronComponentSpec("name",
"classpath",
True,
1,
inputs=100)
with self.assertRaises(TypeError):
invalid_spec._sanitize_inputs()
# dict <HeronComponentSpec -> Grouping>
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs={from_spec: Grouping.SHUFFLE})
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {GlobalStreamId("spout", "default"): Grouping.SHUFFLE})
from_spec = HeronComponentSpec("spout", "sp_clspath", True, 1)
from_spec.outputs = [Stream(name='another_stream')]
to_spec = HeronComponentSpec(
"bolt",
"bl_clspath",
False,
1,
inputs={from_spec['another_stream']: Grouping.ALL})
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {GlobalStreamId("spout", "another_stream"): Grouping.ALL})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs={from_spec: Grouping.ALL})
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# dict <GlobalStreamId -> Grouping>
inputs_dict = {
GlobalStreamId("some_spout", "some_stream"):
Grouping.NONE,
GlobalStreamId("another_spout", "default"):
Grouping.fields(['word', 'count'])
}
spec = HeronComponentSpec("bolt",
"classpath",
False,
1,
inputs=inputs_dict)
ret = spec._sanitize_inputs()
self.assertEqual(ret, inputs_dict)
# list of HeronComponentSpec
from_spec1 = HeronComponentSpec("spout1", "sp1_cls", True, 1)
from_spec2 = HeronComponentSpec("spout2", "sp2_cls", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=[from_spec1, from_spec2])
ret = to_spec._sanitize_inputs()
self.assertEqual(
ret, {
GlobalStreamId("spout1", "default"): Grouping.SHUFFLE,
GlobalStreamId("spout2", "default"): Grouping.SHUFFLE
})
# HeronComponentSpec's name attribute not set
from_spec = HeronComponentSpec(None, "sp_clspath", True, 1)
to_spec = HeronComponentSpec("bolt",
"bl_clspath",
False,
1,
inputs=[from_spec])
with self.assertRaises(RuntimeError):
to_spec._sanitize_inputs()
# list of GlobalStreamId
inputs_list = [
GlobalStreamId("spout1", "default"),
GlobalStreamId("spout2", "some_stream")
]
spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=inputs_list)
ret = spec._sanitize_inputs()
self.assertEqual(ret, dict(zip(inputs_list, [Grouping.SHUFFLE] * 2)))
# list of neither GlobalStreamId nor HeronComponentSpec
inputs_list = [None, 123, "string", [GlobalStreamId("sp", "default")]]
spec = HeronComponentSpec("bolt",
"bl_cls",
False,
1,
inputs=inputs_list)
with self.assertRaises(ValueError):
spec._sanitize_inputs()
