def test_class_dict_to_specs(self):
# duplicate component name
class_dict = {
"spout": HeronComponentSpec("same_name", "sp_cls", True, 1),
"bolt": HeronComponentSpec("same_name", "bl_cls", False, 2)
}
with self.assertRaises(ValueError):
TopologyType.class_dict_to_specs(class_dict)
def test_get_heron_options_from_env(self):
test_value = "cmdline.key.1=/tmp/directory,cmdline.with.space=hello%%%%world"
expecting = {"cmdline.key.1": "/tmp/directory", "cmdline.with.space": "hello world"}
os.environ["HERON_OPTIONS"] = test_value
ret = TopologyType.get_heron_options_from_env()
self.assertEqual(ret, expecting)
# error
os.environ.pop("HERON_OPTIONS")
with self.assertRaises(RuntimeError):
TopologyType.get_heron_options_from_env()
def test_get_heron_options_from_env(self):
test_value = "cmdline.key.1=/tmp/directory,cmdline.with.space=hello%%%%world"
expecting = {
"cmdline.key.1": "/tmp/directory",
"cmdline.with.space": "hello world"
}
os.environ["HERON_OPTIONS"] = test_value
ret = TopologyType.get_heron_options_from_env()
self.assertEqual(ret, expecting)
# error
os.environ.pop("HERON_OPTIONS")
with self.assertRaises(RuntimeError):
TopologyType.get_heron_options_from_env()
def test_sanitize_config(self):
# non-string key
with self.assertRaises(TypeError):
TopologyType._sanitize_config({['k', 'e', 'y']: "value"})
with self.assertRaises(TypeError):
TopologyType._sanitize_config({None: "value"})
# convert boolean value
ret = TopologyType._sanitize_config({"key": True})
self.assertEqual(ret["key"], "true")
ret = TopologyType._sanitize_config({"key": False})
self.assertEqual(ret["key"], "false")
# convert int and float
ret = TopologyType._sanitize_config({"key": 10})
self.assertEqual(ret["key"], "10")
ret = TopologyType._sanitize_config({"key": -2400000})
self.assertEqual(ret["key"], "-2400000")
ret = TopologyType._sanitize_config({"key": 0.0000001})
self.assertEqual(ret["key"], "1e-07")
ret = TopologyType._sanitize_config({"key": -15.33333})
self.assertEqual(ret["key"], "-15.33333")
# non-string value -> should expect the same object
ret = TopologyType._sanitize_config({"key": ['v', 'a', 'l', 'u', 'e']})
self.assertEqual(ret["key"], ['v', 'a', 'l', 'u', 'e'])
ret = TopologyType._sanitize_config({"key": None})
self.assertEqual(ret["key"], None)
def test_add_bolt_specs(self):
spec = HeronComponentSpec("bolt", "bl_cls", False, 1)
with self.assertRaises(ValueError):
TopologyType.add_bolt_specs(spec, {})
def test_add_spout_specs(self):
# spout with no output
spec = HeronComponentSpec("spout", "sp_cls", True, 1)
with self.assertRaises(ValueError):
TopologyType.add_spout_specs(spec, {})
def test_sanitize_config(self):
# non-string key
with self.assertRaises(TypeError):
TopologyType._sanitize_config({['k', 'e', 'y']: "value"})
with self.assertRaises(TypeError):
TopologyType._sanitize_config({None: "value"})
# convert boolean value
ret = TopologyType._sanitize_config({"key": True})
self.assertEqual(ret["key"], "true")
ret = TopologyType._sanitize_config({"key": False})
self.assertEqual(ret["key"], "false")
# convert int and float
ret = TopologyType._sanitize_config({"key": 10})
self.assertEqual(ret["key"], "10")
ret = TopologyType._sanitize_config({"key": -2400000})
self.assertEqual(ret["key"], "-2400000")
ret = TopologyType._sanitize_config({"key": 0.0000001})
self.assertEqual(ret["key"], "1e-07")
ret = TopologyType._sanitize_config({"key": -15.33333})
self.assertEqual(ret["key"], "-15.33333")
# non-string value -> should expect the same object
ret = TopologyType._sanitize_config({"key": ['v', 'a', 'l', 'u', 'e']})
self.assertEqual(ret["key"], ['v', 'a', 'l', 'u', 'e'])
ret = TopologyType._sanitize_config({"key": None})
self.assertEqual(ret["key"], None)
def test_add_bolt_specs(self):
spec = HeronComponentSpec("bolt", "bl_cls", False, 1)
with self.assertRaises(ValueError):
TopologyType.add_bolt_specs(spec, {})
def test_add_spout_specs(self):
# spout with no output
spec = HeronComponentSpec("spout", "sp_cls", True, 1)
with self.assertRaises(ValueError):
TopologyType.add_spout_specs(spec, {})
def test_class_dict_to_specs(self):
# duplicate component name
class_dict = {"spout": HeronComponentSpec("same_name", "sp_cls", True, 1),
"bolt": HeronComponentSpec("same_name", "bl_cls", False, 2)}
with self.assertRaises(ValueError):
TopologyType.class_dict_to_specs(class_dict)
def create_topology(self):
"""Creates an integration_test topology class"""
# first add the aggregation_bolt
# inputs will be updated later
aggregator_config = {integ_const.HTTP_POST_URL_KEY: self.output_location}
self.add_bolt(self.TERMINAL_BOLT_NAME, self.TERMINAL_BOLT_CLASS, 1,
inputs={}, config=aggregator_config)
# building a graph directed from children to parents, by looking only on bolts
# since spouts don't have parents
for name, bolt_spec in self.bolts.iteritems():
if name == self.TERMINAL_BOLT_NAME:
continue
bolt_protobuf = bolt_spec.get_protobuf()
for istream in bolt_protobuf.inputs:
parent = istream.stream.component_name
if name in self.prev:
self.prev[name].add(parent)
else:
parents = set()
parents.add(parent)
self.prev[name] = parents
# Find the terminal bolts defined by users and link them with "AggregatorBolt".
# set of terminal component names
terminals = set()
# set of non-terminal component names
non_terminals = set()
# 1. terminal bolts need upstream components, because we don't want isolated bolts
# 2. terminal bolts should not exist in the prev.values(), meaning that no downstream
for parent_set in self.prev.values():
non_terminals.update(parent_set)
for bolt_name in self.prev.keys():
if bolt_name not in non_terminals:
terminals.add(bolt_name)
# will also consider the cases with spouts without children
for spout_name in self.spouts.keys():
if spout_name not in non_terminals:
terminals.add(spout_name)
# add all grouping to components
for child in self.prev.keys():
for parent in self.prev[child]:
self._add_all_grouping(child, parent, integ_const.INTEGRATION_TEST_CONTROL_STREAM_ID)
# then connect aggregator bolt with user's terminal components
# terminal_outputs are output fields for terminals, list of either str or Stream
for terminal in terminals:
if terminal in self.bolts:
terminal_outputs = self.bolts[terminal].outputs
else:
terminal_outputs = self.spouts[terminal].outputs
# now get a set of stream ids
stream_ids = ["default" if isinstance(out, str) else out.stream_id
for out in terminal_outputs]
for stream_id in set(stream_ids):
self._add_all_grouping(self.TERMINAL_BOLT_NAME, terminal, stream_id)
# create topology class
class_dict = self._construct_topo_class_dict()
return TopologyType(self.topology_name, (Topology,), class_dict)