def test_class_dict_to_specs(self): # duplicate component name class_dict = { "spout": HeronComponentSpec("same_name", "sp_cls", True, 1), "bolt": HeronComponentSpec("same_name", "bl_cls", False, 2) } with self.assertRaises(ValueError): TopologyType.class_dict_to_specs(class_dict)
def test_get_heron_options_from_env(self): test_value = "cmdline.key.1=/tmp/directory,cmdline.with.space=hello%%%%world" expecting = {"cmdline.key.1": "/tmp/directory", "cmdline.with.space": "hello world"} os.environ["HERON_OPTIONS"] = test_value ret = TopologyType.get_heron_options_from_env() self.assertEqual(ret, expecting) # error os.environ.pop("HERON_OPTIONS") with self.assertRaises(RuntimeError): TopologyType.get_heron_options_from_env()
def test_get_heron_options_from_env(self): test_value = "cmdline.key.1=/tmp/directory,cmdline.with.space=hello%%%%world" expecting = {"cmdline.key.1": "/tmp/directory", "cmdline.with.space": "hello world"} os.environ["HERON_OPTIONS"] = test_value ret = TopologyType.get_heron_options_from_env() self.assertEqual(ret, expecting) # error os.environ.pop("HERON_OPTIONS") with self.assertRaises(RuntimeError): TopologyType.get_heron_options_from_env()
def create_topology(self): """Creates an integration_test topology class""" # first add the aggregation_bolt # inputs will be updated later aggregator_config = { integ_const.HTTP_POST_URL_KEY: self.output_location } self.add_bolt(self.TERMINAL_BOLT_NAME, self.TERMINAL_BOLT_CLASS, 1, inputs={}, config=aggregator_config) # building a graph directed from children to parents, by looking only on bolts # since spouts don't have parents for name, bolt_spec in self.bolts.items(): if name == self.TERMINAL_BOLT_NAME: continue bolt_protobuf = bolt_spec.get_protobuf() for istream in bolt_protobuf.inputs: parent = istream.stream.component_name if name in self.prev: self.prev[name].add(parent) else: parents = set() parents.add(parent) self.prev[name] = parents # Find the terminal bolts defined by users and link them with "AggregatorBolt". # set of terminal component names terminals = set() # set of non-terminal component names non_terminals = set() # 1. terminal bolts need upstream components, because we don't want isolated bolts # 2. terminal bolts should not exist in the prev.values(), meaning that no downstream for parent_set in list(self.prev.values()): non_terminals.update(parent_set) for bolt_name in list(self.prev.keys()): if bolt_name not in non_terminals: terminals.add(bolt_name) # will also consider the cases with spouts without children for spout_name in list(self.spouts.keys()): if spout_name not in non_terminals: terminals.add(spout_name) # add all grouping to components for child in list(self.prev.keys()): for parent in self.prev[child]: self._add_all_grouping( child, parent, integ_const.INTEGRATION_TEST_CONTROL_STREAM_ID) # then connect aggregator bolt with user's terminal components # terminal_outputs are output fields for terminals, list of either str or Stream for terminal in terminals: if terminal in self.bolts: terminal_outputs = self.bolts[terminal].outputs else: terminal_outputs = self.spouts[terminal].outputs # now get a set of stream ids stream_ids = [ "default" if isinstance(out, str) else out.stream_id for out in terminal_outputs ] for stream_id in set(stream_ids): self._add_all_grouping(self.TERMINAL_BOLT_NAME, terminal, stream_id) # create topology class class_dict = self._construct_topo_class_dict() return TopologyType(self.topology_name, (Topology, ), class_dict)
def test_sanitize_config(self): # non-string key with self.assertRaises(TypeError): TopologyType._sanitize_config({['k', 'e', 'y']: "value"}) with self.assertRaises(TypeError): TopologyType._sanitize_config({None: "value"}) # convert boolean value ret = TopologyType._sanitize_config({"key": True}) self.assertEqual(ret["key"], "true") ret = TopologyType._sanitize_config({"key": False}) self.assertEqual(ret["key"], "false") # convert int and float ret = TopologyType._sanitize_config({"key": 10}) self.assertEqual(ret["key"], "10") ret = TopologyType._sanitize_config({"key": -2400000}) self.assertEqual(ret["key"], "-2400000") ret = TopologyType._sanitize_config({"key": 0.0000001}) self.assertEqual(ret["key"], "1e-07") ret = TopologyType._sanitize_config({"key": -15.33333}) self.assertEqual(ret["key"], "-15.33333") # non-string value -> should expect the same object ret = TopologyType._sanitize_config({"key": ['v', 'a', 'l', 'u', 'e']}) self.assertEqual(ret["key"], ['v', 'a', 'l', 'u', 'e']) ret = TopologyType._sanitize_config({"key": None}) self.assertEqual(ret["key"], None)
def test_add_bolt_specs(self): spec = HeronComponentSpec("bolt", "bl_cls", False, 1) with self.assertRaises(ValueError): TopologyType.add_bolt_specs(spec, {})
def test_add_spout_specs(self): # spout with no output spec = HeronComponentSpec("spout", "sp_cls", True, 1) with self.assertRaises(ValueError): TopologyType.add_spout_specs(spec, {})
def test_sanitize_config(self): # non-string key with self.assertRaises(TypeError): TopologyType._sanitize_config({['k', 'e', 'y']: "value"}) with self.assertRaises(TypeError): TopologyType._sanitize_config({None: "value"}) # convert boolean value ret = TopologyType._sanitize_config({"key": True}) self.assertEqual(ret["key"], "true") ret = TopologyType._sanitize_config({"key": False}) self.assertEqual(ret["key"], "false") # convert int and float ret = TopologyType._sanitize_config({"key": 10}) self.assertEqual(ret["key"], "10") ret = TopologyType._sanitize_config({"key": -2400000}) self.assertEqual(ret["key"], "-2400000") ret = TopologyType._sanitize_config({"key": 0.0000001}) self.assertEqual(ret["key"], "1e-07") ret = TopologyType._sanitize_config({"key": -15.33333}) self.assertEqual(ret["key"], "-15.33333") # non-string value -> should expect the same object ret = TopologyType._sanitize_config({"key": ['v', 'a', 'l', 'u', 'e']}) self.assertEqual(ret["key"], ['v', 'a', 'l', 'u', 'e']) ret = TopologyType._sanitize_config({"key": None}) self.assertEqual(ret["key"], None)
def test_add_bolt_specs(self): spec = HeronComponentSpec("bolt", "bl_cls", False, 1) with self.assertRaises(ValueError): TopologyType.add_bolt_specs(spec, {})
def test_add_spout_specs(self): # spout with no output spec = HeronComponentSpec("spout", "sp_cls", True, 1) with self.assertRaises(ValueError): TopologyType.add_spout_specs(spec, {})
def test_class_dict_to_specs(self): # duplicate component name class_dict = {"spout": HeronComponentSpec("same_name", "sp_cls", True, 1), "bolt": HeronComponentSpec("same_name", "bl_cls", False, 2)} with self.assertRaises(ValueError): TopologyType.class_dict_to_specs(class_dict)