def _get_comp_config(self):
"""Returns component-specific Config protobuf message
It first adds ``topology.component.parallelism``, and is overriden by
a user-defined component-specific configuration, specified by spec().
"""
proto_config = topology_pb2.Config()
# first add parallelism
key = proto_config.kvs.add()
key.key = TOPOLOGY_COMPONENT_PARALLELISM
key.value = str(self.parallelism)
key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
for key, value in list(sanitized.items()):
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value(
"STRING_VALUE")
else:
# need to serialize
kvs = proto_config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value(
"PYTHON_SERIALIZED_VALUE")
return proto_config
def _get_comp_config(self):
"""Returns component-specific Config protobuf message
It first adds ``topology.component.parallelism``, and is overriden by
a user-defined component-specific configuration, specified by spec().
"""
proto_config = topology_pb2.Config()
# first add parallelism
key = proto_config.kvs.add()
key.key = TOPOLOGY_COMPONENT_PARALLELISM
key.value = str(self.parallelism)
key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
for key, value in sanitized.items():
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = proto_config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")
return proto_config
def custom(cls, customgrouper):
"""Custom grouping from a given implementation of ICustomGrouping
:param customgrouper: The ICustomGrouping implemention to use
"""
if customgrouper is None:
raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
if not isinstance(customgrouper, ICustomGrouping) and not isinstance(customgrouper, str):
raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
serialized = default_serializer.serialize(customgrouper)
return cls.custom_serialized(serialized, is_java=False)
def custom(cls, customgrouper):
"""Custom grouping from a given implementation of ICustomGrouping
:param customgrouper: The ICustomGrouping implemention to use
"""
if customgrouper is None:
raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
if not isinstance(customgrouper, ICustomGrouping) and not isinstance(customgrouper, str):
raise TypeError("Argument to custom() must be ICustomGrouping instance or classpath")
serialized = default_serializer.serialize(customgrouper)
return cls.custom_serialized(serialized, is_java=False)
def get_topology_config_protobuf(mcs, class_dict):
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
for key, value in conf_dict.items():
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")
return config
def get_topology_config_protobuf(mcs, class_dict):
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
for key, value in conf_dict.items():
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")
return config
def test_sane_topology(self):
self.assertEqual(TestSane.topology_name, "TestSane")
# topology-wide config
expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG
expecting_topo_config.update({
"topology.wide.config.1": "value",
"spout.overriden.config": "true"
})
self.assertEqual(TestSane._topo_config, expecting_topo_config)
self.assertEqual(len(TestSane._protobuf_bolts), 1)
self.assertEqual(len(TestSane._protobuf_spouts), 1)
self.assertEqual(len(TestSane._heron_specs), 2)
for spec in TestSane._heron_specs:
if spec.is_spout:
self.assertEqual(spec.name, "spout")
self.assertEqual(spec.python_class_path, "sp_class")
self.assertEqual(spec.parallelism, 3)
else:
self.assertEqual(spec.name, "bolt")
self.assertEqual(spec.python_class_path, "bl_class")
self.assertEqual(spec.parallelism, 4)
self.assertTrue(
isinstance(TestSane.protobuf_topology, topology_pb2.Topology))
proto_topo = TestSane.protobuf_topology
### spout protobuf ###
self.assertEqual(len(proto_topo.spouts), 1)
spout = proto_topo.spouts[0]
self.assertEqual(spout.comp.name, "spout")
self.assertEqual(
spout.comp.spec,
topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(spout.comp.class_name, "sp_class")
expecting_spout_config = {
"topology.component.parallelism": "3",
"spout.specific.config.1": "value",
"spout.specific.config.2": "true",
"spout.specific.config.3": "-12.4",
"spout.specific.config.4": default_serializer.serialize([1, 2, 3]),
"spout.overriden.config": "false"
}
self.assertEqual(len(spout.comp.config.kvs),
len(expecting_spout_config))
for conf in spout.comp.config.kvs:
value = expecting_spout_config[conf.key]
if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"):
self.assertEqual(value, conf.value)
elif conf.type == topology_pb2.ConfigValueType.Value(
"PYTHON_SERIALIZED_VALUE"):
self.assertEqual(value, conf.serialized_value)
else:
self.fail()
# output stream
self.assertEqual(len(spout.outputs), 2)
for out_stream in spout.outputs:
if out_stream.stream.id == "default":
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 2)
else:
self.assertEqual(out_stream.stream.id, "error_stream")
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 1)
### bolt protobuf ###
self.assertEqual(len(proto_topo.bolts), 1)
bolt = proto_topo.bolts[0]
self.assertEqual(bolt.comp.name, "bolt")
self.assertEqual(
bolt.comp.spec,
topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(bolt.comp.class_name, "bl_class")
expecting_bolt_config = {"topology.component.parallelism": "4"}
self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config))
conf = bolt.comp.config.kvs[0]
self.assertEqual(conf.type,
topology_pb2.ConfigValueType.Value("STRING_VALUE"))
self.assertEqual(conf.value, expecting_bolt_config[conf.key])
# out stream
self.assertEqual(len(bolt.outputs), 0)
# in stream
self.assertEqual(len(bolt.inputs), 2)
for in_stream in bolt.inputs:
if in_stream.stream.id == "default":
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype,
topology_pb2.Grouping.Value("SHUFFLE"))
else:
self.assertEqual(in_stream.stream.id, "error_stream")
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype,
topology_pb2.Grouping.Value("ALL"))
self.assertEqual(proto_topo.state,
topology_pb2.TopologyState.Value("RUNNING"))
def test_sane_topology(self):
self.assertEqual(TestSane.topology_name, "TestSane")
# topology-wide config
expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG
expecting_topo_config.update({"topology.wide.config.1": "value",
"spout.overriden.config": "true"})
self.assertEqual(TestSane._topo_config, expecting_topo_config)
self.assertEqual(len(TestSane._protobuf_bolts), 1)
self.assertEqual(len(TestSane._protobuf_spouts), 1)
self.assertEqual(len(TestSane._heron_specs), 2)
for spec in TestSane._heron_specs:
if spec.is_spout:
self.assertEqual(spec.name, "spout")
self.assertEqual(spec.python_class_path, "sp_class")
self.assertEqual(spec.parallelism, 3)
else:
self.assertEqual(spec.name, "bolt")
self.assertEqual(spec.python_class_path, "bl_class")
self.assertEqual(spec.parallelism, 4)
self.assertTrue(isinstance(TestSane.protobuf_topology, topology_pb2.Topology))
proto_topo = TestSane.protobuf_topology
### spout protobuf ###
self.assertEqual(len(proto_topo.spouts), 1)
spout = proto_topo.spouts[0]
self.assertEqual(spout.comp.name, "spout")
self.assertEqual(spout.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(spout.comp.class_name, "sp_class")
expecting_spout_config = {"topology.component.parallelism": "3",
"spout.specific.config.1": "value",
"spout.specific.config.2": "true",
"spout.specific.config.3": "-12.4",
"spout.specific.config.4": default_serializer.serialize([1, 2, 3]),
"spout.overriden.config": "false"}
self.assertEqual(len(spout.comp.config.kvs), len(expecting_spout_config))
for conf in spout.comp.config.kvs:
value = expecting_spout_config[conf.key]
if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"):
self.assertEqual(value, conf.value)
elif conf.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
self.assertEqual(value, conf.serialized_value)
else:
self.fail()
# output stream
self.assertEqual(len(spout.outputs), 2)
for out_stream in spout.outputs:
if out_stream.stream.id == "default":
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 2)
else:
self.assertEqual(out_stream.stream.id, "error_stream")
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 1)
### bolt protobuf ###
self.assertEqual(len(proto_topo.bolts), 1)
bolt = proto_topo.bolts[0]
self.assertEqual(bolt.comp.name, "bolt")
self.assertEqual(bolt.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(bolt.comp.class_name, "bl_class")
expecting_bolt_config = {"topology.component.parallelism": "4"}
self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config))
conf = bolt.comp.config.kvs[0]
self.assertEqual(conf.type, topology_pb2.ConfigValueType.Value("STRING_VALUE"))
self.assertEqual(conf.value, expecting_bolt_config[conf.key])
# out stream
self.assertEqual(len(bolt.outputs), 0)
# in stream
self.assertEqual(len(bolt.inputs), 2)
for in_stream in bolt.inputs:
if in_stream.stream.id == "default":
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("SHUFFLE"))
else:
self.assertEqual(in_stream.stream.id, "error_stream")
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("ALL"))
self.assertEqual(proto_topo.state, topology_pb2.TopologyState.Value("RUNNING"))
