def _get_comp_config(self):
"""Returns component-specific Config protobuf message
It first adds ``topology.component.parallelism``, and is overriden by
a user-defined component-specific configuration, specified by spec().
"""
proto_config = topology_pb2.Config()
# first add parallelism
key = proto_config.kvs.add()
key.key = constants.TOPOLOGY_COMPONENT_PARALLELISM
key.value = str(self.parallelism)
key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
for key, value in sanitized.iteritems():
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = proto_config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")
return proto_config
def _get_comp_config(self):
"""Returns component-specific Config protobuf message
It first adds ``topology.component.parallelism``, and is overriden by
a user-defined component-specific configuration, specified by spec().
"""
proto_config = topology_pb2.Config()
# first add parallelism
key = proto_config.kvs.add()
key.key = constants.TOPOLOGY_COMPONENT_PARALLELISM
key.value = str(self.parallelism)
key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
for key, value in sanitized.iteritems():
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value(
"STRING_VALUE")
else:
# need to serialize
kvs = proto_config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value(
"PYTHON_SERIALIZED_VALUE")
return proto_config
def custom(cls, classpath):
"""Custom grouping from a given classpath
:param classpath: classpath to the ICustomGrouping class to use
"""
if classpath is None or not isinstance(classpath, str):
raise TypeError("Argument to custom() must be classpath string to custom grouping, given: "
"%s" % str(classpath))
serialized = default_serializer.serialize(classpath)
return cls.custom_serialized(serialized, is_java=False)
def custom(cls, classpath):
"""Custom grouping from a given classpath to an implementation of ICustomGrouping
This method does not exist in the Streamparse API.
:param classpath: classpath to the ICustomGrouping class to use
"""
if classpath is None or not isinstance(classpath, str):
raise TypeError("Argument to custom() must be classpath string to custom grouping, given: "
"%s" % str(classpath))
serialized = default_serializer.serialize(classpath)
return cls.custom_serialized(serialized, is_java=False)
def custom(cls, classpath):
"""Custom grouping from a given classpath to an implementation of ICustomGrouping
This method does not exist in the Streamparse API.
:param classpath: classpath to the ICustomGrouping class to use
"""
if classpath is None or not isinstance(classpath, str):
raise TypeError(
"Argument to custom() must be classpath string to custom grouping, given: "
"%s" % str(classpath))
serialized = default_serializer.serialize(classpath)
return cls.custom_serialized(serialized, is_java=False)
def get_topology_config_protobuf(mcs, class_dict):
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
for key, value in conf_dict.iteritems():
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")
return config
def get_topology_config_protobuf(mcs, class_dict):
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
for key, value in conf_dict.iteritems():
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
kvs.value = value
kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
else:
# need to serialize
kvs = config.kvs.add()
kvs.key = key
kvs.serialized_value = default_serializer.serialize(value)
kvs.type = topology_pb2.ConfigValueType.Value(
"PYTHON_SERIALIZED_VALUE")
return config
def test_sane_topology(self):
self.assertEqual(TestSane.topology_name, "TestSane")
# topology-wide config
expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG
expecting_topo_config.update({"topology.wide.config.1": "value",
"spout.overriden.config": "true"})
self.assertEqual(TestSane._topo_config, expecting_topo_config)
self.assertEqual(len(TestSane._protobuf_bolts), 1)
self.assertEqual(len(TestSane._protobuf_spouts), 1)
self.assertEqual(len(TestSane._heron_specs), 2)
for spec in TestSane._heron_specs:
if spec.is_spout:
self.assertEqual(spec.name, "spout")
self.assertEqual(spec.python_class_path, "sp_class")
self.assertEqual(spec.parallelism, 3)
else:
self.assertEqual(spec.name, "bolt")
self.assertEqual(spec.python_class_path, "bl_class")
self.assertEqual(spec.parallelism, 4)
self.assertTrue(isinstance(TestSane.protobuf_topology, topology_pb2.Topology))
proto_topo = TestSane.protobuf_topology
### spout protobuf ###
self.assertEqual(len(proto_topo.spouts), 1)
spout = proto_topo.spouts[0]
self.assertEqual(spout.comp.name, "spout")
self.assertEqual(spout.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(spout.comp.class_name, "sp_class")
expecting_spout_config = {"topology.component.parallelism": "3",
"spout.specific.config.1": "value",
"spout.specific.config.2": "true",
"spout.specific.config.3": "-12.4",
"spout.specific.config.4": default_serializer.serialize([1, 2, 3]),
"spout.overriden.config": "false"}
self.assertEqual(len(spout.comp.config.kvs), len(expecting_spout_config))
for conf in spout.comp.config.kvs:
value = expecting_spout_config[conf.key]
if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"):
self.assertEqual(value, conf.value)
elif conf.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
self.assertEqual(value, conf.serialized_value)
else:
self.fail()
# output stream
self.assertEqual(len(spout.outputs), 2)
for out_stream in spout.outputs:
if out_stream.stream.id == "default":
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 2)
else:
self.assertEqual(out_stream.stream.id, "error_stream")
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 1)
### bolt protobuf ###
self.assertEqual(len(proto_topo.bolts), 1)
bolt = proto_topo.bolts[0]
self.assertEqual(bolt.comp.name, "bolt")
self.assertEqual(bolt.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(bolt.comp.class_name, "bl_class")
expecting_bolt_config = {"topology.component.parallelism": "4"}
self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config))
conf = bolt.comp.config.kvs[0]
self.assertEqual(conf.type, topology_pb2.ConfigValueType.Value("STRING_VALUE"))
self.assertEqual(conf.value, expecting_bolt_config[conf.key])
# out stream
self.assertEqual(len(bolt.outputs), 0)
# in stream
self.assertEqual(len(bolt.inputs), 2)
for in_stream in bolt.inputs:
if in_stream.stream.id == "default":
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("SHUFFLE"))
else:
self.assertEqual(in_stream.stream.id, "error_stream")
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("ALL"))
self.assertEqual(proto_topo.state, topology_pb2.TopologyState.Value("RUNNING"))
def test_sane_topology(self):
self.assertEqual(TestSane.topology_name, "TestSane")
# topology-wide config
expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG
expecting_topo_config.update({"topology.wide.config.1": "value",
"spout.overriden.config": "true"})
self.assertEqual(TestSane._topo_config, expecting_topo_config)
self.assertEqual(len(TestSane._protobuf_bolts), 1)
self.assertEqual(len(TestSane._protobuf_spouts), 1)
self.assertEqual(len(TestSane._heron_specs), 2)
for spec in TestSane._heron_specs:
if spec.is_spout:
self.assertEqual(spec.name, "spout")
self.assertEqual(spec.python_class_path, "sp_class")
self.assertEqual(spec.parallelism, 3)
else:
self.assertEqual(spec.name, "bolt")
self.assertEqual(spec.python_class_path, "bl_class")
self.assertEqual(spec.parallelism, 4)
self.assertTrue(isinstance(TestSane.protobuf_topology, topology_pb2.Topology))
proto_topo = TestSane.protobuf_topology
### spout protobuf ###
self.assertEqual(len(proto_topo.spouts), 1)
spout = proto_topo.spouts[0]
self.assertEqual(spout.comp.name, "spout")
self.assertEqual(spout.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(spout.comp.class_name, "sp_class")
expecting_spout_config = {"topology.component.parallelism": "3",
"spout.specific.config.1": "value",
"spout.specific.config.2": "true",
"spout.specific.config.3": "-12.4",
"spout.specific.config.4": default_serializer.serialize([1, 2, 3]),
"spout.overriden.config": "false"}
self.assertEqual(len(spout.comp.config.kvs), len(expecting_spout_config))
for conf in spout.comp.config.kvs:
value = expecting_spout_config[conf.key]
if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"):
self.assertEqual(value, conf.value)
elif conf.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
self.assertEqual(value, conf.serialized_value)
else:
self.fail()
# output stream
self.assertEqual(len(spout.outputs), 2)
for out_stream in spout.outputs:
if out_stream.stream.id == "default":
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 2)
else:
self.assertEqual(out_stream.stream.id, "error_stream")
self.assertEqual(out_stream.stream.component_name, "spout")
self.assertEqual(len(out_stream.schema.keys), 1)
### bolt protobuf ###
self.assertEqual(len(proto_topo.bolts), 1)
bolt = proto_topo.bolts[0]
self.assertEqual(bolt.comp.name, "bolt")
self.assertEqual(bolt.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME"))
self.assertEqual(bolt.comp.class_name, "bl_class")
expecting_bolt_config = {"topology.component.parallelism": "4"}
self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config))
conf = bolt.comp.config.kvs[0]
self.assertEqual(conf.type, topology_pb2.ConfigValueType.Value("STRING_VALUE"))
self.assertEqual(conf.value, expecting_bolt_config[conf.key])
# out stream
self.assertEqual(len(bolt.outputs), 0)
# in stream
self.assertEqual(len(bolt.inputs), 2)
for in_stream in bolt.inputs:
if in_stream.stream.id == "default":
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("SHUFFLE"))
else:
self.assertEqual(in_stream.stream.id, "error_stream")
self.assertEqual(in_stream.stream.component_name, "spout")
self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("ALL"))
self.assertEqual(proto_topo.state, topology_pb2.TopologyState.Value("RUNNING"))
