def _get_comp_config(self): """Returns component-specific Config protobuf message It first adds ``topology.component.parallelism``, and is overriden by a user-defined component-specific configuration, specified by spec(). """ proto_config = topology_pb2.Config() # first add parallelism key = proto_config.kvs.add() key.key = constants.TOPOLOGY_COMPONENT_PARALLELISM key.value = str(self.parallelism) key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE") # iterate through self.custom_config if self.custom_config is not None: sanitized = self._sanitize_config(self.custom_config) for key, value in sanitized.iteritems(): if isinstance(value, str): kvs = proto_config.kvs.add() kvs.key = key kvs.value = value kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE") else: # need to serialize kvs = proto_config.kvs.add() kvs.key = key kvs.serialized_value = default_serializer.serialize(value) kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE") return proto_config
def _get_comp_config(self): """Returns component-specific Config protobuf message It first adds ``topology.component.parallelism``, and is overriden by a user-defined component-specific configuration, specified by spec(). """ proto_config = topology_pb2.Config() # first add parallelism key = proto_config.kvs.add() key.key = constants.TOPOLOGY_COMPONENT_PARALLELISM key.value = str(self.parallelism) key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE") # iterate through self.custom_config if self.custom_config is not None: sanitized = self._sanitize_config(self.custom_config) for key, value in sanitized.iteritems(): if isinstance(value, str): kvs = proto_config.kvs.add() kvs.key = key kvs.value = value kvs.type = topology_pb2.ConfigValueType.Value( "STRING_VALUE") else: # need to serialize kvs = proto_config.kvs.add() kvs.key = key kvs.serialized_value = default_serializer.serialize(value) kvs.type = topology_pb2.ConfigValueType.Value( "PYTHON_SERIALIZED_VALUE") return proto_config
def custom(cls, classpath): """Custom grouping from a given classpath :param classpath: classpath to the ICustomGrouping class to use """ if classpath is None or not isinstance(classpath, str): raise TypeError("Argument to custom() must be classpath string to custom grouping, given: " "%s" % str(classpath)) serialized = default_serializer.serialize(classpath) return cls.custom_serialized(serialized, is_java=False)
def custom(cls, classpath): """Custom grouping from a given classpath to an implementation of ICustomGrouping This method does not exist in the Streamparse API. :param classpath: classpath to the ICustomGrouping class to use """ if classpath is None or not isinstance(classpath, str): raise TypeError("Argument to custom() must be classpath string to custom grouping, given: " "%s" % str(classpath)) serialized = default_serializer.serialize(classpath) return cls.custom_serialized(serialized, is_java=False)
def custom(cls, classpath): """Custom grouping from a given classpath to an implementation of ICustomGrouping This method does not exist in the Streamparse API. :param classpath: classpath to the ICustomGrouping class to use """ if classpath is None or not isinstance(classpath, str): raise TypeError( "Argument to custom() must be classpath string to custom grouping, given: " "%s" % str(classpath)) serialized = default_serializer.serialize(classpath) return cls.custom_serialized(serialized, is_java=False)
def get_topology_config_protobuf(mcs, class_dict): config = topology_pb2.Config() conf_dict = class_dict['_topo_config'] for key, value in conf_dict.iteritems(): if isinstance(value, str): kvs = config.kvs.add() kvs.key = key kvs.value = value kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE") else: # need to serialize kvs = config.kvs.add() kvs.key = key kvs.serialized_value = default_serializer.serialize(value) kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE") return config
def get_topology_config_protobuf(mcs, class_dict): config = topology_pb2.Config() conf_dict = class_dict['_topo_config'] for key, value in conf_dict.iteritems(): if isinstance(value, str): kvs = config.kvs.add() kvs.key = key kvs.value = value kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE") else: # need to serialize kvs = config.kvs.add() kvs.key = key kvs.serialized_value = default_serializer.serialize(value) kvs.type = topology_pb2.ConfigValueType.Value( "PYTHON_SERIALIZED_VALUE") return config
def test_sane_topology(self): self.assertEqual(TestSane.topology_name, "TestSane") # topology-wide config expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG expecting_topo_config.update({"topology.wide.config.1": "value", "spout.overriden.config": "true"}) self.assertEqual(TestSane._topo_config, expecting_topo_config) self.assertEqual(len(TestSane._protobuf_bolts), 1) self.assertEqual(len(TestSane._protobuf_spouts), 1) self.assertEqual(len(TestSane._heron_specs), 2) for spec in TestSane._heron_specs: if spec.is_spout: self.assertEqual(spec.name, "spout") self.assertEqual(spec.python_class_path, "sp_class") self.assertEqual(spec.parallelism, 3) else: self.assertEqual(spec.name, "bolt") self.assertEqual(spec.python_class_path, "bl_class") self.assertEqual(spec.parallelism, 4) self.assertTrue(isinstance(TestSane.protobuf_topology, topology_pb2.Topology)) proto_topo = TestSane.protobuf_topology ### spout protobuf ### self.assertEqual(len(proto_topo.spouts), 1) spout = proto_topo.spouts[0] self.assertEqual(spout.comp.name, "spout") self.assertEqual(spout.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")) self.assertEqual(spout.comp.class_name, "sp_class") expecting_spout_config = {"topology.component.parallelism": "3", "spout.specific.config.1": "value", "spout.specific.config.2": "true", "spout.specific.config.3": "-12.4", "spout.specific.config.4": default_serializer.serialize([1, 2, 3]), "spout.overriden.config": "false"} self.assertEqual(len(spout.comp.config.kvs), len(expecting_spout_config)) for conf in spout.comp.config.kvs: value = expecting_spout_config[conf.key] if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"): self.assertEqual(value, conf.value) elif conf.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"): self.assertEqual(value, conf.serialized_value) else: self.fail() # output stream self.assertEqual(len(spout.outputs), 2) for out_stream in spout.outputs: if out_stream.stream.id == "default": self.assertEqual(out_stream.stream.component_name, "spout") self.assertEqual(len(out_stream.schema.keys), 2) else: self.assertEqual(out_stream.stream.id, "error_stream") self.assertEqual(out_stream.stream.component_name, "spout") self.assertEqual(len(out_stream.schema.keys), 1) ### bolt protobuf ### self.assertEqual(len(proto_topo.bolts), 1) bolt = proto_topo.bolts[0] self.assertEqual(bolt.comp.name, "bolt") self.assertEqual(bolt.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")) self.assertEqual(bolt.comp.class_name, "bl_class") expecting_bolt_config = {"topology.component.parallelism": "4"} self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config)) conf = bolt.comp.config.kvs[0] self.assertEqual(conf.type, topology_pb2.ConfigValueType.Value("STRING_VALUE")) self.assertEqual(conf.value, expecting_bolt_config[conf.key]) # out stream self.assertEqual(len(bolt.outputs), 0) # in stream self.assertEqual(len(bolt.inputs), 2) for in_stream in bolt.inputs: if in_stream.stream.id == "default": self.assertEqual(in_stream.stream.component_name, "spout") self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("SHUFFLE")) else: self.assertEqual(in_stream.stream.id, "error_stream") self.assertEqual(in_stream.stream.component_name, "spout") self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("ALL")) self.assertEqual(proto_topo.state, topology_pb2.TopologyState.Value("RUNNING"))
def test_sane_topology(self): self.assertEqual(TestSane.topology_name, "TestSane") # topology-wide config expecting_topo_config = TopologyType.DEFAULT_TOPOLOGY_CONFIG expecting_topo_config.update({"topology.wide.config.1": "value", "spout.overriden.config": "true"}) self.assertEqual(TestSane._topo_config, expecting_topo_config) self.assertEqual(len(TestSane._protobuf_bolts), 1) self.assertEqual(len(TestSane._protobuf_spouts), 1) self.assertEqual(len(TestSane._heron_specs), 2) for spec in TestSane._heron_specs: if spec.is_spout: self.assertEqual(spec.name, "spout") self.assertEqual(spec.python_class_path, "sp_class") self.assertEqual(spec.parallelism, 3) else: self.assertEqual(spec.name, "bolt") self.assertEqual(spec.python_class_path, "bl_class") self.assertEqual(spec.parallelism, 4) self.assertTrue(isinstance(TestSane.protobuf_topology, topology_pb2.Topology)) proto_topo = TestSane.protobuf_topology ### spout protobuf ### self.assertEqual(len(proto_topo.spouts), 1) spout = proto_topo.spouts[0] self.assertEqual(spout.comp.name, "spout") self.assertEqual(spout.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")) self.assertEqual(spout.comp.class_name, "sp_class") expecting_spout_config = {"topology.component.parallelism": "3", "spout.specific.config.1": "value", "spout.specific.config.2": "true", "spout.specific.config.3": "-12.4", "spout.specific.config.4": default_serializer.serialize([1, 2, 3]), "spout.overriden.config": "false"} self.assertEqual(len(spout.comp.config.kvs), len(expecting_spout_config)) for conf in spout.comp.config.kvs: value = expecting_spout_config[conf.key] if conf.type == topology_pb2.ConfigValueType.Value("STRING_VALUE"): self.assertEqual(value, conf.value) elif conf.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"): self.assertEqual(value, conf.serialized_value) else: self.fail() # output stream self.assertEqual(len(spout.outputs), 2) for out_stream in spout.outputs: if out_stream.stream.id == "default": self.assertEqual(out_stream.stream.component_name, "spout") self.assertEqual(len(out_stream.schema.keys), 2) else: self.assertEqual(out_stream.stream.id, "error_stream") self.assertEqual(out_stream.stream.component_name, "spout") self.assertEqual(len(out_stream.schema.keys), 1) ### bolt protobuf ### self.assertEqual(len(proto_topo.bolts), 1) bolt = proto_topo.bolts[0] self.assertEqual(bolt.comp.name, "bolt") self.assertEqual(bolt.comp.spec, topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")) self.assertEqual(bolt.comp.class_name, "bl_class") expecting_bolt_config = {"topology.component.parallelism": "4"} self.assertEqual(len(bolt.comp.config.kvs), len(expecting_bolt_config)) conf = bolt.comp.config.kvs[0] self.assertEqual(conf.type, topology_pb2.ConfigValueType.Value("STRING_VALUE")) self.assertEqual(conf.value, expecting_bolt_config[conf.key]) # out stream self.assertEqual(len(bolt.outputs), 0) # in stream self.assertEqual(len(bolt.inputs), 2) for in_stream in bolt.inputs: if in_stream.stream.id == "default": self.assertEqual(in_stream.stream.component_name, "spout") self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("SHUFFLE")) else: self.assertEqual(in_stream.stream.id, "error_stream") self.assertEqual(in_stream.stream.component_name, "spout") self.assertEqual(in_stream.gtype, topology_pb2.Grouping.Value("ALL")) self.assertEqual(proto_topo.state, topology_pb2.TopologyState.Value("RUNNING"))