def _setup_custom_grouping(self, topology): """Checks whether there are any bolts that consume any of my streams using custom grouping""" for i in range(len(topology.bolts)): for in_stream in topology.bolts[i].inputs: if in_stream.stream.component_name == self.my_component_name and \ in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"): # this bolt takes my output in custom grouping manner if in_stream.type == topology_pb2.CustomGroupingObjectType.Value( "PYTHON_OBJECT"): custom_grouping_obj = default_serializer.deserialize( in_stream.custom_grouping_object) if isinstance(custom_grouping_obj, str): pex_loader.load_pex(self.topology_pex_abs_path) grouping_cls = \ pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj) custom_grouping_obj = grouping_cls() assert isinstance(custom_grouping_obj, ICustomGrouping) self.custom_grouper.add( in_stream.stream.id, self._get_taskids_for_component( topology.bolts[i].comp.name), custom_grouping_obj, self.my_component_name) elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value( "JAVA_OBJECT"): raise NotImplementedError( "Java-serialized custom grouping is not yet supported " "for python topology") else: raise ValueError( "Unrecognized custom grouping type found: %s" % str(in_stream.type))
def _get_dict_from_config(topology_config): """Converts Config protobuf message to python dictionary Values are converted according to the rules below: - Number string (e.g. "12" or "1.2") is appropriately converted to ``int`` or ``float`` - Boolean string ("true", "True", "false" or "False") is converted to built-in boolean type (i.e. ``True`` or ``False``) - Normal string is inserted to dict as is - Serialized value is deserialized and inserted as a corresponding Python object """ config = {} for kv in topology_config.kvs: if kv.HasField("value"): assert kv.type == topology_pb2.ConfigValueType.Value("STRING_VALUE") # value is string if PhysicalPlanHelper._is_number(kv.value): config[kv.key] = PhysicalPlanHelper._get_number(kv.value) elif kv.value.lower() in ("true", "false"): config[kv.key] = True if kv.value.lower() == "true" else False else: config[kv.key] = kv.value elif kv.HasField("serialized_value") and \ kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"): # deserialize that config[kv.key] = default_serializer.deserialize(kv.serialized_value) else: assert kv.HasField("type") Log.error("Unsupported config <key:value> found: %s, with type: %s" % (str(kv), str(kv.type))) continue return config
def _setup_custom_grouping(self, topology): """Checks whether there are any bolts that consume any of my streams using custom grouping""" for i in range(len(topology.bolts)): for in_stream in topology.bolts[i].inputs: if in_stream.stream.component_name == self.my_component_name and \ in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"): # this bolt takes my output in custom grouping manner if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT"): custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object) if isinstance(custom_grouping_obj, str): pex_loader.load_pex(self.topology_pex_abs_path) grouping_cls = \ pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj) custom_grouping_obj = grouping_cls() assert isinstance(custom_grouping_obj, ICustomGrouping) self.custom_grouper.add(in_stream.stream.id, self._get_taskids_for_component(topology.bolts[i].comp.name), custom_grouping_obj, self.my_component_name) elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("JAVA_OBJECT"): raise NotImplementedError("Java-serialized custom grouping is not yet supported " "for python topology") else: raise ValueError("Unrecognized custom grouping type found: %s" % str(in_stream.type))
def _get_dict_from_config(topology_config): """Converts Config protobuf message to python dictionary Values are converted according to the rules below: - Number string (e.g. "12" or "1.2") is appropriately converted to ``int`` or ``float`` - Boolean string ("true", "True", "false" or "False") is converted to built-in boolean type (i.e. ``True`` or ``False``) - Normal string is inserted to dict as is - Serialized value is deserialized and inserted as a corresponding Python object """ config = {} for kv in topology_config.kvs: if kv.HasField("value"): assert kv.type == topology_pb2.ConfigValueType.Value("STRING_VALUE") # value is string if PhysicalPlanHelper._is_number(kv.value): config[kv.key] = PhysicalPlanHelper._get_number(kv.value) elif kv.value.lower() in ("true", "false"): config[kv.key] = True if kv.value.lower() == "true" else False else: config[kv.key] = kv.value elif kv.HasField("serialized_value") and \ kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"): # deserialize that config[kv.key] = default_serializer.deserialize(kv.serialized_value) else: assert kv.HasField("type") Log.error("Unsupported config <key:value> found: %s, with type: %s" % (str(kv), str(kv.type))) continue return config