def huawei_processing(grpcPeer, new_msg): PMGRPCDLOG.debug("Huawei: Received GRPC-Data") # dump the raw data if lib_pmgrpcd.OPTIONS.rawdatafile: PMGRPCDLOG.debug("Write rawdatafile: %s" % (lib_pmgrpcd.OPTIONS.rawdatafile)) with open(lib_pmgrpcd.OPTIONS.rawdatafile, "a") as rawdatafile: rawdatafile.write(base64.b64encode(new_msg.data).decode()) rawdatafile.write("\n") try: telemetry_msg = huawei_telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) except Exception as e: PMGRPCDLOG.error( "instancing or parsing data failed with huawei_telemetry_pb2.Telemetry" ) PMGRPCDLOG.error("ERROR: %s" % (e)) raise try: telemetry_msg_dict = MessageToDict( telemetry_msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True, ) except Exception as e: PMGRPCDLOG.error( "instancing or parsing data failed with huawei_telemetry_pb2.Telemetry" ) raise PMGRPCDLOG.debug("Huawei: Received GPB-Data as JSON") # TODO: Do we really need this? it can be expensive PMGRPCDLOG.debug(json.dumps(telemetry_msg_dict, indent=2, sort_keys=True)) message_header_dict = telemetry_msg_dict.copy() if "data_gpb" in message_header_dict: del message_header_dict["data_gpb"] (proto, path) = message_header_dict["sensor_path"].split(":") (node_id_str) = message_header_dict["node_id_str"] (node_ip) = grpcPeer["telemetry_node"] (ne_vendor) = grpcPeer["ne_vendor"] # Get the maching L3-Methode msg = select_gbp_methode(proto) if msg: elem = len(telemetry_msg.data_gpb.row) epochmillis = int(round(time.time() * 1000)) PMGRPCDLOG.info( "EPOCH=%-10s NIP=%-15s NID=%-20s VEN=%-7s PT=%-22s ET=%-12s ELEM:%s" % (epochmillis, node_ip, node_id_str, ne_vendor, proto, "GPB", elem)) # L2: for new_row in telemetry_msg.data_gpb.row: # PMGRPCDLOG.info("NEW_ROW: %s" % (new_row)) new_row_header_dict = MessageToDict( new_row, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True, ) if "content" in new_row_header_dict: del new_row_header_dict["content"] # L3: msg.ParseFromString(new_row.content) content = MessageToDict( msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True, ) message_dict = {} message_dict.update({ "collector": { "grpc": { "grpcPeer": grpcPeer["telemetry_node"], "ne_vendor": grpcPeer["ne_vendor"], } } }) message_dict["collector"].update( {"data": message_header_dict.copy()}) message_dict["collector"]["data"].update(new_row_header_dict) message_dict.update(content) allkeys = parse_dict(content, ret="", level=0) PMGRPCDLOG.debug("Huawei: %s: %s" % (proto, allkeys)) try: returned = FinalizeTelemetryData(message_dict) except Exception as e: PMGRPCDLOG.error("Error finalazing message: %s", e)
def dataPublish(self, message, context): global options grpcPeer = {} grpcPeerStr = context.peer() (grpcPeer['telemetry_proto'], grpcPeer['telemetry_node'], grpcPeer['telemetry_node_port']) = grpcPeerStr.split(":") grpcPeer['vendor'] = 'Huawei' if options.debug: metadata = dict(context.invocation_metadata()) grpcPeer['user-agent'] = metadata['user-agent'] #Example of grpcPeerStr -> 'ipv4:10.215.133.23:57775' grpcPeer['grpc_processing'] = 'huawei_grpc_dialout_pb2_grpc' grpcPeer['grpc_ulayer'] = 'GPB Telemetry' jsonTelemetryNode = json.dumps(grpcPeer, indent=2, sort_keys=True) logging.debug(jsonTelemetryNode) for new_msg in message: if options.verbose: logging.info('Huawei: Received GRPC-Data') elif options.debug: logging.debug('Huawei: Received GRPC-Data') logging.debug(new_msg.data) telemetry_msg = huawei_telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) telemetry_msg_dict = MessageToDict( telemetry_msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) if options.debug: logging.debug("Huawei: Received GPB-Data as JSON") logging.debug( json.dumps(telemetry_msg_dict, indent=2, sort_keys=True)) message_header_dict = telemetry_msg_dict.copy() if 'data_gpb' in message_header_dict: del message_header_dict['data_gpb'] (proto, path) = message_header_dict['sensor_path'].split(":") if options.debug: logging.debug("PROTOTYP=%s" % proto) for new_row in telemetry_msg.data_gpb.row: new_row_header_dict = MessageToDict( new_row, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) if 'content' in new_row_header_dict: del new_row_header_dict['content'] msg = select_gbp_methode(proto) msg.ParseFromString(new_row.content) content = MessageToDict(msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) message_dict = {} message_dict.update( {'grpc': { 'grpcPeer': grpcPeer['telemetry_node'] }}) message_dict.update({'data': {'content': content}}) message_dict['data'].update(message_header_dict) message_dict['data'].update(new_row_header_dict) if options.verbose: allkeys = parse_dict(content, ret='', level=0) logging.info("Huawei: %s: %s" % (proto, allkeys)) sendJsonTelemetryData(message_dict, grpcPeer['vendor'], message_header_dict['sensor_path'])
def huawei_processing(grpcPeer, new_msg): pmgrpcdlog.debug('Huawei: Received GRPC-Data') try: telemetry_msg = huawei_telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) except Exception as e: pmgrpcdlog.error( 'instancing or parsing data failed with huawei_telemetry_pb2.Telemetry' ) pmgrpcdlog.error("ERROR: %s" % (e)) try: telemetry_msg_dict = MessageToDict(telemetry_msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) except Exception as e: pmgrpcdlog.error( 'instancing or parsing data failed with huawei_telemetry_pb2.Telemetry' ) pmgrpcdlog.debug("Huawei: Received GPB-Data as JSON") pmgrpcdlog.debug(json.dumps(telemetry_msg_dict, indent=2, sort_keys=True)) message_header_dict = telemetry_msg_dict.copy() if 'data_gpb' in message_header_dict: del message_header_dict['data_gpb'] (proto, path) = message_header_dict['sensor_path'].split(":") (node_id_str) = message_header_dict['node_id_str'] (node_ip) = grpcPeer['telemetry_node'] (ne_vendor) = grpcPeer['ne_vendor'] #Get the maching L3-Methode msg = select_gbp_methode(proto) if msg: elem = len(telemetry_msg.data_gpb.row) epochmillis = int(round(time.time() * 1000)) pmgrpcdlog.info( "EPOCH=%-10s NIP=%-15s NID=%-20s VEN=%-7s PT=%-22s ET=%-12s ELEM:%s" % (epochmillis, node_ip, node_id_str, ne_vendor, proto, 'GPB', elem)) #L2: for new_row in telemetry_msg.data_gpb.row: #pmgrpcdlog.info("NEW_ROW: %s" % (new_row)) new_row_header_dict = MessageToDict( new_row, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) if 'content' in new_row_header_dict: del new_row_header_dict['content'] #L3: msg.ParseFromString(new_row.content) content = MessageToDict(msg, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) message_dict = {} message_dict.update({ 'collector': { 'grpc': { 'grpcPeer': grpcPeer['telemetry_node'], 'ne_vendor': grpcPeer['ne_vendor'] } } }) message_dict['collector'].update( {'data': message_header_dict.copy()}) message_dict['collector']['data'].update(new_row_header_dict) message_dict.update(content) if options.rawdatafile: pmgrpcdlog.debug("Write rawdatafile: %s" % (options.rawdatafile)) rawjsonTelemetryData = json.dumps(message_dict, indent=2, sort_keys=True) with open(options.rawdatafile, 'a') as rawdatafile: rawdatafile.write(rawjsonTelemetryData) rawdatafile.write("\n") allkeys = parse_dict(content, ret='', level=0) pmgrpcdlog.debug("Huawei: %s: %s" % (proto, allkeys)) FinalizeTelemetryData(message_dict)