def main(): for line in sys.stdin: deviceJson = json.loads(line.strip()) #print(deviceJson) if "telemetry_data" not in deviceJson: print("No telemetry_data") print(deviceJson) continue telemetry_data = deviceJson["telemetry_data"] try: s = base64.b64decode(telemetry_data) #print(s) except Exception as e: print("Failed b64 decoding:", e) continue try: d = telemetry_pb2.Telemetry() # d = telemetry_top_pb2.TelemetryStream() <----- JunOS Native Telemetry d.ParseFromString(s) except Exception as e: print("Failed GPB parsing:", len(telemetry_data), e) try: jsonStrTelemetry = MessageToJson(d) print(jsonStrTelemetry) except Exception as e: print("Failed Conversion to JSON:", len(telemetry_data), e)
def listenToClient(self, client, address): while True: try: data = client.recv(12) if data: print 'getting some data' # Set the response to echo back the recieved data header = data msg_type, encode_type, msg_version, flags, msg_length = struct.unpack( '>hhhhi', header) msg_data = b'' print msg_type, encode_type, msg_version, flags, msg_length if encode_type == 1: while len(msg_data) < msg_length: msg_data += client.recv(msg_length - len(msg_data)) gpb_parser = telemetry_pb2.Telemetry() gpb_data = gpb_parser.ParseFromString(msg_data) if gpb_parser.encoding_path == 'Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters': row_key = ifstatsbag_generic_pb2.ifstatsbag_generic_KEYS( ) row_data = ifstatsbag_generic_pb2.ifstatsbag_generic( ) for new_row in gpb_parser.data_gpb.row: row_data.ParseFromString(new_row.content) row_key.ParseFromString(new_row.keys) if kafka_msg: kafka_msg_parse = create_kafka_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data) producer.send_messages( b'ios_xr_interface_counters', kafka_msg_parse) logging.info( 'Write {} to kafka topic'.format( gpb_parser.encoding_path)) elif influx_msg: influx_msg_parse = create_influx_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data) client_influxdb.write_points( [influx_msg_parse], time_precision='s') logging.info('Write {} to InfluxDB'.format( gpb_parser.encoding_path)) else: print('Row_key:{}\n,Row_data:{}').format( row_key, row_data) else: raise error('Client disconnected') except Exception as e: print 'Error:{}'.format(e) client.close() return False
def MdtDialout(self, message, context): grpcPeerStr = context.peer() grpcPeer = {} (grpcPeerProto, grpcPeer['telemetry_node'], grpcPeer['telemetry_node_port']) = grpcPeerStr.split(":") jsonTelemetryNode = json.dumps(grpcPeer) print(jsonTelemetryNode) for new_msg in message: telemetry_msg = telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) #print(telemetry_msg) #print(type(telemetry_msg)) #print(telemetry_msg.data_gpb.row[0].content) jsonStrTelemetry = MessageToJson(telemetry_msg) dictTelemetry = json.loads(jsonStrTelemetry) #print telemetry json message print(jsonStrTelemetry) print("Message Length {}".format(len(jsonStrTelemetry))) print("=" * 40) print(dictTelemetry["encodingPath"]) if "dataGpb" in dictTelemetry: print("Message in GPB compact mode") if "dataGpbkv" in dictTelemetry: print("message in GPB-kv mode") # according to encoding path and dataGpb OR dataGpbkv to select which gpb-compact pb2 to be used if dictTelemetry[ "encodingPath"] == "Cisco-IOS-XR-shellutil-oper:system-time/uptime" and "dataGpb" in dictTelemetry: gpb_compact_content = telemetry_msg.data_gpb.row[ 0].content # should be use list method tohandle it #TBD Telemetry_row_content = uptime_pb2.system_uptime() Telemetry_row_content.ParseFromString(gpb_compact_content) print(Telemetry_row_content) print("=" * 40) #json_dict = proto_to_dict(Telemetry_row_content) #print(json_dict) return cisco_grpc_dialout_pb2.MdtDialoutArgs( ) # no return should be ok , if get telemetry stream only
def MdtDialout(self, message, context): grpcPeerStr = context.peer() grpcPeer = {} (grpcPeerProto, grpcPeer['telemetry_node'], grpcPeer['telemetry_node_port']) = grpcPeerStr.split(":") jsonTelemetryNode = json.dumps(grpcPeer) print(jsonTelemetryNode) for new_msg in message: telemetry_msg = telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) jsonStrTelemetry = MessageToJson(telemetry_msg) #dictTelemetry = json.loads(jsonStrTelemetry) #print telemetry json message print(jsonStrTelemetry) return cisco_grpc_dialout_pb2.MdtDialoutArgs()
def getsubscription(self, sub_id, unmarshal=True): """Telemetry subscription function :param sub_id: Subscription ID :type: string :return: Returns discrete values emitted by telemetry stream :rtype: JSON formatted string """ sub_args = mdt_grpc_dialin_pb2.CreateSubsArgs(ReqId=1, encode=3, subidstr=sub_id) stream = self._stub.CreateSubs(sub_args, timeout=self._timeout, metadata=self._metadata) for segment in stream: if not unmarshal: yield segment else: # Go straight for telemetry data telemetry_pb = telemetry_pb2.Telemetry() telemetry_pb.ParseFromString(segment.data) # Return in JSON format instead of protobuf. yield MessageToJson(telemetry_pb)
def MdtDialout(self, message, context): grpcPeerStr = context.peer() grpcPeer = {} (grpcPeerProto, grpcPeer['telemetry_node'], grpcPeer['telemetry_node_port']) = grpcPeerStr.split(":") jsonTelemetryNode = json.dumps(grpcPeer) print(jsonTelemetryNode) for new_msg in message: telemetry_msg = telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) #print("RAW message") print(telemetry_msg) print("="*100) #print(type(telemetry_msg)) #print(telemetry_msg.data_gpb.row[0].content) jsonStrTelemetry = MessageToJson(telemetry_msg) dictTelemetry = json.loads(jsonStrTelemetry) #print telemetry json message print(jsonStrTelemetry) print("Message Length {}".format(len(jsonStrTelemetry))) print("="*40) print(dictTelemetry["encodingPath"]) if "dataGpb" in dictTelemetry: print("Message in GPB compact mode") if "dataGpbkv" in dictTelemetry: print("message in GPB-kv mode") # according to encoding path and dataGpb OR dataGpbkv to select which gpb-compact pb2 to be used if dictTelemetry["encodingPath"] == "Cisco-IOS-XR-shellutil-oper:system-time/uptime" and "dataGpb" in dictTelemetry: gpb_compact_content = telemetry_msg.data_gpb.row[0].content # should be use list method tohandle it #TBD Telemetry_row_content = uptime_pb2.system_uptime() Telemetry_row_content.ParseFromString(gpb_compact_content) print(Telemetry_row_content) print("="*40) #if dictTelemetry["encodingPath"] == "analytics:test_query" and "dataGpb" in dictTelemetry: if dictTelemetry["encodingPath"] == "analytics:dcnminitITL" and "dataGpb" in dictTelemetry: ''' MDS 97 32G line card SAN Analytics feature, encoding path should be predefined push analytics query name. ''' gpb_compact_content = telemetry_msg.data_gpb.row[0].content # should be use list method tohandle it #gpb_compact_content = telemetry_msg.data_gpb.row[0] Telemetry_row_content = fabric_telemetry_pb2.FlowRecordsTable() #Telemetry_row_content = fabric_telemetry_pb2.FlowRecordRow() Telemetry_row_content.ParseFromString(gpb_compact_content) fabric_jsonStrTelemetry = MessageToJson(Telemetry_row_content) print(fabric_jsonStrTelemetry) print("=" * 40) if dictTelemetry["encodingPath"] == "show_stats_fc2/2" and "dataGpb" in dictTelemetry: ''' MDS 97 32G line card SAN Analytics feature, encoding path should be predefined push analytics query name. ''' gpb_compact_content = telemetry_msg.data_gpb.row[0].content # should be use list method tohandle it # gpb_compact_content = telemetry_msg.data_gpb.row[0] Telemetry_row_content = fabric_telemetry_pb2.FlowRecordsTable() # Telemetry_row_content = fabric_telemetry_pb2.FlowRecordRow() Telemetry_row_content.ParseFromString(gpb_compact_content) fabric_jsonStrTelemetry = MessageToJson(Telemetry_row_content) print(fabric_jsonStrTelemetry) print("=" * 40) #json_dict = proto_to_dict(Telemetry_row_content) #print(json_dict) return cisco_grpc_dialout_pb2.MdtDialoutArgs() # no return should be ok , if get telemetry stream only
host = '10.75.58.60' port = 57400 options = 'ems.cisco.com' ca_cert = 'ems.pem' # credential file scp from devices creds = open(ca_cert).read() target = '%s:%d' % (host, port) creds = implementations.ssl_channel_credentials(creds.encode( ('utf-8'))) # args with byte type channel = grpc.secure_channel(target, creds, (( 'grpc.ssl_target_name_override', options, ), )) channel = implementations.Channel(channel) stub = ems_grpc_pb2.beta_create_gRPCConfigOper_stub(channel) sub_id = 'test_sub' # Telemetry MDT subscribtion sub_args = ems_grpc_pb2.CreateSubsArgs(ReqId=1, encode=3, subidstr=sub_id) timeout = float(100000) metadata = [('username', 'cisco'), ('password', 'cisco')] stream = stub.CreateSubs(sub_args, timeout=timeout, metadata=metadata) for segment in stream: telemetry_pb = telemetry_pb2.Telemetry() t = telemetry_pb.ParseFromString(segment.data) # Print Json Message print(MessageToJson(telemetry_pb))
# Bind Socket UDP port 57500 as Telemetry recevice server sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(('0.0.0.0', 57501)) count = 0 start_time = time.time() while True: count += 1 buf, addr = sock.recvfrom(65535) Telemetry_content = telemetry_pb2.Telemetry() print(buf.hex()) print(buf) print("Message Length {}".format(len(buf))) #print(len(str(buf))) #handle Telemetry UDP GPB kv from NX OS if buf[0:1] == b'\x01': #check the binary daa , no official document print("Telemetry GPB message from NX OS") Telemetry_content.ParseFromString(buf[6:]) print('Node :'+Telemetry_content.node_id_str) print('IP Address (source port) :'+str(addr)) print('Encodig Path :'+Telemetry_content.encoding_path) Top_Fields_List = Telemetry_content.data_gpbkv[0].fields #print(Top_Fields_List)
def MdtDialout(self, message, context): grpcPeerStr = context.peer() grpcPeer = {} (grpcPeerProto, grpcPeer['telemetry_node'], grpcPeer['telemetry_node_port']) = grpcPeerStr.split(":") jsonTelemetryNode = json.dumps(grpcPeer) print(jsonTelemetryNode) for new_msg in message: telemetry_msg = telemetry_pb2.Telemetry() telemetry_msg.ParseFromString(new_msg.data) #print("RAW message") #print(telemetry_msg) print("=" * 100) #print(type(telemetry_msg)) #print(telemetry_msg.data_gpb.row[0].content) jsonStrTelemetry = MessageToJson(telemetry_msg) dictTelemetry = json.loads(jsonStrTelemetry) #print telemetry json message print(jsonStrTelemetry) print("Message Length {}".format(len(jsonStrTelemetry))) print("=" * 40) print(dictTelemetry["encodingPath"]) if "dataGpb" in dictTelemetry: print("Message in GPB compact mode") if "dataGpbkv" in dictTelemetry: print("message in GPB-kv mode") # according to encoding path and dataGpb OR dataGpbkv to select which gpb-compact pb2 to be used if dictTelemetry[ "encodingPath"] == "Cisco-IOS-XR-shellutil-oper:system-time/uptime" and "dataGpb" in dictTelemetry: gpb_compact_content = telemetry_msg.data_gpb.row[ 0].content # should be use list method tohandle it #TBD Telemetry_row_content = uptime_pb2.system_uptime() Telemetry_row_content.ParseFromString(gpb_compact_content) print(Telemetry_row_content) print("=" * 40) if dictTelemetry[ "encodingPath"] == "analytics:test_query" and "dataGpb" in dictTelemetry: ''' MDS 97 32G line card SAN Analytics feature, encoding path should be predefined push analytics query name. So far , NX OS 8.4.1 only support gRPC GPB/GPB-kv for GPB-kv encoding with fabrc_telemetry.proto file. MDS 9710 sample configuration: telemetry sensor-group 1 path analytics:test_query path show_stats_fc2/1 path show_stats_fc2/2 sensor-group 2 path analytics:dcnminitITL destination-group 1 ip address 10.79.98.77 port 50051 protocol gRPC encoding GPB-compact destination-group 2 ip address 10.124.2.116 port 57500 protocol gRPC encoding GPB-compact subscription 1 snsr-grp 1 sample-interval 30000 dst-grp 1 subscription 2 snsr-grp 2 sample-interval 30000 dst-grp 2 sw-core1-9710# sh analytics query all Total queries:2 ============================ Query Name :test_query Query String :select all from fc-scsi.port Query Type :periodic, interval 30 Query Name :dcnminitITL Query String :select port, vsan, app_id, initiator_id, target_id, lun, active_io_read_count, active_io_write_count, total_read_io_count, total_write _io_count, total_time_metric_based_read_io_count, total_time_metric_based_write_io_count,total_read_io_time, total_write_io_time, total_read_io_initiat ion_time, total_write_io_initiation_time,total_read_io_bytes, total_write_io_bytes, total_time_metric_based_read_io_bytes, total_time_metric_based_writ e_io_bytes, read_io_rate, write_io_rate, read_io_bandwidth, write_io_bandwidth,read_io_size_min, read_io_size_max, write_io_size_min, write_io_size_max ,read_io_completion_time_min, read_io_completion_time_max, write_io_completion_time_min, write_io_completion_time_max,read_io_initiation_time_max, writ e_io_initiation_time_max, read_io_aborts, write_io_aborts,read_io_failures, write_io_failures, read_io_timeouts, write_io_timeouts from fc-scsi.scsi_in itiator_itl_flow Query Type :periodic, interval 30 Query Options :differential ''' gpb_compact_content = telemetry_msg.data_gpb.row[ 0].content # should be use list method tohandle it #gpb_compact_content = telemetry_msg.data_gpb.row[0] Telemetry_row_content = fabric_telemetry_pb2.FlowRecordsTable() #Telemetry_row_content = fabric_telemetry_pb2.FlowRecordRow() Telemetry_row_content.ParseFromString(gpb_compact_content) fabric_jsonStrTelemetry = MessageToJson(Telemetry_row_content) print(fabric_jsonStrTelemetry) print("=" * 40) #json_dict = proto_to_dict(Telemetry_row_content) #print(json_dict) return cisco_grpc_dialout_pb2.MdtDialoutArgs( ) # no return should be ok , if get telemetry stream only
def listenToClient(self, client, address): while True: try: data = client.recv(12) if data: header = data msg_type, encode_type, msg_version, flags, msg_length = struct.unpack( '>hhhhi', header) msg_data = b'' if encode_type == 1: while len(msg_data) < msg_length: msg_data += client.recv(msg_length - len(msg_data)) gpb_parser = telemetry_pb2.Telemetry() gpb_data = gpb_parser.ParseFromString(msg_data) if gpb_parser.encoding_path == 'Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters': row_key = ifstatsbag_generic_pb2.ifstatsbag_generic_KEYS( ) row_data = ifstatsbag_generic_pb2.ifstatsbag_generic( ) for new_row in gpb_parser.data_gpb.row: row_data.ParseFromString(new_row.content) row_key.ParseFromString(new_row.keys) if kafka_msg: kafka_msg_parse = create_kafka_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data) producer.send_messages( b'ios_xr_interface_counters', kafka_msg_parse) logging.info('Write %s to kafka topic' % gpb_parser.encoding_path) elif influx_msg: influx_msg_parse = create_influx_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data, new_row.timestamp) client_influxdb.write_points( [influx_msg_parse], time_precision='ms') logging.info( 'Write interface counters from %s to InfluxDB' % address[0]) else: print('Row_key:{}\n,Row_data:{}').format( row_key, row_data) elif gpb_parser.encoding_path == 'Cisco-IOS-XR-pfi-im-cmd-oper:interfaces/interface-xr/interface': row_key = ifstatsbag_generic_pb2.ifstatsbag_generic_KEYS( ) row_data = im_cmd_info_pb2.im_cmd_info() for new_row in gpb_parser.data_gpb.row: row_data.ParseFromString(new_row.content) row_key.ParseFromString(new_row.keys) if kafka_msg: kafka_msg_parse = create_kafka_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data) producer.send_messages( b'ios_xr_interface_info', kafka_msg_parse) logging.info( 'Write {} to kafka topic'.format( gpb_parser.encoding_path)) elif influx_msg: influx_msg_parse = create_influx_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data, new_row.timestamp) client_influxdb.write_points( [influx_msg_parse], time_precision='ms') logging.info( 'Write interface cmd oper from %s to InfluxDB' % address[0]) else: print('Row_key:{}\n,Row_data:{}').format( row_key, row_data) elif gpb_parser.encoding_path == 'Cisco-IOS-XR-snmp-agent-oper:snmp/if-indexes/if-index': row_data = snmp_agen_oper_if_index_pb2.snmp_ifindex_ifname( ) row_key = snmp_agen_oper_if_index_pb2.snmp_ifindex_ifname_KEYS( ) for new_row in gpb_parser.data_gpb.row: row_data.ParseFromString(new_row.content) row_key.ParseFromString(new_row.keys) if kafka_msg: kafka_msg_parse = create_kafka_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data, new_row.timestamp) producer.send_messages( b'ios_xr_interface_snmp', kafka_msg_parse) logging.info( 'Write {} to kafka topic'.format( gpb_parser.encoding_path)) elif influx_msg: influx_msg_parse = create_influx_message( gpb_parser.encoding_path, gpb_parser.node_id_str, row_key, row_data, new_row.timestamp) client_influxdb.write_points( [influx_msg_parse], time_precision='ms') logging.info( 'Write interface snmp from %s to InfluxDB' % address[0]) else: print('Row_key:{}\n,Row_data:{}').format( row_key, row_data) client.send("ack") else: raise error('Client disconnected') except Exception as e: print 'Error:{}'.format(e) client.close() return False
def run(self): server_ip = "192.168.123.12" server_port = "57777" xr_user = "******" xr_passwd = "root" print("Using GRPC Server IP(%s) Port(%s)" % (server_ip, server_port)) channel = grpc.insecure_channel( str(server_ip) + ":" + str(server_port)) stub = mdt_grpc_dialin_pb2_grpc.gRPCConfigOperStub(channel) metadata = [('username', xr_user), ('password', xr_passwd)] Timeout = 3600 * 24 * 365 sub_args = mdt_grpc_dialin_pb2.CreateSubsArgs(ReqId=99, encode=2, subidstr='IPV6') stream = stub.CreateSubs(sub_args, timeout=Timeout, metadata=metadata) print("##########1") for segment in stream: telemetry_pb = telemetry_pb2.Telemetry() telemetry_pb.ParseFromString(segment.data) telemetry_gpb_table = telemetry_pb2.TelemetryGPBTable() telemetry_gpb_table.CopyFrom(telemetry_pb.data_gpb) gpb_rows = [] print("##########2") while (len(telemetry_gpb_table.row)): gpb_row_dict = {} gpb_row_dict["keys"] = {} gpb_row_dict["content"] = {} telemetry_gpb_row = telemetry_pb2.TelemetryRowGPB() telemetry_gpb_row.CopyFrom(telemetry_gpb_table.row.pop()) gpb_row_dict["timestamp"] = telemetry_gpb_row.timestamp ipv6_nd_neighbor_entry_keys = ipv6_nd_neighbor_entry_KEYS() ipv6_nd_neighbor_entry_keys.ParseFromString( telemetry_gpb_row.keys) ipv6_nd_neighbor_entry_content = ipv6_nd_neighbor_entry() ipv6_nd_neighbor_entry_content.ParseFromString( telemetry_gpb_row.content) print("1: {}".format(ipv6_nd_neighbor_entry_content)) print("2: {}".format(ipv6_nd_neighbor_entry_keys)) content_dump = MessageToJson(ipv6_nd_neighbor_entry_content) keys_dump = MessageToJson(ipv6_nd_neighbor_entry_keys) print("3: {}".format(content_dump)) print("4: {}".format(keys_dump)) gpb_row_dict["content"].update(yaml.safe_load(content_dump)) gpb_row_dict["keys"].update(yaml.safe_load(keys_dump)) gpb_rows.append(gpb_row_dict) self.sensor_service.dispatch( trigger='telemetry_collector.event2', payload=gpb_rows)