def influxdb_to_json(self, sol_influxdb): """ Converts an Influxdb query reply into a list of dicts. :param sol_influxdb dict: the result of a database query (sush as SELECT * FROM) :return: a list of JSON SOL objects :rtype: list """ # verify influxdb data if not ("series" in sol_influxdb): raise ValueError("Influxdb data not recognized") # init json_list = [] # remove unused headers for serie in sol_influxdb["series"]: for val in serie['values']: # convert to dict d_influxdb = dict(zip(serie['columns'], val)) # unflat dict obj_value = flatdict.FlatDict(d_influxdb).as_dict() # parse specific HR_NEIGHBORS hr_nghb_name = SolDefines.solTypeToTypeName( SolDefines, SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS) if serie['name'] == hr_nghb_name: for i in range(0, len(obj_value["neighbors"]) + 1): ngbr_id = str(i) # new HR_NGBR parsing if ngbr_id in obj_value["neighbors"]: if obj_value["neighbors"][ngbr_id][ "neighborFlag"] is None: del obj_value["neighbors"][ngbr_id] # old HR_NGBR parsing if ngbr_id in obj_value: if obj_value[ngbr_id]["neighborFlag"] is not None: obj_value["neighbors"][ngbr_id] = obj_value[ ngbr_id] del obj_value[ngbr_id] # time is not passed in the "value" field del obj_value["time"] # create final dict jdic = { 'type': serie['name'], 'mac': serie['tags']['mac'], 'value': obj_value, 'timestamp': d_influxdb['time'], } json_list.append(jdic) return json_list
def influxdb_to_json(self, sol_influxdb): """ Converts an Influxdb query reply into a list of dicts. :param sol_influxdb dict: the result of a database query (sush as SELECT * FROM) :return: a list of JSON SOL objects :rtype: list """ # verify influxdb data if not ("series" in sol_influxdb): raise ValueError("Influxdb data not recognized") # init json_list = [] # remove unused headers for serie in sol_influxdb["series"]: for val in serie['values']: # convert to dict d_influxdb = dict(zip(serie['columns'], val)) # unflat dict obj_value = flatdict.FlatDict(d_influxdb).as_dict() # parse specific HR_NEIGHBORS hr_nghb_name = SolDefines.solTypeToTypeName( SolDefines, SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS) if serie['name'] == hr_nghb_name: for i in range(0,len(obj_value["neighbors"])+1): ngbr_id = str(i) # new HR_NGBR parsing if ngbr_id in obj_value["neighbors"]: if obj_value["neighbors"][ngbr_id]["neighborFlag"] is None: del obj_value["neighbors"][ngbr_id] # old HR_NGBR parsing if ngbr_id in obj_value: if obj_value[ngbr_id]["neighborFlag"] is not None: obj_value["neighbors"][ngbr_id] = obj_value[ngbr_id] del obj_value[ngbr_id] # time is not passed in the "value" field del obj_value["time"] # create final dict jdic = { 'type' : serie['name'], 'mac' : serie['tags']['mac'], 'value' : obj_value, 'timestamp' : d_influxdb['time'], } json_list.append(jdic) return json_list
def _fields_to_json_with_structure(self,sol_type,fields): sol_struct = SolDefines.solStructure(sol_type) returnVal = {} for name in sol_struct['fields']: returnVal[name] = fields[name] if 'extrafields' in sol_struct: returnVal[sol_struct['extrafields']] = fields[sol_struct['extrafields']] for (k,v) in returnVal.items(): if type(v)==tuple: returnVal[k] = [b for b in v] return returnVal
def _fields_to_json_with_structure(self, sol_type, fields): sol_struct = SolDefines.solStructure(sol_type) returnVal = {} for name in sol_struct['fields']: returnVal[name] = fields[name] if 'extrafields' in sol_struct: returnVal[sol_struct['extrafields']] = fields[ sol_struct['extrafields']] for (k, v) in returnVal.items(): if type(v) == tuple: returnVal[k] = [b for b in v] return returnVal
def _fields_to_binary_with_structure(self, sol_type, fields): sol_struct = SolDefines.solStructure(sol_type) pack_format = sol_struct['structure'] pack_values = [fields[name] for name in sol_struct['fields']] # convert [0x01,0x02,0x03] into 0x010203 to be packable for i in range(len(pack_values)): if type(pack_values[i]) == list: pack_values[i] = self._list_to_num(pack_values[i]) returnVal = [ord(b) for b in struct.pack(pack_format, *pack_values)] if 'extrafields' in sol_struct: returnVal += fields[sol_struct['extrafields']] return returnVal
def _fields_to_binary_with_structure(self,sol_type,fields): sol_struct = SolDefines.solStructure(sol_type) pack_format = sol_struct['structure'] pack_values = [fields[name] for name in sol_struct['fields']] # convert [0x01,0x02,0x03] into 0x010203 to be packable for i in range(len(pack_values)): if type(pack_values[i])==list: pack_values[i] = self._list_to_num(pack_values[i]) returnVal = [ord(b) for b in struct.pack(pack_format,*pack_values)] if 'extrafields' in sol_struct: returnVal += fields[sol_struct['extrafields']] return returnVal
def _binary_to_fields_with_structure(self,sol_type,binary): sol_struct = SolDefines.solStructure(sol_type) pack_format = sol_struct['structure'] pack_length = struct.calcsize(pack_format) t = struct.unpack(pack_format,''.join(chr(b) for b in binary[:pack_length])) returnVal = {} for (k,v) in zip(sol_struct['fields'],t): returnVal[k]= v if 'extrafields' in sol_struct: returnVal[sol_struct['extrafields']] = binary[pack_length:] for (k,v) in returnVal.items(): if k in ['macAddress','source','dest']: returnVal[k] = self._num_to_list(v,8) elif k in ['sol_version','sdk_version','solmanager_version']: returnVal[k] = self._num_to_list(v,4) return returnVal
def _binary_to_fields_with_structure(self, sol_type, binary): sol_struct = SolDefines.solStructure(sol_type) pack_format = sol_struct['structure'] pack_length = struct.calcsize(pack_format) t = struct.unpack(pack_format, ''.join(chr(b) for b in binary[:pack_length])) returnVal = {} for (k, v) in zip(sol_struct['fields'], t): returnVal[k] = v if 'extrafields' in sol_struct: returnVal[sol_struct['extrafields']] = binary[pack_length:] for (k, v) in returnVal.items(): if k in ['macAddress', 'source', 'dest']: returnVal[k] = self._num_to_list(v, 8) elif k in ['sol_version', 'sdk_version', 'solmanager_version']: returnVal[k] = self._num_to_list(v, 4) return returnVal
def _split_dust_notif(self, notif_name, dust_notif): """ Split a single Dust serial API notification into a list of Dust notifications :param dict dust_notif: The Dust serial API notification as created by the SmartMesh SDK :return: A list of Dust notifications :rtype: list """ notif_list = [] if notif_name == IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFHEALTHREPORT: hr_exists = True dust_notifs = [] hr_currptr = 0 hr_nextptr = dust_notif.payload[1] + 2 while hr_exists: # add HR notification to list notif_list.append( IpMgrConnectorSerial.IpMgrConnectorSerial. Tuple_notifHealthReport( macAddress=dust_notif.macAddress, payload=dust_notif.payload[hr_currptr:hr_nextptr], )) # check if other notifs are present hr_currptr = hr_nextptr if len(dust_notif.payload) > (hr_currptr + 2): hr_nextptr = hr_currptr + dust_notif.payload[hr_currptr + 1] + 2 if hr_nextptr < len(dust_notif.payload): hr_exists = False else: hr_exists = False elif notif_name == IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFDATA: # parse header sol_header = dust_notif.data[0] header_V = sol_header >> SolDefines.SOL_HDR_V_OFFSET & 0x03 header_T = sol_header >> SolDefines.SOL_HDR_T_OFFSET & 0x01 header_S = sol_header >> SolDefines.SOL_HDR_S_OFFSET & 0x01 header_Y = sol_header >> SolDefines.SOL_HDR_Y_OFFSET & 0x01 header_L = sol_header >> SolDefines.SOL_HDR_L_OFFSET & 0x03 if header_T == 0: # single object notif_list = [dust_notif] else: # multiple objects # reset header Type bit sol_header = dust_notif.data[0] & 0xdf # get structure size solheader_size = SolDefines.SOL_HEADER_SIZE ts_size = SolDefines.SOL_TIMESTAMP_SIZE objnum_size = SolDefines.SOL_OBJNUMBER_SIZE # get time ts_sec = 0 ts_usec = 0 if header_T == 0: # timestamp from obj ts_sec = dust_notif.data[0:ts_size] ts_usec = 0 else: # timestamp from dust notif ts_sec = dust_notif.utcSecs ts_user = dust_notif.utcUsecs # get number of objects obj_number = dust_notif.data[ts_size + objnum_size] curr_ptr = solheader_size + ts_size + objnum_size for i in range(0, obj_number): obj_type = dust_notif.data[curr_ptr] sol_item = SolDefines.solStructure(obj_type) obj_size = struct.calcsize(sol_item['structure']) notif_list.append( IpMgrConnectorSerial.IpMgrConnectorSerial. Tuple_notifData( utcSecs=ts_sec, utcUsecs=ts_usec, macAddress=dust_notif.macAddress, srcPort=dust_notif.srcPort, dstPort=dust_notif.dstPort, # data = solheader + timestamp + object data=dust_notif.data[:solheader_size + ts_size] + dust_notif.data[curr_ptr:curr_ptr + obj_size + 1])) curr_ptr += obj_size + 1 else: notif_list = [dust_notif] return notif_list
def json_to_influxdb(self, sol_json, tags): """ Convert a JSON SOL object into a InfluxDB point :param list sol_json: JSON SOL object :return: InfluxDB point :rtpe: list """ # fields if sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS: fields = {} for n in sol_json["value"]['neighbors']: fields["neighbors:" + str(n['neighborId'])] = n fields['numItems'] = sol_json["value"]['numItems'] elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED: fields = sol_json["value"] elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_SNAPSHOT: fields = {} fields["mote"] = [] for mote in sol_json["value"]: mote["macAddress"] = FormatUtils.formatBuffer( mote["macAddress"]) for path in mote["paths"]: path["macAddress"] = FormatUtils.formatBuffer( path["macAddress"]) fields["mote"].append(mote) elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_EVENTNETWORKRESET: fields = {'value': 'dummy'} else: fields = sol_json["value"] for (k, v) in fields.items(): if type(v) == list: # mac if k in ['macAddress', 'source', 'dest']: fields[k] = FormatUtils.formatBuffer(v) elif k in [ 'sol_version', 'sdk_version', 'solmanager_version' ]: fields[k] = ".".join(str(i) for i in v) f = flatdict.FlatDict(fields) fields = {} for (k, v) in f.items(): fields[k] = v # add additionnal fiel if apply_function exists try: obj_struct = SolDefines.solStructure(sol_json['type']) if 'apply' in obj_struct: for ap in obj_struct['apply']: arg_list = [fields[arg] for arg in ap["args"]] fields[ap["name"]] = ap["function"](*arg_list) except ValueError: pass # get SOL type measurement = SolDefines.solTypeToTypeName(SolDefines, sol_json['type']) # convert SOL timestamp to UTC utc_time = sol_json["timestamp"] * 1000000000 sol_influxdb = { "time": utc_time, "tags": tags, "measurement": measurement, "fields": fields, } return sol_influxdb
def bin_to_json(self, sol_bin, mac=None): """ Convert a binary SOL object into a JSON SOL Object. :param list sol_bin: binary SOL object :return: JSON SOL Objects :rtpe: list """ sol_json = {} # header h = sol_bin[0] h_V = (h >> SolDefines.SOL_HDR_V_OFFSET) & 0x03 assert h_V == SolDefines.SOL_HDR_V h_H = (h >> SolDefines.SOL_HDR_T_OFFSET) & 0x01 assert h_H == SolDefines.SOL_HDR_T_SINGLE h_M = (h >> SolDefines.SOL_HDR_M_OFFSET) & 0x01 h_S = (h >> SolDefines.SOL_HDR_S_OFFSET) & 0x01 h_Y = (h >> SolDefines.SOL_HDR_Y_OFFSET) & 0x01 h_L = (h >> SolDefines.SOL_HDR_L_OFFSET) & 0x03 sol_bin = sol_bin[1:] # mac if h_M == SolDefines.SOL_HDR_M_NOMAC: assert mac is not None sol_json['mac'] = mac else: assert len(sol_bin) >= 8 sol_json['mac'] = sol_bin[:8] sol_bin = sol_bin[8:] # timestamp assert h_S == SolDefines.SOL_HDR_S_EPOCH assert len(sol_bin) >= 4 sol_json['timestamp'] = self._list_to_num(sol_bin[:4]) sol_bin = sol_bin[4:] # type assert h_Y == SolDefines.SOL_HDR_Y_1B sol_json['type'] = sol_bin[0] sol_bin = sol_bin[1:] # length if h_L == SolDefines.SOL_HDR_L_WK: sol_item = SolDefines.solStructure(sol_json['type']) obj_size = struct.calcsize(sol_item['structure']) elif h_L == SolDefines.SOL_HDR_L_1B: sol_json['length'] = sol_bin[0] obj_size = sol_bin[0] sol_bin = sol_bin[1:] elif h_L == SolDefines.SOL_HDR_L_2B: sol_json['length'] = sol_bin[:2] obj_size = sol_bin[:2] sol_bin = sol_bin[2:] elif h_L == SolDefines.SOL_HDR_L_ELIDED: obj_size = len(sol_bin) # value assert len(sol_bin) == obj_size if sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS: sol_json['value'] = self.hrParser.parseHr( [self.hrParser.HR_ID_NEIGHBORS, len(sol_bin)] + sol_bin, )['Neighbors'] elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED: sol_json['value'] = self.hrParser.parseHr( [self.hrParser.HR_ID_DISCOVERED, len(sol_bin)] + sol_bin, )['Discovered'] elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_SNAPSHOT: sol_json['value'] = self._binary_to_fields_snapshot(sol_bin) else: sol_json['value'] = self._binary_to_fields_with_structure( sol_json['type'], sol_bin, ) return sol_json
def _split_dust_notif(self, notif_name, dust_notif): """ Split a single Dust serial API notification into a list of Dust notifications :param dict dust_notif: The Dust serial API notification as created by the SmartMesh SDK :return: A list of Dust notifications :rtype: list """ notif_list = [] if notif_name==IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFHEALTHREPORT: hr_exists = True dust_notifs = [] hr_currptr = 0 hr_nextptr = dust_notif.payload[1]+2 while hr_exists: # add HR notification to list notif_list.append( IpMgrConnectorSerial.IpMgrConnectorSerial.Tuple_notifHealthReport( macAddress = dust_notif.macAddress, payload = dust_notif.payload[hr_currptr:hr_nextptr], ) ) # check if other notifs are present hr_currptr = hr_nextptr if len(dust_notif.payload) > (hr_currptr+2): hr_nextptr = hr_currptr + dust_notif.payload[hr_currptr+1] + 2 if hr_nextptr < len(dust_notif.payload): hr_exists = False else: hr_exists = False elif notif_name==IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFDATA: # parse header sol_header = dust_notif.data[0] header_V = sol_header >> SolDefines.SOL_HDR_V_OFFSET & 0x03 header_T = sol_header >> SolDefines.SOL_HDR_T_OFFSET & 0x01 header_S = sol_header >> SolDefines.SOL_HDR_S_OFFSET & 0x01 header_Y = sol_header >> SolDefines.SOL_HDR_Y_OFFSET & 0x01 header_L = sol_header >> SolDefines.SOL_HDR_L_OFFSET & 0x03 if header_T == 0: # single object notif_list = [dust_notif] else: # multiple objects # reset header Type bit sol_header = dust_notif.data[0] & 0xdf # get structure size solheader_size = SolDefines.SOL_HEADER_SIZE ts_size = SolDefines.SOL_TIMESTAMP_SIZE objnum_size = SolDefines.SOL_OBJNUMBER_SIZE # get time ts_sec = 0 ts_usec = 0 ts_offset = 0 if header_S == 0: # timestamp from smip header ts_sec = dust_notif.data[0:ts_size] ts_usec = 0 ts_offset = ts_size else: # timestamp from dust notif ts_sec = dust_notif.utcSecs ts_user = dust_notif.utcUsecs # get number of objects obj_number = dust_notif.data[ts_offset+objnum_size] curr_ptr = solheader_size + ts_offset + objnum_size for i in range(0,obj_number): obj_type = dust_notif.data[curr_ptr] sol_item = SolDefines.solStructure(obj_type) obj_size = struct.calcsize(sol_item['structure']) notif_list.append( IpMgrConnectorSerial.IpMgrConnectorSerial.Tuple_notifData( utcSecs = ts_sec, utcUsecs = ts_usec, macAddress = dust_notif.macAddress, srcPort = dust_notif.srcPort, dstPort = dust_notif.dstPort, # data = solheader + timestamp + object data = tuple([sol_header])+ tuple(dust_notif.data[solheader_size:solheader_size+ts_offset])+ tuple(dust_notif.data[curr_ptr:curr_ptr+obj_size+1]) ) ) curr_ptr += obj_size+1 else: notif_list = [dust_notif] return notif_list
def json_to_influxdb(self,sol_json,tags): """ Convert a JSON SOL object into a InfluxDB point :param list sol_json: JSON SOL object :return: InfluxDB point :rtpe: list """ # tags obj_tags = copy.deepcopy(tags) # fields if sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS: fields = {} for n in sol_json["value"]['neighbors']: fields["neighbors:" + str(n['neighborId'])] = n fields['numItems'] = sol_json["value"]['numItems'] elif sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED: fields = sol_json["value"] elif sol_json['type']==SolDefines.SOL_TYPE_DUST_SNAPSHOT: fields = {} fields["mote"] = [] for mote in sol_json["value"]: mote["macAddress"] = FormatUtils.formatBuffer(mote["macAddress"]) for path in mote["paths"]: path["macAddress"] = FormatUtils.formatBuffer(path["macAddress"]) fields["mote"].append(mote) elif sol_json['type']==SolDefines.SOL_TYPE_DUST_EVENTNETWORKRESET: fields = {'value':'dummy'} else: fields = sol_json["value"] for (k,v) in fields.items(): if type(v)==list: # mac if k in ['macAddress','source','dest']: fields[k] = FormatUtils.formatBuffer(v) elif k in ['sol_version','sdk_version','solmanager_version']: fields[k] = ".".join(str(i) for i in v) f = flatdict.FlatDict(fields) fields = {} for (k,v) in f.items(): fields[k] = v # add additionnal field or tag if apply_function exists try: obj_struct = SolDefines.solStructure(sol_json['type']) if 'apply' in obj_struct: for ap in obj_struct['apply']: arg_list = [fields[arg] for arg in ap["args"]] if "field" in ap: fields[ap["field"]] = ap["function"](*arg_list) if "tag" in ap: obj_tags[ap["tag"]] = ap["function"](*arg_list) except ValueError: pass # get SOL type measurement = SolDefines.solTypeToTypeName(SolDefines,sol_json['type']) # convert SOL timestamp to UTC utc_time = sol_json["timestamp"]*1000000000 sol_influxdb = { "time" : utc_time, "tags" : obj_tags, "measurement" : measurement, "fields" : fields, } return sol_influxdb
def bin_to_json(self, sol_bin, mac=None): """ Convert a binary SOL object into a JSON SOL Object. :param list sol_bin: binary SOL object :return: JSON SOL Objects :rtpe: list """ sol_json = {} # header h = sol_bin[0] h_V = (h>>SolDefines.SOL_HDR_V_OFFSET)&0x03 assert h_V==SolDefines.SOL_HDR_V h_H = (h>>SolDefines.SOL_HDR_T_OFFSET)&0x01 assert h_H==SolDefines.SOL_HDR_T_SINGLE h_M = (h>>SolDefines.SOL_HDR_M_OFFSET)&0x01 h_S = (h>>SolDefines.SOL_HDR_S_OFFSET)&0x01 h_Y = (h>>SolDefines.SOL_HDR_Y_OFFSET)&0x01 h_L = (h>>SolDefines.SOL_HDR_L_OFFSET)&0x03 sol_bin = sol_bin[1:] # mac if h_M==SolDefines.SOL_HDR_M_NOMAC: assert mac is not None sol_json['mac'] = mac else: assert len(sol_bin)>=8 sol_json['mac'] = sol_bin[:8] sol_bin = sol_bin[8:] # timestamp assert h_S==SolDefines.SOL_HDR_S_EPOCH assert len(sol_bin)>=4 sol_json['timestamp'] = self._list_to_num(sol_bin[:4]) sol_bin = sol_bin[4:] # type assert h_Y==SolDefines.SOL_HDR_Y_1B sol_json['type'] = sol_bin[0] sol_bin = sol_bin[1:] # length if h_L==SolDefines.SOL_HDR_L_WK: sol_item = SolDefines.solStructure(sol_json['type']) obj_size = struct.calcsize(sol_item['structure']) elif h_L==SolDefines.SOL_HDR_L_1B: sol_json['length'] = sol_bin[0] obj_size = sol_bin[0] sol_bin = sol_bin[1:] elif h_L==SolDefines.SOL_HDR_L_2B: sol_json['length'] = sol_bin[:2] obj_size = sol_bin[:2] sol_bin = sol_bin[2:] elif h_L==SolDefines.SOL_HDR_L_ELIDED: obj_size = len(sol_bin) # value assert len(sol_bin)==obj_size if sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS: sol_json['value'] = self.hrParser.parseHr( [self.hrParser.HR_ID_NEIGHBORS,len(sol_bin)]+sol_bin, )['Neighbors'] elif sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED: sol_json['value'] = self.hrParser.parseHr( [self.hrParser.HR_ID_DISCOVERED,len(sol_bin)]+sol_bin, )['Discovered'] elif sol_json['type']==SolDefines.SOL_TYPE_DUST_SNAPSHOT: sol_json['value'] = self._binary_to_fields_snapshot(sol_bin) else: sol_json['value'] = self._binary_to_fields_with_structure( sol_json['type'], sol_bin, ) return sol_json