def influxdb_to_json(self, sol_influxdb):
"""
Converts an Influxdb query reply into a list of dicts.
:param sol_influxdb dict: the result of a database query (sush as SELECT * FROM)
:return: a list of JSON SOL objects
:rtype: list
"""
# verify influxdb data
if not ("series" in sol_influxdb):
raise ValueError("Influxdb data not recognized")
# init
json_list = []
# remove unused headers
for serie in sol_influxdb["series"]:
for val in serie['values']:
# convert to dict
d_influxdb = dict(zip(serie['columns'], val))
# unflat dict
obj_value = flatdict.FlatDict(d_influxdb).as_dict()
# parse specific HR_NEIGHBORS
hr_nghb_name = SolDefines.solTypeToTypeName(
SolDefines, SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS)
if serie['name'] == hr_nghb_name:
for i in range(0, len(obj_value["neighbors"]) + 1):
ngbr_id = str(i)
# new HR_NGBR parsing
if ngbr_id in obj_value["neighbors"]:
if obj_value["neighbors"][ngbr_id][
"neighborFlag"] is None:
del obj_value["neighbors"][ngbr_id]
# old HR_NGBR parsing
if ngbr_id in obj_value:
if obj_value[ngbr_id]["neighborFlag"] is not None:
obj_value["neighbors"][ngbr_id] = obj_value[
ngbr_id]
del obj_value[ngbr_id]
# time is not passed in the "value" field
del obj_value["time"]
# create final dict
jdic = {
'type': serie['name'],
'mac': serie['tags']['mac'],
'value': obj_value,
'timestamp': d_influxdb['time'],
}
json_list.append(jdic)
return json_list
def influxdb_to_json(self, sol_influxdb):
"""
Converts an Influxdb query reply into a list of dicts.
:param sol_influxdb dict: the result of a database query (sush as SELECT * FROM)
:return: a list of JSON SOL objects
:rtype: list
"""
# verify influxdb data
if not ("series" in sol_influxdb):
raise ValueError("Influxdb data not recognized")
# init
json_list = []
# remove unused headers
for serie in sol_influxdb["series"]:
for val in serie['values']:
# convert to dict
d_influxdb = dict(zip(serie['columns'], val))
# unflat dict
obj_value = flatdict.FlatDict(d_influxdb).as_dict()
# parse specific HR_NEIGHBORS
hr_nghb_name = SolDefines.solTypeToTypeName(
SolDefines,
SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS)
if serie['name'] == hr_nghb_name:
for i in range(0,len(obj_value["neighbors"])+1):
ngbr_id = str(i)
# new HR_NGBR parsing
if ngbr_id in obj_value["neighbors"]:
if obj_value["neighbors"][ngbr_id]["neighborFlag"] is None:
del obj_value["neighbors"][ngbr_id]
# old HR_NGBR parsing
if ngbr_id in obj_value:
if obj_value[ngbr_id]["neighborFlag"] is not None:
obj_value["neighbors"][ngbr_id] = obj_value[ngbr_id]
del obj_value[ngbr_id]
# time is not passed in the "value" field
del obj_value["time"]
# create final dict
jdic = {
'type' : serie['name'],
'mac' : serie['tags']['mac'],
'value' : obj_value,
'timestamp' : d_influxdb['time'],
}
json_list.append(jdic)
return json_list
def _fields_to_json_with_structure(self,sol_type,fields):
sol_struct = SolDefines.solStructure(sol_type)
returnVal = {}
for name in sol_struct['fields']:
returnVal[name] = fields[name]
if 'extrafields' in sol_struct:
returnVal[sol_struct['extrafields']] = fields[sol_struct['extrafields']]
for (k,v) in returnVal.items():
if type(v)==tuple:
returnVal[k] = [b for b in v]
return returnVal
def _fields_to_json_with_structure(self, sol_type, fields):
sol_struct = SolDefines.solStructure(sol_type)
returnVal = {}
for name in sol_struct['fields']:
returnVal[name] = fields[name]
if 'extrafields' in sol_struct:
returnVal[sol_struct['extrafields']] = fields[
sol_struct['extrafields']]
for (k, v) in returnVal.items():
if type(v) == tuple:
returnVal[k] = [b for b in v]
return returnVal
def _fields_to_binary_with_structure(self, sol_type, fields):
sol_struct = SolDefines.solStructure(sol_type)
pack_format = sol_struct['structure']
pack_values = [fields[name] for name in sol_struct['fields']]
# convert [0x01,0x02,0x03] into 0x010203 to be packable
for i in range(len(pack_values)):
if type(pack_values[i]) == list:
pack_values[i] = self._list_to_num(pack_values[i])
returnVal = [ord(b) for b in struct.pack(pack_format, *pack_values)]
if 'extrafields' in sol_struct:
returnVal += fields[sol_struct['extrafields']]
return returnVal
def _fields_to_binary_with_structure(self,sol_type,fields):
sol_struct = SolDefines.solStructure(sol_type)
pack_format = sol_struct['structure']
pack_values = [fields[name] for name in sol_struct['fields']]
# convert [0x01,0x02,0x03] into 0x010203 to be packable
for i in range(len(pack_values)):
if type(pack_values[i])==list:
pack_values[i] = self._list_to_num(pack_values[i])
returnVal = [ord(b) for b in struct.pack(pack_format,*pack_values)]
if 'extrafields' in sol_struct:
returnVal += fields[sol_struct['extrafields']]
return returnVal
def _binary_to_fields_with_structure(self,sol_type,binary):
sol_struct = SolDefines.solStructure(sol_type)
pack_format = sol_struct['structure']
pack_length = struct.calcsize(pack_format)
t = struct.unpack(pack_format,''.join(chr(b) for b in binary[:pack_length]))
returnVal = {}
for (k,v) in zip(sol_struct['fields'],t):
returnVal[k]= v
if 'extrafields' in sol_struct:
returnVal[sol_struct['extrafields']] = binary[pack_length:]
for (k,v) in returnVal.items():
if k in ['macAddress','source','dest']:
returnVal[k] = self._num_to_list(v,8)
elif k in ['sol_version','sdk_version','solmanager_version']:
returnVal[k] = self._num_to_list(v,4)
return returnVal
def _binary_to_fields_with_structure(self, sol_type, binary):
sol_struct = SolDefines.solStructure(sol_type)
pack_format = sol_struct['structure']
pack_length = struct.calcsize(pack_format)
t = struct.unpack(pack_format,
''.join(chr(b) for b in binary[:pack_length]))
returnVal = {}
for (k, v) in zip(sol_struct['fields'], t):
returnVal[k] = v
if 'extrafields' in sol_struct:
returnVal[sol_struct['extrafields']] = binary[pack_length:]
for (k, v) in returnVal.items():
if k in ['macAddress', 'source', 'dest']:
returnVal[k] = self._num_to_list(v, 8)
elif k in ['sol_version', 'sdk_version', 'solmanager_version']:
returnVal[k] = self._num_to_list(v, 4)
return returnVal
def _split_dust_notif(self, notif_name, dust_notif):
"""
Split a single Dust serial API notification into a list of Dust notifications
:param dict dust_notif: The Dust serial API notification as
created by the SmartMesh SDK
:return: A list of Dust notifications
:rtype: list
"""
notif_list = []
if notif_name == IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFHEALTHREPORT:
hr_exists = True
dust_notifs = []
hr_currptr = 0
hr_nextptr = dust_notif.payload[1] + 2
while hr_exists:
# add HR notification to list
notif_list.append(
IpMgrConnectorSerial.IpMgrConnectorSerial.
Tuple_notifHealthReport(
macAddress=dust_notif.macAddress,
payload=dust_notif.payload[hr_currptr:hr_nextptr],
))
# check if other notifs are present
hr_currptr = hr_nextptr
if len(dust_notif.payload) > (hr_currptr + 2):
hr_nextptr = hr_currptr + dust_notif.payload[hr_currptr +
1] + 2
if hr_nextptr < len(dust_notif.payload):
hr_exists = False
else:
hr_exists = False
elif notif_name == IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFDATA:
# parse header
sol_header = dust_notif.data[0]
header_V = sol_header >> SolDefines.SOL_HDR_V_OFFSET & 0x03
header_T = sol_header >> SolDefines.SOL_HDR_T_OFFSET & 0x01
header_S = sol_header >> SolDefines.SOL_HDR_S_OFFSET & 0x01
header_Y = sol_header >> SolDefines.SOL_HDR_Y_OFFSET & 0x01
header_L = sol_header >> SolDefines.SOL_HDR_L_OFFSET & 0x03
if header_T == 0: # single object
notif_list = [dust_notif]
else: # multiple objects
# reset header Type bit
sol_header = dust_notif.data[0] & 0xdf
# get structure size
solheader_size = SolDefines.SOL_HEADER_SIZE
ts_size = SolDefines.SOL_TIMESTAMP_SIZE
objnum_size = SolDefines.SOL_OBJNUMBER_SIZE
# get time
ts_sec = 0
ts_usec = 0
if header_T == 0: # timestamp from obj
ts_sec = dust_notif.data[0:ts_size]
ts_usec = 0
else: # timestamp from dust notif
ts_sec = dust_notif.utcSecs
ts_user = dust_notif.utcUsecs
# get number of objects
obj_number = dust_notif.data[ts_size + objnum_size]
curr_ptr = solheader_size + ts_size + objnum_size
for i in range(0, obj_number):
obj_type = dust_notif.data[curr_ptr]
sol_item = SolDefines.solStructure(obj_type)
obj_size = struct.calcsize(sol_item['structure'])
notif_list.append(
IpMgrConnectorSerial.IpMgrConnectorSerial.
Tuple_notifData(
utcSecs=ts_sec,
utcUsecs=ts_usec,
macAddress=dust_notif.macAddress,
srcPort=dust_notif.srcPort,
dstPort=dust_notif.dstPort,
# data = solheader + timestamp + object
data=dust_notif.data[:solheader_size + ts_size] +
dust_notif.data[curr_ptr:curr_ptr + obj_size + 1]))
curr_ptr += obj_size + 1
else:
notif_list = [dust_notif]
return notif_list
def json_to_influxdb(self, sol_json, tags):
"""
Convert a JSON SOL object into a InfluxDB point
:param list sol_json: JSON SOL object
:return: InfluxDB point
:rtpe: list
"""
# fields
if sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS:
fields = {}
for n in sol_json["value"]['neighbors']:
fields["neighbors:" + str(n['neighborId'])] = n
fields['numItems'] = sol_json["value"]['numItems']
elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED:
fields = sol_json["value"]
elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_SNAPSHOT:
fields = {}
fields["mote"] = []
for mote in sol_json["value"]:
mote["macAddress"] = FormatUtils.formatBuffer(
mote["macAddress"])
for path in mote["paths"]:
path["macAddress"] = FormatUtils.formatBuffer(
path["macAddress"])
fields["mote"].append(mote)
elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_EVENTNETWORKRESET:
fields = {'value': 'dummy'}
else:
fields = sol_json["value"]
for (k, v) in fields.items():
if type(v) == list: # mac
if k in ['macAddress', 'source', 'dest']:
fields[k] = FormatUtils.formatBuffer(v)
elif k in [
'sol_version', 'sdk_version', 'solmanager_version'
]:
fields[k] = ".".join(str(i) for i in v)
f = flatdict.FlatDict(fields)
fields = {}
for (k, v) in f.items():
fields[k] = v
# add additionnal fiel if apply_function exists
try:
obj_struct = SolDefines.solStructure(sol_json['type'])
if 'apply' in obj_struct:
for ap in obj_struct['apply']:
arg_list = [fields[arg] for arg in ap["args"]]
fields[ap["name"]] = ap["function"](*arg_list)
except ValueError:
pass
# get SOL type
measurement = SolDefines.solTypeToTypeName(SolDefines,
sol_json['type'])
# convert SOL timestamp to UTC
utc_time = sol_json["timestamp"] * 1000000000
sol_influxdb = {
"time": utc_time,
"tags": tags,
"measurement": measurement,
"fields": fields,
}
return sol_influxdb
def bin_to_json(self, sol_bin, mac=None):
"""
Convert a binary SOL object into a JSON SOL Object.
:param list sol_bin: binary SOL object
:return: JSON SOL Objects
:rtpe: list
"""
sol_json = {}
# header
h = sol_bin[0]
h_V = (h >> SolDefines.SOL_HDR_V_OFFSET) & 0x03
assert h_V == SolDefines.SOL_HDR_V
h_H = (h >> SolDefines.SOL_HDR_T_OFFSET) & 0x01
assert h_H == SolDefines.SOL_HDR_T_SINGLE
h_M = (h >> SolDefines.SOL_HDR_M_OFFSET) & 0x01
h_S = (h >> SolDefines.SOL_HDR_S_OFFSET) & 0x01
h_Y = (h >> SolDefines.SOL_HDR_Y_OFFSET) & 0x01
h_L = (h >> SolDefines.SOL_HDR_L_OFFSET) & 0x03
sol_bin = sol_bin[1:]
# mac
if h_M == SolDefines.SOL_HDR_M_NOMAC:
assert mac is not None
sol_json['mac'] = mac
else:
assert len(sol_bin) >= 8
sol_json['mac'] = sol_bin[:8]
sol_bin = sol_bin[8:]
# timestamp
assert h_S == SolDefines.SOL_HDR_S_EPOCH
assert len(sol_bin) >= 4
sol_json['timestamp'] = self._list_to_num(sol_bin[:4])
sol_bin = sol_bin[4:]
# type
assert h_Y == SolDefines.SOL_HDR_Y_1B
sol_json['type'] = sol_bin[0]
sol_bin = sol_bin[1:]
# length
if h_L == SolDefines.SOL_HDR_L_WK:
sol_item = SolDefines.solStructure(sol_json['type'])
obj_size = struct.calcsize(sol_item['structure'])
elif h_L == SolDefines.SOL_HDR_L_1B:
sol_json['length'] = sol_bin[0]
obj_size = sol_bin[0]
sol_bin = sol_bin[1:]
elif h_L == SolDefines.SOL_HDR_L_2B:
sol_json['length'] = sol_bin[:2]
obj_size = sol_bin[:2]
sol_bin = sol_bin[2:]
elif h_L == SolDefines.SOL_HDR_L_ELIDED:
obj_size = len(sol_bin)
# value
assert len(sol_bin) == obj_size
if sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS:
sol_json['value'] = self.hrParser.parseHr(
[self.hrParser.HR_ID_NEIGHBORS,
len(sol_bin)] + sol_bin, )['Neighbors']
elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED:
sol_json['value'] = self.hrParser.parseHr(
[self.hrParser.HR_ID_DISCOVERED,
len(sol_bin)] + sol_bin, )['Discovered']
elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_SNAPSHOT:
sol_json['value'] = self._binary_to_fields_snapshot(sol_bin)
else:
sol_json['value'] = self._binary_to_fields_with_structure(
sol_json['type'],
sol_bin,
)
return sol_json
def _split_dust_notif(self, notif_name, dust_notif):
"""
Split a single Dust serial API notification into a list of Dust notifications
:param dict dust_notif: The Dust serial API notification as
created by the SmartMesh SDK
:return: A list of Dust notifications
:rtype: list
"""
notif_list = []
if notif_name==IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFHEALTHREPORT:
hr_exists = True
dust_notifs = []
hr_currptr = 0
hr_nextptr = dust_notif.payload[1]+2
while hr_exists:
# add HR notification to list
notif_list.append(
IpMgrConnectorSerial.IpMgrConnectorSerial.Tuple_notifHealthReport(
macAddress = dust_notif.macAddress,
payload = dust_notif.payload[hr_currptr:hr_nextptr],
)
)
# check if other notifs are present
hr_currptr = hr_nextptr
if len(dust_notif.payload) > (hr_currptr+2):
hr_nextptr = hr_currptr + dust_notif.payload[hr_currptr+1] + 2
if hr_nextptr < len(dust_notif.payload):
hr_exists = False
else:
hr_exists = False
elif notif_name==IpMgrConnectorSerial.IpMgrConnectorSerial.NOTIFDATA:
# parse header
sol_header = dust_notif.data[0]
header_V = sol_header >> SolDefines.SOL_HDR_V_OFFSET & 0x03
header_T = sol_header >> SolDefines.SOL_HDR_T_OFFSET & 0x01
header_S = sol_header >> SolDefines.SOL_HDR_S_OFFSET & 0x01
header_Y = sol_header >> SolDefines.SOL_HDR_Y_OFFSET & 0x01
header_L = sol_header >> SolDefines.SOL_HDR_L_OFFSET & 0x03
if header_T == 0: # single object
notif_list = [dust_notif]
else: # multiple objects
# reset header Type bit
sol_header = dust_notif.data[0] & 0xdf
# get structure size
solheader_size = SolDefines.SOL_HEADER_SIZE
ts_size = SolDefines.SOL_TIMESTAMP_SIZE
objnum_size = SolDefines.SOL_OBJNUMBER_SIZE
# get time
ts_sec = 0
ts_usec = 0
ts_offset = 0
if header_S == 0: # timestamp from smip header
ts_sec = dust_notif.data[0:ts_size]
ts_usec = 0
ts_offset = ts_size
else: # timestamp from dust notif
ts_sec = dust_notif.utcSecs
ts_user = dust_notif.utcUsecs
# get number of objects
obj_number = dust_notif.data[ts_offset+objnum_size]
curr_ptr = solheader_size + ts_offset + objnum_size
for i in range(0,obj_number):
obj_type = dust_notif.data[curr_ptr]
sol_item = SolDefines.solStructure(obj_type)
obj_size = struct.calcsize(sol_item['structure'])
notif_list.append(
IpMgrConnectorSerial.IpMgrConnectorSerial.Tuple_notifData(
utcSecs = ts_sec,
utcUsecs = ts_usec,
macAddress = dust_notif.macAddress,
srcPort = dust_notif.srcPort,
dstPort = dust_notif.dstPort,
# data = solheader + timestamp + object
data = tuple([sol_header])+
tuple(dust_notif.data[solheader_size:solheader_size+ts_offset])+
tuple(dust_notif.data[curr_ptr:curr_ptr+obj_size+1])
)
)
curr_ptr += obj_size+1
else:
notif_list = [dust_notif]
return notif_list
def json_to_influxdb(self,sol_json,tags):
"""
Convert a JSON SOL object into a InfluxDB point
:param list sol_json: JSON SOL object
:return: InfluxDB point
:rtpe: list
"""
# tags
obj_tags = copy.deepcopy(tags)
# fields
if sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS:
fields = {}
for n in sol_json["value"]['neighbors']:
fields["neighbors:" + str(n['neighborId'])] = n
fields['numItems'] = sol_json["value"]['numItems']
elif sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED:
fields = sol_json["value"]
elif sol_json['type']==SolDefines.SOL_TYPE_DUST_SNAPSHOT:
fields = {}
fields["mote"] = []
for mote in sol_json["value"]:
mote["macAddress"] = FormatUtils.formatBuffer(mote["macAddress"])
for path in mote["paths"]:
path["macAddress"] = FormatUtils.formatBuffer(path["macAddress"])
fields["mote"].append(mote)
elif sol_json['type']==SolDefines.SOL_TYPE_DUST_EVENTNETWORKRESET:
fields = {'value':'dummy'}
else:
fields = sol_json["value"]
for (k,v) in fields.items():
if type(v)==list: # mac
if k in ['macAddress','source','dest']:
fields[k] = FormatUtils.formatBuffer(v)
elif k in ['sol_version','sdk_version','solmanager_version']:
fields[k] = ".".join(str(i) for i in v)
f = flatdict.FlatDict(fields)
fields = {}
for (k,v) in f.items():
fields[k] = v
# add additionnal field or tag if apply_function exists
try:
obj_struct = SolDefines.solStructure(sol_json['type'])
if 'apply' in obj_struct:
for ap in obj_struct['apply']:
arg_list = [fields[arg] for arg in ap["args"]]
if "field" in ap:
fields[ap["field"]] = ap["function"](*arg_list)
if "tag" in ap:
obj_tags[ap["tag"]] = ap["function"](*arg_list)
except ValueError:
pass
# get SOL type
measurement = SolDefines.solTypeToTypeName(SolDefines,sol_json['type'])
# convert SOL timestamp to UTC
utc_time = sol_json["timestamp"]*1000000000
sol_influxdb = {
"time" : utc_time,
"tags" : obj_tags,
"measurement" : measurement,
"fields" : fields,
}
return sol_influxdb
def bin_to_json(self, sol_bin, mac=None):
"""
Convert a binary SOL object into a JSON SOL Object.
:param list sol_bin: binary SOL object
:return: JSON SOL Objects
:rtpe: list
"""
sol_json = {}
# header
h = sol_bin[0]
h_V = (h>>SolDefines.SOL_HDR_V_OFFSET)&0x03
assert h_V==SolDefines.SOL_HDR_V
h_H = (h>>SolDefines.SOL_HDR_T_OFFSET)&0x01
assert h_H==SolDefines.SOL_HDR_T_SINGLE
h_M = (h>>SolDefines.SOL_HDR_M_OFFSET)&0x01
h_S = (h>>SolDefines.SOL_HDR_S_OFFSET)&0x01
h_Y = (h>>SolDefines.SOL_HDR_Y_OFFSET)&0x01
h_L = (h>>SolDefines.SOL_HDR_L_OFFSET)&0x03
sol_bin = sol_bin[1:]
# mac
if h_M==SolDefines.SOL_HDR_M_NOMAC:
assert mac is not None
sol_json['mac'] = mac
else:
assert len(sol_bin)>=8
sol_json['mac'] = sol_bin[:8]
sol_bin = sol_bin[8:]
# timestamp
assert h_S==SolDefines.SOL_HDR_S_EPOCH
assert len(sol_bin)>=4
sol_json['timestamp'] = self._list_to_num(sol_bin[:4])
sol_bin = sol_bin[4:]
# type
assert h_Y==SolDefines.SOL_HDR_Y_1B
sol_json['type'] = sol_bin[0]
sol_bin = sol_bin[1:]
# length
if h_L==SolDefines.SOL_HDR_L_WK:
sol_item = SolDefines.solStructure(sol_json['type'])
obj_size = struct.calcsize(sol_item['structure'])
elif h_L==SolDefines.SOL_HDR_L_1B:
sol_json['length'] = sol_bin[0]
obj_size = sol_bin[0]
sol_bin = sol_bin[1:]
elif h_L==SolDefines.SOL_HDR_L_2B:
sol_json['length'] = sol_bin[:2]
obj_size = sol_bin[:2]
sol_bin = sol_bin[2:]
elif h_L==SolDefines.SOL_HDR_L_ELIDED:
obj_size = len(sol_bin)
# value
assert len(sol_bin)==obj_size
if sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS:
sol_json['value'] = self.hrParser.parseHr(
[self.hrParser.HR_ID_NEIGHBORS,len(sol_bin)]+sol_bin,
)['Neighbors']
elif sol_json['type']==SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED:
sol_json['value'] = self.hrParser.parseHr(
[self.hrParser.HR_ID_DISCOVERED,len(sol_bin)]+sol_bin,
)['Discovered']
elif sol_json['type']==SolDefines.SOL_TYPE_DUST_SNAPSHOT:
sol_json['value'] = self._binary_to_fields_snapshot(sol_bin)
else:
sol_json['value'] = self._binary_to_fields_with_structure(
sol_json['type'],
sol_bin,
)
return sol_json
