def make_tick(self):
updates = OrderedDict()
updates.update(self.update())
self.theano_tick = theano.function([], [], updates=updates)
# introduce 1-time-tick delay
for o in self.origin.values():
if o.func is not None and self.mode == 'direct': continue
o.tick()
def update(self, dt):
"""Update the input and output of all the theano variables.
"""
updates = OrderedDict()
for input in self.input.values():
updates.update(input.update(dt))
for origin in self.origin.values():
if origin.func is None:
updates.update({origin.decoded_output: origin.method()})
return updates
def update(self, dt):
"""Update the input and output of all the theano variables.
"""
updates = OrderedDict()
for input in self.input.values():
updates.update(input.update(dt))
for origin in self.origin.values():
if origin.func is None:
updates.update(
{origin.decoded_output: origin.method()})
return updates
def make_theano_tick(self):
"""Generate the theano function for running the network simulation.
:returns: theano function
"""
# dictionary for all variables
# and the theano description of how to compute them
updates = OrderedDict()
# for every node in the network
for node in self.nodes.values():
# if there is some variable to update
if hasattr(node, 'update'):
# add it to the list of variables to update every time step
updates.update(node.update(self.dt))
# create graph and return optimized update function
return theano.function([], [], updates=updates.items())#, mode='ProfileMode')
def schema2dict(xmlcode: str) -> Dict:
elements_dict = OrderedDict()
tree = etree.fromstring(xmlcode)
ns_dict = {_globals.schemainfo.ns_prefix: _globals.schemainfo.ns}
_globals.schemainfo.ns_dict = ns_dict
# resource (entity)
resource, resource_dict = parse_resource(
tree=tree,
resource_root_xpath='.//xs:element[@name="resource"]'
)
elements_dict.update(resource_dict)
# pprint(elements_dict)
# properties
for prop in resource.findall('./xs:complexType/xs:all/xs:element',
namespaces=_globals.schemainfo.ns_dict):
prop_dict = parse_property(tree=tree, prop_el=prop)
elements_dict.update(prop_dict)
return elements_dict
def make_theano_tick(self):
"""Generate the theano function for running the network simulation.
:returns: theano function
"""
# dictionary for all variables
# and the theano description of how to compute them
updates = OrderedDict()
# for every node in the network
for node in self.nodes.values():
# if there is some variable to update
if hasattr(node, 'update'):
# add it to the list of variables to update every time step
updates.update(node.update(self.dt))
# create graph and return optimized update function
return theano.function([], [], updates=updates.items())
def writer(cls, argv):
conf_file_path = get_env_file_path(cls)
if os.path.exists(conf_file_path):
raise Exception(f"Found {conf_file_path} existing already")
vals = OrderedDict()
vals.update(_normalize_prefix(cls._default_prefix))
for k in cls.__dict__.keys():
if k in os.environ:
vals[k] = os.environ[k]
for arg in argv:
if "=" in arg:
k, val = arg.split("=")
vals[k] = val
dump = os.environ.get("VARS_DUMP",
cls._dump) # type: Union[bool, List[str]]
if dump:
for k in cls.__dict__.keys():
if isinstance(dump, (list, tuple)) and k not in dump:
continue
if k not in vals and not k.startswith("_"):
vals[k] = getattr(cls, k)
if vals:
ordered_vals = OrderedDict()
for k in cls.__dict__.keys():
if k in vals:
ordered_vals[k] = vals.pop(k)
ordered_vals.update(vals)
_write(conf_file_path, ordered_vals)
else:
log.info("Dynamic-Conf: No variables available.") # pragma: no cover
return vals
def update(self, dt):
"""Compute the set of theano updates needed for this ensemble.
Returns a dictionary with new neuron state,
termination, and origin values.
:param float dt: the timestep of the update
"""
### find the total input current to this population of neurons
# set up matrix to store accumulated decoded input
X = None
# updates is an ordered dictionary of theano variables to update
updates = OrderedDict()
for ii, di in enumerate(self.decoded_input.values()):
# add its values to the total decoded input
if ii == 0: X = di.value
else: X += di.value
updates.update(di.update(dt))
# if we're in spiking mode, then look at the input current and
# calculate new neuron activities for output
if self.mode == 'spiking':
# apply respective biases to neurons in the population
J = TT.as_tensor_variable(np.array(self.bias))
for ei in self.encoded_input.values():
# add its values directly to the input current
J += (ei.value.T * self.alpha.T).T
updates.update(ei.update(dt))
# only do this if there is decoded_input
if X is not None:
# add to input current for each neuron as
# represented input signal x preferred direction
for i in range(self.array_size): #len(self.bias)):
J = TT.basic.inc_subtensor(J[i],
TT.dot(X[i], self.shared_encoders[i].T))
# if noise has been specified for this neuron,
if self.noise:
# generate random noise values, one for each input_current element,
# with standard deviation = sqrt(self.noise=std**2)
# When simulating white noise, the noise process must be scaled by
# sqrt(dt) instead of dt. Hence, we divide the std by sqrt(dt).
if self.noise_type.lower() == 'gaussian':
J += self.srng.normal(
size=self.bias.shape, std=np.sqrt(self.noise/dt))
elif self.noise_type.lower() == 'uniform':
J += self.srng.uniform(
size=self.bias.shape,
low=-self.noise/np.sqrt(dt),
high=self.noise/np.sqrt(dt))
# pass that total into the neuron model to produce
# the main theano computation
updates.update(self.neurons.update(J, dt))
for l in self.learned_terminations:
# also update the weight matrices on learned terminations
updates.update(l.update(dt))
# and compute the decoded origin decoded_input from the neuron output
for o in self.origin.values():
updates.update(o.update(dt, updates[self.neurons.output]))
if self.mode == 'direct':
# if we're in direct mode then just directly pass the decoded_input
# to the origins for decoded_output
for o in self.origin.values():
if o.func is None:
if len(self.decoded_input) > 0:
updates.update(OrderedDict({o.decoded_output:
TT.flatten(X).astype('float32')}))
return updates
def update(self):
"""Compute the set of theano updates needed for this ensemble.
Returns a dictionary with new neuron state,
termination, and origin values.
"""
### find the total input current to this population of neurons
# set up matrix to store accumulated decoded input
X = None
# updates is an ordered dictionary of theano variables to update
updates = OrderedDict()
for ii, di in enumerate(self.decoded_input.values()):
# add its values to the total decoded input
if ii == 0:
X = di.value
else:
X += di.value
updates.update(di.update(self.dt))
# if we're in spiking mode, then look at the input current and
# calculate new neuron activities for output
if self.mode == 'spiking':
# apply respective biases to neurons in the population
J = TT.as_tensor_variable(np.array(self.bias))
for ei in self.encoded_input.values():
# add its values directly to the input current
J += (ei.value.T * self.alpha.T).T
updates.update(ei.update(self.dt))
# only do this if there is decoded_input
if X is not None:
# add to input current for each neuron as
# represented input signal x preferred direction
J = map_gemv(1.0, self.shared_encoders, X, 1.0, J)
# if noise has been specified for this neuron,
if self.noise:
# generate random noise values, one for each input_current element,
# with standard deviation = sqrt(self.noise=std**2)
# When simulating white noise, the noise process must be scaled by
# sqrt(dt) instead of dt. Hence, we divide the std by sqrt(dt).
if self.noise_type.lower() == 'gaussian':
J += self.srng.normal(
size=self.bias.shape, std=np.sqrt(self.noise/self.dt))
elif self.noise_type.lower() == 'uniform':
J += self.srng.uniform(
size=self.bias.shape,
low=-self.noise/np.sqrt(self.dt),
high=self.noise/np.sqrt(self.dt))
# pass that total into the neuron model to produce
# the main theano computation
updates.update(self.neurons.update(J, self.dt))
for l in self.learned_terminations:
# also update the weight matrices on learned terminations
updates.update(l.update(self.dt))
# and compute the decoded origin decoded_input from the neuron output
for o in self.origin.values():
updates.update(o.update(self.dt, updates[self.neurons.output]))
if self.mode == 'direct':
# if we're in direct mode then just directly pass the decoded_input
# to the origins for decoded_output
for o in self.origin.values():
if o.func is None:
if len(self.decoded_input) > 0:
updates.update(OrderedDict({o.decoded_output:
TT.flatten(X).astype(FLOAT_TYPE)}))
return updates
# Имеется log-файл с ip-адресами, проанализировать последние N адресов и
# сохранить в новый файл пары значений "ip-адрес - кол-во запросов"
from _collections import OrderedDict, defaultdict, deque
N = 3000
with open('big_log.txt', 'r', encoding='utf-8') as f:
log = deque(f, N)
print(log)
data = OrderedDict()
spam = defaultdict(int)
for item in log:
ip = item[:-1]
if not ip.startswith('192.168'):
spam[ip] += 1
data[ip] = 1
print(spam)
print(data)
data.update(spam)
print(data)
with open('data.txt', 'w', encoding='utf-8') as f:
for key, value in data.items()():
f.write(f'{key} - {value}\n')
class ParseDownlink:
def __init__(self, hexstr='', dlim=''):
from _collections import OrderedDict
self._errmsg = ''
self.compileddata = OrderedDict()
self._parse(hexstr, dlim)
@classmethod
def parse(cls, hexstr='', dlim=''):
obj = cls(hexstr, dlim)
return obj.compileddata
@classmethod
def parserecord(cls, hexstr='', logpath='[$HOME]/ss2logs/ss2beacon_parsed_[$TIMESTAMP].json', dlim=''):
obj = cls(hexstr, dlim)
if obj._errmsg == '':
obj.record(logpath)
return obj.compileddata
def display(self):
import json
print(json.dumps(self.compileddata, indent=4))
def get(self):
return self.compileddata
def record(self, logpath='[$HOME]/ss2logs/ss2beacon_parsed_[$TIMESTAMP].json'):
import json
import datetime
import os
# Replace the $HOME placeholder with the OS/user corrected home folder
logpath = logpath.replace(os.path.normcase('[$HOME]'), os.path.expanduser('~'))
# Add a timestamp to the file if the "[$TIMESTAMP]" placeholder exists
logpath = logpath.replace(os.path.normcase('[$TIMESTAMP]'), datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
# Create path directory tree if it doesn't already exist
try:
os.makedirs(os.path.split(logpath)[0], exist_ok=True)
except OSError:
pass
# Create file
with open(logpath, 'at', encoding='utf-8') as lfile:
json.dump(self.compileddata, lfile, indent=4)
lfile.write('\n')
def _parse(self, hexstr, dlim=''):
from collections import OrderedDict
import datetime
# Expected packet lengths
packetlens = {'eps': 116, 'battery': 15, 'vutrx': 28, 'ants': 4, 'stx': 22}
# Get timestamp
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S ') + datetime.datetime.now().astimezone().tzname()
# Clean input
hexstr_cleaned, errmsg = ParseDownlink._cleaninput(hexstr, dlim)
length = len(hexstr_cleaned)
if length == 0:
self._errmsg = errmsg
# Check if the downlink is contains the acknowledgement
elif ''.join(['47', '61', '74', '6f', '72', '20', '4e', '61',
'74', '69', '6f', '6e', '20', '49', '73', '20',
'45', '76', '65', '72', '79', '77', '68', '65',
'72', '65', '21', '20', '46', '72', '6f', '6d',
'20', '53', '77', '61', '6d', '70', '53', '61',
'74', '20', '49', '49']) in ''.join(hexstr_cleaned):
self.compileddata = OrderedDict()
self.compileddata['timestamp'] = timestamp
self.compileddata['msgtype'] = 0
self.compileddata['messagenum'] = 1
self.compileddata['messagetotal'] = 1
self.compileddata['message'] = 'Gator Nation Is Everywhere! From SwampSat II'
# Flight mode 1 second beacon
elif length == 163:
# Separate data packets
epslist = hexstr_cleaned[:packetlens['eps']]
batterylist = hexstr_cleaned[
packetlens['eps']:
packetlens['eps'] +
packetlens['battery']
]
vutrxlist = hexstr_cleaned[
packetlens['eps'] +
packetlens['battery']:
packetlens['eps'] +
packetlens['battery'] +
packetlens['vutrx']
]
antslist = hexstr_cleaned[packetlens['eps'] + packetlens['battery'] + packetlens['vutrx']:]
epsdata = self._eps(epslist)
batterydata = self._battery(batterylist)
vutrxdata = self._vutrx(vutrxlist)
antsdata = self._ants(antslist)
self.compileddata = OrderedDict()
self.compileddata['timestamp'] = timestamp
self.compileddata['msgtype'] = 3
self.compileddata['messagenum'] = 2
self.compileddata['messagetotal'] = 2
self.compileddata.update(epsdata)
self.compileddata.update(batterydata)
self.compileddata.update(vutrxdata)
self.compileddata.update(antsdata)
# Flight mode 2 second beacon
elif length == 185:
# Separate data packets
epslist = hexstr_cleaned[:packetlens['eps']]
batterylist = hexstr_cleaned[
packetlens['eps']:
packetlens['eps'] +
packetlens['battery']
]
vutrxlist = hexstr_cleaned[
packetlens['eps'] +
packetlens['battery']:
packetlens['eps'] +
packetlens['battery'] +
packetlens['vutrx']
]
antslist = hexstr_cleaned[
packetlens['eps'] +
packetlens['battery'] +
packetlens['vutrx']:
packetlens['eps'] +
packetlens['battery'] +
packetlens['vutrx'] +
packetlens['ants']
]
stxlist = hexstr_cleaned[
packetlens['eps'] +
packetlens['battery'] +
packetlens['vutrx'] +
packetlens['ants']:
]
epsdata = self._eps(epslist)
batterydata = self._battery(batterylist)
vutrxdata = self._vutrx(vutrxlist)
antsdata = self._ants(antslist)
stxdata = self._stx(stxlist)
self.compileddata = OrderedDict()
self.compileddata['timestamp'] = timestamp
self.compileddata['msgtype'] = 4
self.compileddata['messagenum'] = 2
self.compileddata['messagetotal'] = 2
self.compileddata.update(epsdata)
self.compileddata.update(batterydata)
self.compileddata.update(vutrxdata)
self.compileddata.update(antsdata)
self.compileddata.update(stxdata)
else:
self._errmsg = '\t\t Not a valid SS2 beacon'
if self._errmsg != '':
print(self._errmsg)
self.compileddata = OrderedDict()
return self.compileddata
@staticmethod
def _cleaninput(hstr, dlim):
# Clean input
hstr_cleaned = hstr.lower().strip().replace(' ', '').replace(dlim, '').replace('\t', '').replace('\r', '').replace('\n', '')
# Check length
if len(hstr_cleaned) == 0:
return [], '\t\t String is empty'
# Check if the string contains anything except for hex values and the delimiter
if ParseDownlink._validatehex(hstr_cleaned, dlim):
return [], '\t\t Invalid character found in string'
# Split hex string into list of bytes
return [hstr_cleaned[i:i + 2] for i in range(0, len(hstr_cleaned), 2)], ''
@staticmethod
def _validatehex(hstr, dl=''):
validhex = '0123456789abcdef' + dl.lower()
return any(c not in validhex for c in hstr) # Returns true if the str is not valid
@staticmethod
def _parsebinary(data, dtype, numbytes=1):
# Bytes are read assuming little endian
dtype = dtype.lower()
if dtype == 'uint8':
hexnum = ''.join([data[i] for i in reversed(range(1))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(1))] # Remove used elements
return int(hexnum, 16) # Convert hex to numeric
elif dtype == 'uint16':
hexnum = ''.join([data[i] for i in reversed(range(2))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(2))] # Remove used elements
return int(hexnum, 16) # Convert hex to numeric
elif dtype == 'uint32':
hexnum = ''.join([data[i] for i in reversed(range(4))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(4))] # Remove used elements
return int(hexnum, 16) # Convert hex to numeric
elif dtype == 'uint': # Allows for a specified number of bytes (defaults to 1 byte bool)
hexnum = ''.join(
[data[i] for i in reversed(range(numbytes))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(numbytes))] # Remove used elements
return int(hexnum, 16) # Convert hex to numeric
elif dtype == 'int8':
hexnum = ''.join([data[i] for i in reversed(range(1))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(1))] # Remove used elements
num = int(hexnum, 16) # Convert hex to integer
if num >= 2 ** 7: # Check for the sign
num -= 2 ** 8 # Account for the sign
return num
elif dtype == 'int16':
hexnum = ''.join([data[i] for i in reversed(range(2))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(2))] # Remove used elements
num = int(hexnum, 16) # Convert hex to integer
if num >= 2 ** 15: # Check for the sign
num -= 2 ** 16 # Account for the sign
return num
elif dtype == 'int32':
hexnum = ''.join([data[i] for i in reversed(range(4))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(4))] # Remove used elements
num = int(hexnum, 16) # Convert hex to integer
if num >= 2 ** 31: # Check for the sign
num -= 2 ** 32 # Account for the sign
return num
elif dtype == 'int': # Allows for a specified number of bytes (defaults to 1 byte bool)
hexnum = ''.join(
[data[i] for i in reversed(range(numbytes))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(numbytes))] # Remove used elements
num = int(hexnum, 16) # Convert hex to integer
if num >= 2 ** (numbytes * 8 - 1): # Check for the sign
num -= 2 ** (numbytes * 8) # Account for the sign
return num
elif dtype == 'bool8':
hexnum = ''.join([data[i] for i in reversed(range(1))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(1))] # Remove used elements
num = int(hexnum, 16) # Convert hex to numeric
nleadingzeros = numbytes * 8 + 2 - len(
bin(num)) # Count the number of leading zeros necessary to complete the list of bits
return [int(char) for char in
list('0' * nleadingzeros + bin(num)[2:])[::-1]] # Place bit flag values in a list
elif dtype == 'bool16':
hexnum = ''.join([data[i] for i in reversed(range(2))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(2))] # Remove used elements
num = int(hexnum, 16) # Convert hex to numeric
nleadingzeros = numbytes * 8 + 2 - len(
bin(num)) # Count the number of leading zeros necessary to complete the list of bits
return [int(char) for char in
list('0' * nleadingzeros + bin(num)[2:])[::-1]] # Place bit flag values in a list
elif dtype == 'bool32':
hexnum = ''.join([data[i] for i in reversed(range(4))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(4))] # Remove used elements
num = int(hexnum, 16) # Convert hex to numeric
nleadingzeros = numbytes * 8 + 2 - len(
bin(num)) # Count the number of leading zeros necessary to complete the list of bits
return [int(char) for char in
list('0' * nleadingzeros + bin(num)[2:])[::-1]] # Place bit flag values in a list
elif dtype == 'bool': # Allows for a specified number of bytes (defaults to 1 byte bool)
hexnum = ''.join(
[data[i] for i in reversed(range(numbytes))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(numbytes))] # Remove used elements
num = int(hexnum, 16) # Convert hex to numeric
nleadingzeros = numbytes * 8 + 2 - len(
bin(num)) # Count the number of leading zeros necessary to complete the list of bits
return [int(char) for char in
list('0' * nleadingzeros + bin(num)[2:])[::-1]] # Place bit flag values in a list
elif dtype == 'single':
hexnum = ''.join([data[i] for i in reversed(range(4))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(4))] # Remove used elements
return ParseDownlink.hextofloat(hexnum) # Convert hex to numeric
elif dtype == 'double':
hexnum = ''.join([data[i] for i in reversed(range(8))]) # Read and concatenate hex bytes in reverse order
[data.pop(i) for i in reversed(range(8))] # Remove used elements
return ParseDownlink.hextodouble(hexnum) # Convert hex to numeric
@staticmethod
def _eps(hexarray):
from collections import OrderedDict
ordict = OrderedDict()
ordict['eps_output_current_bcr'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 14.662757
ordict['eps_output_voltage_bcr'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.008993157
ordict['eps_output_current_12v'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.00207
ordict['eps_output_voltage_12v'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.01349
ordict['eps_output_current_bat'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005237
ordict['eps_output_voltage_bat'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.008978
ordict['eps_output_current_5v'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005237
ordict['eps_output_voltage_5v'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005865
ordict['eps_output_current_3v3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005237
ordict['eps_output_voltage_3v3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.004311
ordict['eps_temperature_motherboard'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.372434 - 273.15
ordict['eps_temperature_daughterboard'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.372434 - 273.15
ordict['eps_currentdraw_3v3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001327547
ordict['eps_currentdraw_5v'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001327547
ordict['eps_switchbus_voltage_motor'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.01349
ordict['eps_switchbus_current_motor'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_hstx'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.008993
ordict['eps_switchbus_current_hstx'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.006239
ordict['eps_switchbus_voltage_camera'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005865
ordict['eps_switchbus_current_camera'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_adac5'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005865
ordict['eps_switchbus_current_adac5'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_vlf'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.005865
ordict['eps_switchbus_current_vlf'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_ants'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.004311
ordict['eps_switchbus_current_ants'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_adac3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.004311
ordict['eps_switchbus_current_adac3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_switchbus_voltage_gps'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.004311
ordict['eps_switchbus_current_gps'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.001328
ordict['eps_bcr1_temperature_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr1_temperature_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr1_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0322581
ordict['eps_bcr1_current'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr2_temperature_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr2_temperature_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr2_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0322581
ordict['eps_bcr2_current_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr2_current_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr3_temperature_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr3_temperature_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.4963 - 273.15
ordict['eps_bcr3_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0099706
ordict['eps_bcr3_current_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr3_current_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr4_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0322581
ordict['eps_bcr4_current_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr4_current_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr6_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0322581
ordict['eps_bcr6_current_a'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
ordict['eps_bcr6_current_b'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.0009775
# 2 Byte of bit flags
bitflags_pdmstate = ParseDownlink._parsebinary(hexarray, 'bool', 2)
ordict['eps_pdmstate_vlf_12v'] = bitflags_pdmstate[1]
ordict['eps_pdmstate_stx_bat'] = bitflags_pdmstate[3]
ordict['eps_pdmstate_camera'] = bitflags_pdmstate[5]
ordict['eps_pdmstate_adac_5v'] = bitflags_pdmstate[6]
ordict['eps_pdmstate_vlf_5v'] = bitflags_pdmstate[7]
ordict['eps_pdmstate_ants'] = bitflags_pdmstate[8]
ordict['eps_pdmstate_adac_3v3'] = bitflags_pdmstate[9]
ordict['eps_pdmstate_gps_3v3'] = bitflags_pdmstate[10]
ordict['eps_reset_brownout_motherboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_brownout_daughterboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_software_motherboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_software_daughterboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_manual_motherboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_manual_daughterboard'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['eps_reset_watchdog'] = ParseDownlink._parsebinary(hexarray, 'uint16')
return ordict
@staticmethod
def _battery(hexarray):
from collections import OrderedDict
ordict = OrderedDict()
ordict['battery_voltage'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.008993
ordict['battery_current'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 14.662757 / 1000
ordict['battery_temperature_motherboard'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.372434 - 273.15
ordict['battery_temperature_daughterboard_1'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.3976 - 238.57
ordict['battery_temperature_daughterboard_2'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.3976 - 238.57
ordict['battery_temperature_daughterboard_3'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.3976 - 238.57
ordict['battery_temperature_daughterboard_4'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 0.3976 - 238.57
# 1 Byte of bit flags
bitflags_heaterstatus = ParseDownlink._parsebinary(hexarray, 'bool', 1)
ordict['battery_heaterstatus_1'] = bitflags_heaterstatus[0]
ordict['battery_heaterstatus_2'] = bitflags_heaterstatus[1]
ordict['battery_heaterstatus_3'] = bitflags_heaterstatus[2]
ordict['battery_heaterstatus_4'] = bitflags_heaterstatus[3]
return ordict
@staticmethod
def _vutrx(hexarray):
from collections import OrderedDict
def getkbits8(num, k, p):
binary = bin(num)[2:] # convert number into binary first
leadingzeros = 8 - len(binary) # Count the necessary leading zeros to fill byte
binary = '0' * leadingzeros + binary # Fill byte with leading zeros
end = 8 - p - 1
start = end - k + 1
k_bit_sub_str = binary[start: end + 1] # extract k bit sub-string
return int(k_bit_sub_str, 2) # convert extracted sub-string into decimal again
ordict = OrderedDict()
ordict['vutrx_rx_failedpackage'] = ParseDownlink._parsebinary(hexarray, 'uint8')
ordict['vutrx_rx_crcfailedpackage'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['vutrx_rx_packagecounter'] = ParseDownlink._parsebinary(hexarray, 'uint16')
frequentlock = ParseDownlink._parsebinary(hexarray, 'uint8') # Get whole register
ordict['vutrx_rx_frequentlock'] = getkbits8(frequentlock, 1, 0) # Split register by bit position
ordict['vutrx_tx_frequentlock'] = getkbits8(frequentlock, 1, 1)
ordict['vutrx_rssi'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 3 / 4096
ordict['vutrx_smps_temperature'] = ParseDownlink._parsebinary(hexarray, 'int8')
ordict['vutrx_poweramplifier_temperature'] = ParseDownlink._parsebinary(hexarray, 'int8')
ordict['vutrx_poweramplifier_power'] = ParseDownlink._parsebinary(hexarray, 'uint8')
ordict['vutrx_frequencyoffset_tx'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['vutrx_frequencyoffset_rx'] = ParseDownlink._parsebinary(hexarray, 'uint16')
dtmf = ParseDownlink._parsebinary(hexarray, 'uint8') # Get whole register
ordict['vutrx_dtmf_tone'] = getkbits8(dtmf, 4, 0) # Split register by bit position
ordict['vutrx_dtmf_counter'] = getkbits8(dtmf, 4, 4)
ordict['vutrx_current_3v3'] = ParseDownlink._parsebinary(hexarray, 'int16') * 3e-6
ordict['vutrx_current_5v'] = ParseDownlink._parsebinary(hexarray, 'int16') * 62e-6
ordict['vutrx_voltage_3v3'] = ParseDownlink._parsebinary(hexarray, 'int16') * 4e-3
ordict['vutrx_voltage_5v'] = ParseDownlink._parsebinary(hexarray, 'int16') * 4e-3
ordict['vutrx_poweramplifier_forwardpower'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 3 / 4096
ordict['vutrx_poweramplifier_reversepower'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 3 / 4096
return ordict
@staticmethod
def _stx(hexarray):
from collections import OrderedDict
def getkbits8(num, k, p):
binary = bin(num)[2:] # convert number into binary first
leadingzeros = 8 - len(binary) # Count the necessary leading zeros to fill byte
binary = '0' * leadingzeros + binary # Fill byte with leading zeros
end = 8 - p - 1
start = end - k + 1
k_bit_sub_str = binary[start: end + 1] # extract k bit sub-string
return int(k_bit_sub_str, 2) # convert extracted sub-string into decimal again
ordict = OrderedDict()
ordict['stx_voltage_battery'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 4e-3
ordict['stx_current_battery'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 40e-6
ordict['stx_voltage_poweramplifier'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 4e-3
ordict['stx_current_poweramplifier'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 40e-6
ordict['stx_temperature_top'] = getkbits8(ParseDownlink._parsebinary(hexarray, 'int16'), 12, 4) * 0.0625
ordict['stx_temperature_bottom'] = getkbits8(ParseDownlink._parsebinary(hexarray, 'int16'), 12, 4) * 0.0625
ordict['stx_temperature_poweramplifier'] = \
((ParseDownlink._parsebinary(hexarray, 'uint8') * 3 / 4096) - 0.5) * 100
ordict['stx_synth_offset'] = ParseDownlink._parsebinary(hexarray, 'uint8') * 0.5 + 2400
ordict['stx_buffer_overrun'] = ParseDownlink._parsebinary(hexarray, 'uint16')
ordict['stx_buffer_underrun'] = ParseDownlink._parsebinary(hexarray, 'uint16')
# 1 Byte of bit flags
bitflags_pastatus = ParseDownlink._parsebinary(hexarray, 'bool', 1)
ordict['stx_poweramplifier_status_frequencylock'] = bitflags_pastatus[0]
ordict['stx_poweramplifier_status_powergood'] = bitflags_pastatus[1]
ordict['stx_rf_poweroutput'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 3 * 28 / 4096 / 18
return ordict
@staticmethod
def _ants(hexarray):
from collections import OrderedDict
ordict = OrderedDict()
ordict['ants_temperature'] = ParseDownlink._parsebinary(hexarray, 'uint16') * 3.3 / 1023
# 2 Bytes of bit flags
bitflags_antsstatus = ParseDownlink._parsebinary(hexarray, 'bool', 2)
ordict['ants_status_armed'] = bitflags_antsstatus[0]
ordict['ants_status_deploymentactive_4'] = bitflags_antsstatus[1]
ordict['ants_status_stopcriteria_4'] = bitflags_antsstatus[2]
ordict['ants_status_deploymentflag_4'] = bitflags_antsstatus[3]
ordict['ants_status_independentburn'] = bitflags_antsstatus[4]
ordict['ants_status_deploymentactive_3'] = bitflags_antsstatus[5]
ordict['ants_status_stopcriteria_3'] = bitflags_antsstatus[6]
ordict['ants_status_deploymentflag_3'] = bitflags_antsstatus[7]
ordict['ants_status_ignoreswitches'] = bitflags_antsstatus[8]
ordict['ants_status_deploymentactive_2'] = bitflags_antsstatus[9]
ordict['ants_status_stopcriteria_2'] = bitflags_antsstatus[10]
ordict['ants_status_deploymentflag_2'] = bitflags_antsstatus[11]
ordict['ants_status_deploymentactive_1'] = bitflags_antsstatus[13]
ordict['ants_status_stopcriteria_1'] = bitflags_antsstatus[14]
ordict['ants_status_deploymentflag_1'] = bitflags_antsstatus[15]
return ordict
@staticmethod
def hextofloat(h, swap=False):
import string
if not isinstance(h, str):
raise TypeError
if not h.startswith('0x'):
h = '0x' + h
if not all(c in string.hexdigits.lower() for c in h[2:]):
raise ValueError
if swap:
h = '0x' + ''.join(reversed([h[2:][i:i + 2] for i in range(0, len(h[2:]), 2)]))
i = int(h[2:], 16) # Convert hex to int
b = bin(i) # Convert int to binary
b = b[2:].zfill(32) # Pad binary to fill all 32 bits
sign = 1 if int(b[0], 2) == 0 else -1
exponent = int(b[1:9], 2)
mantissa = b[9:]
mval = 1
for i in range(len(mantissa)):
if mantissa[i] == '1':
mval += 2 ** (-(i + 1))
return sign * 2 ** (exponent - 127) * mval
@staticmethod
def hextodouble(h, swap=False):
import string
if not isinstance(h, str):
raise TypeError
if not h.startswith('0x'):
h = '0x' + h
if not all(c in string.hexdigits.lower() for c in h[2:]):
raise ValueError
if swap:
h = '0x' + ''.join(reversed([h[2:][i:i + 2] for i in range(0, len(h[2:]), 2)]))
i = int(h[2:], 16) # Convert hex to int
b = bin(i) # Convert int to binary
b = b[2:].zfill(64) # Pad binary to fill all 32 bits
sign = 1 if int(b[0], 2) == 0 else -1
exponent = int(b[1:12], 2)
mantissa = b[12:]
mval = 1
for i in range(len(mantissa)):
if mantissa[i] == '1':
mval += 2 ** (-(i + 1))
return sign * 2 ** (exponent - 1023) * mval
def result_summary(res_dict,
args_dict,
TNR_target=0.05,
skip_pattern=None,
include_pattern='.*',
pvalue_record=None):
from utils.meters import simple_auc
from _collections import OrderedDict
## if not configured setup logging for external caller
if not logging.getLogger('').handlers:
setup_logging()
in_dist = args_dict['dataset']
alphas = args_dict['alphas']
logging.info(f'Report for {args_dict["model"]} - {in_dist}')
logging.info(f'Tag: {args_dict["tag"]}')
result_dict = OrderedDict(
model=args_dict["model"],
in_dist=args_dict['dataset'],
LDA=args_dict.get('LDA'),
joint=args_dict['measure_joint_distribution'],
tag=args_dict['tag'],
channles_sellect=args_dict.get('channel_selection_fn'))
# read indist results to calibrate alpha value for target TNR
rows = []
accuracies = {'model': {}}
for reduction_name, reduction_metrics in res_dict[in_dist].items():
accuracies[reduction_name] = {}
if reduction_name.endswith('_acc'):
acc = reduction_metrics.mean.cpu().numpy()
std = reduction_metrics.std.cpu().numpy()
acc_name = reduction_name.replace('_acc', '')
if acc_name == 'model':
reduction_name = 'model'
if acc_name.endswith('rescaled-smx'):
reduction_name = acc_name[:-13]
acc_name = 'model_rescaled_smx'
elif acc_name.endswith('-pval'):
reduction_name = acc_name[:-5]
acc_name = 'pval'
accuracies[reduction_name][f'{acc_name}_t1'] = acc[0]
accuracies[reduction_name][f'{acc_name}_t5'] = acc[1]
accuracies[reduction_name][f'{acc_name}_std_t1'] = std[0]
for reduction_name, reduction_metrics in res_dict[in_dist].items():
if skip_pattern and bool(re.match(
skip_pattern, reduction_name)) or include_pattern and not bool(
re.match(include_pattern, reduction_name)):
continue
result_dict['reduction'] = reduction_name
result_dict.update(**accuracies['model'])
result_dict.update(**accuracies[reduction_name])
logging.info(reduction_name)
if type(reduction_metrics) != dict:
# report simple metric
logging.info(
f'\t{reduction_metrics.mean}\t({reduction_metrics.std})')
continue
# report reduction specific metrics
for metric_name, meter_object in reduction_metrics.items():
metric_stats = MeterDict()
if not metric_name.endswith('_roc'):
logging.info(
f'\t{metric_name}: {meter_object.mean.numpy():0.3}')
continue
FPR = meter_object.mean.numpy()
calibrated_alpha_id = min((FPR < TNR_target).sum() - 1, len(FPR))
if calibrated_alpha_id == -1:
# all pvalues are larger than alpha
fpr_under_target_alpha = meter_object.mean[0]
interp_alpha = FPR[0]
calibrated_alpha_id = 0
else:
fpr_under_target_alpha = FPR[calibrated_alpha_id]
# actual rejection threshold to use for TNR 95%
interp_alpha = np.interp(0.05, FPR.squeeze(), alphas)
result_dict.update(
dict(metric_name=metric_name,
FPR_strict=fpr_under_target_alpha,
FPR_over=FPR[calibrated_alpha_id + 1],
chosen_alpha=interp_alpha))
logging.info(
f'\t{metric_name} - in-dist rejected: '
# f'alpha-{indist_pvalues_roc[alphas.index(TNR_target)]:0.3f} ({TNR_target:0.3f}), '
f'under-{fpr_under_target_alpha:0.3f} ({alphas[calibrated_alpha_id]:0.3f}), '
f'interp-{TNR_target:0.3f} ({interp_alpha:0.3f}), '
f'over-{FPR[calibrated_alpha_id + 1]:0.3f} ({alphas[calibrated_alpha_id + 1]})'
)
if pvalue_record and reduction_name in pvalue_record[in_dist]:
if metric_name.startswith(
'class_cond'
) and 'predicted_id' in pvalue_record[in_dist]:
predicted_ids = pvalue_record[in_dist]['predicted_id']
in_cc_pval_pred = pvalue_record[in_dist][reduction_name][
th.arange(predicted_ids.shape[0]), predicted_ids]
else:
in_cc_pval_pred = pvalue_record[in_dist][
reduction_name].max(1)[0]
for target_dataset_name, reduction_metrics in res_dict.items():
if target_dataset_name != in_dist and metric_name in reduction_metrics[
reduction_name]:
interp_rejected = np.interp(
interp_alpha, alphas, reduction_metrics[reduction_name]
[metric_name].mean.numpy())
TPR = reduction_metrics[reduction_name][
metric_name].mean.numpy()
raw_rejected = TPR[alphas.index(TNR_target)]
auroc = simple_auc(TPR, FPR)
logging.info(
f'\t\t{target_dataset_name}:\traw-{raw_rejected:0.3f}\tinterp-{interp_rejected:0.3f}\tAUROC:{auroc:0.3f}'
)
if pvalue_record and reduction_name in pvalue_record[
target_dataset_name]:
if metric_name.startswith(
'class_cond'
) and 'predicted_id' in pvalue_record[
target_dataset_name]:
predicted_ids = pvalue_record[target_dataset_name][
'predicted_id']
out_cc_pval_pred = pvalue_record[
target_dataset_name][reduction_name][
th.arange(predicted_ids.shape[0]),
predicted_ids]
else:
out_cc_pval_pred = pvalue_record[
target_dataset_name][reduction_name].max(1)[0]
m = metric(in_cc_pval_pred.numpy(),
out_cc_pval_pred.numpy())
logging.info(f'\t\t\tbenchmark metrics: {m}')
result_dict.update(**m)
result_dict.update(
dict(out_dist=target_dataset_name,
TPR95_raw=raw_rejected,
TPR95_interp=interp_rejected,
AUROC=auroc))
rows.append(result_dict.copy())
if in_dist.startswith(
'cifar') and target_dataset_name.startswith(
'cifar'):
continue
metric_stats.update(
dict(TPR95_raw=th.tensor([raw_rejected]),
TPR95_interp=th.tensor([interp_rejected]),
AUROC=th.tensor([auroc])))
if target_dataset_name != in_dist and metric_name in reduction_metrics[
reduction_name]:
result_dict['out_dist'] = 'avg'
logging.info(
f'\tmetric avg stats: {[k + " " + str(float(v)) for k, v in metric_stats.get_mean_dict().items()]}'
)
result_dict.update(**metric_stats.get_mean_dict())
rows.append(result_dict.copy())
return rows
