def testArgDefn():
name = "SEQ_ENABLE_DISABLE"
defn = cmd.CmdDict(CMDDICT_TEST)[name]
arg = defn.argdefns[0]
assert arg.bytes == [0, 1]
assert arg.desc == None
assert arg.enum == None
assert arg.fixed == False
assert arg.name == "sequence_id"
assert arg.nbytes == 2
assert arg.range == None
assert arg.slice() == slice(0, 2)
assert arg.type == dtype.get("MSB_U16")
assert arg.units == None
assert arg.value == None
assert type(repr(arg)) is str
arg = defn.argdefns[1]
assert arg.bytes == 2
assert arg.desc == None
assert arg.enum == {"DISABLED": 0, "ENABLED": 1}
assert arg.fixed == False
assert arg.name == "enable"
assert arg.nbytes == 1
assert arg.range == None
assert arg.slice() == slice(2, 3)
assert arg.type == dtype.get("U8")
assert arg.units == None
assert arg.value == None
assert type(repr(arg)) is str
def testString():
dt = dtype.get("S16")
assert isinstance(dt, dtype.PrimitiveType)
assert dt.string
assert not dt.float
assert not dt.signed
assert dt.name == "S16"
assert dt.nbytes == 16
assert dt.nbits == (16 * 8)
dt = dtype.get("S32")
assert isinstance(dt, dtype.PrimitiveType)
assert dt.string
assert not dt.float
assert not dt.signed
assert dt.name == "S32"
assert dt.nbytes == 32
assert dt.nbits == (32 * 8)
ival = 1
errmsgs = []
assert not dt.validate(ival, errmsgs)
assert errmsgs
fval = 1.1
errmsgs = []
assert not dt.validate(fval, errmsgs)
assert errmsgs
sval = "1"
errmsgs = []
assert dt.validate(sval)
assert not errmsgs
def testArgDefn():
name = 'SEQ_ENABLE_DISABLE'
defn = cmd.CmdDict(CMDDICT_TEST)[name]
arg = defn.argdefns[0]
assert arg.bytes == [0, 1]
assert arg.desc == None
assert arg.enum == None
assert arg.fixed == False
assert arg.name == 'sequence_id'
assert arg.nbytes == 2
assert arg.range == None
assert arg.slice() == slice(0, 2)
assert arg.type == dtype.get('MSB_U16')
assert arg.units == None
assert arg.value == None
assert type(repr(arg)) is str
arg = defn.argdefns[1]
assert arg.bytes == 2
assert arg.desc == None
assert arg.enum == {'DISABLED': 0, 'ENABLED': 1}
assert arg.fixed == False
assert arg.name == 'enable'
assert arg.nbytes == 1
assert arg.range == None
assert arg.slice() == slice(2, 3)
assert arg.type == dtype.get('U8')
assert arg.units == None
assert arg.value == None
assert type(repr(arg)) is str
def testLSB_D64():
val = 1.2
bytes = struct.pack("<d", val)
LSB_D64 = dtype.get("LSB_D64")
assert fpeq(LSB_D64.decode(bytes), val)
assert fpeq(LSB_D64.decode(bytes, raw=True), val)
def testLSB_D64():
val = 1.2
bytes = struct.pack('<d', val)
LSB_D64 = dtype.get('LSB_D64')
assert fpeq(LSB_D64.decode(bytes), val)
assert fpeq(LSB_D64.decode(bytes, raw=True), val)
def testMSB_D64():
val = 3.4
bytes = struct.pack('>d', val)
MSB_D64 = dtype.get('MSB_D64')
assert fpeq(MSB_D64.decode(bytes), val)
assert fpeq(MSB_D64.decode(bytes, raw=True), val)
def testLSB_F32():
val = 5.6
bytes = struct.pack('<f', val)
LSB_F32 = dtype.get('LSB_F32')
assert fpeq(LSB_F32.decode(bytes), val)
assert fpeq(LSB_F32.decode(bytes, raw=True), val)
def testMSB_F32():
val = 7.8
bytes = struct.pack('>f', val)
MSB_F32 = dtype.get('MSB_F32')
assert fpeq(MSB_F32.decode(bytes), val)
assert fpeq(MSB_F32.decode(bytes, raw=True), val)
def testLSB_F32():
val = 5.6
bytes = struct.pack("<f", val)
LSB_F32 = dtype.get("LSB_F32")
assert fpeq(LSB_F32.decode(bytes), val)
assert fpeq(LSB_F32.decode(bytes, raw=True), val)
def testMSB_D64():
val = 3.4
bytes = struct.pack(">d", val)
MSB_D64 = dtype.get("MSB_D64")
assert fpeq(MSB_D64.decode(bytes), val)
assert fpeq(MSB_D64.decode(bytes, raw=True), val)
def testMSB_F32():
val = 7.8
bytes = struct.pack(">f", val)
MSB_F32 = dtype.get("MSB_F32")
enc_ba = MSB_F32.encode(val)
assert enc_ba
assert fpeq(MSB_F32.decode(bytes), val)
assert fpeq(MSB_F32.decode(bytes, raw=True), val)
def testget():
assert isinstance(dtype.get("U8"), dtype.PrimitiveType)
assert isinstance(dtype.get("S40"), dtype.PrimitiveType)
assert isinstance(dtype.get("TIME32"), dtype.Time32Type)
assert dtype.get('LSB_U32[10]') == dtype.ArrayType('LSB_U32', 10)
with nose.tools.assert_raises(ValueError):
dtype.get('U8["foo"]')
with nose.tools.assert_raises(ValueError):
dtype.get('U8[-42]')
def testget():
assert isinstance(dtype.get("U8"), dtype.PrimitiveType)
assert isinstance(dtype.get("S40"), dtype.PrimitiveType)
assert isinstance(dtype.get("TIME32"), dtype.Time32Type)
assert dtype.get("LSB_U32[10]") == dtype.ArrayType("LSB_U32", 10)
with pytest.raises(ValueError):
dtype.get('U8["foo"]')
with pytest.raises(ValueError):
dtype.get("U8[-42]")
def handle():
"""Set playback query with packet name, start time, and end time from form"""
global playback
if not playback.enabled:
return HttpResponse(status=404,
body='Historic data playback is disabled')
tlm_dict = tlm.getDefaultDict()
# Get values from form
packet = bottle.request.forms.get('packet')
start_time = bottle.request.forms.get('startTime')
end_time = bottle.request.forms.get('endTime')
uid = tlm_dict[packet].uid
# Query packet and time range from database
point_query = 'SELECT * FROM "{}" WHERE time >= \'{}\' AND time <= \'{}\''.format(
packet, start_time, end_time)
points = list(playback.dbconn.query(point_query).get_points())
pkt = tlm_dict[packet]
fields = pkt.fields
# Build field names list from tlm dictionary for sorting data query
field_names = []
# Build field types list from tlm dictionary for packing data
field_formats = []
for i in range(len(fields)):
field_names.append(fields[i].name)
field_type = str(fields[i].type).split("'")[1]
field_formats.append(dtype.get(field_type).format)
# Put query into a map of {timestamp: list of (uid, data)}
for i in range(len(points)):
# Round time down to nearest 0.1 second
timestamp = str(points[i]['time'][:21] + 'Z')
data = ''
for j in range(len(field_names)):
data += struct.pack(field_formats[j], points[i][field_names[j]])
if playback.query.has_key(timestamp):
playback.query[timestamp].append((uid, data))
else:
playback.query[timestamp] = [(uid, data)]
def encode(self, value):
"""Encode a columns value according to its data type
Arguments:
value: The value to encode provided as either a string or
the appropriate type for the column's data type.
"""
dt = dtype.get(self.type)
if self._enum_rev is not None:
value = self._enum_rev.get(value, value)
value = self._parse_column_value_from_string(value)
# For some reason ArrayType.encode expects to receive the values
# for encoding in *args instead of, you know, an iterable ...
if isinstance(dt, dtype.ArrayType):
# `value` needs to be an unpackable iterable or this is going
# to explode. More than likely this'll be a bytearray.
return dt.encode(*value)
else:
return dt.encode(value)
def _parse_column_value_from_string(self, value):
"""Parse strings into an appropriate type for a given table column
Attempt to cast a string value into the appropriate data type for a
column to use during encoding. If the column's type is a "Primitive"
type as defined in ait.core.dtype then we simple cast to float or int
depending on the definition. ArrayType values must be passed as a
string which can be encoded into a binary string. All other AIT
data types are not supported in the table module.
Arguments:
value: The string from which to extract an appropriate data type
value for future processing. If this is not a string it is
returned without modification.
"""
if not isinstance(value, str):
return value
col_defn = dtype.get(self.type)
if isinstance(col_defn, dtype.ArrayType):
# Value is expected to be a string which can be encoded to a
# binary string for use by a bytes-like object.
#
# E.g., for a type of U8[2] the following is valid:
# value = '\x01\x02'
return bytearray(value.encode("ascii"))
elif (isinstance(col_defn, dtype.Time64Type)
or isinstance(col_defn, dtype.Time40Type)
or isinstance(col_defn, dtype.Time32Type)):
return datetime.datetime.strptime(value, dmc.RFC3339_Format)
else:
if col_defn.float:
return float(value)
else:
return int(value, base=0)
def decode(self, in_stream, raw=False):
"""Decode a column's value according to its data type
Read bytes equal to this column's data type from the input stream
and decode it into a value per that data type's definition.
Arguments:
in_stream: A file-like object from which to read data.
raw: Flag denoting whether raw values or enumerate values
(if present for this column) should be returned.
Raises:
EOFError: If the number of bytes read from the input stream
is less than the length of the data type.
"""
val = None
dt = dtype.get(self.type)
if dt is not None:
data = in_stream.read(dt.nbytes)
if len(data) != dt.nbytes:
raise EOFError
if (isinstance(dt, dtype.Time64Type)
or isinstance(dt, dtype.Time40Type)
or isinstance(dt, dtype.Time32Type)):
val = dt.decode(data).strftime(dmc.RFC3339_Format)
else:
val = dt.decode(data, raw=True)
if self.enum and not raw:
val = self.enum.get(val, val)
return val
def type(self, value):
from ait.core import dtype
self._type = dtype.get(value) if type(value) is str else value
def toBinary(self, tabfile, stream, fswbin_f, verbose, version):
#print "self.name: "+self.name
#print "stream name: "+stream.name
size = os.path.getsize(stream.name)
#print "stream len: " + str(size)
#print "self.size: " + str(self.size)
no_lines = 0
for line in stream:
no_lines += 1
stream.seek(0)
fsw_header = bytearray(32)
# sha1 = hash_file(tabfile)
sha1 = 0
# Write magic number
fswbin_f.write( struct.pack('>H', self.MagicNumber ) )
# Write upload type
fswbin_f.write( struct.pack('B', self.uptype ) )
# Write version
fswbin_f.write( struct.pack('B', int(version,16)&255 ) )
# Write number of lines
if self.name == "memory":
fswbin_f.write( struct.pack('>H', 0 ) )
else:
fswbin_f.write( struct.pack('>H', no_lines ) )
# Write ID (0)
fswbin_f.write( struct.pack('>H', 0) )
# # Write CRC placeholder
fswbin_f.write( struct.pack('>I', 0) )
# SHA as 0
pad = struct.pack('B', 0)
for n in range(20):
fswbin_f.write(pad)
# data = bytearray(20)
# i = 0
# tmpbytes = list(sha1)
# for x in range(0, len(sha1)/2):
# tmp = ((int(tmpbytes[x],16)&255)<<4) + (int(tmpbytes[x+1],16)&255)
# #print "tmp: "+ str(tmp)
# data[i] = tmp&0xFF
# i += 1
# fswbin_f.write(data)
for line in stream:
#print "line: "+line
idx = 0
line = line.replace("\n","")
allcols = line.split(self.delimiter)
if self.name == "memory":
for val in allcols:
if val != "":
#print "val: "+val
data = bytearray(2)
tmpbytes = list(val)
data[0] = ((int(tmpbytes[0],16)&0xF)<<4) + (int(tmpbytes[1],16)&0xF)
data[1] = ((int(tmpbytes[2],16)&0xF)<<4) + (int(tmpbytes[3],16)&0xF)
#print "tmp byte1: "+ str(int(tmpbytes[0],16)&0xF)
#print "tmp byte2: "+ str(int(tmpbytes[1],16)&0xF)
#print "tmp byte3: "+ str(int(tmpbytes[2],16)&0xF)
#print "tmp byte4: "+ str(int(tmpbytes[3],16)&0xF)
fswbin_f.write(data)
else:
idx = 0
#this is how to step into table definitions
for coldef in enumerate(self.coldefns):
# print "column definition: " + str(coldef[1])
fswcoldefn = coldef[1]
name = fswcoldefn.name
#print "name: " + name
colpk = dtype.get(fswcoldefn.type)
#print "packing: " + str(colpk)
units = fswcoldefn.units
#print "units: " + units
enum = fswcoldefn.enum
if isinstance(fswcoldefn.bytes,list):
nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1
else:
nobytes = 1
# print "bytes: " + str(fswcoldefn.bytes)
# print "nobytes: " + str(nobytes)
if name == 'RESERVED':
#add reserved bytes
fswbin_f.write(colpk.encode(0))
if coldef[0] != len(self.coldefns)-1:
continue
else:
break
items = fswcoldefn.items
if items is not None:
#print "items: " + str(items)
for i in range(items):
val = allcols[i]
#print "item col val: "+str(val)
if units != 'none':
val = val.strip()
val = val.split(" ")[0]
else:
val = val.replace(" ","")
#print "item col val: "+str(val)
fswbin_f.write(colpk.encode(self.convertValue(val)))
else:
val = allcols[idx]
val = val.replace("\n","")
#print "else col val 1: "+str(val)
if enum is not None:
if enum is not None:
for enumkey in enumerate(enum.keys()):
enumval = enum[enumkey[1]]
#print "enumkey: " + str(enumkey[1]) + ", enumval: " + str(enum[enumkey[1]])
if enumval == val:
val = str(enumkey[1])
#print "XXXX colpk.type: "+colpk.format
fswbin_f.write(colpk.encode(self.convertValue(val)))
else:
#print "XXXX colpk.type: "+colpk.format
#print "fswcoldefn.bytes: "+str(fswcoldefn.bytes)
#print "xxxx val: "+str(val)
#print "units: "+str(units)
if units != 'none':
val = val.strip()
val = val.split(" ")[0]
#print "else col val 2a: "+str(val)
fswbin_f.write(colpk.encode(self.convertValue(val)))
#fswbin_f.write(colpk.encode(float(val)))
elif str(colpk) == "PrimitiveType('U8')" and nobytes>1:
strval = ""
for c in list(val):
#print "xxx c: "+str(c)
tmp = (int(c,16))&255
#print "tmp: "+str(tmp)
fswbin_f.write(colpk.encode(tmp))
else:
val = val.replace(" ","")
#print "else col val 2b: "+str(val)
fswbin_f.write(colpk.encode(self.convertValue(val)))
#fswbin_f.write(colpk.encode(float(val)))
idx += 1
written = fswbin_f.tell()
#print "written: "+str(written)
# print str(self.size) + ", " + str(written)
if self.size > written:
padding = bytearray(self.size - (written))
fswbin_f.write(padding)
#Now calculate and update CRC field in the FSW header
fswbin_f.close()
fname = fswbin_f.name
crc32 = util.crc32File(fname, 0)
fswbin_f = open(fname, 'r+b')
fswbin_f.seek(28)
crcbuf = bytearray(4)
crcbuf[0:4] = struct.pack('>L',crc32)
fswbin_f.write(crcbuf)
#version = "0"
if verbose is not None and verbose != 0:
log.info("CRC: %x" % crc32)
print "MAGIC_NUMBER: %x" % self.MagicNumber
print "UPLOAD_TYPE: " + str(self.uptype)
print "VERSION: " + str(version)
print "NUMBER_ENTRIES: " + str(no_lines)
# print "SHA-1: "+sha1
#print "fname: "+fname+", crc32: "+str(crc32)
# Once we are done appending all the columns to the row
# strip off last comma and append a \r
# Note: Since it is Access, \n alone does not work
return
def toText(self, stream, fswtab_f, verbose, version):
out = ""
size = os.path.getsize(stream.name)
#print "table name: " + self.name
#print "stream len: " + str(size)
noentries = 0
if self.name != "memory":
for fswheaderdef in enumerate(self.fswheaderdefns):
#print "header definition: " + str(fswheaderdef[1])
fswcoldefn = fswheaderdef[1]
name = fswcoldefn.name
#print "name: " + name
colfmt = str(fswcoldefn.format)
#print "format: " + colfmt
colpk = dtype.get(fswcoldefn.type)
#print "packing: " + str(colpk)
coltype = fswcoldefn.type
#print "type: " + coltype
if isinstance(fswcoldefn.bytes,list):
nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1
else:
nobytes = 1
#print "bytes: " + str(fswcoldefn.bytes)
#print "nobytes: " + str(nobytes)
strval = ""
if str(colpk) == "PrimitiveType('U8')" and nobytes>1:
strval = ""
for i in range(nobytes):
value = colpk.decode(stream.read(1))
strval += str(colfmt % value)
#print(colfmt % value)
else:
value = colpk.decode(stream.read(nobytes))
#print(colfmt % value)
if (str(colpk) != "PrimitiveType('U8')") or (str(colpk) == "PrimitiveType('U8')" and nobytes == 1):
strval = str(colfmt % value)
#print "strval: " + strval
if name == "NUMBER_ENTRIES":
noentries = strval
#print "noentries: " + strval
if self.name != "keep_out_zones" and self.name != "line_of_sight":
# Append the value to table row
out += name+': %s\n' % strval
if verbose is not None and verbose != 0:
print
print out
#fswtab_f.write(out)
size = size - 32
out = ""
if self.name.startswith("log_"):
norows = self.rows
#print "norows: " + str(norows)
else:
rowbytes = 0
items = None
for coldef in enumerate(self.coldefns):
fswcoldefn = coldef[1]
items = fswcoldefn.items
if isinstance(fswcoldefn.bytes,list):
nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1
else:
nobytes = 1
rowbytes = rowbytes + nobytes
#print "Row bytes: " + str(rowbytes)
if items is not None:
rowbytes = rowbytes * items
norows = size / rowbytes
else:
norows = int(noentries)
#print "norows: " + str(norows)
#print "items: " + str(items)
if norows == 0:
idx = 1
for i in range(size):
byte = stream.read(1)
value = binascii.hexlify(byte)
fswtab_f.write(value)
if (idx%2) == 0:
fswtab_f.write(" ")
if (idx%16) == 0:
fswtab_f.write("\n")
idx += 1
return
for i in range(norows):
condition = None
#this is how to step into table definitions
for coldef in enumerate(self.coldefns):
#print "column definition: " + str(coldef[1])
fswcoldefn = coldef[1]
name = fswcoldefn.name
#print "name: " + name
colfmt = str(fswcoldefn.format)
#print "format: " + colfmt
colpk = dtype.get(fswcoldefn.type)
#print "packing: " + str(colpk)
coltype = fswcoldefn.type
#print "type: " + coltype
if isinstance(fswcoldefn.bytes,list):
nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1
else:
nobytes = 1
#print "bytes: " + str(fswcoldefn.bytes)
#print "nobytes: " + str(nobytes)
units = fswcoldefn.units
#print "units: " + units
enum = fswcoldefn.enum
items = fswcoldefn.items
if items is not None:
#print "items: " + str(items)
for i in range(items):
value = colpk.decode(stream.read(nobytes))
strval = str(colfmt % value)
out += strval + self.delimiter
else:
strval = ""
if str(colpk) == "PrimitiveType('U8')" and nobytes>1:
strval = ""
for i in range(nobytes):
value = colpk.decode(stream.read(1))
if name == "RESERVED":
continue
strval += str(colfmt % value)
#print(colfmt % value)
else:
value = colpk.decode(stream.read(nobytes))
#print(colfmt % value)
if enum is not None:
if enum is not None:
for enumkey in enumerate(enum.keys()):
#print "enumkey: " + str(enumkey[1]) + ", enumval: " + str(enum[enumkey[1]])
if enumkey[1] == value:
strval = str(enum[enumkey[1]])
else:
if units != 'none':
strval = str(colfmt % value) + " " + units
#print "units: " + units
else:
if (str(colpk) != "PrimitiveType('U8')") or (str(colpk) == "PrimitiveType('U8')" and nobytes == 1):
strval = str(colfmt % value)
if self.name == "response" and "CONSTANT" in name and condition > 6:
strval = str('%d' % value)
#print "strval: " + strval
if self.name == "response" and name == "CONDITION_TYPE":
#print "value: "+str(value)
condition = value
#print "condition: "+str(condition)
# Append the value to table row
if name == "RESERVED":
continue
out += strval + self.delimiter
out = out[:-1] + "\n"
#print
#print out
fswtab_f.write(out)
# Once we are done appending all the columns to the row
# strip off last comma and append a \r
# Note: Since it is Access, \n alone does not work
return
def type(self, value):
if type(value) is str and dtype.get(value) is not None:
self._type = dtype.get(value)
else:
self._type = value
log.error("Invalid field type '%s' " % value)
def get_historical_tlm_for_packet_fields(self, ait_pkt_id, ait_field_names,
start_millis, end_millis):
"""
Perform a historical query for a particular AIT packet type
:param ait_pkt_id: AIT Packet definition Id
:param ait_field_names: List of field names to include, use None to include all fields
:param start_millis: Start time, milliseconds since UNIX epoch
:param end_millis: End time, milliseconds since UNIX epoch
:return: List of OpenMct measurements that satisfy query
"""
if not self._database and False:
return None
result_list = []
ait_pkt_def = self._aitTlmDict[ait_pkt_id]
ait_field_defs = ait_pkt_def.fields
# Build field names list from tlm dictionary for sorting data query
field_names = []
# Build field types list from tlm dictionary for packing data
field_formats = []
# Collect the field type information (prolly dont need dtype)
for i in range(len(ait_field_defs)):
field_def = ait_field_defs[i]
# if no request-list or current field is in request list
if (not ait_field_names) or (field_def.name in ait_field_names):
field_names.append(field_def.name)
field_type = str(field_def.type).split("'")[1]
field_formats.append(dtype.get(field_type).format)
# A list with single entry of pkt id
packet_ids = [ait_pkt_id]
# Convert unix timestamp to UTC datetime for time range
start_timestamp_secs = start_millis / 1000.0
start_date = datetime.datetime.fromtimestamp(start_timestamp_secs,
tz=datetime.timezone.utc)
end_timestamp_secs = end_millis / 1000.0
end_date = datetime.datetime.fromtimestamp(end_timestamp_secs,
tz=datetime.timezone.utc)
query_args_str = f"Packets = {packet_ids}; Start = {start_date};" \
f" End = {end_date}"
self.dbg_message(f"Query args : {query_args_str}")
# default response is empty
res_pkts = list()
# Query packet and time range from database
try:
if self._database:
ait_db_result = self._database.query_packets(
packets=packet_ids,
start_time=start_date,
end_time=end_date,
yield_packet_time=True,
)
if ait_db_result.errors is not None:
log.error("[OpenMCT] Database query for packets " +
str(packet_ids) + " resulted in errors: ")
for db_err in ait_db_result.errors:
log.error("[OpenMCT] Error: " + str(db_err))
elif ait_db_result.has_packets:
res_pkts = list(ait_db_result.get_packets())
# Debug result size
self.dbg_message(f"Number of results for query "
f"{query_args_str} : {len(res_pkts)}")
except Exception as e:
log.error("[OpenMCT] Database query failed. Error: " + str(e))
return None
for cur_pkt_time, cur_pkt in res_pkts:
# Convert datetime to Javascript timestamp (in milliseconds)
cur_timestamp_sec = datetime.datetime.timestamp(cur_pkt_time)
unix_timestamp_msec = int(cur_timestamp_sec) * 1000
# Add a record for each requested field for this timestamp
for cur_field_name in field_names:
record = {"timestamp": unix_timestamp_msec}
record["id"] = DictUtils.create_mct_pkt_id(
ait_pkt_id, cur_field_name)
record["value"] = getattr(cur_pkt, cur_field_name)
result_list.append(record)
return result_list
def testgetdtype():
dt = dtype.get("TIME32")
assert isinstance(dt, dtype.Time32Type)
assert dt.name == "TIME32"
assert dt.pdt == "MSB_U32"
assert dt.max == 4294967295
