def testArgDefn(): name = "SEQ_ENABLE_DISABLE" defn = cmd.CmdDict(CMDDICT_TEST)[name] arg = defn.argdefns[0] assert arg.bytes == [0, 1] assert arg.desc == None assert arg.enum == None assert arg.fixed == False assert arg.name == "sequence_id" assert arg.nbytes == 2 assert arg.range == None assert arg.slice() == slice(0, 2) assert arg.type == dtype.get("MSB_U16") assert arg.units == None assert arg.value == None assert type(repr(arg)) is str arg = defn.argdefns[1] assert arg.bytes == 2 assert arg.desc == None assert arg.enum == {"DISABLED": 0, "ENABLED": 1} assert arg.fixed == False assert arg.name == "enable" assert arg.nbytes == 1 assert arg.range == None assert arg.slice() == slice(2, 3) assert arg.type == dtype.get("U8") assert arg.units == None assert arg.value == None assert type(repr(arg)) is str
def testString(): dt = dtype.get("S16") assert isinstance(dt, dtype.PrimitiveType) assert dt.string assert not dt.float assert not dt.signed assert dt.name == "S16" assert dt.nbytes == 16 assert dt.nbits == (16 * 8) dt = dtype.get("S32") assert isinstance(dt, dtype.PrimitiveType) assert dt.string assert not dt.float assert not dt.signed assert dt.name == "S32" assert dt.nbytes == 32 assert dt.nbits == (32 * 8) ival = 1 errmsgs = [] assert not dt.validate(ival, errmsgs) assert errmsgs fval = 1.1 errmsgs = [] assert not dt.validate(fval, errmsgs) assert errmsgs sval = "1" errmsgs = [] assert dt.validate(sval) assert not errmsgs
def testArgDefn(): name = 'SEQ_ENABLE_DISABLE' defn = cmd.CmdDict(CMDDICT_TEST)[name] arg = defn.argdefns[0] assert arg.bytes == [0, 1] assert arg.desc == None assert arg.enum == None assert arg.fixed == False assert arg.name == 'sequence_id' assert arg.nbytes == 2 assert arg.range == None assert arg.slice() == slice(0, 2) assert arg.type == dtype.get('MSB_U16') assert arg.units == None assert arg.value == None assert type(repr(arg)) is str arg = defn.argdefns[1] assert arg.bytes == 2 assert arg.desc == None assert arg.enum == {'DISABLED': 0, 'ENABLED': 1} assert arg.fixed == False assert arg.name == 'enable' assert arg.nbytes == 1 assert arg.range == None assert arg.slice() == slice(2, 3) assert arg.type == dtype.get('U8') assert arg.units == None assert arg.value == None assert type(repr(arg)) is str
def testLSB_D64(): val = 1.2 bytes = struct.pack("<d", val) LSB_D64 = dtype.get("LSB_D64") assert fpeq(LSB_D64.decode(bytes), val) assert fpeq(LSB_D64.decode(bytes, raw=True), val)
def testLSB_D64(): val = 1.2 bytes = struct.pack('<d', val) LSB_D64 = dtype.get('LSB_D64') assert fpeq(LSB_D64.decode(bytes), val) assert fpeq(LSB_D64.decode(bytes, raw=True), val)
def testMSB_D64(): val = 3.4 bytes = struct.pack('>d', val) MSB_D64 = dtype.get('MSB_D64') assert fpeq(MSB_D64.decode(bytes), val) assert fpeq(MSB_D64.decode(bytes, raw=True), val)
def testLSB_F32(): val = 5.6 bytes = struct.pack('<f', val) LSB_F32 = dtype.get('LSB_F32') assert fpeq(LSB_F32.decode(bytes), val) assert fpeq(LSB_F32.decode(bytes, raw=True), val)
def testMSB_F32(): val = 7.8 bytes = struct.pack('>f', val) MSB_F32 = dtype.get('MSB_F32') assert fpeq(MSB_F32.decode(bytes), val) assert fpeq(MSB_F32.decode(bytes, raw=True), val)
def testLSB_F32(): val = 5.6 bytes = struct.pack("<f", val) LSB_F32 = dtype.get("LSB_F32") assert fpeq(LSB_F32.decode(bytes), val) assert fpeq(LSB_F32.decode(bytes, raw=True), val)
def testMSB_D64(): val = 3.4 bytes = struct.pack(">d", val) MSB_D64 = dtype.get("MSB_D64") assert fpeq(MSB_D64.decode(bytes), val) assert fpeq(MSB_D64.decode(bytes, raw=True), val)
def testMSB_F32(): val = 7.8 bytes = struct.pack(">f", val) MSB_F32 = dtype.get("MSB_F32") enc_ba = MSB_F32.encode(val) assert enc_ba assert fpeq(MSB_F32.decode(bytes), val) assert fpeq(MSB_F32.decode(bytes, raw=True), val)
def testget(): assert isinstance(dtype.get("U8"), dtype.PrimitiveType) assert isinstance(dtype.get("S40"), dtype.PrimitiveType) assert isinstance(dtype.get("TIME32"), dtype.Time32Type) assert dtype.get('LSB_U32[10]') == dtype.ArrayType('LSB_U32', 10) with nose.tools.assert_raises(ValueError): dtype.get('U8["foo"]') with nose.tools.assert_raises(ValueError): dtype.get('U8[-42]')
def testget(): assert isinstance(dtype.get("U8"), dtype.PrimitiveType) assert isinstance(dtype.get("S40"), dtype.PrimitiveType) assert isinstance(dtype.get("TIME32"), dtype.Time32Type) assert dtype.get("LSB_U32[10]") == dtype.ArrayType("LSB_U32", 10) with pytest.raises(ValueError): dtype.get('U8["foo"]') with pytest.raises(ValueError): dtype.get("U8[-42]")
def handle(): """Set playback query with packet name, start time, and end time from form""" global playback if not playback.enabled: return HttpResponse(status=404, body='Historic data playback is disabled') tlm_dict = tlm.getDefaultDict() # Get values from form packet = bottle.request.forms.get('packet') start_time = bottle.request.forms.get('startTime') end_time = bottle.request.forms.get('endTime') uid = tlm_dict[packet].uid # Query packet and time range from database point_query = 'SELECT * FROM "{}" WHERE time >= \'{}\' AND time <= \'{}\''.format( packet, start_time, end_time) points = list(playback.dbconn.query(point_query).get_points()) pkt = tlm_dict[packet] fields = pkt.fields # Build field names list from tlm dictionary for sorting data query field_names = [] # Build field types list from tlm dictionary for packing data field_formats = [] for i in range(len(fields)): field_names.append(fields[i].name) field_type = str(fields[i].type).split("'")[1] field_formats.append(dtype.get(field_type).format) # Put query into a map of {timestamp: list of (uid, data)} for i in range(len(points)): # Round time down to nearest 0.1 second timestamp = str(points[i]['time'][:21] + 'Z') data = '' for j in range(len(field_names)): data += struct.pack(field_formats[j], points[i][field_names[j]]) if playback.query.has_key(timestamp): playback.query[timestamp].append((uid, data)) else: playback.query[timestamp] = [(uid, data)]
def encode(self, value): """Encode a columns value according to its data type Arguments: value: The value to encode provided as either a string or the appropriate type for the column's data type. """ dt = dtype.get(self.type) if self._enum_rev is not None: value = self._enum_rev.get(value, value) value = self._parse_column_value_from_string(value) # For some reason ArrayType.encode expects to receive the values # for encoding in *args instead of, you know, an iterable ... if isinstance(dt, dtype.ArrayType): # `value` needs to be an unpackable iterable or this is going # to explode. More than likely this'll be a bytearray. return dt.encode(*value) else: return dt.encode(value)
def _parse_column_value_from_string(self, value): """Parse strings into an appropriate type for a given table column Attempt to cast a string value into the appropriate data type for a column to use during encoding. If the column's type is a "Primitive" type as defined in ait.core.dtype then we simple cast to float or int depending on the definition. ArrayType values must be passed as a string which can be encoded into a binary string. All other AIT data types are not supported in the table module. Arguments: value: The string from which to extract an appropriate data type value for future processing. If this is not a string it is returned without modification. """ if not isinstance(value, str): return value col_defn = dtype.get(self.type) if isinstance(col_defn, dtype.ArrayType): # Value is expected to be a string which can be encoded to a # binary string for use by a bytes-like object. # # E.g., for a type of U8[2] the following is valid: # value = '\x01\x02' return bytearray(value.encode("ascii")) elif (isinstance(col_defn, dtype.Time64Type) or isinstance(col_defn, dtype.Time40Type) or isinstance(col_defn, dtype.Time32Type)): return datetime.datetime.strptime(value, dmc.RFC3339_Format) else: if col_defn.float: return float(value) else: return int(value, base=0)
def decode(self, in_stream, raw=False): """Decode a column's value according to its data type Read bytes equal to this column's data type from the input stream and decode it into a value per that data type's definition. Arguments: in_stream: A file-like object from which to read data. raw: Flag denoting whether raw values or enumerate values (if present for this column) should be returned. Raises: EOFError: If the number of bytes read from the input stream is less than the length of the data type. """ val = None dt = dtype.get(self.type) if dt is not None: data = in_stream.read(dt.nbytes) if len(data) != dt.nbytes: raise EOFError if (isinstance(dt, dtype.Time64Type) or isinstance(dt, dtype.Time40Type) or isinstance(dt, dtype.Time32Type)): val = dt.decode(data).strftime(dmc.RFC3339_Format) else: val = dt.decode(data, raw=True) if self.enum and not raw: val = self.enum.get(val, val) return val
def type(self, value): from ait.core import dtype self._type = dtype.get(value) if type(value) is str else value
def toBinary(self, tabfile, stream, fswbin_f, verbose, version): #print "self.name: "+self.name #print "stream name: "+stream.name size = os.path.getsize(stream.name) #print "stream len: " + str(size) #print "self.size: " + str(self.size) no_lines = 0 for line in stream: no_lines += 1 stream.seek(0) fsw_header = bytearray(32) # sha1 = hash_file(tabfile) sha1 = 0 # Write magic number fswbin_f.write( struct.pack('>H', self.MagicNumber ) ) # Write upload type fswbin_f.write( struct.pack('B', self.uptype ) ) # Write version fswbin_f.write( struct.pack('B', int(version,16)&255 ) ) # Write number of lines if self.name == "memory": fswbin_f.write( struct.pack('>H', 0 ) ) else: fswbin_f.write( struct.pack('>H', no_lines ) ) # Write ID (0) fswbin_f.write( struct.pack('>H', 0) ) # # Write CRC placeholder fswbin_f.write( struct.pack('>I', 0) ) # SHA as 0 pad = struct.pack('B', 0) for n in range(20): fswbin_f.write(pad) # data = bytearray(20) # i = 0 # tmpbytes = list(sha1) # for x in range(0, len(sha1)/2): # tmp = ((int(tmpbytes[x],16)&255)<<4) + (int(tmpbytes[x+1],16)&255) # #print "tmp: "+ str(tmp) # data[i] = tmp&0xFF # i += 1 # fswbin_f.write(data) for line in stream: #print "line: "+line idx = 0 line = line.replace("\n","") allcols = line.split(self.delimiter) if self.name == "memory": for val in allcols: if val != "": #print "val: "+val data = bytearray(2) tmpbytes = list(val) data[0] = ((int(tmpbytes[0],16)&0xF)<<4) + (int(tmpbytes[1],16)&0xF) data[1] = ((int(tmpbytes[2],16)&0xF)<<4) + (int(tmpbytes[3],16)&0xF) #print "tmp byte1: "+ str(int(tmpbytes[0],16)&0xF) #print "tmp byte2: "+ str(int(tmpbytes[1],16)&0xF) #print "tmp byte3: "+ str(int(tmpbytes[2],16)&0xF) #print "tmp byte4: "+ str(int(tmpbytes[3],16)&0xF) fswbin_f.write(data) else: idx = 0 #this is how to step into table definitions for coldef in enumerate(self.coldefns): # print "column definition: " + str(coldef[1]) fswcoldefn = coldef[1] name = fswcoldefn.name #print "name: " + name colpk = dtype.get(fswcoldefn.type) #print "packing: " + str(colpk) units = fswcoldefn.units #print "units: " + units enum = fswcoldefn.enum if isinstance(fswcoldefn.bytes,list): nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1 else: nobytes = 1 # print "bytes: " + str(fswcoldefn.bytes) # print "nobytes: " + str(nobytes) if name == 'RESERVED': #add reserved bytes fswbin_f.write(colpk.encode(0)) if coldef[0] != len(self.coldefns)-1: continue else: break items = fswcoldefn.items if items is not None: #print "items: " + str(items) for i in range(items): val = allcols[i] #print "item col val: "+str(val) if units != 'none': val = val.strip() val = val.split(" ")[0] else: val = val.replace(" ","") #print "item col val: "+str(val) fswbin_f.write(colpk.encode(self.convertValue(val))) else: val = allcols[idx] val = val.replace("\n","") #print "else col val 1: "+str(val) if enum is not None: if enum is not None: for enumkey in enumerate(enum.keys()): enumval = enum[enumkey[1]] #print "enumkey: " + str(enumkey[1]) + ", enumval: " + str(enum[enumkey[1]]) if enumval == val: val = str(enumkey[1]) #print "XXXX colpk.type: "+colpk.format fswbin_f.write(colpk.encode(self.convertValue(val))) else: #print "XXXX colpk.type: "+colpk.format #print "fswcoldefn.bytes: "+str(fswcoldefn.bytes) #print "xxxx val: "+str(val) #print "units: "+str(units) if units != 'none': val = val.strip() val = val.split(" ")[0] #print "else col val 2a: "+str(val) fswbin_f.write(colpk.encode(self.convertValue(val))) #fswbin_f.write(colpk.encode(float(val))) elif str(colpk) == "PrimitiveType('U8')" and nobytes>1: strval = "" for c in list(val): #print "xxx c: "+str(c) tmp = (int(c,16))&255 #print "tmp: "+str(tmp) fswbin_f.write(colpk.encode(tmp)) else: val = val.replace(" ","") #print "else col val 2b: "+str(val) fswbin_f.write(colpk.encode(self.convertValue(val))) #fswbin_f.write(colpk.encode(float(val))) idx += 1 written = fswbin_f.tell() #print "written: "+str(written) # print str(self.size) + ", " + str(written) if self.size > written: padding = bytearray(self.size - (written)) fswbin_f.write(padding) #Now calculate and update CRC field in the FSW header fswbin_f.close() fname = fswbin_f.name crc32 = util.crc32File(fname, 0) fswbin_f = open(fname, 'r+b') fswbin_f.seek(28) crcbuf = bytearray(4) crcbuf[0:4] = struct.pack('>L',crc32) fswbin_f.write(crcbuf) #version = "0" if verbose is not None and verbose != 0: log.info("CRC: %x" % crc32) print "MAGIC_NUMBER: %x" % self.MagicNumber print "UPLOAD_TYPE: " + str(self.uptype) print "VERSION: " + str(version) print "NUMBER_ENTRIES: " + str(no_lines) # print "SHA-1: "+sha1 #print "fname: "+fname+", crc32: "+str(crc32) # Once we are done appending all the columns to the row # strip off last comma and append a \r # Note: Since it is Access, \n alone does not work return
def toText(self, stream, fswtab_f, verbose, version): out = "" size = os.path.getsize(stream.name) #print "table name: " + self.name #print "stream len: " + str(size) noentries = 0 if self.name != "memory": for fswheaderdef in enumerate(self.fswheaderdefns): #print "header definition: " + str(fswheaderdef[1]) fswcoldefn = fswheaderdef[1] name = fswcoldefn.name #print "name: " + name colfmt = str(fswcoldefn.format) #print "format: " + colfmt colpk = dtype.get(fswcoldefn.type) #print "packing: " + str(colpk) coltype = fswcoldefn.type #print "type: " + coltype if isinstance(fswcoldefn.bytes,list): nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1 else: nobytes = 1 #print "bytes: " + str(fswcoldefn.bytes) #print "nobytes: " + str(nobytes) strval = "" if str(colpk) == "PrimitiveType('U8')" and nobytes>1: strval = "" for i in range(nobytes): value = colpk.decode(stream.read(1)) strval += str(colfmt % value) #print(colfmt % value) else: value = colpk.decode(stream.read(nobytes)) #print(colfmt % value) if (str(colpk) != "PrimitiveType('U8')") or (str(colpk) == "PrimitiveType('U8')" and nobytes == 1): strval = str(colfmt % value) #print "strval: " + strval if name == "NUMBER_ENTRIES": noentries = strval #print "noentries: " + strval if self.name != "keep_out_zones" and self.name != "line_of_sight": # Append the value to table row out += name+': %s\n' % strval if verbose is not None and verbose != 0: print print out #fswtab_f.write(out) size = size - 32 out = "" if self.name.startswith("log_"): norows = self.rows #print "norows: " + str(norows) else: rowbytes = 0 items = None for coldef in enumerate(self.coldefns): fswcoldefn = coldef[1] items = fswcoldefn.items if isinstance(fswcoldefn.bytes,list): nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1 else: nobytes = 1 rowbytes = rowbytes + nobytes #print "Row bytes: " + str(rowbytes) if items is not None: rowbytes = rowbytes * items norows = size / rowbytes else: norows = int(noentries) #print "norows: " + str(norows) #print "items: " + str(items) if norows == 0: idx = 1 for i in range(size): byte = stream.read(1) value = binascii.hexlify(byte) fswtab_f.write(value) if (idx%2) == 0: fswtab_f.write(" ") if (idx%16) == 0: fswtab_f.write("\n") idx += 1 return for i in range(norows): condition = None #this is how to step into table definitions for coldef in enumerate(self.coldefns): #print "column definition: " + str(coldef[1]) fswcoldefn = coldef[1] name = fswcoldefn.name #print "name: " + name colfmt = str(fswcoldefn.format) #print "format: " + colfmt colpk = dtype.get(fswcoldefn.type) #print "packing: " + str(colpk) coltype = fswcoldefn.type #print "type: " + coltype if isinstance(fswcoldefn.bytes,list): nobytes = fswcoldefn.bytes[1] - fswcoldefn.bytes[0] + 1 else: nobytes = 1 #print "bytes: " + str(fswcoldefn.bytes) #print "nobytes: " + str(nobytes) units = fswcoldefn.units #print "units: " + units enum = fswcoldefn.enum items = fswcoldefn.items if items is not None: #print "items: " + str(items) for i in range(items): value = colpk.decode(stream.read(nobytes)) strval = str(colfmt % value) out += strval + self.delimiter else: strval = "" if str(colpk) == "PrimitiveType('U8')" and nobytes>1: strval = "" for i in range(nobytes): value = colpk.decode(stream.read(1)) if name == "RESERVED": continue strval += str(colfmt % value) #print(colfmt % value) else: value = colpk.decode(stream.read(nobytes)) #print(colfmt % value) if enum is not None: if enum is not None: for enumkey in enumerate(enum.keys()): #print "enumkey: " + str(enumkey[1]) + ", enumval: " + str(enum[enumkey[1]]) if enumkey[1] == value: strval = str(enum[enumkey[1]]) else: if units != 'none': strval = str(colfmt % value) + " " + units #print "units: " + units else: if (str(colpk) != "PrimitiveType('U8')") or (str(colpk) == "PrimitiveType('U8')" and nobytes == 1): strval = str(colfmt % value) if self.name == "response" and "CONSTANT" in name and condition > 6: strval = str('%d' % value) #print "strval: " + strval if self.name == "response" and name == "CONDITION_TYPE": #print "value: "+str(value) condition = value #print "condition: "+str(condition) # Append the value to table row if name == "RESERVED": continue out += strval + self.delimiter out = out[:-1] + "\n" #print #print out fswtab_f.write(out) # Once we are done appending all the columns to the row # strip off last comma and append a \r # Note: Since it is Access, \n alone does not work return
def type(self, value): if type(value) is str and dtype.get(value) is not None: self._type = dtype.get(value) else: self._type = value log.error("Invalid field type '%s' " % value)
def get_historical_tlm_for_packet_fields(self, ait_pkt_id, ait_field_names, start_millis, end_millis): """ Perform a historical query for a particular AIT packet type :param ait_pkt_id: AIT Packet definition Id :param ait_field_names: List of field names to include, use None to include all fields :param start_millis: Start time, milliseconds since UNIX epoch :param end_millis: End time, milliseconds since UNIX epoch :return: List of OpenMct measurements that satisfy query """ if not self._database and False: return None result_list = [] ait_pkt_def = self._aitTlmDict[ait_pkt_id] ait_field_defs = ait_pkt_def.fields # Build field names list from tlm dictionary for sorting data query field_names = [] # Build field types list from tlm dictionary for packing data field_formats = [] # Collect the field type information (prolly dont need dtype) for i in range(len(ait_field_defs)): field_def = ait_field_defs[i] # if no request-list or current field is in request list if (not ait_field_names) or (field_def.name in ait_field_names): field_names.append(field_def.name) field_type = str(field_def.type).split("'")[1] field_formats.append(dtype.get(field_type).format) # A list with single entry of pkt id packet_ids = [ait_pkt_id] # Convert unix timestamp to UTC datetime for time range start_timestamp_secs = start_millis / 1000.0 start_date = datetime.datetime.fromtimestamp(start_timestamp_secs, tz=datetime.timezone.utc) end_timestamp_secs = end_millis / 1000.0 end_date = datetime.datetime.fromtimestamp(end_timestamp_secs, tz=datetime.timezone.utc) query_args_str = f"Packets = {packet_ids}; Start = {start_date};" \ f" End = {end_date}" self.dbg_message(f"Query args : {query_args_str}") # default response is empty res_pkts = list() # Query packet and time range from database try: if self._database: ait_db_result = self._database.query_packets( packets=packet_ids, start_time=start_date, end_time=end_date, yield_packet_time=True, ) if ait_db_result.errors is not None: log.error("[OpenMCT] Database query for packets " + str(packet_ids) + " resulted in errors: ") for db_err in ait_db_result.errors: log.error("[OpenMCT] Error: " + str(db_err)) elif ait_db_result.has_packets: res_pkts = list(ait_db_result.get_packets()) # Debug result size self.dbg_message(f"Number of results for query " f"{query_args_str} : {len(res_pkts)}") except Exception as e: log.error("[OpenMCT] Database query failed. Error: " + str(e)) return None for cur_pkt_time, cur_pkt in res_pkts: # Convert datetime to Javascript timestamp (in milliseconds) cur_timestamp_sec = datetime.datetime.timestamp(cur_pkt_time) unix_timestamp_msec = int(cur_timestamp_sec) * 1000 # Add a record for each requested field for this timestamp for cur_field_name in field_names: record = {"timestamp": unix_timestamp_msec} record["id"] = DictUtils.create_mct_pkt_id( ait_pkt_id, cur_field_name) record["value"] = getattr(cur_pkt, cur_field_name) result_list.append(record) return result_list
def testgetdtype(): dt = dtype.get("TIME32") assert isinstance(dt, dtype.Time32Type) assert dt.name == "TIME32" assert dt.pdt == "MSB_U32" assert dt.max == 4294967295