def testMask():
"""
# This test will use the following TLM dictionary definitions.
# The mask 0x0180 singles out the two bits on either MSB_U16
# word:
#
# 0b00000001 0b10000000
- !Packet
name: P
fields:
- !Field
name: M
type: MSB_U16
mask: 0x0180
"""
defn = tlm.TlmDict(testMask.__doc__)["P"]
packet = tlm.Packet(defn)
assert packet.M == 0
assert packet._data == bytearray([0x00, 0x00])
packet.M = 1
assert packet.M == 1
assert packet._data == bytearray([0x00, 0x80])
packet.M = 2
assert packet.M == 2
assert packet._data == bytearray([0x01, 0x00])
packet.M = 3
assert packet.M == 3
assert packet._data == bytearray([0x01, 0x80])
def _run(self):
try:
while True:
gevent.sleep(0)
msg = self._sub.recv_multipart()
topic, message = serv_utils.decode_message(msg)
message = pickle.loads(message)
if topic is None or message is None:
log.error(f"{self} received invalid topic or message. Skipping")
continue
if not isinstance(message, tuple):
log.error(
"TlmMonitor received message that it is unable to process "
"Messages must be tagged packet data tuples (uid, data)."
)
continue
if message[0] not in self._defns:
log.error(f"Skipping packet with id {message[0]}")
continue
pkt = tlm.Packet(defn=self._defns[message[0]], data=message[1])
pkt_name = pkt._defn.name
if pkt_name in self._pktbufs:
self._pktbufs[pkt_name].appendleft(pkt)
except Exception as e:
log.error("Exception raised in TlmMonitor while receiving messages")
log.error(f"API telemetry is no longer being received from server. {e}")
raise e
def test_sqlite_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, 'wt') as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict['Packet1']
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
sqlbackend.insert(pkt)
sqlbackend._conn.execute.assert_called_with(
'INSERT INTO "Packet1" (col1, SampleTime) VALUES (?, ?)',
[1, 33752069.10112411])
os.remove(self.test_yaml_file)
def process(self, input_data, topic=None):
"""
Process received input message.
This plugin should be configured to only receive telemetry.
Received messaged is expected to be a tuple of the form produced
by AITPacketHandler.
"""
processed = False
try:
pkl_load = pickle.loads(input_data)
pkt_id, pkt_data = int(pkl_load[0]), pkl_load[1]
packet_def = self._get_tlm_packet_def(pkt_id)
if packet_def:
packet_def = self._uidToPktDefMap[pkt_id]
tlm_packet = tlm.Packet(packet_def, data=bytearray(pkt_data))
self._process_telem_msg(tlm_packet)
processed = True
else:
log.error(
"OpenMCT Plugin received telemetry message with unknown "
f"packet id {pkt_id}. Skipping input...")
except Exception as e:
log.error(f"OpenMCT Plugin: {e}")
log.error(
"OpenMCT Plugin received input_data that it is unable to "
"process. Skipping input ...")
return processed
def get_packet_and_defn():
first_stream = ait.config.get('gui.telemetry')[0]
stream = cfg.AitConfig(config=first_stream).get('stream')
name = stream.get('name', '<unnamed>')
pkt_defn = tlm.getDefaultDict().get(name, None)
pkt = tlm.Packet(pkt_defn)
return pkt_defn, pkt
def create_packet_from_result(cls, packet_name, data):
try:
pkt_defn = tlm.getDefaultDict()[packet_name]
except KeyError:
log.error("Unknown packet name {}. Unable to unpack SQLite result".
format(packet_name))
return None
return tlm.Packet(pkt_defn, data=data)
def test_derivation_definition(self):
tlmdict = tlm.TlmDict(self.test_yaml_deriv1)
pktdefn = tlmdict["Test_Packet"]
deriv1 = pktdefn.derivations[0]
assert deriv1.name == "deriv_1"
assert type(deriv1.equation) == tlm.PacketExpression
assert deriv1.equation.toJSON() == "field_1 + field_2"
pkt = tlm.Packet(pktdefn)
pkt.field_1 = 1
pkt.field_2 = 2
assert pkt.deriv_1 == 3
def process(self, input_data, topic=None, **kwargs):
try:
load = pickle.loads(input_data)
pkt_id, pkt_data = int(load[0]), load[1]
packet = self.packet_dict[pkt_id]
decoded = tlm.Packet(packet, data=bytearray(pkt_data))
except Exception as e:
log.error('TelemetryLimitMonitor: {}'.format(e))
log.error(
'TelemetryLimitMonitor received input_data that it is unable to process. Skipping input ...'
)
return
if packet.name in self.limit_dict:
for field, defn in self.limit_dict[packet.name].iteritems():
v = decoded._getattr(field)
if packet.name not in self.limit_trip_repeats.keys():
self.limit_trip_repeats[packet.name] = {}
if field not in self.limit_trip_repeats[packet.name].keys():
self.limit_trip_repeats[packet.name][field] = 0
if defn.error(v):
msg = 'Field {} error out of limit with value {}'.format(
field, v)
log.error(msg)
self.limit_trip_repeats[packet.name][field] += 1
repeats = self.limit_trip_repeats[packet.name][field]
if (repeats == self.notif_thrshld or
(repeats > self.notif_thrshld and
(repeats - self.notif_thrshld) % self.notif_freq
== 0)):
notify.trigger_notification('limit-error', msg)
elif defn.warn(v):
msg = 'Field {} warning out of limit with value {}'.format(
field, v)
log.warn(msg)
self.limit_trip_repeats[packet.name][field] += 1
repeats = self.limit_trip_repeats[packet.name][field]
if (repeats == self.notif_thrshld or
(repeats > self.notif_thrshld and
(repeats - self.notif_thrshld) % self.notif_freq
== 0)):
notify.trigger_notification('limit-warn', msg)
else:
self.limit_trip_repeats[packet.name][field] = 0
def testArray():
"""
# This test will use the following TLM dictionary definitions:
- !Packet
name: P
fields:
- !Field
name: A
type: MSB_U16[3]
"""
defn = tlm.TlmDict(testArray.__doc__)["P"]
packet = tlm.Packet(defn, struct.pack(">HHH", 1, 2, 3))
assert packet.A == [1, 2, 3]
def test_sqlite_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, "wt") as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict["Packet1"]
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
# Note: we can't fully test this call given the modification to
# the packet data on insert. Better than nothing I suppose.
# Test without time
sqlbackend.insert(pkt)
assert ('INSERT INTO "Packet1" (PKTDATA) VALUES (?)'
in sqlbackend._conn.execute.call_args[0])
sqlbackend._conn.reset_mock()
# Test with time
now = dt.datetime.utcnow()
sqlbackend.insert(pkt, time=now)
assert ('INSERT INTO "Packet1" (PKTDATA, time) VALUES (?, ?)'
in sqlbackend._conn.execute.call_args[0])
assert (now.strftime(
dmc.RFC3339_Format) == sqlbackend._conn.execute.call_args[0][1][1])
os.remove(self.test_yaml_file)
def process(self, input_data, topic=None, **kwargs):
"""
Splits tuple received from PacketHandler into packet UID and packet message.
Decodes packet and inserts into database backend.
Logs any exceptions raised.
Params:
input_data: message received from inbound stream through PacketHandler
topic: name of inbound stream message received from
**kwargs: any args required for connected to the backend
"""
try:
load = pickle.loads(input_data)
uid, pkt = int(load[0]), load[1]
defn = self.packet_dict[uid]
decoded = tlm.Packet(defn, data=bytearray(pkt))
self.dbconn.insert(decoded, **kwargs)
except Exception as e:
log.error('Data archival failed with error: {}.'.format(e))
def main():
log.begin()
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--all", action="store_true", help="output all fields/values")
parser.add_argument(
"--csv",
default="output.csv",
metavar="</path/to/output/csv>",
help="Output as CSV with filename",
)
parser.add_argument(
"--fields",
metavar="</path/to/fields/file>",
help="path to the file containing all fields to query, separated by newline.",
)
parser.add_argument(
"--packet", required=True, help="Packet name from telemetry dictionary specified in config file."
)
parser.add_argument(
"--time_field",
help=(
"Time field to use for time range comparisons. Ground receipt time "
"will be used if nothing is specified."
),
)
parser.add_argument(
"--stime",
help=(
"Datetime in file to start collecting the data values. Defaults to "
"beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ"
),
)
parser.add_argument(
"--etime",
help=(
"Datetime in file to end collecting the data values. Defaults to end "
"of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ"
),
)
parser.add_argument(
"pcap", nargs="*", help=("PCAP file(s) containing telemetry packets")
)
args = parser.parse_args()
args.ground_time = True
if args.time_field is not None:
args.ground_time = False
tlmdict = tlm.getDefaultDict()
defn = None
try:
if tlmdict is not None:
defn = tlmdict[args.packet]
except KeyError:
log.error('Packet "%s" not defined in telemetry dictionary.' % args.packet)
log.end()
sys.exit(2)
if not args.all and args.fields is None:
log.error(
"Must provide fields file with --fields or specify that all fields should be queried with --all"
)
log.end()
sys.exit(2)
if args.all:
fields = [flddefn.name for flddefn in defn.fields]
else:
# Parse the fields file into a list
with open(args.fields, "r") as stream:
fields = [fldname.strip() for fldname in stream.readlines()]
not_found = False
# TODO Rework this into the CSV generation. Not here.
# Duplicating effort
for fldname in fields:
raw = fldname.split(".")
if fldname not in defn.fieldmap and (
len(raw) == 2 and raw[0] != "raw" or raw[1] not in defn.fieldmap
):
not_found = True
log.error('No telemetry point named "%s"' % fldname)
if not_found:
log.end()
sys.exit(2)
if args.stime:
start = datetime.strptime(args.stime, dmc.ISO_8601_Format)
else:
start = dmc.GPS_Epoch
if args.etime:
stop = datetime.strptime(args.etime, dmc.ISO_8601_Format)
else:
stop = datetime.utcnow()
# Append time to beginning of each row
if not args.ground_time:
fields.insert(0, args.time_field)
else:
fields.insert(0, "Ground Receipt Time")
csv_file = None
csv_writer = None
npackets = 0
if args.csv:
csv_file = open(args.csv, "w")
csv_writer = csv.writer(csv_file)
output(csv_writer, fields)
# If we're comparing off ground receipt time we need to drop the header label to avoid
# indexing errors when processing the fields.
if args.ground_time:
fields = fields[1:]
rowcnt = 0
for filename in args.pcap:
log.debug("Processing %s" % filename)
with pcap.open(filename, "rb") as stream:
header, data = stream.read()
while data:
packet = tlm.Packet(defn, data)
comp_time = (
header.timestamp
if args.ground_time
else getattr(packet, args.time_field)
)
if start < comp_time < stop:
row = []
for field in fields:
try:
# check if raw value requested
_raw = False
names = field.split(".")
if len(names) == 2 and names[0] == "raw":
field = names[1]
_raw = True
field_val = packet._getattr(field, raw=_raw)
if hasattr(field_val, "name"):
field_val = field_val.name
else:
field_val = str(field_val)
except KeyError:
log.debug("%s not found in Packet" % field)
field_val = None
except ValueError:
# enumeration not found. just get the raw value
field_val = packet._getattr(field, raw=True)
row.append(field_val)
if args.ground_time:
row = [comp_time] + row
rowcnt += 1
output(csv_writer, row)
npackets += 1
header, data = stream.read()
log.debug("Parsed %s packets." % npackets)
csv_file.close()
if rowcnt == 0:
os.remove(args.csv)
log.end()
def mimic_tlm(self, ait_tlm_pkt_name, ait_tlm_pkt_fill=None):
"""Used for debugging, creates an instance of a packet based on
packet name, and fills it with zero data.
Special case for '1553_HS_Packet' which will get random number
data fills.
If HTTP Request query includes a value for 'repeat', then this
will continue emitting telemetry.
"""
# Http query option, if it is set to anything, consider it true
self._debugMimicRepeat = len(str(bottle.request.query.repeat)) > 0
# This will be helpful in testing by simulating TLM
# but by a rest call instead of actual telem
ait_pkt_defn = None
if ait_tlm_pkt_name:
ait_pkt_defn = tlm.getDefaultDict()[ait_tlm_pkt_name]
else:
ait_pkt_defn = tlm.getDefaultDict().values()[0]
# Create the expected message format
pkt_size_bytes = ait_pkt_defn.nbytes
# if self._debugMimicRepeat:
repeat_str = " REPEATED " if self._debugMimicRepeat else " a single "
info_msg = ("Received request to mimic" + repeat_str +
"telemetry packet for " + ait_pkt_defn.name)
self.dbg_message(info_msg)
# Create a binary array of size filled with 0
dummy_data = bytearray(pkt_size_bytes)
info_msg = ""
while True:
# Special handling for simply integer based packet, others will
# have all 0 zero
if ait_pkt_defn.name == "1553_HS_Packet":
hs_packet = struct.Struct(">hhhhh")
random_num = random.randint(1, 100)
dummy_data = hs_packet.pack(random_num, random_num, random_num,
random_num, random_num)
tlm_pkt = tlm.Packet(ait_pkt_defn, data=bytearray(dummy_data))
self._process_telem_msg(tlm_pkt)
info_msg = (
"AIT OpenMct Plugin submitted mimicked telemetry for " +
ait_pkt_defn.name + " (" + str(datetime.datetime.now()) +
") to telem queue")
self.dbg_message(info_msg)
# sleep if mimic on
if self._debugMimicRepeat:
gsleep(5)
# either it was immediate or we woke up, check break condition
if not self._debugMimicRepeat:
break
# Return last status message as result to client
return info_msg
def main():
dn_limits = {}
eu_limits = {}
eu_values = {}
ld = limits.getDefaultDict()
td = tlm.getDefaultDict()
all_vals = [[
"Telem Point",
"EU lower.error",
"EU lower.warn",
"EU upper.warn",
"EU upper.error",
"DN lower.error",
"DN lower.warn",
"DN upper.warn",
"DN upper.error",
"DN to EU LE",
"DN to EU LW",
"DN to EU UW",
"DN to EU UE",
]]
for source in sorted(ld.keys(), key=lambda x: x.split(".")[1]):
log.info(f"Processing {source}")
pkt_name, name = source.split(".")
# Don't support limits specifying individual values. This is usually
# used to specify enumerations that aren't valid and we don't properly
# handle those cases.
if ld[source].value is not None:
log.warn(f'Skipping unsupported "value" limit {source}')
continue
dn_limits.setdefault(name, [None, None, None, None])
eu_limits.setdefault(name, [None, None, None, None])
eu_values.setdefault(name, [None, None, None, None])
if ld[source].lower is not None:
try:
eu_limits[name][0] = ld[source].lower.error
except AttributeError:
pass
try:
eu_limits[name][1] = ld[source].lower.warn
except AttributeError:
pass
if ld[source].upper is not None:
try:
eu_limits[name][2] = ld[source].upper.warn
except AttributeError:
pass
try:
eu_limits[name][3] = ld[source].upper.error
except AttributeError:
pass
values = []
defn = td[pkt_name]
data = bytearray(defn.nbytes)
packet = tlm.Packet(defn, data)
for dn in range(65536):
setattr(packet, name, dn)
eu = getattr(packet, name)
if eu is not None:
values.append((dn, eu))
values.sort(key=lambda pair: pair[1])
for dn, eu in values:
for n in range(4):
if (eu_limits[name][n] is not None
and dn_limits[name][n] is None
and eu > eu_limits[name][n]):
value = dn - 1 if dn > 0 else 0
dn_limits[name][n] = value
setattr(packet, name, value)
eu_values[name][n] = getattr(packet, name)
if all(dn_limits[name][n] is not None for n in range(4)):
break
values = [source]
values.extend(map(str, eu_limits[name]))
values.extend(map(str, dn_limits[name]))
values.extend(map(str, eu_values[name]))
all_vals.append(values)
s = [[str(e) for e in row] for row in all_vals]
lens = [max(map(len, col)) for col in zip(*s)]
fmt = "\t".join("{{:{}}}".format(x) for x in lens)
table = [fmt.format(*row) for row in s]
print("\n".join(table))
def handle (self, data, address):
self._pktbuf.appendleft( tlm.Packet(self._defn, data) )
def create_packets_from_results(self, packet_name, result_set):
''' Generate AIT Packets from a InfluxDB query ResultSet
Extract Influx DB query results into one packet per result entry. This
assumes that telemetry data was inserted in the format generated by
:func:`InfluxDBBackend.insert`. Complex types such as CMD16 and EVR16 are
evaluated if they can be properly encoded from the raw value in the
query result. If there is no opcode / EVR-code for a particular raw
value the value is skipped (and thus defaulted to 0).
Arguments
packet_name (string)
The name of the AIT Packet to create from each result entry
result_set (influxdb.resultset.ResultSet)
The query ResultSet object to convert into packets
Returns
A list of packets extracted from the ResultSet object or None if
an invalid packet name is supplied.
'''
try:
pkt_defn = tlm.getDefaultDict()[packet_name]
except KeyError:
log.error('Unknown packet name {} Unable to unpack ResultSet'.format(packet_name))
return None
pkt = tlm.Packet(pkt_defn)
pkts = []
for r in result_set.get_points():
new_pkt = tlm.Packet(pkt_defn)
for f, f_defn in pkt_defn.fieldmap.iteritems():
field_type_name = f_defn.type.name
if field_type_name == 'CMD16':
if cmd.getDefaultDict().opcodes.get(r[f], None):
setattr(new_pkt, f, cmd_def.name)
elif field_type_name == 'EVR16':
if evr.getDefaultDict().codes.get(r[f], None):
setattr(new_pkt, f, r[f])
elif field_type_name == 'TIME8':
setattr(new_pkt, f, r[f] / 256.0)
elif field_type_name == 'TIME32':
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=r[f])
setattr(new_pkt, f, new_val)
elif field_type_name == 'TIME40':
sec = int(r[f])
microsec = r[f] * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec, microseconds=microsec)
setattr(new_pkt, f, new_val)
elif field_type_name == 'TIME64':
sec = int(r[f])
microsec = r[f] % 1 * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec, microseconds=microsec)
setattr(new_pkt, f, new_val)
else:
try:
setattr(new_pkt, f, r[f])
except KeyError:
log.info('Field not found in query results {} Skipping ...'.format(f))
pkts.append(new_pkt)
return pkts
def main():
tlmdict = tlm.getDefaultDict()
pnames = tlmdict.keys()
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arguments = {
'--packet': {
'type': str,
'choices': pnames,
'default': pnames[0] if len(pnames) > 0 else None,
'help': 'Type of packets (!Packet name in tlm.yaml) in file',
'required': len(pnames) > 1,
},
'--database': {
'default':
ait.config.get('database.name'),
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)'),
'required':
ait.config.get('database.name') is None
},
'--backend': {
'default':
'sqlite',
'choices': ['sqlite', 'influx'],
'action':
'store',
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)')
},
'--use-current-time': {
'action':
'store_true',
'help': ('Use current time stamps when insert packets instead '
'of ground receipt time (or the time written in the '
'PCAP header).')
},
'file': {
'nargs': '+',
'help': 'File(s) containing telemetry packets'
}
}
for name, params in arguments.items():
ap.add_argument(name, **params)
args = ap.parse_args()
log.begin()
try:
npackets = 0
dbconn = None
defn = tlm.getDefaultDict()[args.packet]
nbytes = defn.nbytes
if args.backend == 'sqlite':
dbconn = db.SQLiteBackend()
elif args.backend == 'influx':
dbconn = db.InfluxDBBackend()
if args.backend == 'sqlite' and (args.database == ':memory:'
or not os.path.exists(args.database)):
dbconn.create(database=args.database)
else:
dbconn.connect(database=args.database)
for filename in args.file:
log.info('Processing %s' % filename)
with pcap.open(filename) as stream:
for header, pkt_data in stream:
try:
packet = tlm.Packet(defn, pkt_data)
time = header.timestamp
if args.use_current_time:
time = None
dbconn.insert(packet, time=time)
npackets += 1
except struct.error:
log.error(
"Unable to unpack data into packet. Skipping ...")
except KeyboardInterrupt:
log.info('Received Ctrl-C. Stopping database insert.')
except IOError as e:
log.error(str(e))
finally:
dbconn.close()
values = npackets, args.packet, args.database
log.info('Inserted %d %s packets into database %s.' % values)
log.end()
def main():
log.begin()
description = """Parses 1553 telemetry into CSV file."""
arguments = {
'--all': {
'action': 'store_true',
'help': 'output all fields/values',
},
'--csv': {
'type': str,
'default': 'output.csv',
'metavar': '</path/to/output/csv>',
'help': 'Output as CSV with filename'
},
'--fields': {
'type': str,
'metavar': '</path/to/fields/file>',
'help':
'file containing all fields to query, separated by newline.'
},
'--packet': {
'type': str,
'required': True,
'help': 'field names to query, separated by space'
},
'--time_field': {
'type':
str,
'help':
'Time field to use for time range comparisons. Ground receipt time will be used if nothing is specified.'
},
'--stime': {
'type':
str,
'help':
'Datetime in file to start collecting the data values. Defaults to beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ'
},
'--etime': {
'type':
str,
'help':
'Datetime in file to end collecting the data values. Defaults to end of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ'
}
}
arguments['pcap'] = {
'nargs': '*',
'help': 'PCAP file(s) containing telemetry packets'
}
args = gds.arg_parse(arguments, description)
args.ground_time = True
if args.time_field is not None:
args.ground_time = False
tlmdict = tlm.getDefaultDict()
defn = None
try:
if tlmdict is not None:
defn = tlmdict[args.packet]
except KeyError:
log.error('Packet "%s" not defined in telemetry dictionary.' %
args.packet)
gds.exit(2)
if not args.all and args.fields is None:
log.error(
'Must provide fields file with --fields or specify that all fields should be queried with --all'
)
gds.exit(2)
if args.all:
fields = [flddefn.name for flddefn in defn.fields]
else:
# Parse the fields file into a list
with open(args.fields, 'rb') as stream:
fields = [fldname.strip() for fldname in stream.readlines()]
not_found = False
# TODO Rework this into the CSV generation. Not here.
# Duplicating effort
for fldname in fields:
raw = fldname.split('.')
if fldname not in defn.fieldmap and (len(raw) == 2 and raw[0] != 'raw'
or raw[1] not in defn.fieldmap):
not_found = True
log.error('No telemetry point named "%s"' % fldname)
if not_found:
gds.exit(2)
if args.stime:
start = datetime.strptime(args.stime, dmc.ISO_8601_Format)
else:
start = dmc.GPS_Epoch
if args.etime:
stop = datetime.strptime(args.etime, dmc.ISO_8601_Format)
else:
stop = datetime.utcnow()
# Append time to beginning of each row
if not args.ground_time:
fields.insert(0, args.time_field)
else:
fields.insert(0, 'Ground Receipt Time')
csv_file = None
csv_writer = None
npackets = 0
if args.csv:
csv_file = open(args.csv, 'wb')
csv_writer = csv.writer(csv_file)
output(csv_writer, fields)
# If we're comparing off ground receipt time we need to drop the header label to avoid
# indexing errors when processing the fields.
if args.ground_time:
fields = fields[1:]
rowcnt = 0
for filename in args.pcap:
log.debug('Processing %s' % filename)
with pcap.open(filename, 'rb') as stream:
header, data = stream.read()
while data:
packet = tlm.Packet(defn, data)
comp_time = header.timestamp if args.ground_time else getattr(
packet, args.time_field)
if start < comp_time < stop:
row = []
for field in fields:
try:
# check if raw value requested
_raw = False
names = field.split('.')
if len(names) == 2 and names[0] == 'raw':
field = names[1]
_raw = True
fieldVal = packet._getattr(field, raw=_raw)
if hasattr(fieldVal, 'name'):
fieldVal = fieldVal.name
else:
fieldVal = str(fieldVal)
except KeyError:
log.debug('%s not found in Packet' % field)
fieldVal = None
except ValueError:
# enumeration not found. just get the raw value
fieldVal = packet._getattr(field, raw=True)
row.append(fieldVal)
if args.ground_time:
row = [comp_time] + row
rowcnt += 1
output(csv_writer, row)
npackets += 1
header, data = stream.read()
log.debug('Parsed %s packets.' % npackets)
csv_file.close()
if rowcnt == 0:
os.remove(args.csv)
log.end()
def main():
tlmdict = tlm.getDefaultDict()
pnames = list(tlmdict.keys())
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arguments = {
"--packet": {
"type": str,
"choices": pnames,
"default": pnames[0] if len(pnames) > 0 else None,
"help": "Type of packets (!Packet name in tlm.yaml) in file",
"required": len(pnames) > 1,
},
"--database": {
"default":
ait.config.get("database.dbname"),
"help": ("Name of database in which to insert packets (may "
"also be specified in config.yaml database.name)"),
"required":
ait.config.get("database.dbname") is None,
},
"--backend": {
"default":
"sqlite",
"choices": ["sqlite", "influx"],
"action":
"store",
"help": ("Name of database in which to insert packets (may "
"also be specified in config.yaml database.name)"),
},
"--use-current-time": {
"action":
"store_true",
"help": ("Use current time stamps when insert packets instead "
"of ground receipt time (or the time written in the "
"PCAP header)."),
},
"file": {
"nargs": "+",
"help": "File(s) containing telemetry packets"
},
}
for name, params in arguments.items():
ap.add_argument(name, **params)
args = ap.parse_args()
log.begin()
try:
npackets = 0
dbconn = None
defn = tlm.getDefaultDict()[args.packet]
if args.backend == "sqlite":
dbconn = db.SQLiteBackend()
elif args.backend == "influx":
dbconn = db.InfluxDBBackend()
if args.backend == "sqlite" and (args.database == ":memory:"
or not os.path.exists(args.database)):
dbconn.create(database=args.database)
else:
dbconn.connect(database=args.database)
for filename in args.file:
log.info("Processing %s" % filename)
with pcap.open(filename) as stream:
for header, pkt_data in stream:
try:
packet = tlm.Packet(defn, pkt_data)
time = header.timestamp
if args.use_current_time:
time = None
dbconn.insert(packet, time=time)
npackets += 1
except struct.error:
log.error(
"Unable to unpack data into packet. Skipping ...")
except KeyboardInterrupt:
log.info("Received Ctrl-C. Stopping database insert.")
except IOError as e:
log.error(str(e))
finally:
dbconn.close()
values = npackets, args.packet, args.database
log.info("Inserted %d %s packets into database %s." % values)
log.end()
def test_influx_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
- !Field
name: SampleTime32
type: TIME32
- !Field
name: SampleTime40
type: TIME40
- !Field
name: SampleEvr16
type: EVR16
- !Field
name: SampleCmd16
type: CMD16
"""
with open(self.test_yaml_file, "wt") as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.InfluxDBBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict["Packet1"]
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
now = dt.datetime.utcnow()
sqlbackend.insert(pkt, time=now)
sqlbackend._conn.write_points.assert_called_with([{
"measurement":
"Packet1",
"time":
now.strftime(dmc.RFC3339_Format),
"tags": {},
"fields": {
"col1": 1,
"SampleTime": 33752069.10112411,
"SampleTime32": 168496141,
"SampleTime40": 235868177.0703125,
"SampleCmd16": 5398,
"SampleEvr16": 4884,
},
}])
sqlbackend._conn.reset_mock()
# Insert without a timestamp
sqlbackend.insert(pkt)
sqlbackend._conn.write_points.assert_called_with([{
"measurement": "Packet1",
"tags": {},
"fields": {
"col1": 1,
"SampleTime": 33752069.10112411,
"SampleTime32": 168496141,
"SampleTime40": 235868177.0703125,
"SampleCmd16": 5398,
"SampleEvr16": 4884,
},
}])
sqlbackend._conn.reset_mock()
# Insert with additional tags
sqlbackend.insert(pkt, tags={"testNum": "3"})
sqlbackend._conn.write_points.assert_called_with([{
"measurement": "Packet1",
"tags": {
"testNum": "3"
},
"fields": {
"col1": 1,
"SampleTime": 33752069.10112411,
"SampleTime32": 168496141,
"SampleTime40": 235868177.0703125,
"SampleCmd16": 5398,
"SampleEvr16": 4884,
},
}])
sqlbackend._conn.reset_mock()
os.remove(self.test_yaml_file)
def test_influx_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
- !Field
name: SampleTime32
type: TIME32
- !Field
name: SampleTime40
type: TIME40
- !Field
name: SampleEvr16
type: EVR16
- !Field
name: SampleCmd16
type: CMD16
"""
with open(self.test_yaml_file, 'wt') as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.InfluxDBBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict['Packet1']
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
now = dt.datetime.utcnow()
sqlbackend.insert(pkt, time=now)
sqlbackend._conn.write_points.assert_called_with([{
'measurement':
'Packet1',
'time':
now.strftime('%Y-%m-%dT%H:%M:%S'),
'tags': {},
'fields': {
'col1': 1,
'SampleTime': 33752069.10112411,
'SampleTime32': 168496141,
'SampleTime40': 235868177.0703125,
'SampleCmd16': 5398,
'SampleEvr16': 4884
}
}])
sqlbackend._conn.reset_mock()
# Insert without a timestamp
sqlbackend.insert(pkt)
sqlbackend._conn.write_points.assert_called_with([{
'measurement': 'Packet1',
'tags': {},
'fields': {
'col1': 1,
'SampleTime': 33752069.10112411,
'SampleTime32': 168496141,
'SampleTime40': 235868177.0703125,
'SampleCmd16': 5398,
'SampleEvr16': 4884
}
}])
sqlbackend._conn.reset_mock()
# Insert with additional tags
sqlbackend.insert(pkt, tags={'testNum': '3'})
sqlbackend._conn.write_points.assert_called_with([{
'measurement': 'Packet1',
'tags': {
'testNum': '3'
},
'fields': {
'col1': 1,
'SampleTime': 33752069.10112411,
'SampleTime32': 168496141,
'SampleTime40': 235868177.0703125,
'SampleCmd16': 5398,
'SampleEvr16': 4884
}
}])
sqlbackend._conn.reset_mock()
os.remove(self.test_yaml_file)
def create_packet_from_result(cls, packet_id, result):
"""Create an AIT Packet from an InfluxDB query ResultSet item
Extract Influx DB query results entry into an AIT packet. This
assumes that telemetry data was inserted in the format generated by
:func:`InfluxDBBackend.insert`. Complex types such as CMD16 and EVR16 are
evaluated if they can be properly encoded from the raw value in the
query result. If there is no opcode / EVR-code for a particular raw
value the value is skipped (and thus defaulted to 0).
TODO: Reevaluate this assumption that missing opcodes / EVR-codes should be
left as 0 if a match isn't found in the dictionary.
Arguments
packet_id (string or PacketDefinition)
The "id" for the packet to create. If packet_id is a string it must
name a valid PacketDefintion in the telemetry dictionary. Otherwise,
it must be a PacketDefinition.
result (dict)
The :class:`influxdb.resultset.ResultSet` entry from which values
should be extracted to form the AIT packet
Returns
A :class:`ait.core.tlm.Packet` with values initialized from the values in the
ResultSet entry. If a field cannot be located in the result entry it will left
as the default value in the Packet or set to None if it's a CMD / EVR type.
"""
if isinstance(packet_id, str):
try:
pkt_defn = tlm.getDefaultDict()[packet_id]
except KeyError:
log.error(
f"Unknown packet name {packet_id} Unable to unpack ResultSet"
)
return None
elif isinstance(packet_id, tlm.PacketDefinition):
pkt_defn = packet_id
else:
log.error(
f"Unknown packet id type {packet_id}. Unable to unpack ResultSet"
)
return None
new_pkt = tlm.Packet(pkt_defn)
cmd_dict = cmd.getDefaultDict()
evr_dict = evr.getDefaultDict()
for f, f_defn in pkt_defn.fieldmap.items():
field_type_name = f_defn.type.name
if field_type_name == "CMD16":
if cmd_dict.opcodes.get(result[f], None):
cmd_def = cmd_dict.opcodes.get(result[f])
setattr(new_pkt, f, cmd_def.name)
elif field_type_name == "EVR16":
if evr_dict.codes.get(result[f], None):
evr_def = evr_dict.codes.get(result[f])
setattr(new_pkt, f, evr_def.name)
elif field_type_name == "TIME8":
setattr(new_pkt, f, result[f] / 256.0)
elif field_type_name == "TIME32":
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=result[f])
setattr(new_pkt, f, new_val)
elif field_type_name == "TIME40":
sec = int(result[f])
microsec = result[f] % 1 * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec,
microseconds=microsec)
setattr(new_pkt, f, new_val)
elif field_type_name == "TIME64":
sec = int(result[f])
microsec = result[f] % 1 * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec,
microseconds=microsec)
setattr(new_pkt, f, new_val)
else:
try:
setattr(new_pkt, f, result[f])
except KeyError:
log.info(
"Field not found in query results {} Skipping ...".
format(f))
return new_pkt
