def create_packets_from_results(cls, packet_name, result_set):
"""Generate AIT Packets from a InfluxDB query ResultSet
Extract Influx DB query results into one packet per result entry. This
assumes that telemetry data was inserted in the format generated by
:func:`InfluxDBBackend.insert`. Complex types such as CMD16 and EVR16 are
evaluated if they can be properly encoded from the raw value in the
query result. If there is no opcode / EVR-code for a particular raw
value the value is skipped (and thus defaulted to 0).
Arguments
packet_name (string)
The name of the AIT Packet to create from each result entry
result_set (influxdb.resultset.ResultSet)
The query ResultSet object to convert into packets
Returns
A list of packets extracted from the ResultSet object or None if
an invalid packet name is supplied.
"""
try:
tlm.getDefaultDict()[packet_name]
except KeyError:
log.error(
"Unknown packet name {} Unable to unpack ResultSet".format(
packet_name))
return None
return [
InfluxDBBackend.create_packet_from_result(packet_name, r)
for r in result_set.get_points()
]
def test_query_packets_arg_handling(self, importlib_mock):
sqlbackend = db.InfluxDBBackend()
sqlbackend._conn = mock.MagicMock()
# Test no packet provided handling
#######################################
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
sqlbackend.query_packets(start_time=start, end_time=end)
packets = ", ".join([f'"{i}"' for i in tlm.getDefaultDict().keys()])
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = (
f"SELECT * FROM \"{packets}\" WHERE time >= '{start}' AND time <= '{end}'"
)
assert sqlbackend._conn.query.call_args[0][0] == query
sqlbackend._conn.reset_mock()
# Test no start time handling
#######################################
end = dt.datetime.utcnow()
sqlbackend.query_packets(end_time=end)
packets = ", ".join([f'"{i}"' for i in tlm.getDefaultDict().keys()])
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = (
f"SELECT * FROM \"{packets}\" WHERE time >= '{start}' AND time <= '{end}'"
)
assert sqlbackend._conn.query.call_args[0][0] == query
sqlbackend._conn.reset_mock()
# Test no end time handling
#######################################
sqlbackend.query_packets()
packets = ", ".join([f'"{i}"' for i in tlm.getDefaultDict().keys()])
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = dt.datetime.utcnow()
query = f"SELECT * FROM {packets} WHERE time >= '{start}' AND time <= '{end}'"
exec_end_time = dt.datetime.strptime(
sqlbackend._conn.query.call_args[0][0].split("'")[-2],
dmc.RFC3339_Format)
assert (end - exec_end_time).seconds < 1
sqlbackend.query_packets()
# Test bad packet name exception
#######################################
with pytest.raises(ValueError):
sqlbackend.query_packets(packets=["not_a_valid_packet"])
def __init__(self, input_type=None, output_type=None, **kwargs):
"""
Params:
input_type: (optional) Specifies expected input type, used to
validate handler workflow. Defaults to None.
output_type: (optional) Specifies expected output type, used to
validate handler workflow. Defaults to None
**kwargs:
packet: (required) Name of packet, present in default tlm dict.
Raises:
ValueError: If packet is not present in kwargs.
If packet is specified but not present in default tlm dict.
"""
super(PacketHandler, self).__init__(input_type, output_type)
self.packet = kwargs.get("packet", None)
if not self.packet:
msg = 'PacketHandler: No packet name provided in handler config as key "packet"'
raise ValueError(msg)
tlm_dict = tlm.getDefaultDict()
if self.packet not in tlm_dict:
msg = "PacketHandler: Packet name {} not present in telemetry dictionary".format(
self.packet)
msg += " Available packet types are {}".format(tlm_dict.keys())
raise ValueError(msg)
self._pkt_defn = tlm_dict[self.packet]
def handle():
"""Return JSON Telemetry dictionary
**Example Response**:
.. sourcecode: json
{
ExaplePacket1: {
uid: 1,
fields: {
Voltage_B: {
type: "MSB_U16",
bytes: [2, 3],
name: "Voltage_B",
desc: "Voltage B as a 14-bit DN. Conversion to engineering units is TBD."
},
Voltage_C: {
type: "MSB_U16",
bytes: [4, 5],
name: "Voltage_C",
desc: "Voltage C as a 14-bit DN. Conversion to engineering units is TBD."
},
...
}
},
ExamplePacket2: {
...
}
}
"""
return json.dumps(tlm.getDefaultDict().toJSON())
def create(self, **kwargs):
''' Create and connect to a SQLite instance
Since creation and connection are intertwined in SQLite,
a pre-existing connection is not required unlike InfluxDB.
A skeleton database is built with the telemetry dictionary
since tables must be pre-created in SQLite.
**Configuration Parameters**
database name
The database name to create. Passed as either the config
key **database.dbname** or the kwargs argument
**database**. Defaults to **ait.db**.
tlmdict
The :class:`ait.core.tlm.TlmDict` instance to use. Defaults to
the currently configured telemetry dictionary.
'''
tlmdict = kwargs.get('tlmdict', tlm.getDefaultDict())
dbname = ait.config.get('database.dbname', kwargs.get('database', None))
if dbname:
self._conn = self._backend.connect(dbname)
else:
self._conn = self._backend.connect('ait.db')
for name, defn in tlmdict.items():
self._create_table(defn)
def __init__(self, input_type=None, output_type=None, **kwargs):
"""
Params:
input_type: (optional) Specifies expected input type, used to
validate handler workflow. Defaults to None.
output_type: (optional) Specifies expected output type, used to
validate handler workflow. Defaults to None
packet_types: (required) APID value (string) : packet name (string) pairs
APID value can use 'X' to mask out a bit
For example, 'XXXXX1011XXX' means only bits 6-9 represent the APID
packet_secondary_header_length: (optional) Length of secondary header in octets.
Defaults to 0.
Raises:
ValueError: If packet in config is not present in default tlm dict.
"""
super(CCSDSPacketHandler, self).__init__(input_type, output_type)
self.packet_types = kwargs['packet_types']
self.packet_secondary_header_length = kwargs.get(
'packet_secondary_header_length', 0)
# Check if all packet names in config are in telemetry dictionary
tlm_dict = tlm.getDefaultDict()
for packet_name in self.packet_types.values():
if packet_name not in tlm_dict.keys():
msg = 'CCSDSPacketHandler: Packet name {} not present in telemetry dictionary.'.format(
packet_name)
msg += ' Available packet types are {}'.format(tlm_dict.keys())
raise ValueError(msg)
def get_packet_and_defn():
first_stream = ait.config.get('gui.telemetry')[0]
stream = cfg.AitConfig(config=first_stream).get('stream')
name = stream.get('name', '<unnamed>')
pkt_defn = tlm.getDefaultDict().get(name, None)
pkt = tlm.Packet(pkt_defn)
return pkt_defn, pkt
def main():
try:
log.begin()
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
# Add optional command line arguments
parser.add_argument("--port", default=3076, type=int)
parser.add_argument("--host", default="127.0.0.1", type=str)
parser.add_argument("--packetName", default=None)
parser.add_argument("--packetFill", default=None)
# Get command line arguments
args = vars(parser.parse_args())
port = args["port"]
host = args["host"]
fill = args["packetFill"]
name = args["packetName"]
if name:
defn = tlm.getDefaultDict()[name]
else:
defn = list(tlm.getDefaultDict().values())[0]
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
packet = defn.simulate(fill=fill)
while True:
sock.sendto(packet._data, (host, port))
log.info("Sent telemetry (%d bytes) to %s:%d" %
(packet.nbytes, host, port))
time.sleep(1)
except KeyboardInterrupt:
log.info("Received Ctrl-C. Stopping telemetry stream.")
except Exception as e:
log.error("TLM send error: %s" % str(e))
log.end()
def test_ccsds_packet_uid(self):
handler = CCSDSPacketHandler(
packet_types={"01011100111": "CCSDS_HEADER"})
data = bytearray(b"\x02\xE7\x40\x00\x00\x00\x01")
tlm_dict = tlm.getDefaultDict()
packet_uid = tlm_dict["CCSDS_HEADER"].uid
result = handler.handle(data)
self.assertEqual(packet_uid, pickle.loads(result)[0])
def create_packet_from_result(cls, packet_name, data):
try:
pkt_defn = tlm.getDefaultDict()[packet_name]
except KeyError:
log.error("Unknown packet name {}. Unable to unpack SQLite result".
format(packet_name))
return None
return tlm.Packet(pkt_defn, data=data)
def create(self, **kwargs):
"""Create packet tables in the connected database
**Configuration Parameters**
tlmdict
The :class:`ait.core.tlm.TlmDict` instance to use. Defaults to
the currently configured telemetry dictionary.
"""
tlmdict = kwargs.get("tlmdict", tlm.getDefaultDict())
for _name, defn in tlmdict.items():
self._create_table(defn)
def create(database, tlmdict=None):
"""Creates a new database for the given Telemetry Dictionary and
returns a connection to it.
"""
if tlmdict is None:
tlmdict = tlm.getDefaultDict()
dbconn = connect(database)
for name, defn in tlmdict.items():
createTable(dbconn, defn)
return dbconn
def handle(self, input_data):
"""
Params:
packet: CCSDS packet
Returns:
tuple of packet UID and packet data field
"""
# Check if packet length is at least 7 bytes
primary_header_length = 6
if len(input_data) < primary_header_length + 1:
ait.core.log.info(
'CCSDSPacketHandler: Received packet length is less than minimum of 7 bytes.'
)
return
# Extract APID from packet
packet_apid = str(
bin(int(binascii.hexlify(input_data[0:2]), 16)
& 0x07FF))[2:].zfill(11)
# Check if packet_apid matches with an APID in the config
config_apid = self.comp_apid(packet_apid)
if not config_apid:
msg = 'CCSDSPacketHandler: Packet APID {} not present in config.'.format(
packet_apid)
msg += ' Available packet APIDs are {}'.format(
self.packet_types.keys())
ait.core.log.info(msg)
return
# Map APID to packet name in config to get UID from telemetry dictionary
packet_name = self.packet_types[config_apid]
tlm_dict = tlm.getDefaultDict()
packet_uid = tlm_dict[packet_name].uid
# Extract user data field from packet
packet_data_length = int(binascii.hexlify(input_data[4:6]), 16) + 1
if len(input_data) < primary_header_length + packet_data_length:
ait.core.log.info(
'CCSDSPacketHandler: Packet data length is less than stated length in packet primary header.'
)
return
udf_length = packet_data_length - self.packet_secondary_header_length
udf_start = primary_header_length + self.packet_secondary_header_length
user_data_field = input_data[udf_start:udf_start + udf_length + 1]
return pickle.dumps((packet_uid, user_data_field), 2)
def test_query_packets_calldown(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets,
start_time=start,
end_time=end)
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
for i, pkt in enumerate(packets):
query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC'
assert sqlbackend._conn.execute.call_args[i][0] == query
def __init__(self, inputs, outputs, **kwargs):
super(TelemetryLimitMonitor, self).__init__(inputs, outputs, **kwargs)
self.limit_dict = defaultdict(dict)
for k, v in limits.getDefaultDict().items():
packet, field = k.split('.')
self.limit_dict[packet][field] = v
self.packet_dict = defaultdict(dict)
for k, v in tlm.getDefaultDict().items():
self.packet_dict[v.uid] = v
self.notif_thrshld = ait.config.get('notifications.options.threshold', 1)
self.notif_freq = ait.config.get('notifications.options.frequency', float('inf'))
self.limit_trip_repeats = {}
log.info('Starting telemetry limit monitoring')
def __init__(self, inputs, outputs, zmq_args=None, **kwargs):
"""
Params:
inputs: names of inbound streams plugin receives data from
outputs: names of outbound streams plugin sends its data to
zmq_args: dict containing the follow keys:
zmq_context
zmq_proxy_xsub_url
zmq_proxy_xpub_url
Defaults to empty dict. Default values
assigned during instantiation of parent class.
**kwargs: (optional) Dependent on requirements of child class.
"""
super(AITOpenMctPlugin, self).__init__(inputs, outputs, zmq_args,
**kwargs)
log.info('Running AIT OpenMCT Plugin')
# Initialize state fields
# Debug state fields
self._debugEnabled = AITOpenMctPlugin.DEFAULT_DEBUG
self._debugMimicRepeat = False
# Port value for the server
self._servicePort = AITOpenMctPlugin.DEFAULT_PORT
# Setup server state
self._app = bottle.Bottle()
self._servers = []
# Queues for AIT events events
self._tlmQueue = api.GeventDeque(maxlen=100)
self._logQueue = api.GeventDeque(maxlen=100)
# Load AIT tlm dict and create OpenMCT format of it
self._aitTlmDict = tlm.getDefaultDict()
self._mctTlmDict = self.format_tlmdict_for_openmct(self._aitTlmDict)
# Create lookup from packet-uid to packet def
self._uidToPktDefMap = self.create_uid_pkt_map(self._aitTlmDict)
# Check for AIT config overrides
self._checkConfig()
gevent.spawn(self.init)
def __init__(self,
inputs,
outputs,
datastore='ait.core.db.InfluxDBBackend',
**kwargs):
"""
Attempts to connect to database backend. Plugin will not be created if
connection fails.
Creates base packet dictionary for decoding packets with packet UIDs as
keys and packet definitions as values.
Params:
inputs: list of names of input streams to plugin
outputs: list of names of plugin output streams
datastore: path to database backend to use
**kwargs: any args required for connecting to backend database
Raises:
ImportError: raised if provided database backend does not exist or
cannot be imported
Exception: raised if the backened database cannot be connected to
for any reason
"""
super(DataArchive, self).__init__(inputs, outputs, **kwargs)
self.datastore = datastore
self.packet_dict = defaultdict(dict)
for k, v in tlm.getDefaultDict().iteritems():
self.packet_dict[v.uid] = v
try:
mod, cls = self.datastore.rsplit('.', 1)
self.dbconn = getattr(importlib.import_module(mod), cls)()
self.dbconn.connect(**kwargs)
log.info('Starting telemetry data archiving')
except ImportError as e:
log.error("Could not import specified datastore {}".format(
self.datastore))
raise (e)
except Exception as e:
log.error(
"Unable to connect to {} backend. Disabling data archive.".
format(self.datastore))
raise (e)
def __init__(self,
inputs=None,
outputs=None,
zmq_args=None,
routing_table=None,
default_topic=None):
super().__init__(inputs, outputs, zmq_args)
self.default_topic = default_topic
if 'path' in routing_table:
self.routing_table_object = self.load_table_yaml(
routing_table['path'], tlm.getDefaultDict())
else:
self.routing_table_object = None
log.error("no path specified for routing table")
if self.routing_table_object is None:
log.error("Unable to load routing table .yaml file")
def create(self, **kwargs):
''' Create a database for the current telemetry dictionary
Connects to a SQLite instance via :func:`connect` and creates a
skeleton database for future data inserts.
**Configuration Parameters**
tlmdict
The :class:`ait.core.tlm.TlmDict` instance to use. Defaults to
the currently configured telemetry dictionary.
'''
tlmdict = kwargs.get('tlmdict', tlm.getDefaultDict())
self.connect(**kwargs)
for name, defn in tlmdict.items():
self._create_table(defn)
def handle():
"""Set playback query with packet name, start time, and end time from form"""
global playback
if not playback.enabled:
return HttpResponse(status=404,
body='Historic data playback is disabled')
tlm_dict = tlm.getDefaultDict()
# Get values from form
packet = bottle.request.forms.get('packet')
start_time = bottle.request.forms.get('startTime')
end_time = bottle.request.forms.get('endTime')
uid = tlm_dict[packet].uid
# Query packet and time range from database
point_query = 'SELECT * FROM "{}" WHERE time >= \'{}\' AND time <= \'{}\''.format(
packet, start_time, end_time)
points = list(playback.dbconn.query(point_query).get_points())
pkt = tlm_dict[packet]
fields = pkt.fields
# Build field names list from tlm dictionary for sorting data query
field_names = []
# Build field types list from tlm dictionary for packing data
field_formats = []
for i in range(len(fields)):
field_names.append(fields[i].name)
field_type = str(fields[i].type).split("'")[1]
field_formats.append(dtype.get(field_type).format)
# Put query into a map of {timestamp: list of (uid, data)}
for i in range(len(points)):
# Round time down to nearest 0.1 second
timestamp = str(points[i]['time'][:21] + 'Z')
data = ''
for j in range(len(field_names)):
data += struct.pack(field_formats[j], points[i][field_names[j]])
if playback.query.has_key(timestamp):
playback.query[timestamp].append((uid, data))
else:
playback.query[timestamp] = [(uid, data)]
def __init__(self,
cmdport=ait.config.get('command.port', ait.DEFAULT_CMD_PORT),
tlmport=3076, defn=None):
if defn is None:
tlmdict = tlm.getDefaultDict()
names = sorted( tlmdict.keys() )
if len(names) == 0:
msg = 'No packets defined in default TLM dictionary.'
raise TypeError(msg)
defn = tlmdict[ names[0] ]
self._packets = PacketBuffers()
self._cmd = CmdAPI(cmdport)
self._packets.create(defn.name)
pktbuf = self._packets[defn.name]
self._tlm = UdpTelemetryServer(tlmport, pktbuf, defn)
self._tlm.start()
def test_query_packets_calldown(self, importlib_mock):
sqlbackend = db.InfluxDBBackend()
sqlbackend._conn = mock.MagicMock()
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets,
start_time=start,
end_time=end)
packets = ", ".join(packets)
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = (
f"SELECT * FROM \"{packets}\" WHERE time >= '{start}' AND time <= '{end}'"
)
assert sqlbackend._conn.query.call_args[0][0] == query
def __init__(self, cmdport=None, packets=None):
""""""
tlmdict = tlm.getDefaultDict()
if packets is None:
packets = list(tlmdict.keys())
else:
if not isinstance(packets, collections.Iterable):
packets = [packets]
cln_pkts = []
for pkt in packets:
if pkt not in tlmdict:
log.error(
f"Instrument passed invalid packet name {pkt}. Skipping ..."
)
else:
cln_pkts.append(pkt)
packets = cln_pkts
defns = {tlmdict[k].uid: tlmdict[k] for k in packets}
if len(defns.keys()) == 0:
msg = (
"No packets available to Instrument. Ensure your dictionary "
"is valid and contains Packets. If you passed packet names to "
"Instrument ensure at least one is valid."
)
raise TypeError(msg)
self._pkt_buffs = PacketBuffers()
for _, pkt_defn in defns.items():
self._pkt_buffs.create(pkt_defn.name)
self._cmd = CmdAPI(cmdport)
self._monitor = TlmMonitor(self._pkt_buffs, defns)
self._monitor.start()
gevent.sleep(0)
def main():
log.begin()
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--all", action="store_true", help="output all fields/values")
parser.add_argument(
"--csv",
default="output.csv",
metavar="</path/to/output/csv>",
help="Output as CSV with filename",
)
parser.add_argument(
"--fields",
metavar="</path/to/fields/file>",
help="path to the file containing all fields to query, separated by newline.",
)
parser.add_argument(
"--packet", required=True, help="Packet name from telemetry dictionary specified in config file."
)
parser.add_argument(
"--time_field",
help=(
"Time field to use for time range comparisons. Ground receipt time "
"will be used if nothing is specified."
),
)
parser.add_argument(
"--stime",
help=(
"Datetime in file to start collecting the data values. Defaults to "
"beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ"
),
)
parser.add_argument(
"--etime",
help=(
"Datetime in file to end collecting the data values. Defaults to end "
"of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ"
),
)
parser.add_argument(
"pcap", nargs="*", help=("PCAP file(s) containing telemetry packets")
)
args = parser.parse_args()
args.ground_time = True
if args.time_field is not None:
args.ground_time = False
tlmdict = tlm.getDefaultDict()
defn = None
try:
if tlmdict is not None:
defn = tlmdict[args.packet]
except KeyError:
log.error('Packet "%s" not defined in telemetry dictionary.' % args.packet)
log.end()
sys.exit(2)
if not args.all and args.fields is None:
log.error(
"Must provide fields file with --fields or specify that all fields should be queried with --all"
)
log.end()
sys.exit(2)
if args.all:
fields = [flddefn.name for flddefn in defn.fields]
else:
# Parse the fields file into a list
with open(args.fields, "r") as stream:
fields = [fldname.strip() for fldname in stream.readlines()]
not_found = False
# TODO Rework this into the CSV generation. Not here.
# Duplicating effort
for fldname in fields:
raw = fldname.split(".")
if fldname not in defn.fieldmap and (
len(raw) == 2 and raw[0] != "raw" or raw[1] not in defn.fieldmap
):
not_found = True
log.error('No telemetry point named "%s"' % fldname)
if not_found:
log.end()
sys.exit(2)
if args.stime:
start = datetime.strptime(args.stime, dmc.ISO_8601_Format)
else:
start = dmc.GPS_Epoch
if args.etime:
stop = datetime.strptime(args.etime, dmc.ISO_8601_Format)
else:
stop = datetime.utcnow()
# Append time to beginning of each row
if not args.ground_time:
fields.insert(0, args.time_field)
else:
fields.insert(0, "Ground Receipt Time")
csv_file = None
csv_writer = None
npackets = 0
if args.csv:
csv_file = open(args.csv, "w")
csv_writer = csv.writer(csv_file)
output(csv_writer, fields)
# If we're comparing off ground receipt time we need to drop the header label to avoid
# indexing errors when processing the fields.
if args.ground_time:
fields = fields[1:]
rowcnt = 0
for filename in args.pcap:
log.debug("Processing %s" % filename)
with pcap.open(filename, "rb") as stream:
header, data = stream.read()
while data:
packet = tlm.Packet(defn, data)
comp_time = (
header.timestamp
if args.ground_time
else getattr(packet, args.time_field)
)
if start < comp_time < stop:
row = []
for field in fields:
try:
# check if raw value requested
_raw = False
names = field.split(".")
if len(names) == 2 and names[0] == "raw":
field = names[1]
_raw = True
field_val = packet._getattr(field, raw=_raw)
if hasattr(field_val, "name"):
field_val = field_val.name
else:
field_val = str(field_val)
except KeyError:
log.debug("%s not found in Packet" % field)
field_val = None
except ValueError:
# enumeration not found. just get the raw value
field_val = packet._getattr(field, raw=True)
row.append(field_val)
if args.ground_time:
row = [comp_time] + row
rowcnt += 1
output(csv_writer, row)
npackets += 1
header, data = stream.read()
log.debug("Parsed %s packets." % npackets)
csv_file.close()
if rowcnt == 0:
os.remove(args.csv)
log.end()
def mimic_tlm(self, ait_tlm_pkt_name, ait_tlm_pkt_fill=None):
"""Used for debugging, creates an instance of a packet based on
packet name, and fills it with zero data.
Special case for '1553_HS_Packet' which will get random number
data fills.
If HTTP Request query includes a value for 'repeat', then this
will continue emitting telemetry.
"""
# Http query option, if it is set to anything, consider it true
self._debugMimicRepeat = len(str(bottle.request.query.repeat)) > 0
# This will be helpful in testing by simulating TLM
# but by a rest call instead of actual telem
ait_pkt_defn = None
if ait_tlm_pkt_name:
ait_pkt_defn = tlm.getDefaultDict()[ait_tlm_pkt_name]
else:
ait_pkt_defn = tlm.getDefaultDict().values()[0]
# Create the expected message format
pkt_size_bytes = ait_pkt_defn.nbytes
# if self._debugMimicRepeat:
repeat_str = " REPEATED " if self._debugMimicRepeat else " a single "
info_msg = ("Received request to mimic" + repeat_str +
"telemetry packet for " + ait_pkt_defn.name)
self.dbg_message(info_msg)
# Create a binary array of size filled with 0
dummy_data = bytearray(pkt_size_bytes)
info_msg = ""
while True:
# Special handling for simply integer based packet, others will
# have all 0 zero
if ait_pkt_defn.name == "1553_HS_Packet":
hs_packet = struct.Struct(">hhhhh")
random_num = random.randint(1, 100)
dummy_data = hs_packet.pack(random_num, random_num, random_num,
random_num, random_num)
tlm_pkt = tlm.Packet(ait_pkt_defn, data=bytearray(dummy_data))
self._process_telem_msg(tlm_pkt)
info_msg = (
"AIT OpenMct Plugin submitted mimicked telemetry for " +
ait_pkt_defn.name + " (" + str(datetime.datetime.now()) +
") to telem queue")
self.dbg_message(info_msg)
# sleep if mimic on
if self._debugMimicRepeat:
gsleep(5)
# either it was immediate or we woke up, check break condition
if not self._debugMimicRepeat:
break
# Return last status message as result to client
return info_msg
def __init__(self,
inputs,
outputs,
zmq_args=None,
datastore="ait.core.db.InfluxDBBackend",
**kwargs):
"""
Params:
inputs: names of inbound streams plugin receives data from
outputs: names of outbound streams plugin sends its data to
zmq_args: dict containing the follow keys:
zmq_context
zmq_proxy_xsub_url
zmq_proxy_xpub_url
Defaults to empty dict. Default values
assigned during instantiation of parent class.
datastore: path to database backend to use
**kwargs: (optional) Dependent on requirements of child class.
"""
super(AITOpenMctPlugin, self).__init__(inputs, outputs, zmq_args,
**kwargs)
log.info("Running AIT OpenMCT Plugin")
self._datastore = datastore
# Initialize state fields
# Debug state fields
self._debugEnabled = AITOpenMctPlugin.DEFAULT_DEBUG
self._debugMimicRepeat = False
# Port value for the server
self._servicePort = AITOpenMctPlugin.DEFAULT_PORT
# Flag indicating if we should create a database connection for historical queries
self._databaseEnabled = AITOpenMctPlugin.DEFAULT_DATABASE_ENABLED
# Check for AIT config overrides
self._check_config()
# Setup server state
self._app = bottle.Bottle()
self._servers = []
# Queues for AIT events events
self._tlmQueue = api.GeventDeque(maxlen=100)
# Load AIT tlm dict and create OpenMCT format of it
self._aitTlmDict = tlm.getDefaultDict()
self._mctTlmDict = DictUtils.format_tlmdict_for_openmct(
self._aitTlmDict)
# Create lookup from packet-uid to packet def
self._uidToPktDefMap = DictUtils.create_uid_pkt_map(self._aitTlmDict)
# Attempt to initialize database, None if no DB
self._database = self.load_database(**kwargs)
# Maintains a set of active websocket structs
self._socket_set = set()
# Spawn greenlets to poll telemetry
self.tlm_poll_greenlet = Greenlet.spawn(
self.poll_telemetry_periodically)
gevent.spawn(self.init)
def test_query_packets_arg_handling(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
query_string = (
'SELECT * FROM "{}" WHERE time >= "{}" AND time <= "{}" ORDER BY time ASC'
)
# Test no packet provided handling
#######################################
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
res = sqlbackend.query_packets(start_time=start, end_time=end)
packets = list(tlm.getDefaultDict().keys())
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = query_string.format(packets, start, end)
for i, pkt in enumerate(packets):
query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC'
assert sqlbackend._conn.execute.call_args_list[i][0][0] == query
sqlbackend._conn.reset_mock()
# Test no start time handling
#######################################
end = dt.datetime.utcnow()
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets, end_time=end)
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = query_string.format(packets[0], start, end)
assert sqlbackend._conn.execute.call_args[0][0] == query
sqlbackend._conn.reset_mock()
# Test no end time handling
#######################################
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets)
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = dt.datetime.utcnow()
query = query_string.format(packets, start, end)
exec_end_time = dt.datetime.strptime(
sqlbackend._conn.execute.call_args[0][0].split('"')[-2],
dmc.RFC3339_Format)
assert (end - exec_end_time).seconds < 1
sqlbackend.query_packets()
# Test bad packet name exception
#######################################
with pytest.raises(ValueError):
sqlbackend.query_packets(packets=["not_a_valid_packet"])
def create_packets_from_results(self, packet_name, result_set):
''' Generate AIT Packets from a InfluxDB query ResultSet
Extract Influx DB query results into one packet per result entry. This
assumes that telemetry data was inserted in the format generated by
:func:`InfluxDBBackend.insert`. Complex types such as CMD16 and EVR16 are
evaluated if they can be properly encoded from the raw value in the
query result. If there is no opcode / EVR-code for a particular raw
value the value is skipped (and thus defaulted to 0).
Arguments
packet_name (string)
The name of the AIT Packet to create from each result entry
result_set (influxdb.resultset.ResultSet)
The query ResultSet object to convert into packets
Returns
A list of packets extracted from the ResultSet object or None if
an invalid packet name is supplied.
'''
try:
pkt_defn = tlm.getDefaultDict()[packet_name]
except KeyError:
log.error('Unknown packet name {} Unable to unpack ResultSet'.format(packet_name))
return None
pkt = tlm.Packet(pkt_defn)
pkts = []
for r in result_set.get_points():
new_pkt = tlm.Packet(pkt_defn)
for f, f_defn in pkt_defn.fieldmap.iteritems():
field_type_name = f_defn.type.name
if field_type_name == 'CMD16':
if cmd.getDefaultDict().opcodes.get(r[f], None):
setattr(new_pkt, f, cmd_def.name)
elif field_type_name == 'EVR16':
if evr.getDefaultDict().codes.get(r[f], None):
setattr(new_pkt, f, r[f])
elif field_type_name == 'TIME8':
setattr(new_pkt, f, r[f] / 256.0)
elif field_type_name == 'TIME32':
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=r[f])
setattr(new_pkt, f, new_val)
elif field_type_name == 'TIME40':
sec = int(r[f])
microsec = r[f] * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec, microseconds=microsec)
setattr(new_pkt, f, new_val)
elif field_type_name == 'TIME64':
sec = int(r[f])
microsec = r[f] % 1 * 1e6
new_val = dmc.GPS_Epoch + dt.timedelta(seconds=sec, microseconds=microsec)
setattr(new_pkt, f, new_val)
else:
try:
setattr(new_pkt, f, r[f])
except KeyError:
log.info('Field not found in query results {} Skipping ...'.format(f))
pkts.append(new_pkt)
return pkts
def main():
tlmdict = tlm.getDefaultDict()
pnames = tlmdict.keys()
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arguments = {
'--packet': {
'type': str,
'choices': pnames,
'default': pnames[0] if len(pnames) > 0 else None,
'help': 'Type of packets (!Packet name in tlm.yaml) in file',
'required': len(pnames) > 1,
},
'--database': {
'default':
ait.config.get('database.name'),
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)'),
'required':
ait.config.get('database.name') is None
},
'--backend': {
'default':
'sqlite',
'choices': ['sqlite', 'influx'],
'action':
'store',
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)')
},
'--use-current-time': {
'action':
'store_true',
'help': ('Use current time stamps when insert packets instead '
'of ground receipt time (or the time written in the '
'PCAP header).')
},
'file': {
'nargs': '+',
'help': 'File(s) containing telemetry packets'
}
}
for name, params in arguments.items():
ap.add_argument(name, **params)
args = ap.parse_args()
log.begin()
try:
npackets = 0
dbconn = None
defn = tlm.getDefaultDict()[args.packet]
nbytes = defn.nbytes
if args.backend == 'sqlite':
dbconn = db.SQLiteBackend()
elif args.backend == 'influx':
dbconn = db.InfluxDBBackend()
if args.backend == 'sqlite' and (args.database == ':memory:'
or not os.path.exists(args.database)):
dbconn.create(database=args.database)
else:
dbconn.connect(database=args.database)
for filename in args.file:
log.info('Processing %s' % filename)
with pcap.open(filename) as stream:
for header, pkt_data in stream:
try:
packet = tlm.Packet(defn, pkt_data)
time = header.timestamp
if args.use_current_time:
time = None
dbconn.insert(packet, time=time)
npackets += 1
except struct.error:
log.error(
"Unable to unpack data into packet. Skipping ...")
except KeyboardInterrupt:
log.info('Received Ctrl-C. Stopping database insert.')
except IOError as e:
log.error(str(e))
finally:
dbconn.close()
values = npackets, args.packet, args.database
log.info('Inserted %d %s packets into database %s.' % values)
log.end()
def main():
log.begin()
description = """Parses 1553 telemetry into CSV file."""
arguments = {
'--all': {
'action': 'store_true',
'help': 'output all fields/values',
},
'--csv': {
'type': str,
'default': 'output.csv',
'metavar': '</path/to/output/csv>',
'help': 'Output as CSV with filename'
},
'--fields': {
'type': str,
'metavar': '</path/to/fields/file>',
'help':
'file containing all fields to query, separated by newline.'
},
'--packet': {
'type': str,
'required': True,
'help': 'field names to query, separated by space'
},
'--time_field': {
'type':
str,
'help':
'Time field to use for time range comparisons. Ground receipt time will be used if nothing is specified.'
},
'--stime': {
'type':
str,
'help':
'Datetime in file to start collecting the data values. Defaults to beginning of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ'
},
'--etime': {
'type':
str,
'help':
'Datetime in file to end collecting the data values. Defaults to end of pcap. Expected format: YYYY-MM-DDThh:mm:ssZ'
}
}
arguments['pcap'] = {
'nargs': '*',
'help': 'PCAP file(s) containing telemetry packets'
}
args = gds.arg_parse(arguments, description)
args.ground_time = True
if args.time_field is not None:
args.ground_time = False
tlmdict = tlm.getDefaultDict()
defn = None
try:
if tlmdict is not None:
defn = tlmdict[args.packet]
except KeyError:
log.error('Packet "%s" not defined in telemetry dictionary.' %
args.packet)
gds.exit(2)
if not args.all and args.fields is None:
log.error(
'Must provide fields file with --fields or specify that all fields should be queried with --all'
)
gds.exit(2)
if args.all:
fields = [flddefn.name for flddefn in defn.fields]
else:
# Parse the fields file into a list
with open(args.fields, 'rb') as stream:
fields = [fldname.strip() for fldname in stream.readlines()]
not_found = False
# TODO Rework this into the CSV generation. Not here.
# Duplicating effort
for fldname in fields:
raw = fldname.split('.')
if fldname not in defn.fieldmap and (len(raw) == 2 and raw[0] != 'raw'
or raw[1] not in defn.fieldmap):
not_found = True
log.error('No telemetry point named "%s"' % fldname)
if not_found:
gds.exit(2)
if args.stime:
start = datetime.strptime(args.stime, dmc.ISO_8601_Format)
else:
start = dmc.GPS_Epoch
if args.etime:
stop = datetime.strptime(args.etime, dmc.ISO_8601_Format)
else:
stop = datetime.utcnow()
# Append time to beginning of each row
if not args.ground_time:
fields.insert(0, args.time_field)
else:
fields.insert(0, 'Ground Receipt Time')
csv_file = None
csv_writer = None
npackets = 0
if args.csv:
csv_file = open(args.csv, 'wb')
csv_writer = csv.writer(csv_file)
output(csv_writer, fields)
# If we're comparing off ground receipt time we need to drop the header label to avoid
# indexing errors when processing the fields.
if args.ground_time:
fields = fields[1:]
rowcnt = 0
for filename in args.pcap:
log.debug('Processing %s' % filename)
with pcap.open(filename, 'rb') as stream:
header, data = stream.read()
while data:
packet = tlm.Packet(defn, data)
comp_time = header.timestamp if args.ground_time else getattr(
packet, args.time_field)
if start < comp_time < stop:
row = []
for field in fields:
try:
# check if raw value requested
_raw = False
names = field.split('.')
if len(names) == 2 and names[0] == 'raw':
field = names[1]
_raw = True
fieldVal = packet._getattr(field, raw=_raw)
if hasattr(fieldVal, 'name'):
fieldVal = fieldVal.name
else:
fieldVal = str(fieldVal)
except KeyError:
log.debug('%s not found in Packet' % field)
fieldVal = None
except ValueError:
# enumeration not found. just get the raw value
fieldVal = packet._getattr(field, raw=True)
row.append(fieldVal)
if args.ground_time:
row = [comp_time] + row
rowcnt += 1
output(csv_writer, row)
npackets += 1
header, data = stream.read()
log.debug('Parsed %s packets.' % npackets)
csv_file.close()
if rowcnt == 0:
os.remove(args.csv)
log.end()
