def test_sqlite_create_table(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, 'wt') as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
sqlbackend._create_table(tlmdict['Packet1'])
sqlbackend._conn.execute.assert_called_with(
'CREATE TABLE IF NOT EXISTS "Packet1" (time DATETIME DEFAULT(STRFTIME(\'%Y-%m-%dT%H:%M:%fZ\', \'NOW\')), col1 INTEGER, SampleTime REAL)'
)
os.remove(self.test_yaml_file)
def test_sqlite_create(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime64
type: TIME64
"""
with open(self.test_yaml_file, 'wt') as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend.connect = mock.MagicMock()
sqlbackend._create_table = mock.MagicMock()
sqlbackend.create(tlmdict=tlmdict)
sqlbackend._create_table.assert_called_with(tlmdict['Packet1'])
os.remove(self.test_yaml_file)
def test_sqlite_create_table(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, "wt") as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
sqlbackend._create_table(tlmdict["Packet1"])
sqlbackend._conn.execute.assert_called_with(
"CREATE TABLE IF NOT EXISTS \"Packet1\" (time DATETIME DEFAULT(STRFTIME('%Y-%m-%dT%H:%M:%fZ', 'NOW')), PKTDATA BLOB NOT NULL)"
)
os.remove(self.test_yaml_file)
def test_sqlite_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, 'wt') as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict['Packet1']
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
sqlbackend.insert(pkt)
sqlbackend._conn.execute.assert_called_with(
'INSERT INTO "Packet1" (col1, SampleTime) VALUES (?, ?)',
[1, 33752069.10112411])
os.remove(self.test_yaml_file)
def test_query_fail_handling(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
sqlbackend._query = mock.MagicMock()
sqlbackend._query.side_effect = sqlbackend._backend.OperationalError(
"foo")
res = sqlbackend.query_packets()
for e in res.errors:
assert e == "foo"
def test_sqlite_connect(self, importlib_mock):
sqlbackend = db.SQLiteBackend()
sqlbackend._backend = mock.MagicMock()
sqlbackend.connect()
assert sqlbackend._backend.connect.called
sqlbackend._backend.connect.assert_called_with('ait.db')
sqlbackend._backend.reset_mock()
sqlbackend.connect(database='foo.db')
assert sqlbackend._backend.connect.called
sqlbackend._backend.connect.assert_called_with('foo.db')
def test_sqlite_insert(self, importlib_mock):
yaml_doc = """
- !Packet
name: Packet1
history:
- col1
fields:
- !Field
name: col1
desc: test column 1
type: MSB_U16
enum:
a: testa
- !Field
name: SampleTime
type: TIME64
"""
with open(self.test_yaml_file, "wt") as out:
out.write(yaml_doc)
tlmdict = tlm.TlmDict(self.test_yaml_file)
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
pkt_defn = tlmdict["Packet1"]
pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes)))
# Note: we can't fully test this call given the modification to
# the packet data on insert. Better than nothing I suppose.
# Test without time
sqlbackend.insert(pkt)
assert ('INSERT INTO "Packet1" (PKTDATA) VALUES (?)'
in sqlbackend._conn.execute.call_args[0])
sqlbackend._conn.reset_mock()
# Test with time
now = dt.datetime.utcnow()
sqlbackend.insert(pkt, time=now)
assert ('INSERT INTO "Packet1" (PKTDATA, time) VALUES (?, ?)'
in sqlbackend._conn.execute.call_args[0])
assert (now.strftime(
dmc.RFC3339_Format) == sqlbackend._conn.execute.call_args[0][1][1])
os.remove(self.test_yaml_file)
def test_query_packets_calldown(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets,
start_time=start,
end_time=end)
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
for i, pkt in enumerate(packets):
query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC'
assert sqlbackend._conn.execute.call_args[i][0] == query
def test_query_return_types(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
sqlbackend._query = mock.MagicMock()
# Check that a successful query returns a properly formatted AITDBResult
ret_val = [1, 2, 3]
query_string = "select * from table"
sqlbackend._query.return_value = ret_val
results = sqlbackend.query(query_string)
assert isinstance(results, db.AITDBResult)
assert results.query == query_string
assert results.results == ret_val
# Check that a failed query returns a properly formatted AITDBResult
sqlbackend._query.side_effect = sqlbackend._backend.OperationalError(
"foo")
results = sqlbackend.query(query_string)
assert results.query == query_string
assert results.errors == ["foo"]
def test_sqlite_connect(self, importlib_mock):
sqlbackend = db.SQLiteBackend()
sqlbackend._backend = mock.MagicMock()
# Check default database naming
sqlbackend.connect()
assert sqlbackend._backend.connect.called
sqlbackend._backend.connect.assert_called_with("ait.db")
sqlbackend._backend.reset_mock()
# Check custom database naming
sqlbackend.connect(database="foo.db")
assert sqlbackend._backend.connect.called
sqlbackend._backend.connect.assert_called_with("foo.db")
# Backend should only call self.create if a database doesn't exist
#
# Mock return_value handling wasn't cooperating with decorators
# so we'll handle it manually instead ...
import os.path
isfile_mock = mock.MagicMock(return_value=True)
isfile_orig = os.path.isfile
os.path.isfile = isfile_mock
sqlbackend.create = mock.MagicMock()
assert isfile_mock is os.path.isfile
isfile_mock.return_value = True
sqlbackend.connect()
assert sqlbackend.create.called is False
isfile_mock.return_value = False
sqlbackend.connect()
assert sqlbackend.create.called
os.path.isfile = isfile_orig
def test_query_success_handling(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
res_mock = mock.MagicMock()
res_mock.return_value = [
("2020-12-02T00:41:43.199Z",
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"),
("2020-12-02T00:41:44.200Z",
b"\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01"),
("2020-12-02T00:41:45.205Z",
b"\x00\x02\x00\x02\x00\x02\x00\x02\x00\x02"),
("2020-12-02T00:41:46.211Z",
b"\x00\x03\x00\x03\x00\x03\x00\x03\x00\x03"),
("2020-12-02T00:41:47.216Z",
b"\x00\x04\x00\x04\x00\x04\x00\x04\x00\x04"),
("2020-12-02T00:41:48.221Z",
b"\x00\x05\x00\x05\x00\x05\x00\x05\x00\x05"),
]
sqlbackend._query = res_mock
res = sqlbackend.query_packets(packets=["1553_HS_Packet"])
assert isinstance(res, db.AITDBResult)
assert res._packets is not None
res_pkts = list(res.get_packets())
assert len(res_pkts) == 6
assert isinstance(res_pkts[0], tlm.Packet)
assert res_pkts[0].Voltage_A == 0
assert res_pkts[1].Voltage_A == 1
assert res_pkts[2].Voltage_A == 2
assert res_pkts[3].Voltage_A == 3
assert res_pkts[4].Voltage_A == 4
assert res_pkts[5].Voltage_A == 5
def test_query_packet_time_inclusion(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
ret_data = [
("2020-12-02T00:41:43.199Z",
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"),
("2020-12-02T00:41:44.200Z",
b"\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01"),
("2020-12-02T00:41:45.205Z",
b"\x00\x02\x00\x02\x00\x02\x00\x02\x00\x02"),
("2020-12-02T00:41:46.211Z",
b"\x00\x03\x00\x03\x00\x03\x00\x03\x00\x03"),
("2020-12-02T00:41:47.216Z",
b"\x00\x04\x00\x04\x00\x04\x00\x04\x00\x04"),
("2020-12-02T00:41:48.221Z",
b"\x00\x05\x00\x05\x00\x05\x00\x05\x00\x05"),
]
res_mock = mock.MagicMock()
res_mock.return_value = ret_data
sqlbackend._query = res_mock
res = sqlbackend.query_packets(packets=["1553_HS_Packet"],
yield_packet_time=True)
assert isinstance(res, db.AITDBResult)
assert res._packets is not None
res_pkts = list(res.get_packets())
assert len(res_pkts) == 6
assert isinstance(res_pkts[0], tuple)
for i, test_data in enumerate(ret_data):
assert dmc.rfc3339_str_to_datetime(
ret_data[i][0]) == res_pkts[i][0]
assert res_pkts[i][1].Voltage_A == i
def main():
tlmdict = tlm.getDefaultDict()
pnames = list(tlmdict.keys())
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arguments = {
"--packet": {
"type": str,
"choices": pnames,
"default": pnames[0] if len(pnames) > 0 else None,
"help": "Type of packets (!Packet name in tlm.yaml) in file",
"required": len(pnames) > 1,
},
"--database": {
"default":
ait.config.get("database.dbname"),
"help": ("Name of database in which to insert packets (may "
"also be specified in config.yaml database.name)"),
"required":
ait.config.get("database.dbname") is None,
},
"--backend": {
"default":
"sqlite",
"choices": ["sqlite", "influx"],
"action":
"store",
"help": ("Name of database in which to insert packets (may "
"also be specified in config.yaml database.name)"),
},
"--use-current-time": {
"action":
"store_true",
"help": ("Use current time stamps when insert packets instead "
"of ground receipt time (or the time written in the "
"PCAP header)."),
},
"file": {
"nargs": "+",
"help": "File(s) containing telemetry packets"
},
}
for name, params in arguments.items():
ap.add_argument(name, **params)
args = ap.parse_args()
log.begin()
try:
npackets = 0
dbconn = None
defn = tlm.getDefaultDict()[args.packet]
if args.backend == "sqlite":
dbconn = db.SQLiteBackend()
elif args.backend == "influx":
dbconn = db.InfluxDBBackend()
if args.backend == "sqlite" and (args.database == ":memory:"
or not os.path.exists(args.database)):
dbconn.create(database=args.database)
else:
dbconn.connect(database=args.database)
for filename in args.file:
log.info("Processing %s" % filename)
with pcap.open(filename) as stream:
for header, pkt_data in stream:
try:
packet = tlm.Packet(defn, pkt_data)
time = header.timestamp
if args.use_current_time:
time = None
dbconn.insert(packet, time=time)
npackets += 1
except struct.error:
log.error(
"Unable to unpack data into packet. Skipping ...")
except KeyboardInterrupt:
log.info("Received Ctrl-C. Stopping database insert.")
except IOError as e:
log.error(str(e))
finally:
dbconn.close()
values = npackets, args.packet, args.database
log.info("Inserted %d %s packets into database %s." % values)
log.end()
def test_sqlite_query_calldown(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
sqlbackend.query("SELECT * FROM table")
sqlbackend._conn.execute.assert_called_with("SELECT * FROM table")
def test_sqlite_backend_init(self, importlib_mock):
sqlbackend = db.SQLiteBackend()
importlib_mock.assert_called_with('sqlite3')
def main():
tlmdict = tlm.getDefaultDict()
pnames = tlmdict.keys()
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arguments = {
'--packet': {
'type': str,
'choices': pnames,
'default': pnames[0] if len(pnames) > 0 else None,
'help': 'Type of packets (!Packet name in tlm.yaml) in file',
'required': len(pnames) > 1,
},
'--database': {
'default':
ait.config.get('database.name'),
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)'),
'required':
ait.config.get('database.name') is None
},
'--backend': {
'default':
'sqlite',
'choices': ['sqlite', 'influx'],
'action':
'store',
'help': ('Name of database in which to insert packets (may '
'also be specified in config.yaml database.name)')
},
'--use-current-time': {
'action':
'store_true',
'help': ('Use current time stamps when insert packets instead '
'of ground receipt time (or the time written in the '
'PCAP header).')
},
'file': {
'nargs': '+',
'help': 'File(s) containing telemetry packets'
}
}
for name, params in arguments.items():
ap.add_argument(name, **params)
args = ap.parse_args()
log.begin()
try:
npackets = 0
dbconn = None
defn = tlm.getDefaultDict()[args.packet]
nbytes = defn.nbytes
if args.backend == 'sqlite':
dbconn = db.SQLiteBackend()
elif args.backend == 'influx':
dbconn = db.InfluxDBBackend()
if args.backend == 'sqlite' and (args.database == ':memory:'
or not os.path.exists(args.database)):
dbconn.create(database=args.database)
else:
dbconn.connect(database=args.database)
for filename in args.file:
log.info('Processing %s' % filename)
with pcap.open(filename) as stream:
for header, pkt_data in stream:
try:
packet = tlm.Packet(defn, pkt_data)
time = header.timestamp
if args.use_current_time:
time = None
dbconn.insert(packet, time=time)
npackets += 1
except struct.error:
log.error(
"Unable to unpack data into packet. Skipping ...")
except KeyboardInterrupt:
log.info('Received Ctrl-C. Stopping database insert.')
except IOError as e:
log.error(str(e))
finally:
dbconn.close()
values = npackets, args.packet, args.database
log.info('Inserted %d %s packets into database %s.' % values)
log.end()
def test_query_packets_arg_handling(self):
sqlbackend = db.SQLiteBackend()
sqlbackend._conn = mock.MagicMock()
query_string = (
'SELECT * FROM "{}" WHERE time >= "{}" AND time <= "{}" ORDER BY time ASC'
)
# Test no packet provided handling
#######################################
start = dmc.GPS_Epoch
end = dt.datetime.utcnow()
res = sqlbackend.query_packets(start_time=start, end_time=end)
packets = list(tlm.getDefaultDict().keys())
start = start.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = query_string.format(packets, start, end)
for i, pkt in enumerate(packets):
query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC'
assert sqlbackend._conn.execute.call_args_list[i][0][0] == query
sqlbackend._conn.reset_mock()
# Test no start time handling
#######################################
end = dt.datetime.utcnow()
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets, end_time=end)
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = end.strftime(dmc.RFC3339_Format)
query = query_string.format(packets[0], start, end)
assert sqlbackend._conn.execute.call_args[0][0] == query
sqlbackend._conn.reset_mock()
# Test no end time handling
#######################################
packets = [list(tlm.getDefaultDict().keys())[0]]
sqlbackend.query_packets(packets=packets)
start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format)
end = dt.datetime.utcnow()
query = query_string.format(packets, start, end)
exec_end_time = dt.datetime.strptime(
sqlbackend._conn.execute.call_args[0][0].split('"')[-2],
dmc.RFC3339_Format)
assert (end - exec_end_time).seconds < 1
sqlbackend.query_packets()
# Test bad packet name exception
#######################################
with pytest.raises(ValueError):
sqlbackend.query_packets(packets=["not_a_valid_packet"])
