def test_sqlite_create_table(self, importlib_mock): yaml_doc = """ - !Packet name: Packet1 history: - col1 fields: - !Field name: col1 desc: test column 1 type: MSB_U16 enum: a: testa - !Field name: SampleTime type: TIME64 """ with open(self.test_yaml_file, 'wt') as out: out.write(yaml_doc) tlmdict = tlm.TlmDict(self.test_yaml_file) sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() sqlbackend._create_table(tlmdict['Packet1']) sqlbackend._conn.execute.assert_called_with( 'CREATE TABLE IF NOT EXISTS "Packet1" (time DATETIME DEFAULT(STRFTIME(\'%Y-%m-%dT%H:%M:%fZ\', \'NOW\')), col1 INTEGER, SampleTime REAL)' ) os.remove(self.test_yaml_file)
def test_sqlite_create(self, importlib_mock): yaml_doc = """ - !Packet name: Packet1 fields: - !Field name: col1 desc: test column 1 type: MSB_U16 enum: a: testa - !Field name: SampleTime64 type: TIME64 """ with open(self.test_yaml_file, 'wt') as out: out.write(yaml_doc) tlmdict = tlm.TlmDict(self.test_yaml_file) sqlbackend = db.SQLiteBackend() sqlbackend.connect = mock.MagicMock() sqlbackend._create_table = mock.MagicMock() sqlbackend.create(tlmdict=tlmdict) sqlbackend._create_table.assert_called_with(tlmdict['Packet1']) os.remove(self.test_yaml_file)
def test_sqlite_create_table(self, importlib_mock): yaml_doc = """ - !Packet name: Packet1 history: - col1 fields: - !Field name: col1 desc: test column 1 type: MSB_U16 enum: a: testa - !Field name: SampleTime type: TIME64 """ with open(self.test_yaml_file, "wt") as out: out.write(yaml_doc) tlmdict = tlm.TlmDict(self.test_yaml_file) sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() sqlbackend._create_table(tlmdict["Packet1"]) sqlbackend._conn.execute.assert_called_with( "CREATE TABLE IF NOT EXISTS \"Packet1\" (time DATETIME DEFAULT(STRFTIME('%Y-%m-%dT%H:%M:%fZ', 'NOW')), PKTDATA BLOB NOT NULL)" ) os.remove(self.test_yaml_file)
def test_sqlite_insert(self, importlib_mock): yaml_doc = """ - !Packet name: Packet1 history: - col1 fields: - !Field name: col1 desc: test column 1 type: MSB_U16 enum: a: testa - !Field name: SampleTime type: TIME64 """ with open(self.test_yaml_file, 'wt') as out: out.write(yaml_doc) tlmdict = tlm.TlmDict(self.test_yaml_file) sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() pkt_defn = tlmdict['Packet1'] pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes))) sqlbackend.insert(pkt) sqlbackend._conn.execute.assert_called_with( 'INSERT INTO "Packet1" (col1, SampleTime) VALUES (?, ?)', [1, 33752069.10112411]) os.remove(self.test_yaml_file)
def test_query_fail_handling(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() sqlbackend._query = mock.MagicMock() sqlbackend._query.side_effect = sqlbackend._backend.OperationalError( "foo") res = sqlbackend.query_packets() for e in res.errors: assert e == "foo"
def test_sqlite_connect(self, importlib_mock): sqlbackend = db.SQLiteBackend() sqlbackend._backend = mock.MagicMock() sqlbackend.connect() assert sqlbackend._backend.connect.called sqlbackend._backend.connect.assert_called_with('ait.db') sqlbackend._backend.reset_mock() sqlbackend.connect(database='foo.db') assert sqlbackend._backend.connect.called sqlbackend._backend.connect.assert_called_with('foo.db')
def test_sqlite_insert(self, importlib_mock): yaml_doc = """ - !Packet name: Packet1 history: - col1 fields: - !Field name: col1 desc: test column 1 type: MSB_U16 enum: a: testa - !Field name: SampleTime type: TIME64 """ with open(self.test_yaml_file, "wt") as out: out.write(yaml_doc) tlmdict = tlm.TlmDict(self.test_yaml_file) sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() pkt_defn = tlmdict["Packet1"] pkt = tlm.Packet(pkt_defn, bytearray(range(pkt_defn.nbytes))) # Note: we can't fully test this call given the modification to # the packet data on insert. Better than nothing I suppose. # Test without time sqlbackend.insert(pkt) assert ('INSERT INTO "Packet1" (PKTDATA) VALUES (?)' in sqlbackend._conn.execute.call_args[0]) sqlbackend._conn.reset_mock() # Test with time now = dt.datetime.utcnow() sqlbackend.insert(pkt, time=now) assert ('INSERT INTO "Packet1" (PKTDATA, time) VALUES (?, ?)' in sqlbackend._conn.execute.call_args[0]) assert (now.strftime( dmc.RFC3339_Format) == sqlbackend._conn.execute.call_args[0][1][1]) os.remove(self.test_yaml_file)
def test_query_packets_calldown(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() start = dmc.GPS_Epoch end = dt.datetime.utcnow() packets = [list(tlm.getDefaultDict().keys())[0]] sqlbackend.query_packets(packets=packets, start_time=start, end_time=end) start = start.strftime(dmc.RFC3339_Format) end = end.strftime(dmc.RFC3339_Format) for i, pkt in enumerate(packets): query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC' assert sqlbackend._conn.execute.call_args[i][0] == query
def test_query_return_types(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() sqlbackend._query = mock.MagicMock() # Check that a successful query returns a properly formatted AITDBResult ret_val = [1, 2, 3] query_string = "select * from table" sqlbackend._query.return_value = ret_val results = sqlbackend.query(query_string) assert isinstance(results, db.AITDBResult) assert results.query == query_string assert results.results == ret_val # Check that a failed query returns a properly formatted AITDBResult sqlbackend._query.side_effect = sqlbackend._backend.OperationalError( "foo") results = sqlbackend.query(query_string) assert results.query == query_string assert results.errors == ["foo"]
def test_sqlite_connect(self, importlib_mock): sqlbackend = db.SQLiteBackend() sqlbackend._backend = mock.MagicMock() # Check default database naming sqlbackend.connect() assert sqlbackend._backend.connect.called sqlbackend._backend.connect.assert_called_with("ait.db") sqlbackend._backend.reset_mock() # Check custom database naming sqlbackend.connect(database="foo.db") assert sqlbackend._backend.connect.called sqlbackend._backend.connect.assert_called_with("foo.db") # Backend should only call self.create if a database doesn't exist # # Mock return_value handling wasn't cooperating with decorators # so we'll handle it manually instead ... import os.path isfile_mock = mock.MagicMock(return_value=True) isfile_orig = os.path.isfile os.path.isfile = isfile_mock sqlbackend.create = mock.MagicMock() assert isfile_mock is os.path.isfile isfile_mock.return_value = True sqlbackend.connect() assert sqlbackend.create.called is False isfile_mock.return_value = False sqlbackend.connect() assert sqlbackend.create.called os.path.isfile = isfile_orig
def test_query_success_handling(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() res_mock = mock.MagicMock() res_mock.return_value = [ ("2020-12-02T00:41:43.199Z", b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"), ("2020-12-02T00:41:44.200Z", b"\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01"), ("2020-12-02T00:41:45.205Z", b"\x00\x02\x00\x02\x00\x02\x00\x02\x00\x02"), ("2020-12-02T00:41:46.211Z", b"\x00\x03\x00\x03\x00\x03\x00\x03\x00\x03"), ("2020-12-02T00:41:47.216Z", b"\x00\x04\x00\x04\x00\x04\x00\x04\x00\x04"), ("2020-12-02T00:41:48.221Z", b"\x00\x05\x00\x05\x00\x05\x00\x05\x00\x05"), ] sqlbackend._query = res_mock res = sqlbackend.query_packets(packets=["1553_HS_Packet"]) assert isinstance(res, db.AITDBResult) assert res._packets is not None res_pkts = list(res.get_packets()) assert len(res_pkts) == 6 assert isinstance(res_pkts[0], tlm.Packet) assert res_pkts[0].Voltage_A == 0 assert res_pkts[1].Voltage_A == 1 assert res_pkts[2].Voltage_A == 2 assert res_pkts[3].Voltage_A == 3 assert res_pkts[4].Voltage_A == 4 assert res_pkts[5].Voltage_A == 5
def test_query_packet_time_inclusion(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() ret_data = [ ("2020-12-02T00:41:43.199Z", b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"), ("2020-12-02T00:41:44.200Z", b"\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01"), ("2020-12-02T00:41:45.205Z", b"\x00\x02\x00\x02\x00\x02\x00\x02\x00\x02"), ("2020-12-02T00:41:46.211Z", b"\x00\x03\x00\x03\x00\x03\x00\x03\x00\x03"), ("2020-12-02T00:41:47.216Z", b"\x00\x04\x00\x04\x00\x04\x00\x04\x00\x04"), ("2020-12-02T00:41:48.221Z", b"\x00\x05\x00\x05\x00\x05\x00\x05\x00\x05"), ] res_mock = mock.MagicMock() res_mock.return_value = ret_data sqlbackend._query = res_mock res = sqlbackend.query_packets(packets=["1553_HS_Packet"], yield_packet_time=True) assert isinstance(res, db.AITDBResult) assert res._packets is not None res_pkts = list(res.get_packets()) assert len(res_pkts) == 6 assert isinstance(res_pkts[0], tuple) for i, test_data in enumerate(ret_data): assert dmc.rfc3339_str_to_datetime( ret_data[i][0]) == res_pkts[i][0] assert res_pkts[i][1].Voltage_A == i
def main(): tlmdict = tlm.getDefaultDict() pnames = list(tlmdict.keys()) ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) arguments = { "--packet": { "type": str, "choices": pnames, "default": pnames[0] if len(pnames) > 0 else None, "help": "Type of packets (!Packet name in tlm.yaml) in file", "required": len(pnames) > 1, }, "--database": { "default": ait.config.get("database.dbname"), "help": ("Name of database in which to insert packets (may " "also be specified in config.yaml database.name)"), "required": ait.config.get("database.dbname") is None, }, "--backend": { "default": "sqlite", "choices": ["sqlite", "influx"], "action": "store", "help": ("Name of database in which to insert packets (may " "also be specified in config.yaml database.name)"), }, "--use-current-time": { "action": "store_true", "help": ("Use current time stamps when insert packets instead " "of ground receipt time (or the time written in the " "PCAP header)."), }, "file": { "nargs": "+", "help": "File(s) containing telemetry packets" }, } for name, params in arguments.items(): ap.add_argument(name, **params) args = ap.parse_args() log.begin() try: npackets = 0 dbconn = None defn = tlm.getDefaultDict()[args.packet] if args.backend == "sqlite": dbconn = db.SQLiteBackend() elif args.backend == "influx": dbconn = db.InfluxDBBackend() if args.backend == "sqlite" and (args.database == ":memory:" or not os.path.exists(args.database)): dbconn.create(database=args.database) else: dbconn.connect(database=args.database) for filename in args.file: log.info("Processing %s" % filename) with pcap.open(filename) as stream: for header, pkt_data in stream: try: packet = tlm.Packet(defn, pkt_data) time = header.timestamp if args.use_current_time: time = None dbconn.insert(packet, time=time) npackets += 1 except struct.error: log.error( "Unable to unpack data into packet. Skipping ...") except KeyboardInterrupt: log.info("Received Ctrl-C. Stopping database insert.") except IOError as e: log.error(str(e)) finally: dbconn.close() values = npackets, args.packet, args.database log.info("Inserted %d %s packets into database %s." % values) log.end()
def test_sqlite_query_calldown(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() sqlbackend.query("SELECT * FROM table") sqlbackend._conn.execute.assert_called_with("SELECT * FROM table")
def test_sqlite_backend_init(self, importlib_mock): sqlbackend = db.SQLiteBackend() importlib_mock.assert_called_with('sqlite3')
def main(): tlmdict = tlm.getDefaultDict() pnames = tlmdict.keys() ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) arguments = { '--packet': { 'type': str, 'choices': pnames, 'default': pnames[0] if len(pnames) > 0 else None, 'help': 'Type of packets (!Packet name in tlm.yaml) in file', 'required': len(pnames) > 1, }, '--database': { 'default': ait.config.get('database.name'), 'help': ('Name of database in which to insert packets (may ' 'also be specified in config.yaml database.name)'), 'required': ait.config.get('database.name') is None }, '--backend': { 'default': 'sqlite', 'choices': ['sqlite', 'influx'], 'action': 'store', 'help': ('Name of database in which to insert packets (may ' 'also be specified in config.yaml database.name)') }, '--use-current-time': { 'action': 'store_true', 'help': ('Use current time stamps when insert packets instead ' 'of ground receipt time (or the time written in the ' 'PCAP header).') }, 'file': { 'nargs': '+', 'help': 'File(s) containing telemetry packets' } } for name, params in arguments.items(): ap.add_argument(name, **params) args = ap.parse_args() log.begin() try: npackets = 0 dbconn = None defn = tlm.getDefaultDict()[args.packet] nbytes = defn.nbytes if args.backend == 'sqlite': dbconn = db.SQLiteBackend() elif args.backend == 'influx': dbconn = db.InfluxDBBackend() if args.backend == 'sqlite' and (args.database == ':memory:' or not os.path.exists(args.database)): dbconn.create(database=args.database) else: dbconn.connect(database=args.database) for filename in args.file: log.info('Processing %s' % filename) with pcap.open(filename) as stream: for header, pkt_data in stream: try: packet = tlm.Packet(defn, pkt_data) time = header.timestamp if args.use_current_time: time = None dbconn.insert(packet, time=time) npackets += 1 except struct.error: log.error( "Unable to unpack data into packet. Skipping ...") except KeyboardInterrupt: log.info('Received Ctrl-C. Stopping database insert.') except IOError as e: log.error(str(e)) finally: dbconn.close() values = npackets, args.packet, args.database log.info('Inserted %d %s packets into database %s.' % values) log.end()
def test_query_packets_arg_handling(self): sqlbackend = db.SQLiteBackend() sqlbackend._conn = mock.MagicMock() query_string = ( 'SELECT * FROM "{}" WHERE time >= "{}" AND time <= "{}" ORDER BY time ASC' ) # Test no packet provided handling ####################################### start = dmc.GPS_Epoch end = dt.datetime.utcnow() res = sqlbackend.query_packets(start_time=start, end_time=end) packets = list(tlm.getDefaultDict().keys()) start = start.strftime(dmc.RFC3339_Format) end = end.strftime(dmc.RFC3339_Format) query = query_string.format(packets, start, end) for i, pkt in enumerate(packets): query = f'SELECT * FROM "{pkt}" WHERE time >= "{start}" AND time <= "{end}" ORDER BY time ASC' assert sqlbackend._conn.execute.call_args_list[i][0][0] == query sqlbackend._conn.reset_mock() # Test no start time handling ####################################### end = dt.datetime.utcnow() packets = [list(tlm.getDefaultDict().keys())[0]] sqlbackend.query_packets(packets=packets, end_time=end) start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format) end = end.strftime(dmc.RFC3339_Format) query = query_string.format(packets[0], start, end) assert sqlbackend._conn.execute.call_args[0][0] == query sqlbackend._conn.reset_mock() # Test no end time handling ####################################### packets = [list(tlm.getDefaultDict().keys())[0]] sqlbackend.query_packets(packets=packets) start = dmc.GPS_Epoch.strftime(dmc.RFC3339_Format) end = dt.datetime.utcnow() query = query_string.format(packets, start, end) exec_end_time = dt.datetime.strptime( sqlbackend._conn.execute.call_args[0][0].split('"')[-2], dmc.RFC3339_Format) assert (end - exec_end_time).seconds < 1 sqlbackend.query_packets() # Test bad packet name exception ####################################### with pytest.raises(ValueError): sqlbackend.query_packets(packets=["not_a_valid_packet"])