def write_sensor_log(
start_time: float,
outfile,
data_queue: Queue,
continue_running: AtomicValue,
continue_logging: AtomicValue,
):
"""Write the queue to the log until told to stop"""
lines_written = 0
last_queue_check = time.time()
while continue_running.get_value() and continue_logging.get_value():
try:
new_lines_written = write_queue_log(outfile, data_queue, 300)
if new_lines_written > 0:
lines_written += new_lines_written
if last_queue_check + 10.0 < time.time():
last_queue_check = time.time()
elapsed = last_queue_check - start_time
logging.info(
"Lines written: %d in %s seconds with %d ready",
lines_written,
elapsed,
data_queue.qsize(),
)
time.sleep(7)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Telemetry log line writing failure", ex)
def test_gps(configuration: Configuration):
"""Take a reading from the gps to test its functionality"""
try:
logging.info("Testing gps ...")
gps = init_gps(configuration)
if gps:
gps_value = AtomicValue()
start_time = time.time()
readings = 0
while readings < 10 or time.time() - start_time < TEST_TIME_LENGTH:
try:
if take_gps_reading(gps, gps_value):
readings += 1
except Exception as ex: # pylint: disable=broad-except:
handle_exception("Reading failure", ex)
total_time = time.time() - start_time
logging.info(
"Last GPS reading: %f, %f, %f, %f out of %d readings at an average rate of %f/sec",
*gps_value.get_value(),
readings,
float(readings) / total_time,
)
else:
logging.info("No GPS hardware configured")
except Exception as ex: # pylint: disable=broad-except
handle_exception("GPS failure", ex)
def test_atomic_value_try_update_fail():
raw_value = random.random()
value = AtomicValue(raw_value)
value.lock.acquire()
assert not value.try_update(random.random())
value.lock.release()
assert raw_value == value.get_value()
def test_atomic_value_try_update_succeed():
raw_value = random.random()
value = AtomicValue(raw_value)
value.lock.acquire()
value.lock.release()
assert value.try_update(random.random())
assert raw_value != value.get_value()
def test_digest_next_ground_reading():
values = [random.random() for _ in range(TELEMETRY_TUPLE_LENGTH)]
rfm9x = MockRFM9x(values)
new_data_queue = Queue()
gps_value = AtomicValue([random.random() for _ in range(4)])
digest_next_ground_reading(rfm9x, new_data_queue, gps_value)
assert new_data_queue.qsize() == 1
expected = (*values, rfm9x.last_rssi, *gps_value.get_value())
assert new_data_queue.get() == expected
def __init__(self,
configuration: Configuration,
sql_file: str = "sessionstore.sqlite3"):
"""Create a new session store"""
self.buffer = SafeBuffer()
self.current_session = AtomicValue()
self.output_dir = configuration.get("output_directory")
self.sql_file = sql_file
self.connection = None
self.cursor = None
def altimeter_reading_loop(configuration: Configuration,
altimeter_value: AtomicValue):
"""Continually read the altimeter and set it to the atomic var"""
try:
bmp = init_altimeter(configuration)
while bmp:
try:
altimeter_value.update(bmp._read()) # pylint: disable=protected-access
except Exception as ex: # pylint: disable=broad-except
handle_exception("Altimeter reading failure", ex)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Altimeter setup failure", ex)
class BufferSessionStore:
"""Data capture session storage engine"""
def __init__(self,
configuration: Configuration,
sql_file: str = "sessionstore.sqlite3"):
"""Create a new session store"""
self.buffer = SafeBuffer()
self.current_session = AtomicValue()
self.output_dir = configuration.get("output_directory")
self.sql_file = sql_file
self.connection = None
self.cursor = None
def initialize(self):
"""Initialize the store. (Pins the SQLite connection to the calling thread)"""
self.connection = sqlite3.connect(
path.join(self.output_dir, self.sql_file))
self.cursor = self.connection.cursor()
self.cursor.execute(
"CREATE TABLE IF NOT EXISTS sessions (timestamp NUMBER UNIQUE NOT NULL)"
)
self.connection.commit()
self.create_new_session()
def get_sessions(self) -> List[float]:
"""Get a list of saved sessions"""
self.cursor.execute(
"SELECT timestamp FROM sessions WHERE timestamp != ? ORDER BY timestamp DESC",
(self.current_session.get_value(), ),
)
return [row[0] for row in self.cursor.fetchall()]
def create_new_session(self) -> float:
"""Create and save a new session"""
timestamp = int(time.time())
self.current_session.update(timestamp)
self.cursor.execute("INSERT INTO sessions (timestamp) VALUES (?)",
(timestamp, ))
self.connection.commit()
self.buffer.purge()
return timestamp
def load_session(self, session) -> List[List[float]]:
"""Load data from a given session"""
with open(self.data_path_for_session(session), "r") as csvfile:
reader = csv.reader(csvfile)
return [[float(v) for v in r] for r in reader]
def data_path_for_session(self, session=None) -> str:
"""Get the path to the given session's data"""
if not session:
session = self.current_session.get_value()
return path.join(self.output_dir, f"telemetry_log_{session}.csv")
def take_gps_reading(sio, gps_value: AtomicValue) -> bool:
"""Grab the most recent data from GPS feed"""
line = sio.readline()
if line[0:6] == "$GPGGA":
gps = pynmea2.parse(line)
gps_value.update((
gps.latitude if gps else 0.0,
gps.longitude if gps else 0.0,
gps.gps_qual if gps else 0.0,
float(gps.num_sats) if gps else 0.0,
))
return True
return False
def magnetometer_accelerometer_reading_loop(
configuration: Configuration,
magnetometer_accelerometer_value: AtomicValue):
"""Continually read the accelerometer/magnetometer and set it to the atomic var"""
try:
mag, accel = init_magnetometer_accelerometer(configuration)
while mag and accel:
try:
magnetometer_accelerometer_value.update(
(*accel.acceleration, *mag.magnetic))
except Exception as ex: # pylint: disable=broad-except
handle_exception("Magnetometer/Accelerometer reading failure",
ex)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Magnetometer/Accelerometer setup failure", ex)
def take_gps_reading(sio, gps_value: AtomicValue) -> bool:
"""Grab the most recent data from GPS feed"""
line = sio.readline()
gps = pynmea2.parse(line)
if isinstance(gps, pynmea2.types.talker.GGA):
gps_value.update(
(
gps.latitude if gps else 0.0,
gps.longitude if gps else 0.0,
float(gps.gps_qual) if gps else 0.0,
float(gps.num_sats) if gps else 0.0,
)
)
return True
return False
def transmitter_thread(
configuration: Configuration,
start_time: float,
current_readings: AtomicBuffer,
pcnt_to_limit: AtomicValue,
continue_running: AtomicValue,
):
"""Transmit the latest data"""
try:
rfm9x = init_radio(configuration)
if not rfm9x:
return
last_check = time.time()
readings_sent = 0
while continue_running.get_value():
try:
readings_sent, last_check = transmit_latest_readings(
pcnt_to_limit,
rfm9x,
last_check,
readings_sent,
start_time,
current_readings,
)
time.sleep(0)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Transmitter failure", ex)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Transmitter failure", ex)
def transmit_latest_readings(
pcnt_to_limit: AtomicValue,
rfm9x,
last_check: float,
readings_sent: int,
start_time: float,
current_readings: AtomicBuffer,
) -> Tuple[int, float]:
"""Get the latest value from the sensor store and transmit it as a byte array"""
infos = current_readings.read()
if len(infos) < 2:
return readings_sent, last_check
info1 = infos[0]
info2 = infos[int(math.ceil(len(infos) / 2))]
if not info1 or not info2:
return readings_sent, last_check
info = (*info1, *info2)
clean_info = [float(i) for i in info]
encoded = struct.pack(
"d" + TELEMETRY_STRUCT_STRING + TELEMETRY_STRUCT_STRING,
*(pcnt_to_limit.get_value(), *clean_info)
)
current_readings.clear()
logging.debug("Transmitting %d bytes", len(encoded))
rfm9x.send(encoded)
readings_sent += 1
if last_check > 0 and last_check + 10.0 < time.time():
last_check = time.time()
logging.info(
"Transmit rate: %f/s",
float(readings_sent) / float(last_check - start_time),
)
return readings_sent, last_check
def test_rfm9x(configuration: Configuration):
"""Transmit on the rfm9x to test its functionality"""
try:
logging.info("Testing rfm9x ...")
rfm9x = init_radio(configuration)
if rfm9x:
camera_is_running = AtomicValue(0.0)
current_reading = AtomicBuffer(2)
current_reading.put([0.0 for _ in range(TELEMETRY_TUPLE_LENGTH)])
current_reading.put([0.0 for _ in range(TELEMETRY_TUPLE_LENGTH)])
start_time = time.time()
transmissions = 0
while time.time() - start_time < TEST_TIME_LENGTH:
transmit_latest_readings(camera_is_running, rfm9x, 0, 0, 0,
current_reading)
transmissions += 1
total_time = time.time() - start_time
logging.info(
"Transmitted %d messages at an average rate of %f/sec",
transmissions,
float(transmissions) / total_time,
)
else:
logging.info("No rfm9x hardware configured")
except Exception as ex: # pylint: disable=broad-except
handle_exception("rfm9x failure", ex)
def test_write_sensor_log():
start_time = time.time()
outfile = io.StringIO("")
data_queue = Queue()
while data_queue.qsize() < 100:
data_queue.put((random.random(), random.random(), random.random()))
continue_running = AtomicValue(True)
thread = Thread(
target=write_sensor_log,
args=(start_time, outfile, data_queue, continue_running,
continue_running),
)
thread.start()
time.sleep(5)
continue_running.update(False)
thread.join()
contents = outfile.getvalue()
assert contents
assert len(contents.split("\n")) > 0
def gps_reception_loop(sio, gps_value: AtomicValue, continue_running: AtomicValue):
"""Loop forever reading GPS data and passing it to an atomic value"""
if not sio:
return
while continue_running.get_value():
try:
take_gps_reading(sio, gps_value)
except Exception as ex: # pylint: disable=broad-except
handle_exception("GPS reading failure", ex)
time.sleep(0)
def digest_next_sensor_reading(
start_time: float,
data_queue: Queue,
current_reading: AtomicValue,
gps_value: AtomicValue,
altimeter_value: AtomicValue,
magnetometer_accelerometer_value: AtomicValue,
) -> float:
"""Grab the latest values from all sensors and put the data in the queue and atomic store"""
now = time.time()
info = (
now - start_time,
*altimeter_value.get_value(),
*magnetometer_accelerometer_value.get_value(),
*gps_value.get_value(),
)
data_queue.put(info)
current_reading.try_update(info)
return now
def camera_thread(
configuration: Configuration,
start_time: float,
continue_running: AtomicValue,
continue_logging: AtomicValue,
):
"""Start the camera and log the video"""
try:
logging.info("Starting video capture")
output_directory = configuration.get("output_directory")
camera = picamera.PiCamera(framerate=90)
camera.start_recording(
os.path.join(output_directory, f"video_{int(start_time)}.h264")
)
while continue_running.get_value() and continue_logging.get_value():
time.sleep(5)
camera.stop_recording()
logging.info("Video capture complete")
except Exception as ex: # pylint: disable=broad-except
handle_exception("Video capture failure", ex)
def sensor_reading_loop(
configuration: Configuration,
start_time: float,
data_queue: Queue,
current_readings: AtomicBuffer,
gps_value: AtomicValue,
continue_running: AtomicValue,
):
"""Read from the sensors on and infinite loop and queue it for transmission and logging"""
try:
logging.info("Starting sensor measurement loop")
bmp = init_altimeter(configuration)
mag, accel = init_magnetometer_accelerometer(configuration)
start = time.time()
readings = 0.0
while continue_running.get_value():
try:
digest_next_sensor_reading(
start_time,
data_queue,
current_readings,
gps_value.get_value(),
bmp._read(), # pylint: disable=protected-access
(*accel.acceleration, *mag.magnetic),
)
readings += 1.0
now = time.time()
if now - start > 10:
readings_per_second = readings / (now - start)
logging.info("Sample rate: %f/s", readings_per_second)
start = time.time()
readings = 0
except Exception as ex: # pylint: disable=broad-except
handle_exception("Telemetry measurement point reading failure", ex)
except Exception as ex: # pylint: disable=broad-except
handle_exception("Telemetry measurement point reading failure", ex)
def test_transmit_latest_readings():
last_check = 1
readings_sent = 0
start_time = time.time()
rfm9x = MockRFM9X()
current_reading = AtomicValue(
[random.random() for _ in range(TELEMETRY_TUPLE_LENGTH)]
)
readings_sent_1, last_check_1 = transmit_latest_readings(
rfm9x, last_check, readings_sent, start_time, current_reading
)
assert readings_sent_1 > readings_sent
assert last_check < last_check_1
assert last_check_1 <= time.time()
assert len(rfm9x.sent) == (TELEMETRY_TUPLE_LENGTH * 8)
def transmit_latest_readings(
rfm9x,
last_check: float,
readings_sent: int,
start_time: float,
current_reading: AtomicValue,
) -> Tuple[int, float]:
"""Get the latest value from the sensor store and transmit it as a byte array"""
info = current_reading.get_value()
if info:
clean_info = [float(i) for i in info]
encoded = struct.pack(TELEMETRY_STRUCT_STRING, *clean_info)
logging.debug("Transmitting %d bytes", len(encoded))
rfm9x.send(encoded)
readings_sent += 1
if last_check > 0 and last_check + 10.0 < time.time():
last_check = time.time()
logging.info("Readings sent: %d / %d seconds", readings_sent,
last_check - start_time)
return readings_sent, last_check
def main():
"""Inboard data capture and transmission script"""
# Load up the system configuration
configuration = Configuration(os.getenv("AIR_CONFIG_FILE", None),
Configuration.default_air_configuration)
# Queue to manage data synchronization between sensor reading, transmission, and data logging
data_queue = Queue(1000)
# Timestamp to use for log files and log saving cutoff
start_time = time.time()
# Thread safe place to store altitude reading
current_readings = AtomicBuffer(50)
# Holds the most recent GPS data
gps_value = AtomicValue((0.0, 0.0, 0.0, 0.0))
# pcnt counter to runtime limit
pcnt_to_limit = AtomicValue(0.0)
# Thread safe place to store continue value
continue_running = AtomicValue(True)
# Thread safe place to store continue value
continue_logging = AtomicValue(True)
# Setup listener for reset button
init_reset_button(configuration, continue_running)
gps_thread = create_gps_thread(configuration, gps_value, continue_running)
gps_thread.start()
write_thread = Thread(
target=sensor_log_writing_loop,
args=(
configuration,
start_time,
data_queue,
continue_running,
continue_logging,
),
daemon=True,
)
write_thread.start()
camera_thread_handle = Thread(
target=camera_thread,
args=(configuration, start_time, continue_running, continue_logging),
daemon=True,
)
camera_thread_handle.start()
transmitter_thread_handle = Thread(
target=transmitter_thread,
args=(
configuration,
start_time,
current_readings,
pcnt_to_limit,
continue_running,
),
daemon=True,
)
transmitter_thread_handle.start()
sensor_reading_thread = Thread(
target=sensor_reading_loop,
args=(
configuration,
start_time,
data_queue,
current_readings,
gps_value,
continue_running,
),
daemon=True,
)
sensor_reading_thread.start()
runtime_limit = configuration.get("runtime_limit")
while continue_running.get_value(
) and time.time() - start_time <= runtime_limit:
pcnt_to_limit.update((time.time() - start_time) / runtime_limit)
time.sleep(1)
logging.info("Stopping write activities")
continue_logging.update(False)
write_thread.join()
camera_thread_handle.join()
pcnt_to_limit.update(1)
logging.info("Write activities ended")
gps_thread.join()
transmitter_thread_handle.join()
sensor_reading_thread.join()
def main():
"""Inboard data capture and transmission script"""
# Load up the system configuration
configuration = Configuration(
os.getenv("AIR_CONFIG_FILE", None), Configuration.default_air_configuration
)
# Queue to manage data synchronization between sensor reading, transmission, and data logging
data_queue = Queue()
# Timestamp to use for log files and log saving cutoff
start_time = time.time()
# Thread safe place to store altitude reading
current_reading = AtomicValue()
# Holds the most recent GPS data
gps_value = AtomicValue((0.0, 0.0, 0.0, 0.0))
altimeter_value = AtomicValue()
altimeter_value.update((0.0, 0.0))
magnetometer_accelerometer_value = AtomicValue()
magnetometer_accelerometer_value.update((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
gps_thread = create_gps_thread(configuration, gps_value)
gps_thread.start()
write_thread = Thread(
target=sensor_log_writing_loop,
args=(configuration, start_time, data_queue),
daemon=True,
)
write_thread.start()
camera_thread_handle = Thread(
target=camera_thread,
args=(configuration, start_time),
daemon=True,
)
camera_thread_handle.start()
transmitter_thread_handle = Thread(
target=transmitter_thread,
args=(
configuration,
start_time,
current_reading,
),
daemon=True,
)
transmitter_thread_handle.start()
altimeter_thread = Thread(
target=altimeter_reading_loop, args=(configuration, altimeter_value)
)
altimeter_thread.start()
magnetometer_accelerometer_thread = Thread(
target=magnetometer_accelerometer_reading_loop,
args=(configuration, magnetometer_accelerometer_value),
)
magnetometer_accelerometer_thread.start()
sensor_reading_loop(
start_time,
data_queue,
current_reading,
gps_value,
altimeter_value,
magnetometer_accelerometer_value,
)
def test_atomic_value():
raw_value = random.random()
value = AtomicValue(raw_value)
assert raw_value == value.get_value()
value.update(random.random())
assert raw_value != value.get_value()
def test_digest_next_sensor_reading():
start_time = time.time()
altimeter_value = AtomicValue([random.random() for _ in range(2)])
gps_value = AtomicValue([random.random() for _ in range(4)])
magnetometer_accelerometer_value = AtomicValue([random.random() for _ in range(6)])
data_queue = Queue()
current_reading = AtomicValue()
now = digest_next_sensor_reading(
start_time,
data_queue,
current_reading,
gps_value,
altimeter_value,
magnetometer_accelerometer_value,
)
logged = data_queue.get()
expected_tuple = (
now - start_time,
*altimeter_value.get_value(),
*magnetometer_accelerometer_value.get_value(),
*gps_value.get_value(),
)
assert logged
assert logged == expected_tuple
assert current_reading.get_value() == expected_tuple
assert len(logged) == TELEMETRY_TUPLE_LENGTH
def test_take_gps_reading():
line = f"$GPGGA,134658.00,5106.9792,N,11402.3003,W,2,09,1.0,1048.47,M,-16.27,M,08,AAAA*60"
sio = MockSerial(line)
val = AtomicValue()
take_gps_reading(sio, val)
assert val.get_value() == (51.11632, -114.03833833333333, 2, 9)