def merge_summaries_of_type(self, stype: SensorType):
"""
combines and replaces multiple summaries of one SensorType into a single one
*caution: using this on an audio sensor may cause data validation issues*
:param stype: the type of sensor to combine
"""
smrs = []
other_smrs = []
for smry in self.summaries:
if smry.stype == stype:
smrs.append(smry)
else:
other_smrs.append(smry)
first_summary = smrs.pop(0)
tbl = first_summary.data()
if not first_summary.check_data():
os.makedirs(first_summary.fdir, exist_ok=True)
for smrys in smrs:
tbl = pa.concat_tables([first_summary.data(), smrys.data()])
if first_summary.check_data():
first_summary._data = tbl
else:
pq.write_table(tbl, first_summary.file_name())
os.remove(smrys.file_name())
mnint = dtu.microseconds_to_seconds(float(np.mean(np.diff(tbl["timestamps"].to_numpy()))))
stdint = dtu.microseconds_to_seconds(float(np.std(np.diff(tbl["timestamps"].to_numpy()))))
single_smry = PyarrowSummary(first_summary.name, first_summary.stype, first_summary.start,
1 / mnint, first_summary.fdir, tbl.num_rows, mnint, stdint,
first_summary.data() if first_summary.check_data() else None
)
self.summaries = other_smrs
self.summaries.append(single_smry)
def update_data_timestamps(self,
offset_model: om.OffsetModel,
use_model_function: bool = True):
"""
updates the timestamps of the data points
:param offset_model: model used to update the timestamps
:param use_model_function: if True, use the offset model's correction function to correct time,
otherwise use best offset (model's intercept value). default True
"""
slope = dtu.seconds_to_microseconds(self.sample_interval_s) * (1 + offset_model.slope) \
if use_model_function else dtu.seconds_to_microseconds(self.sample_interval_s)
if self.type == SensorType.AUDIO:
# use the model to update the first timestamp or add the best offset (model's intercept value)
self.data_df["timestamps"] = \
calc_evenly_sampled_timestamps(offset_model.update_time(self.first_data_timestamp(),
use_model_function),
self.num_samples(),
slope)
else:
self.data_df["timestamps"] = offset_model.update_timestamps(
self.data_timestamps(), use_model_function)
time_diffs = np.floor(np.diff(self.data_timestamps()))
if len(time_diffs) > 1:
self.sample_interval_s = dtu.microseconds_to_seconds(slope)
if self.sample_interval_s > 0:
self.sample_rate_hz = 1 / self.sample_interval_s
self.sample_interval_std_s = dtu.microseconds_to_seconds(
np.std(time_diffs))
self.timestamps_altered = True
def test_organize_and_update_stats(self):
self.even_sensor.organize_and_update_stats(self.even_sensor.pyarrow_table())
self.assertEqual(self.even_sensor.data_timestamps()[1], 40)
self.assertAlmostEqual(self.even_sensor.sample_rate_hz(), 50000.0, 1)
self.assertEqual(self.even_sensor.sample_interval_std_s(), 0)
timestamps = [120, 111, 97, 83, 74, 65, 31, 25, 14]
test_data = [75, 12, 86, 22, 200, 52, 99, 188, 121]
sample_interval = dtu.microseconds_to_seconds(
float(np.mean(np.diff(timestamps)))
)
sample_interval_std = dtu.microseconds_to_seconds(
float(np.std(np.diff(timestamps)))
)
uneven_sensor = SensorData.from_dict(
"test",
dict(zip(["timestamps", "test_data"], [timestamps, test_data])),
SensorType.UNKNOWN_SENSOR,
1 / sample_interval,
sample_interval,
sample_interval_std,
False,
)
uneven_sensor.organize_and_update_stats(uneven_sensor.pyarrow_table())
self.assertAlmostEqual(uneven_sensor.sample_interval_s(), 0.000013, 6)
self.assertAlmostEqual(uneven_sensor.sample_interval_std_s(), 0.000008, 6)
def setUp(self):
timestamps = [120., 60., 80., 100., 40., 140., 20., 160., 180.]
sensor_data = [-20., 15., 50., -5., 20., -15., 10., 74., 111.]
test_data = [75., 12., 86., 22., 200., 52., 99., 188., 121.]
self.even_sensor = SensorData.from_dict(
"test",
dict(zip(["timestamps", "unaltered_timestamps", "microphone", "test_data"],
[timestamps, timestamps, sensor_data, test_data])),
SensorType.AUDIO,
1 / dtu.microseconds_to_seconds(20),
dtu.microseconds_to_seconds(20),
0,
True,
)
timestamps = [14., 25., 31., 65., 74., 83., 97., 111., 120.]
sample_interval = dtu.microseconds_to_seconds(
float(np.mean(np.diff(timestamps)))
)
sample_interval_std = dtu.microseconds_to_seconds(
float(np.std(np.diff(timestamps)))
)
self.uneven_sensor = SensorData.from_dict(
"test",
dict(zip(["timestamps", "unaltered_timestamps", "barometer", "test_data"],
[timestamps, timestamps, sensor_data, test_data])),
SensorType.PRESSURE,
1 / sample_interval,
sample_interval,
sample_interval_std,
False,
)
def validate_time_gaps(self,
gap_duration_s: float,
debug: bool = False) -> bool:
"""
confirms there are no data gaps between packets
outputs warning if a gap is detected
:param gap_duration_s: length of time in seconds to be detected as a gap
:param debug: if True, output warning message, default False
:return: True if no gap
"""
if self.get_num_packets() < 2:
self.errors.append(
"Less than 2 timesync data objects to evaluate gaps with")
if debug:
self.errors.print()
else:
for index in range(1, self.get_num_packets()):
# compare last packet's end timestamp with current start timestamp
if (dt.microseconds_to_seconds(
self.timesync_data[index].packet_start_timestamp -
self.timesync_data[index - 1].packet_end_timestamp) >
gap_duration_s):
self.errors.append(
f"Gap detected at packet number: {index}")
if debug:
self.errors.print()
return False
# if here, no gaps
return True
def get_sample_statistics(data_df: pa.Table) -> Tuple[float, float, float]:
"""
calculate the sample rate, interval and interval std dev using the timestamps in the dataframe
:param data_df: the dataframe containing timestamps to calculate statistics from
:return: a Tuple containing the sample rate, interval and interval std dev
"""
sample_interval: float
sample_interval_std: float
timestamps: np.array = data_df["timestamps"].to_numpy()
if timestamps.size > 1:
sample_interval = dtu.microseconds_to_seconds(
float(np.mean(np.diff(timestamps))))
sample_interval_std = dtu.microseconds_to_seconds(
float(np.std(np.diff(timestamps))))
else:
sample_interval = np.nan
sample_interval_std = np.nan
return 1.0 / sample_interval, sample_interval, sample_interval_std
def merge_non_audio_summaries(self):
"""
combines and replaces all summaries per type except for audio summaries
"""
smrs_dict = {}
for smry in self.summaries:
if smry.stype != SensorType.AUDIO:
if smry.stype in smrs_dict.keys():
smrs_dict[smry.stype].append(smry)
else:
smrs_dict[smry.stype] = [smry]
self.summaries = self.get_audio()
for styp, smrys in smrs_dict.items():
first_summary = smrys.pop(0)
tbl = first_summary.data()
combined_mint = np.mean([smrs.smint_s for smrs in smrys])
combined_std = np.mean([smrs.sstd_s for smrs in smrys])
if not first_summary.check_data():
os.makedirs(first_summary.fdir, exist_ok=True)
for smrs in smrys:
tbl = pa.concat_tables([tbl, smrs.data()])
if not first_summary.check_data():
os.remove(smrs.file_name())
if first_summary.check_data():
first_summary._data = tbl
else:
pq.write_table(tbl, first_summary.file_name())
mnint = dtu.microseconds_to_seconds(float(np.mean(np.diff(tbl["timestamps"].to_numpy()))))
stdint = dtu.microseconds_to_seconds(float(np.std(np.diff(tbl["timestamps"].to_numpy()))))
if not combined_mint + combined_std > mnint > combined_mint - combined_std:
self.errors.append(f"Mean interval s of combined {styp.name} sensor does not match the "
f"compilation of individual mean interval s per packet. Will use compilation of "
f"individual values.")
mnint = combined_mint
stdint = combined_std
single_smry = PyarrowSummary(first_summary.name, styp, first_summary.start,
1 / mnint, first_summary.fdir, tbl.num_rows, mnint, stdint,
first_summary.data() if first_summary.check_data() else None
)
self.summaries.append(single_smry)
def load_apim_location(packet: RedvoxPacketM) -> Optional[PyarrowSummary]:
"""
load location data from a single packet
:param packet: packet with data to load
:return: location sensor data if it exists, None otherwise
"""
if srupa.__has_sensor(packet, srupa.__LOCATION_FIELD_NAME):
loc: RedvoxPacketM.Sensors.Location = packet.sensors.location
timestamps = loc.timestamps.timestamps
if len(timestamps) > 0:
if len(timestamps) > 1:
m_intv = dtu.microseconds_to_seconds(float(np.mean(np.diff(timestamps))))
intv_std = dtu.microseconds_to_seconds(float(np.std(np.diff(timestamps))))
else:
m_intv = srupa.__packet_duration_s(packet)
intv_std = 0.
return PyarrowSummary(
loc.sensor_description, srupa.SensorType.LOCATION, np.nan, np.nan, "",
len(timestamps), m_intv, intv_std, srupa.apim_location_to_pyarrow(loc)
)
return None
def load_xyz(
packet: RedvoxPacketM,
sensor_type: srupa.SensorType,
) -> Optional[PyarrowSummary]:
field_name: str = srupa.__SENSOR_TYPE_TO_FIELD_NAME[sensor_type]
sensor_fn: Optional[
Callable[[RedvoxPacketM], srupa.Sensor]
] = srupa.__SENSOR_TYPE_TO_SENSOR_FN[sensor_type]
if srupa.__has_sensor(packet, field_name) and sensor_fn is not None:
sensor = sensor_fn(packet)
t = sensor.timestamps.timestamps
if len(t) > 1:
m_intv = dtu.microseconds_to_seconds(float(np.mean(np.diff(t))))
intv_std = dtu.microseconds_to_seconds(float(np.std(np.diff(t))))
else:
m_intv = srupa.__packet_duration_s(packet)
intv_std = 0.
if len(t) > 0:
return PyarrowSummary(
sensor.sensor_description, sensor_type, np.nan, np.nan, "",
len(t), m_intv, intv_std, srupa.read_apim_xyz_sensor(sensor, field_name)
)
return None
def sampling_rate_statistics(timestamps: np.ndarray) -> Tuple[float, float]:
"""
Calculates the mean sample rate in Hz and standard deviation of the sampling rate.
:param timestamps:
:return: A tuple containing (mean_sample_rate, stdev_sample_rate)
"""
sample_interval: np.ndarray = np.diff(timestamps)
if len(sample_interval) < 2:
return 0.0, 0.0
mean_sample_interval: float = sample_interval.mean()
stdev_sample_interval: float = sample_interval.std()
if mean_sample_interval <= 0:
return 0.0, 0.0
mean_sample_rate: float = 1.0 / dt_utils.microseconds_to_seconds(
mean_sample_interval)
stdev_sample_rate: float = mean_sample_rate**2 * dt_utils.microseconds_to_seconds(
stdev_sample_interval)
return mean_sample_rate, stdev_sample_rate
def organize_and_update_stats(self) -> "SensorData":
"""
sorts the data by timestamps, then if the sample rate is not fixed, recalculates the sample rate, interval,
and interval std dev. If there is only one value, sets the sample rate, interval, and interval std dev
to np.nan. Updates the SensorData object with the new values
:return: updated version of self
"""
self.sort_by_data_timestamps()
if not self.is_sample_rate_fixed:
if self.num_samples() > 1:
timestamp_diffs = np.diff(self.data_timestamps())
self.sample_interval_s = dtu.microseconds_to_seconds(
float(np.mean(timestamp_diffs)))
self.sample_interval_std_s = dtu.microseconds_to_seconds(
float(np.std(timestamp_diffs)))
self.sample_rate_hz = (
np.nan if self.is_sample_interval_invalid() else 1 /
self.sample_interval_s)
else:
self.sample_interval_s = np.nan
self.sample_interval_std_s = np.nan
self.sample_rate_hz = np.nan
return self
def __stats_for_sensor_per_packet_per_second(
num_packets: int, packet_dur_s: float,
timestamps: np.array) -> Tuple[float, float, float]:
"""
Sensor being evaluated must either have 1/packet or 1/second sample rate
:param num_packets: number of packets to calculate stats for
:param packet_dur_s: duration of packet in seconds
:param timestamps: timestamps of the samples
:return: sample rate in hz, sample interval in seconds, and sample interval std deviation
"""
if len(timestamps) != num_packets:
sample_rate = 1.0
else:
sample_rate = 1 / packet_dur_s
sample_interval = 1 / sample_rate
sample_interval_std = (dtu.microseconds_to_seconds(
float(np.std(np.diff(timestamps)))) if len(timestamps) > 1 else np.nan)
return sample_rate, sample_interval, sample_interval_std
def identify_time_gaps(wrapped_redvox_packets: List,
allowed_timing_error_s: float = 5.0) -> List[GapResult]:
"""
Identifies time gaps between wrapped packets.
:param wrapped_redvox_packets: Packets to check for a gaps.
:param allowed_timing_error_s: The amount of time in seconds that are provided for timing error.
:return: A list of GapResults.
"""
gaps: List[GapResult] = []
if len(wrapped_redvox_packets) <= 1:
return gaps
packet_len_s: float = _packet_len_s(wrapped_redvox_packets[0])
allowed_gap_s: float = packet_len_s + allowed_timing_error_s
for i in range(1, len(wrapped_redvox_packets)):
prev_packet = wrapped_redvox_packets[i - 1]
next_packet = wrapped_redvox_packets[i]
prev_timestamp = prev_packet.microphone_sensor(
).first_sample_timestamp_epoch_microseconds_utc()
next_timestamp = next_packet.microphone_sensor(
).first_sample_timestamp_epoch_microseconds_utc()
time_diff_us: int = next_timestamp - prev_timestamp
time_diff_s: float = date_time_utils.microseconds_to_seconds(
time_diff_us)
if time_diff_s > allowed_gap_s:
description: str = f"Time gap. Expected packet length s={packet_len_s}. " \
f"Allowed timing error s={allowed_timing_error_s}. " \
f"Allowed gap s={allowed_gap_s}. " \
f"Actual time gap s={time_diff_s}"
gaps.append(GapResult(i, description))
packet_len_s = _packet_len_s(wrapped_redvox_packets[i])
allowed_gap_s = packet_len_s + allowed_timing_error_s
return gaps
def plot_summarized_data(
summarized_data: typing.Dict[str,
typing.List[WrappedRedvoxPacketSummary]]):
"""
Given summarized data, plot the data to display sensor activity for all passed in summarized data.
:param summarized_data: Data to plot.
"""
try:
import matplotlib.pyplot as plt
all_summaries = list(itertools.chain(*summarized_data.values()))
start_timestamp_us = numpy.min(
list(
map(lambda packet_summary: packet_summary.start_timestamp_us,
all_summaries)))
end_timestamp_us = numpy.max(
list(
map(lambda packet_summary: packet_summary.end_timestamp_us,
all_summaries)))
start_s = int(
date_time_utils.microseconds_to_seconds(start_timestamp_us))
end_s = int(date_time_utils.microseconds_to_seconds(end_timestamp_us))
timestamps = numpy.arange(start_s, end_s + 1)
# Setup the plot
fig, axes = plt.subplots(nrows=1)
axes.set_xlim([timestamps[0], timestamps[-1]])
axes.set_ylim([0, len(summarized_data.keys()) + 1])
# Not for each device, let's plot its ranges
yticks = []
ylabels = []
i = 1
for redvox_id, summaries in summarized_data.items():
for summary in summaries:
summary_start_s = int(
date_time_utils.microseconds_to_seconds(
summary.start_timestamp_us))
summary_end_s = int(
date_time_utils.microseconds_to_seconds(
summary.end_timestamp_us))
values = numpy.full(len(timestamps), numpy.nan)
values[timestamps.searchsorted(summary_start_s):timestamps.
searchsorted(summary_end_s)] = i
axes.plot(timestamps, values)
yticks.append(i)
ylabels.append(redvox_id.split(":")[0])
i += 1
xticks = _subsample(timestamps, 6)
datetimes = map(datetime.datetime.utcfromtimestamp, xticks)
datetimes = map(lambda dt: dt.strftime("%d %H:%M"), datetimes)
axes.set_xticks(xticks)
axes.set_xticklabels(list(datetimes))
axes.set_yticks(yticks)
axes.set_yticklabels(ylabels)
axes.tick_params(axis='both', which='major', labelsize=8)
axes.tick_params(axis='both', which='minor', labelsize=7)
axes.set_title("RedVox Device Summary")
axes.set_xlabel("Date and Time")
axes.set_ylabel("Device Activity")
fig.text(.5,
.01,
"UTC %s - %s" % (datetime.datetime.utcfromtimestamp(start_s),
datetime.datetime.utcfromtimestamp(end_s)),
ha='center')
plt.show()
except ImportError:
import warnings
warnings.warn(
"GUI dependencies are not installed. Install the 'GUI' extra to enable this functionality."
)
def test_microseconds_to_seconds(self):
self.assertEqual(dt.microseconds_to_seconds(1000000), 1)
def _identify_gaps(wrapped_redvox_packets,
allowed_timing_error_s: float) -> typing.List[int]:
"""
Identifies discontinuities in sensor data by checking if sensors drop in and out and by comparing timing info.
:param wrapped_redvox_packets: Packets to look for gaps in.
:param allowed_timing_error_s: The amount of timing error in seconds.
:return: A list of indices into the original list where gaps were found.
"""
if len(wrapped_redvox_packets) <= 1:
return []
gaps = set()
truth_len = _packet_len_s(wrapped_redvox_packets[0])
for i in range(1, len(wrapped_redvox_packets)):
prev_packet = wrapped_redvox_packets[i - 1]
next_packet = wrapped_redvox_packets[i]
# Sensor discontinuity
prev_sensors = [prev_packet.has_microphone_sensor(),
prev_packet.has_barometer_sensor(),
prev_packet.has_time_synchronization_sensor(),
prev_packet.has_accelerometer_sensor(),
prev_packet.has_gyroscope_sensor(),
prev_packet.has_infrared_sensor(),
prev_packet.has_light_sensor(),
prev_packet.has_image_sensor(),
prev_packet.has_location_sensor(),
prev_packet.has_magnetometer_sensor()]
next_sensors = [next_packet.has_microphone_sensor(),
next_packet.has_barometer_sensor(),
next_packet.has_time_synchronization_sensor(),
next_packet.has_accelerometer_sensor(),
next_packet.has_gyroscope_sensor(),
next_packet.has_infrared_sensor(),
next_packet.has_light_sensor(),
next_packet.has_image_sensor(),
next_packet.has_location_sensor(),
next_packet.has_magnetometer_sensor()]
# pylint: disable=C0200
for j in range(len(prev_sensors)):
if prev_sensors[j] != next_sensors[j]:
gaps.add(i)
# Time based gaps
prev_timestamp = prev_packet.microphone_sensor().first_sample_timestamp_epoch_microseconds_utc()
next_timestamp = next_packet.microphone_sensor().first_sample_timestamp_epoch_microseconds_utc()
# print(next_timestamp - prev_timestamp)
if _date_time_utils.microseconds_to_seconds(next_timestamp - prev_timestamp) > (
truth_len + allowed_timing_error_s):
gaps.add(i)
print("time gap")
truth_len = _packet_len_s(wrapped_redvox_packets[i])
prev_mach_time_zero = prev_packet.mach_time_zero()
next_mach_time_zero = next_packet.mach_time_zero()
if next_mach_time_zero != prev_mach_time_zero:
gaps.add(i)
print("mach time zero gap")
return sorted(list(gaps))
