def _station_by_index(self, findex: io.Index) -> Station:
"""
builds station using the index of files to read
splits the index into smaller chunks if entire record cannot be held in memory
:param findex: index with files to build a station with
:return: Station built from files in findex, without building the data from parquet
"""
split_list = self._split_workload(findex)
use_temp_dir = True if len(split_list) > 1 else False
if len(split_list) > 0:
if self.debug and use_temp_dir:
print(
"Writing data to disk; this may take a few minutes to complete."
)
stpa = Station.create_from_indexes(
split_list,
use_model_correction=self.use_model_correction,
base_out_dir=self.dw_base_dir,
use_temp_dir=use_temp_dir)
if self.debug:
print(f"station {stpa.id()} files read: {len(findex.entries)}")
if len(split_list) > 1:
print(
f"required making {len(split_list)} smaller segments due to memory restraints"
)
if self.dw_save_mode == io.FileSystemSaveMode.MEM and use_temp_dir:
self.dw_save_mode = io.FileSystemSaveMode.TEMP
if self.correct_timestamps:
stpa.set_correct_timestamps()
return stpa
self.errors.append("No files found to create station.")
return Station()
def test_append_station_success(self):
empty_apim_station = Station()
empty_apim_station.set_id(self.apim_station.id()).set_uuid(
self.apim_station.uuid()).set_start_date(
self.apim_station.start_date())
empty_apim_station.set_metadata(self.apim_station.metadata())
empty_apim_station.append_station(self.apim_station)
self.assertEqual(len(empty_apim_station.data()), 3)
self.assertEqual(empty_apim_station.timesync_data().best_latency(),
1296.0)
def from_json_dict(json_dict: Dict) -> "DataWindow":
"""
Reads a JSON dictionary and loads the data into the DataWindow.
If dictionary is improperly formatted, raises a ValueError.
:param json_dict: the dictionary to read
:return: The DataWindow as defined by the JSON
"""
if "out_type" not in json_dict.keys() \
or json_dict["out_type"].upper() not in dw_io.DataWindowOutputType.list_names():
raise ValueError('Dictionary loading type is invalid or unknown. '
'Check the value "out_type"; it must be one of: '
f'{dw_io.DataWindowOutputType.list_non_none_names()}')
else:
out_type = dw_io.DataWindowOutputType.str_to_type(json_dict["out_type"])
if out_type == dw_io.DataWindowOutputType.PARQUET:
dwin = DataWindow(json_dict["event_name"], EventOrigin.from_dict(json_dict["event_origin"]),
None, json_dict["base_dir"], json_dict["out_type"], json_dict["make_runme"],
json_dict["debug"])
dwin._config = DataWindowConfig.from_dict(json_dict["config"])
dwin._errors = RedVoxExceptions.from_dict(json_dict["errors"])
dwin._sdk_version = json_dict["sdk_version"]
for st in json_dict["stations"]:
dwin.add_station(Station.from_json_file(os.path.join(json_dict["base_dir"], st), f"{st}.json"))
elif out_type == dw_io.DataWindowOutputType.LZ4:
dwin = DataWindow.deserialize(os.path.join(json_dict["base_dir"],
f"{json_dict['event_name']}.pkl.lz4"))
else:
dwin = DataWindow()
return dwin
def test_append_station_mismatch(self):
empty_apim_station = Station()
with contextlib.redirect_stdout(None):
empty_apim_station.append_station(self.apim_station)
self.assertEqual(len(empty_apim_station.data()), 0)
def test_check_key(self):
empty_apim_station = Station()
with contextlib.redirect_stdout(None):
self.assertFalse(empty_apim_station.check_key())
empty_apim_station.set_id("1234567890")
self.assertFalse(empty_apim_station.check_key())
empty_apim_station.set_uuid("abcdefghij")
self.assertTrue(empty_apim_station.check_key())
empty_apim_station.set_start_date(1579448154300000)
self.assertTrue(empty_apim_station.check_key())
def test_empty_station_update_timestamp(self):
empty_apim_station = Station()
empty_apim_station.set_start_date(empty_apim_station.start_date() +
100)
self.assertTrue(np.isnan(empty_apim_station.start_date()))
def test_empty_station(self):
empty_apim_station = Station()
self.assertEqual(len(empty_apim_station._data), 0)
self.assertTrue(np.isnan(empty_apim_station.start_date()))
self.assertFalse(empty_apim_station.audio_sensor())
def test_set_sensor(self):
empty_apim_station = Station()
self.assertFalse(empty_apim_station.has_audio_sensor())
self.assertIsNone(empty_apim_station.audio_sensor())
empty_apim_station.set_audio_sensor(
get_empty_sensor("empty mic", SensorType.AUDIO).class_from_type())
self.assertTrue(empty_apim_station.has_audio_sensor())
self.assertFalse(empty_apim_station.has_audio_data())
empty_apim_station.set_audio_sensor()
self.assertFalse(empty_apim_station.has_audio_sensor())
empty_apim_station.set_audio_sensor(self.apim_station.audio_sensor())
self.assertTrue(empty_apim_station.has_audio_sensor())
self.assertTrue(empty_apim_station.has_audio_data())
self.assertEqual(empty_apim_station.audio_sensor().sample_rate_hz(),
48000)
def test_append_sensor(self):
empty_apim_station = Station()
self.assertFalse(empty_apim_station.has_audio_sensor())
empty_apim_station.append_sensor(self.apim_station.audio_sensor())
self.assertEqual(len(empty_apim_station.data()), 1)
self.assertTrue(empty_apim_station.has_audio_sensor())
self.assertEqual(empty_apim_station.audio_sensor().sample_rate_hz(),
48000.0)
self.assertTrue(empty_apim_station.audio_sensor().is_sample_rate_fixed)
empty_apim_station.append_sensor(self.api900_station.pressure_sensor())
self.assertAlmostEqual(
empty_apim_station.pressure_sensor().sample_rate_hz(), 5.01, 2)
def _station_by_index(self, findex: io.Index) -> Station:
"""
:param findex: index with files to build a station with
:return: Station built from files in findex
"""
return Station.create_from_packets(self.read_files_in_index(findex))
def get_station_by_id(self, get_id: str) -> Optional[Station]:
"""
:param get_id: the id to filter on
:return: a station containing the data of the packets with the requested id or None if id can't be found
"""
return Station(self.read_files_by_id(get_id))
def create_window_in_sensors(
self, station: Station, start_datetime: Optional[dtu.datetime] = None,
end_datetime: Optional[dtu.datetime] = None
):
"""
truncate the sensors in the station to only contain data from start_date_timestamp to end_date_timestamp
if the start and/or end are not specified, keeps all audio data that fits and uses it
to truncate the other sensors.
returns nothing, updates the station in place
:param station: station object to truncate sensors of
:param start_datetime: datetime of start of window, default None
:param end_datetime: datetime of end of window, default None
"""
if start_datetime:
start_datetime = dtu.datetime_to_epoch_microseconds_utc(start_datetime)
else:
start_datetime = 0
if end_datetime:
end_datetime = dtu.datetime_to_epoch_microseconds_utc(end_datetime)
else:
end_datetime = dtu.datetime_to_epoch_microseconds_utc(dtu.datetime.max)
self.process_sensor(station.audio_sensor(), station.id(), start_datetime, end_datetime)
for sensor in [s for s in station.data() if s.type() != SensorType.AUDIO]:
self.process_sensor(sensor, station.id(), station.audio_sensor().first_data_timestamp(),
station.audio_sensor().last_data_timestamp())
# recalculate metadata
station.update_first_and_last_data_timestamps()
station.set_packet_metadata([meta for meta in station.packet_metadata()
if meta.packet_start_mach_timestamp < station.last_data_timestamp() and
meta.packet_end_mach_timestamp >= station.first_data_timestamp()])
if self._fs_writer.is_save_disk():
station.set_save_mode(io.FileSystemSaveMode.DISK)
station.set_save_dir(self.save_dir() if self._fs_writer.is_use_disk() else self._fs_writer.get_temp())
self._stations.append(station)