def extract_analog_for_single_dataset(analog_files, continuous_time_file):
single_dataset_data = {}
for analog_file in analog_files:
if not 'timestamps' in analog_file:
analog_data = readTrodesExtractedDataFile(analog_files[analog_file])
values = analog_data['data']
single_dataset_data[analog_data['id']] = values
else:
continuous_time_dict = ContinuousTimeExtractor.get_continuous_time_dict_file(continuous_time_file)
timestamp = readTrodesExtractedDataFile(analog_files[analog_file])
keys = [key[0] for key in timestamp['data']]
single_dataset_data[analog_file] = TimestampConverter.convert_timestamps(continuous_time_dict, keys)
return single_dataset_data
def get_continuous_time_dict_file(file):
logger.info('Reading continuous time dict from: ' + str(file))
continuous_time = readTrodesExtractedDataFile(file)
return {
str(data[0]): float(data[3])
for data in continuous_time['data']
}
def get_column_labels(self):
"""extract column labels from POS files"""
pos_online = readTrodesExtractedDataFile(self.directories[0][0])
column_labels = pd.DataFrame(pos_online['data']).columns
column_labels = column_labels[1:]
column_labels_list = column_labels.values.tolist()
return column_labels_list
def fix_timestamp_lag(continuoustime_filename):
"""
Fix the correspondence between trodestime
and system (wall) time.
There is some jitter in the arrival times of packets from the MCU (as
reflected in the sysclock records in the .rec file. If we assume that
the Trodes clock is actually regular, and that any episodes of lag are
fairly sporadic, we can recover the correspondence between trodestime
and system (wall) time.
Parameters
----------
continuoustime_filename : str
Path to .continuoustime.dat file
"""
data_file = readTrodesExtractedDataFile(continuoustime_filename)
if 'systime' not in data_file['data'].dtype.names:
# logging.warn
new_data = infer_systime(data_file)
else:
new_data = (pd.DataFrame(data_file['data']).assign(
time_chunk_label=lambda df: _label_time_chunks(df.trodestime)).
assign(adjusted_systime=lambda df: _regress_timestamps(
df.trodestime, df.systime)))
new_data_file = _insert_new_data(data_file, new_data)
write_trodes_extracted_datafile(continuoustime_filename, new_data_file)
def read_data(self, dataset_id, file_id):
"""extract data from POS files and build FlPos"""
pos_online = readTrodesExtractedDataFile(
self.directories[dataset_id][file_id])
position = pd.DataFrame(pos_online['data'])
labels = self.get_column_labels()
filtered_position = [position[label] for label in labels]
return filtered_position
def extract_analog_for_single_dataset(analog_files):
single_dataset_data = {}
for analog_file in analog_files:
analog_data = readTrodesExtractedDataFile(
analog_files[analog_file])
values = analog_data['data']
single_dataset_data[analog_data['id']] = values
return single_dataset_data
def retrieve_real_timestamps(self, dataset_id):
"""Gets the corresponding Trodes timestamps from the online position
tracking and matches them to the PTP time in the video file.
Otherwise, we get the corresponding timestamps from
continuous time which corresponds to the neural recording time stamps.
If there is no corresponding timestamp, the result will be NaN.
Parameters
----------
dataset_id : int
Index of the epoch
Returns
-------
timestamps : ndarray, shape (n_online_tracked_positions,)
"""
try:
# Get online position tracking data
pos_online_path = self.directories[dataset_id][0]
pos_online = readTrodesExtractedDataFile(pos_online_path)
pos_online = pd.DataFrame(pos_online['data'])
# Make sure to get only the unique timestamps because they can
# sometimes repeat after a jump in timestamps
online_timestamps_ind = pos_online.time.unique().astype(np.uint64)
# Get video PTP timestamps
camera_hwsync = readTrodesExtractedDataFile(
pos_online_path.replace('.pos_online.dat',
'.pos_cameraHWFrameCount.dat'))
camera_hwsync = (pd.DataFrame(
camera_hwsync['data']).set_index('PosTimestamp'))
# Find the PTP timestamps that correspond to position tracking
# Convert from nanoseconds to seconds
return (camera_hwsync.loc[online_timestamps_ind, 'HWTimestamp'] /
NANOSECONDS_PER_SECOND).to_numpy()
except KeyError:
# If PTP timestamps do not exist find the corresponding timestamps
# from the neural recording
logger.info('No PTP timestamps found. Using neural timestamps.')
return TimestampManager.retrieve_real_timestamps(
self, dataset_id, convert_timestamps=self.convert_timestamps)
def __extract_single(self, hw_frame_count_filename):
content = readTrodesExtractedDataFile(self.raw_data_path + "/" +
hw_frame_count_filename)["data"]
camera_sample_frame_counts = np.ndarray(shape=(len(content), 2),
dtype='uint32')
for i, record in enumerate(content):
camera_sample_frame_counts[i, 0] = record[1]
camera_sample_frame_counts[i, 1] = record[0]
return camera_sample_frame_counts
def __get_continuous_time_data_from_single_file(self,
continuous_time_file):
continuous_time = readTrodesExtractedDataFile(continuous_time_file)
new_array = np.ndarray(shape=(len(continuous_time['data']), 2),
dtype='int64')
for i, single_timestamp in enumerate(continuous_time['data']):
new_array[i, 0] = single_timestamp[0]
new_array[i, 1] = single_timestamp[3]
return new_array
def find_camera_dio_channel(pos_online_path, pos_online, max_frames_diff=10):
"""Find the camera DIO by looping through all the DIOs
and finding the right number of DIO pulses.
Parameters
----------
pos_online_path : str
pos_online : pd.DataFrame, shape (n_camera_frames, 5)
max_frames_diff : int, optional
maximum difference between camera frames and dio pulses
Returns
-------
camera_dio_times : pd.Series, shape (n_dio_pulse_state_changes,)
Trodes time of dio ticks
"""
head, tail = os.path.split(pos_online_path)
dio_paths = glob.glob(
os.path.join(
os.path.split(head)[0],
tail.split('.')[0] + '.DIO',
tail.split('.')[0] + '*.dat'))
n_ticks = np.asarray([
pd.DataFrame(
readTrodesExtractedDataFile(dio_file)['data']).state.sum()
for dio_file in dio_paths
])
n_camera_frames = pos_online.shape[0]
position_ticks_file_ind = np.argmin(np.abs(n_ticks - n_camera_frames))
camera_ticks_dio = pd.DataFrame(
readTrodesExtractedDataFile(
dio_paths[position_ticks_file_ind])['data'])
return camera_ticks_dio.loc[camera_ticks_dio.state == 1].time
def __extract_single(self, filename):
"""Returns the video frame counts and timestamps for a single epoch."""
content = pd.DataFrame(
readTrodesExtractedDataFile(
os.path.join(self.raw_data_path, filename))["data"])
try:
# columns: frame count, timestamps
return content.iloc[:, [1, 0]].to_numpy()
except IndexError:
return np.vstack((
content.iloc[:, 0].to_numpy(), # frame counts
np.arange(len(content))) # dummy timestamps
).T.astype(np.uint32)
def extract_analog_for_single_dataset(analog_files,
continuous_time_file,
convert_timestamps=True):
single_dataset_data = {}
for analog_sensor in analog_files:
analog_data = readTrodesExtractedDataFile(
analog_files[analog_sensor])
if 'timestamps' not in analog_sensor:
values = analog_data['data']
single_dataset_data[analog_data['id']] = values
else:
timestamps = FlAnalogExtractor._extract_analog_timestamps(
analog_data, continuous_time_file, convert_timestamps)
single_dataset_data[analog_sensor] = timestamps
return single_dataset_data
def extract_dio_for_single_dataset(filtered_files):
single_dataset_data = {}
for dio_file in filtered_files:
try:
dio_data = readTrodesExtractedDataFile(
filtered_files[dio_file])
keys, values = OldDioExtractor.__get_dio_time_series(dio_data)
single_dataset_data[dio_file] = ([keys, values])
except KeyError as error:
message = "there is no " + str(dio_file) + ", error: "
logger.exception(message + str(error))
except TypeError as error:
message = "there is no data for event " + str(
dio_file) + ", error: "
logger.exception(message + str(error))
return single_dataset_data
def extract_video_files(self):
video_files = self.video_files_metadata
extracted_video_files = []
for video_file in video_files:
new_fl_video_file = {
"name":
video_file["name"],
"timestamps":
self.convert_timestamps(
readTrodesExtractedDataFile(
self.raw_data_path + "/" + video_file["name"][:-4] +
"videoTimeStamps.cameraHWSync")["data"]),
"device":
video_file["camera_id"]
}
extracted_video_files.append(new_fl_video_file)
return extracted_video_files
def _read_video_timestamps_hw_framecount(self, video_file):
"""Returns the index of video frames.
If PTP is not in use, only the videoTimeStamps.cameraHWFrameCount
file is generated by the `rec_to_binaries` package.
Parameters
----------
video_file : str
Returns
-------
index : ndarray
"""
return readTrodesExtractedDataFile(
os.path.join(
self.raw_data_path,
os.path.splitext(video_file["name"])[0] +
".videoTimeStamps.cameraHWFrameCount"))['data']['frameCount']
def get_camera_hwsync(pos_online_path):
"""Get video PTP timestamps if they exist.
Parameters
----------
pos_online_path : str
Returns
-------
camera_hwsync : pd.DataFrame, shape (n_camera_frames, 2)
PosTimestamp: unadjusted postimestamps. UINT32
HWframeCount: integer count of frames acquired by camera
(rolls over at 65535; can be used to detect dropped frames). UINT32
HWTimestamp: POSIX time in nanoseconds, synchronized to PC sysclock via PTP. UINT64.
"""
camera_hwsync = readTrodesExtractedDataFile(
pos_online_path.replace('.pos_online.dat',
'.pos_cameraHWFrameCount.dat'))
return (pd.DataFrame(camera_hwsync['data']).set_index('PosTimestamp'))
def get_continuous_time(pos_online_path):
"""Neural timestamps.
Parameters
----------
pos_online_path : str
Returns
-------
continuous_times : pd.DataFrame
trodestime uint32
adjusted_systime int64
"""
head, tail = os.path.split(pos_online_path)
continuous_time_path = os.path.join(
os.path.split(head)[0],
tail.split('.')[0] + '.time',
tail.split('.')[0] + '.continuoustime.dat')
cont_time = readTrodesExtractedDataFile(continuous_time_path)
return (pd.DataFrame(
cont_time['data']).set_index('trodestime').adjusted_systime)
def _read_video_timestamps_hw_sync(self, video_file):
"""Returns video timestamps in unix time which are synchronized to the
Trodes data packets.
videoTimeStamps.cameraHWSync is a file extracted by the python package
`rec_to_binaries` from the .rec file. It only is extracted when using
precision time protocol (PTP) to synchronize the camera clock with
Trodes data packets. The HWTimestamp array in this file are the unix
timestamps relative to seconds since 1/1/1970.
Parameters
----------
video_file : str
Returns
-------
unix_timestamps : ndarray
"""
return readTrodesExtractedDataFile(
os.path.join(
self.raw_data_path,
os.path.splitext(video_file["name"])[0] +
".videoTimeStamps.cameraHWSync"))['data']['HWTimestamp']
def extract_dio_for_single_dataset(filtered_files, continuous_time_file,
convert_timestamps=True):
single_dataset_data = {}
continuous_time = ContinuousTimeExtractor.get_continuous_time_array_file(
continuous_time_file)
for dio_sensor in filtered_files:
try:
dio_data = readTrodesExtractedDataFile(
filtered_files[dio_sensor])
# dio_data['data'] is a labeled array with 'time' and 'state' columns. 'time' corresponds to sample count
single_dataset_data[dio_sensor] = DioExtractor.__get_dio_time_series(
dio_data, continuous_time, convert_timestamps)
# keys, values = DioExtractor.__get_dio_time_series(dio_data, continuous_time_dict
# single_dataset_data[dio_sensor] = ([keys, values])
except KeyError as error:
message = "there is no " + str(dio_sensor) + ", error: "
logger.exception(message + str(error))
except TypeError as error:
message = "there is no data for event " + \
str(dio_sensor) + ", error: "
logger.exception(message + str(error))
return single_dataset_data
def get_position_with_corrected_timestamps(pos_online_path):
logger.info(os.path.split(pos_online_path)[-1])
pos_online = pd.DataFrame(
readTrodesExtractedDataFile(pos_online_path)['data']).set_index(
'time')
camera_hwsync = get_camera_hwsync(pos_online_path)
if pos_online.shape[0] < camera_hwsync.shape[0]:
diff = camera_hwsync.shape[0] - pos_online.shape[0]
camera_hwsync = camera_hwsync.iloc[diff:]
frame_count = np.asarray(camera_hwsync.HWframeCount)
# On AVT cameras, HWFrame counts wraps to 0 above this value.
AVT_camHWframeCount_wrapval = 65535
frame_count = np.unwrap(frame_count,
period=AVT_camHWframeCount_wrapval)
continuous_time = get_continuous_time(pos_online_path)
dio_camera_ticks = find_camera_dio_channel(pos_online_path, pos_online)
dio_systime = np.asarray(continuous_time.loc[dio_camera_ticks])
pause_mid_time = find_acquisition_timing_pause(dio_systime)
ptp_systime = np.asarray(camera_hwsync.HWTimestamp)
frame_rate_from_dio = get_framerate(
dio_systime[dio_systime > pause_mid_time])
logger.info('Camera frame rate estimated from DIO camera ticks:'
f' {frame_rate_from_dio:0.1f} cm/s')
ptp_enabled = detect_ptp(continuous_time, ptp_systime)
if ptp_enabled:
logger.info('PTP detected')
(non_repeat_timestamp_labels, non_repeat_timestamp_labels_id
) = detect_trodes_time_repeats_or_frame_jumps(
camera_hwsync.index[ptp_systime > pause_mid_time],
frame_count[ptp_systime > pause_mid_time])
frame_rate_from_ptp = get_framerate(
ptp_systime[ptp_systime > pause_mid_time])
logger.info('Camera frame rate estimated from ptp:'
f' {frame_rate_from_ptp:0.1f} cm/s')
ptp_timestamps = pd.Index(
ptp_systime[ptp_systime > pause_mid_time] /
NANOSECONDS_PER_SECOND,
name='time')
pos_online = (pos_online.iloc[
ptp_systime > pause_mid_time].set_index(ptp_timestamps))
return pos_online
else:
logger.info('PTP not detected')
camera_systime, is_valid_camera_time = estimate_camera_time_from_mcu_time(
camera_hwsync, continuous_time)
(dio_systime, frame_count, is_valid_camera_time,
camera_systime) = remove_acquisition_timing_pause_non_ptp(
dio_systime, frame_count, camera_systime,
is_valid_camera_time, pause_mid_time)
frame_rate_from_camera_systime = get_framerate(camera_systime)
logger.info('Camera frame rate estimated from mcu timestamps:'
f' {frame_rate_from_camera_systime:0.1f} cm/s')
(non_repeat_timestamp_labels, non_repeat_timestamp_labels_id
) = detect_trodes_time_repeats_or_frame_jumps(
camera_hwsync.index[is_valid_camera_time], frame_count)
camera_to_mcu_lag = estimate_camera_to_mcu_lag(
camera_systime, dio_systime,
len(non_repeat_timestamp_labels_id))
corrected_camera_systime = []
for id in non_repeat_timestamp_labels_id:
is_chunk = non_repeat_timestamp_labels == id
corrected_camera_systime.append(
correct_timestamps_for_camera_to_mcu_lag(
frame_count[is_chunk], camera_systime[is_chunk],
camera_to_mcu_lag))
corrected_camera_systime = np.concatenate(corrected_camera_systime)
valid_camera_ind = np.nonzero(is_valid_camera_time)[0]
valid_camera_ind = valid_camera_ind[non_repeat_timestamp_labels ==
0]
is_valid_camera_time[valid_camera_ind] = False
return (pos_online.iloc[camera_hwsync.index.isin(pos_online.index)
& is_valid_camera_time].set_index(
pd.Index(corrected_camera_systime,
name='time')))
# visualize timestamps from epoch in function of its indexes from continuoustime.dat files.
from rec_to_binaries.read_binaries import readTrodesExtractedDataFile
import matplotlib.pyplot as plt
import numpy as np
path_epoch1 = 'GitHub/rec_to_nwb/rec_to_nwb/test/test_data/beans/preprocessing/20190718/20190718_beans_01_s1.time/20190718_beans_01_s1.continuoustime.dat'
path_epoch2 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_02_r1.continuoustime.dat'
path_epoch3 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_03_s2.continuoustime.dat'
path_epoch4 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_04_r2.continuoustime.dat'
continuous_time = readTrodesExtractedDataFile(path_epoch1)
continuous_time_dict_epoch1 = {
str(data[0]): float(data[1])
for data in continuous_time['data']
}
print("epoch1 length: " + str(len(continuous_time_dict_epoch1)))
continuous_time = readTrodesExtractedDataFile(path_epoch2)
continuous_time_dict_epoch2 = {
str(data[0]): float(data[1])
for data in continuous_time['data']
}
print("epoch2 length: " + str(len(continuous_time_dict_epoch2)))
continuous_time = readTrodesExtractedDataFile(path_epoch3)
continuous_time_dict_epoch3 = {
str(data[0]): float(data[1])
for data in continuous_time['data']
}
print("epoch3 length: " + str(len(continuous_time_dict_epoch3)))
continuous_time = readTrodesExtractedDataFile(path_epoch4)
def __read_contunious_time_file(self, continuous_time_file):
return readTrodesExtractedDataFile(continuous_time_file)
def _get_timestamps(self, dataset_id):
pos_online = readTrodesExtractedDataFile(
self.directories[dataset_id][0])
position = pd.DataFrame(pos_online['data'])
return position.time.to_numpy(dtype='int64')
def get_continuous_time_array_file(file):
logger.info('Reading continuous time array from: ' + str(file))
continuous_time = readTrodesExtractedDataFile(file)
return np.vstack((continuous_time['data']['trodestime'],
continuous_time['data']['adjusted_systime']))
def read_pos_timestamps(file):
pos_online = readTrodesExtractedDataFile(file)
position = pd.DataFrame(pos_online['data'])
return position.time.to_numpy(dtype='int64')
def _get_timestamps(self, dataset_id):
"""Gets timestamps from the online position tracking"""
pos_online = readTrodesExtractedDataFile(
self.directories[dataset_id][0])
position = pd.DataFrame(pos_online['data'])
return position.time.unique().astype(int)
