def extract_analog_for_single_dataset(analog_files, continuous_time_file): single_dataset_data = {} for analog_file in analog_files: if not 'timestamps' in analog_file: analog_data = readTrodesExtractedDataFile(analog_files[analog_file]) values = analog_data['data'] single_dataset_data[analog_data['id']] = values else: continuous_time_dict = ContinuousTimeExtractor.get_continuous_time_dict_file(continuous_time_file) timestamp = readTrodesExtractedDataFile(analog_files[analog_file]) keys = [key[0] for key in timestamp['data']] single_dataset_data[analog_file] = TimestampConverter.convert_timestamps(continuous_time_dict, keys) return single_dataset_data
def get_continuous_time_dict_file(file): logger.info('Reading continuous time dict from: ' + str(file)) continuous_time = readTrodesExtractedDataFile(file) return { str(data[0]): float(data[3]) for data in continuous_time['data'] }
def get_column_labels(self): """extract column labels from POS files""" pos_online = readTrodesExtractedDataFile(self.directories[0][0]) column_labels = pd.DataFrame(pos_online['data']).columns column_labels = column_labels[1:] column_labels_list = column_labels.values.tolist() return column_labels_list
def fix_timestamp_lag(continuoustime_filename): """ Fix the correspondence between trodestime and system (wall) time. There is some jitter in the arrival times of packets from the MCU (as reflected in the sysclock records in the .rec file. If we assume that the Trodes clock is actually regular, and that any episodes of lag are fairly sporadic, we can recover the correspondence between trodestime and system (wall) time. Parameters ---------- continuoustime_filename : str Path to .continuoustime.dat file """ data_file = readTrodesExtractedDataFile(continuoustime_filename) if 'systime' not in data_file['data'].dtype.names: # logging.warn new_data = infer_systime(data_file) else: new_data = (pd.DataFrame(data_file['data']).assign( time_chunk_label=lambda df: _label_time_chunks(df.trodestime)). assign(adjusted_systime=lambda df: _regress_timestamps( df.trodestime, df.systime))) new_data_file = _insert_new_data(data_file, new_data) write_trodes_extracted_datafile(continuoustime_filename, new_data_file)
def read_data(self, dataset_id, file_id): """extract data from POS files and build FlPos""" pos_online = readTrodesExtractedDataFile( self.directories[dataset_id][file_id]) position = pd.DataFrame(pos_online['data']) labels = self.get_column_labels() filtered_position = [position[label] for label in labels] return filtered_position
def extract_analog_for_single_dataset(analog_files): single_dataset_data = {} for analog_file in analog_files: analog_data = readTrodesExtractedDataFile( analog_files[analog_file]) values = analog_data['data'] single_dataset_data[analog_data['id']] = values return single_dataset_data
def retrieve_real_timestamps(self, dataset_id): """Gets the corresponding Trodes timestamps from the online position tracking and matches them to the PTP time in the video file. Otherwise, we get the corresponding timestamps from continuous time which corresponds to the neural recording time stamps. If there is no corresponding timestamp, the result will be NaN. Parameters ---------- dataset_id : int Index of the epoch Returns ------- timestamps : ndarray, shape (n_online_tracked_positions,) """ try: # Get online position tracking data pos_online_path = self.directories[dataset_id][0] pos_online = readTrodesExtractedDataFile(pos_online_path) pos_online = pd.DataFrame(pos_online['data']) # Make sure to get only the unique timestamps because they can # sometimes repeat after a jump in timestamps online_timestamps_ind = pos_online.time.unique().astype(np.uint64) # Get video PTP timestamps camera_hwsync = readTrodesExtractedDataFile( pos_online_path.replace('.pos_online.dat', '.pos_cameraHWFrameCount.dat')) camera_hwsync = (pd.DataFrame( camera_hwsync['data']).set_index('PosTimestamp')) # Find the PTP timestamps that correspond to position tracking # Convert from nanoseconds to seconds return (camera_hwsync.loc[online_timestamps_ind, 'HWTimestamp'] / NANOSECONDS_PER_SECOND).to_numpy() except KeyError: # If PTP timestamps do not exist find the corresponding timestamps # from the neural recording logger.info('No PTP timestamps found. Using neural timestamps.') return TimestampManager.retrieve_real_timestamps( self, dataset_id, convert_timestamps=self.convert_timestamps)
def __extract_single(self, hw_frame_count_filename): content = readTrodesExtractedDataFile(self.raw_data_path + "/" + hw_frame_count_filename)["data"] camera_sample_frame_counts = np.ndarray(shape=(len(content), 2), dtype='uint32') for i, record in enumerate(content): camera_sample_frame_counts[i, 0] = record[1] camera_sample_frame_counts[i, 1] = record[0] return camera_sample_frame_counts
def __get_continuous_time_data_from_single_file(self, continuous_time_file): continuous_time = readTrodesExtractedDataFile(continuous_time_file) new_array = np.ndarray(shape=(len(continuous_time['data']), 2), dtype='int64') for i, single_timestamp in enumerate(continuous_time['data']): new_array[i, 0] = single_timestamp[0] new_array[i, 1] = single_timestamp[3] return new_array
def find_camera_dio_channel(pos_online_path, pos_online, max_frames_diff=10): """Find the camera DIO by looping through all the DIOs and finding the right number of DIO pulses. Parameters ---------- pos_online_path : str pos_online : pd.DataFrame, shape (n_camera_frames, 5) max_frames_diff : int, optional maximum difference between camera frames and dio pulses Returns ------- camera_dio_times : pd.Series, shape (n_dio_pulse_state_changes,) Trodes time of dio ticks """ head, tail = os.path.split(pos_online_path) dio_paths = glob.glob( os.path.join( os.path.split(head)[0], tail.split('.')[0] + '.DIO', tail.split('.')[0] + '*.dat')) n_ticks = np.asarray([ pd.DataFrame( readTrodesExtractedDataFile(dio_file)['data']).state.sum() for dio_file in dio_paths ]) n_camera_frames = pos_online.shape[0] position_ticks_file_ind = np.argmin(np.abs(n_ticks - n_camera_frames)) camera_ticks_dio = pd.DataFrame( readTrodesExtractedDataFile( dio_paths[position_ticks_file_ind])['data']) return camera_ticks_dio.loc[camera_ticks_dio.state == 1].time
def __extract_single(self, filename): """Returns the video frame counts and timestamps for a single epoch.""" content = pd.DataFrame( readTrodesExtractedDataFile( os.path.join(self.raw_data_path, filename))["data"]) try: # columns: frame count, timestamps return content.iloc[:, [1, 0]].to_numpy() except IndexError: return np.vstack(( content.iloc[:, 0].to_numpy(), # frame counts np.arange(len(content))) # dummy timestamps ).T.astype(np.uint32)
def extract_analog_for_single_dataset(analog_files, continuous_time_file, convert_timestamps=True): single_dataset_data = {} for analog_sensor in analog_files: analog_data = readTrodesExtractedDataFile( analog_files[analog_sensor]) if 'timestamps' not in analog_sensor: values = analog_data['data'] single_dataset_data[analog_data['id']] = values else: timestamps = FlAnalogExtractor._extract_analog_timestamps( analog_data, continuous_time_file, convert_timestamps) single_dataset_data[analog_sensor] = timestamps return single_dataset_data
def extract_dio_for_single_dataset(filtered_files): single_dataset_data = {} for dio_file in filtered_files: try: dio_data = readTrodesExtractedDataFile( filtered_files[dio_file]) keys, values = OldDioExtractor.__get_dio_time_series(dio_data) single_dataset_data[dio_file] = ([keys, values]) except KeyError as error: message = "there is no " + str(dio_file) + ", error: " logger.exception(message + str(error)) except TypeError as error: message = "there is no data for event " + str( dio_file) + ", error: " logger.exception(message + str(error)) return single_dataset_data
def extract_video_files(self): video_files = self.video_files_metadata extracted_video_files = [] for video_file in video_files: new_fl_video_file = { "name": video_file["name"], "timestamps": self.convert_timestamps( readTrodesExtractedDataFile( self.raw_data_path + "/" + video_file["name"][:-4] + "videoTimeStamps.cameraHWSync")["data"]), "device": video_file["camera_id"] } extracted_video_files.append(new_fl_video_file) return extracted_video_files
def _read_video_timestamps_hw_framecount(self, video_file): """Returns the index of video frames. If PTP is not in use, only the videoTimeStamps.cameraHWFrameCount file is generated by the `rec_to_binaries` package. Parameters ---------- video_file : str Returns ------- index : ndarray """ return readTrodesExtractedDataFile( os.path.join( self.raw_data_path, os.path.splitext(video_file["name"])[0] + ".videoTimeStamps.cameraHWFrameCount"))['data']['frameCount']
def get_camera_hwsync(pos_online_path): """Get video PTP timestamps if they exist. Parameters ---------- pos_online_path : str Returns ------- camera_hwsync : pd.DataFrame, shape (n_camera_frames, 2) PosTimestamp: unadjusted postimestamps. UINT32 HWframeCount: integer count of frames acquired by camera (rolls over at 65535; can be used to detect dropped frames). UINT32 HWTimestamp: POSIX time in nanoseconds, synchronized to PC sysclock via PTP. UINT64. """ camera_hwsync = readTrodesExtractedDataFile( pos_online_path.replace('.pos_online.dat', '.pos_cameraHWFrameCount.dat')) return (pd.DataFrame(camera_hwsync['data']).set_index('PosTimestamp'))
def get_continuous_time(pos_online_path): """Neural timestamps. Parameters ---------- pos_online_path : str Returns ------- continuous_times : pd.DataFrame trodestime uint32 adjusted_systime int64 """ head, tail = os.path.split(pos_online_path) continuous_time_path = os.path.join( os.path.split(head)[0], tail.split('.')[0] + '.time', tail.split('.')[0] + '.continuoustime.dat') cont_time = readTrodesExtractedDataFile(continuous_time_path) return (pd.DataFrame( cont_time['data']).set_index('trodestime').adjusted_systime)
def _read_video_timestamps_hw_sync(self, video_file): """Returns video timestamps in unix time which are synchronized to the Trodes data packets. videoTimeStamps.cameraHWSync is a file extracted by the python package `rec_to_binaries` from the .rec file. It only is extracted when using precision time protocol (PTP) to synchronize the camera clock with Trodes data packets. The HWTimestamp array in this file are the unix timestamps relative to seconds since 1/1/1970. Parameters ---------- video_file : str Returns ------- unix_timestamps : ndarray """ return readTrodesExtractedDataFile( os.path.join( self.raw_data_path, os.path.splitext(video_file["name"])[0] + ".videoTimeStamps.cameraHWSync"))['data']['HWTimestamp']
def extract_dio_for_single_dataset(filtered_files, continuous_time_file, convert_timestamps=True): single_dataset_data = {} continuous_time = ContinuousTimeExtractor.get_continuous_time_array_file( continuous_time_file) for dio_sensor in filtered_files: try: dio_data = readTrodesExtractedDataFile( filtered_files[dio_sensor]) # dio_data['data'] is a labeled array with 'time' and 'state' columns. 'time' corresponds to sample count single_dataset_data[dio_sensor] = DioExtractor.__get_dio_time_series( dio_data, continuous_time, convert_timestamps) # keys, values = DioExtractor.__get_dio_time_series(dio_data, continuous_time_dict # single_dataset_data[dio_sensor] = ([keys, values]) except KeyError as error: message = "there is no " + str(dio_sensor) + ", error: " logger.exception(message + str(error)) except TypeError as error: message = "there is no data for event " + \ str(dio_sensor) + ", error: " logger.exception(message + str(error)) return single_dataset_data
def get_position_with_corrected_timestamps(pos_online_path): logger.info(os.path.split(pos_online_path)[-1]) pos_online = pd.DataFrame( readTrodesExtractedDataFile(pos_online_path)['data']).set_index( 'time') camera_hwsync = get_camera_hwsync(pos_online_path) if pos_online.shape[0] < camera_hwsync.shape[0]: diff = camera_hwsync.shape[0] - pos_online.shape[0] camera_hwsync = camera_hwsync.iloc[diff:] frame_count = np.asarray(camera_hwsync.HWframeCount) # On AVT cameras, HWFrame counts wraps to 0 above this value. AVT_camHWframeCount_wrapval = 65535 frame_count = np.unwrap(frame_count, period=AVT_camHWframeCount_wrapval) continuous_time = get_continuous_time(pos_online_path) dio_camera_ticks = find_camera_dio_channel(pos_online_path, pos_online) dio_systime = np.asarray(continuous_time.loc[dio_camera_ticks]) pause_mid_time = find_acquisition_timing_pause(dio_systime) ptp_systime = np.asarray(camera_hwsync.HWTimestamp) frame_rate_from_dio = get_framerate( dio_systime[dio_systime > pause_mid_time]) logger.info('Camera frame rate estimated from DIO camera ticks:' f' {frame_rate_from_dio:0.1f} cm/s') ptp_enabled = detect_ptp(continuous_time, ptp_systime) if ptp_enabled: logger.info('PTP detected') (non_repeat_timestamp_labels, non_repeat_timestamp_labels_id ) = detect_trodes_time_repeats_or_frame_jumps( camera_hwsync.index[ptp_systime > pause_mid_time], frame_count[ptp_systime > pause_mid_time]) frame_rate_from_ptp = get_framerate( ptp_systime[ptp_systime > pause_mid_time]) logger.info('Camera frame rate estimated from ptp:' f' {frame_rate_from_ptp:0.1f} cm/s') ptp_timestamps = pd.Index( ptp_systime[ptp_systime > pause_mid_time] / NANOSECONDS_PER_SECOND, name='time') pos_online = (pos_online.iloc[ ptp_systime > pause_mid_time].set_index(ptp_timestamps)) return pos_online else: logger.info('PTP not detected') camera_systime, is_valid_camera_time = estimate_camera_time_from_mcu_time( camera_hwsync, continuous_time) (dio_systime, frame_count, is_valid_camera_time, camera_systime) = remove_acquisition_timing_pause_non_ptp( dio_systime, frame_count, camera_systime, is_valid_camera_time, pause_mid_time) frame_rate_from_camera_systime = get_framerate(camera_systime) logger.info('Camera frame rate estimated from mcu timestamps:' f' {frame_rate_from_camera_systime:0.1f} cm/s') (non_repeat_timestamp_labels, non_repeat_timestamp_labels_id ) = detect_trodes_time_repeats_or_frame_jumps( camera_hwsync.index[is_valid_camera_time], frame_count) camera_to_mcu_lag = estimate_camera_to_mcu_lag( camera_systime, dio_systime, len(non_repeat_timestamp_labels_id)) corrected_camera_systime = [] for id in non_repeat_timestamp_labels_id: is_chunk = non_repeat_timestamp_labels == id corrected_camera_systime.append( correct_timestamps_for_camera_to_mcu_lag( frame_count[is_chunk], camera_systime[is_chunk], camera_to_mcu_lag)) corrected_camera_systime = np.concatenate(corrected_camera_systime) valid_camera_ind = np.nonzero(is_valid_camera_time)[0] valid_camera_ind = valid_camera_ind[non_repeat_timestamp_labels == 0] is_valid_camera_time[valid_camera_ind] = False return (pos_online.iloc[camera_hwsync.index.isin(pos_online.index) & is_valid_camera_time].set_index( pd.Index(corrected_camera_systime, name='time')))
# visualize timestamps from epoch in function of its indexes from continuoustime.dat files. from rec_to_binaries.read_binaries import readTrodesExtractedDataFile import matplotlib.pyplot as plt import numpy as np path_epoch1 = 'GitHub/rec_to_nwb/rec_to_nwb/test/test_data/beans/preprocessing/20190718/20190718_beans_01_s1.time/20190718_beans_01_s1.continuoustime.dat' path_epoch2 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_02_r1.continuoustime.dat' path_epoch3 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_03_s2.continuoustime.dat' path_epoch4 = 'GitHub/rec_to_nwb/rec_to_nwb/test/20190718_beans_04_r2.continuoustime.dat' continuous_time = readTrodesExtractedDataFile(path_epoch1) continuous_time_dict_epoch1 = { str(data[0]): float(data[1]) for data in continuous_time['data'] } print("epoch1 length: " + str(len(continuous_time_dict_epoch1))) continuous_time = readTrodesExtractedDataFile(path_epoch2) continuous_time_dict_epoch2 = { str(data[0]): float(data[1]) for data in continuous_time['data'] } print("epoch2 length: " + str(len(continuous_time_dict_epoch2))) continuous_time = readTrodesExtractedDataFile(path_epoch3) continuous_time_dict_epoch3 = { str(data[0]): float(data[1]) for data in continuous_time['data'] } print("epoch3 length: " + str(len(continuous_time_dict_epoch3))) continuous_time = readTrodesExtractedDataFile(path_epoch4)
def __read_contunious_time_file(self, continuous_time_file): return readTrodesExtractedDataFile(continuous_time_file)
def _get_timestamps(self, dataset_id): pos_online = readTrodesExtractedDataFile( self.directories[dataset_id][0]) position = pd.DataFrame(pos_online['data']) return position.time.to_numpy(dtype='int64')
def get_continuous_time_array_file(file): logger.info('Reading continuous time array from: ' + str(file)) continuous_time = readTrodesExtractedDataFile(file) return np.vstack((continuous_time['data']['trodestime'], continuous_time['data']['adjusted_systime']))
def read_pos_timestamps(file): pos_online = readTrodesExtractedDataFile(file) position = pd.DataFrame(pos_online['data']) return position.time.to_numpy(dtype='int64')
def _get_timestamps(self, dataset_id): """Gets timestamps from the online position tracking""" pos_online = readTrodesExtractedDataFile( self.directories[dataset_id][0]) position = pd.DataFrame(pos_online['data']) return position.time.unique().astype(int)