def load_eegData(fpath, eegChans, bitVolts=0.195):
# Load EEG data of selected channel
print([hfunct.time_string(), 'DEBUG: load from file'])
continuous_data = NWBio.load_continuous_as_array(fpath, eegChans)
if not (continuous_data is None):
print([hfunct.time_string(), 'DEBUG: extract timestamps'])
timestamps = np.array(continuous_data['timestamps'])
print([hfunct.time_string(), 'DEBUG: transforming into float32'])
data = continuous_data['continuous'].astype(np.float32)
sampling_rate = NWBio.OpenEphys_SamplingRate()
else:
# If no raw data available, load downsampled data for that tetrode
lowpass_data = NWBio.load_tetrode_lowpass(fpath)
timestamps = np.array(lowpass_data['tetrode_lowpass_timestamps'])
sampling_rate = NWBio.OpenEphys_SamplingRate() / float(lowpass_downsampling)
if not isinstance(eegChans, list):
eegChans = [eegChans]
tetrodes = []
for eegChan in eegChans:
tetrodes.append(hfunct.channels_tetrode(eegChan))
data = np.array(lowpass_data['tetrode_lowpass'][:, tetrodes]).astype(np.float32)
# Scale data with bitVolts value to convert from raw data to voltage values
print([hfunct.time_string(), 'DEBUG: converting to bitVolts'])
data = data * bitVolts
return {'data': data, 'timestamps': timestamps, 'sampling_rate': sampling_rate}
def get_first_available_spike_data(OpenEphysDataPath, tetrode_nrs, use_idx_keep, use_badChan,
clustering_name=None):
_, spike_names = NWBio.processing_method_and_spike_name_combinations()
spike_data_available = False
for spike_name in spike_names:
spike_data = NWBio.load_spikes(OpenEphysDataPath, tetrode_nrs=tetrode_nrs,
spike_name=spike_name, use_idx_keep=use_idx_keep,
use_badChan=use_badChan, clustering_name=clustering_name)
if len(spike_data) > 0:
spike_data_available = True
break
if not spike_data_available:
raise Exception('Spike data is not available in file: ' + OpenEphysDataPath + '\nChecked for following spike_name: ' + str(spike_names))
return spike_data
def getExperimentInfo(fpath=None):
'''
Extract experiment info from NWB file and sets to correc tformat for
createAxonaData() function.
If fpath is not provided, sham data is provided that is compatible
with createAxonaData() function.
'''
if not (fpath is None) and NWBio.check_if_settings_available(fpath,'/General/animal/'):
animal = NWBio.load_settings(fpath,'/General/animal/')
else:
animal = 'unknown'
if not (fpath is None) and NWBio.check_if_settings_available(fpath,'/Time/'):
full_timestring = NWBio.load_settings(fpath,'/Time/')
timeobject = datetime.strptime(full_timestring, '%Y-%m-%d_%H-%M-%S')
else:
timeobject = datetime.now()
datestring = timeobject.strftime('%d-%m-%y')
timestring = timeobject.strftime('%H-%M-%S')
experiment_info = {'trial_date': datestring,
'trial_time': timestring,
'animal': animal}
return experiment_info
def main(fpath):
print('Opening NWB file...')
data = NWBio.load_continuous(fpath)
# Load timestamps into memory
timestamps = np.array(data['timestamps'])
timestamps = timestamps - timestamps[0]
# Get user specified start and end time
print('Session length is ' + str(timestamps[-1]) + ' seconds.')
start_time = float(raw_input('Enter segment start time: '))
end_time = float(raw_input('Enter segment end time: '))
start_idx = np.argmin(np.abs(timestamps - start_time))
end_idx = np.argmin(np.abs(timestamps - end_time))
# Get user specified start and end time
print('There are ' + str(data['continuous'].shape[1]) + ' channels.')
first_chan = int(raw_input('Enter first channel nr: ')) - 1
last_chan = int(raw_input('Enter last channel nr: '))
# Load continuous data of specified shape
print('Loading continuous data for segment...')
timestamps = timestamps[start_idx:end_idx]
continuous = np.array(data['continuous'][start_idx:end_idx, first_chan:last_chan])
continuous = continuous.astype(np.float32) * 0.195
n_channels = continuous.shape[1]
n_datapoints = continuous.shape[0]
data['file_handle'].close()
# Convert continuous data such that it can all be show in single plot
print('Converting continuous data for display...')
mean_std = np.mean(np.std(continuous, axis=0))
channel_spacing = mean_std * 3
channel_spacer = np.linspace(0, (n_channels - 1) * channel_spacing, n_channels)
channel_spacer = np.repeat(np.transpose(channel_spacer[:, None]), n_datapoints, axis=0)
continuous = continuous + channel_spacer
# Plot data
print('Plotting...')
fig = plt.figure()
ax = plt.gca()
plt.axis('off')
plt.plot(timestamps, continuous, linewidth=1)
plt.xlim(np.min(timestamps), np.max(timestamps))
plt.ylim(np.min(continuous), np.max(continuous))
plt.subplots_adjust(left=0, right=1, top=1, bottom=0)
plt.gca().invert_yaxis()
plt.show()
def createAxonaData_for_NWBfile(OpenEphysDataPath, spike_name='first_available', channel_map=None,
subfolder='AxonaData', eegChans=None, pixels_per_metre=None,
show_output=False, clustering_name=None):
# Construct path for AxonaData and get experiment info
AxonaDataPath = os.path.join(os.path.dirname(OpenEphysDataPath), subfolder)
experiment_info = getExperimentInfo(OpenEphysDataPath)
# Get channel_map for this dataset
if channel_map is None:
if NWBio.check_if_settings_available(OpenEphysDataPath,'/General/channel_map/'):
channel_map = NWBio.load_settings(OpenEphysDataPath,'/General/channel_map/')
else:
raise ValueError('Channel map could not be generated. Enter channels to process.')
# Get position data for this recording
data_time_edges = NWBio.get_processed_tracking_data_timestamp_edges(OpenEphysDataPath)
if NWBio.check_if_processed_position_data_available(OpenEphysDataPath):
posdata = NWBio.load_processed_tracking_data(OpenEphysDataPath)
else:
posdata = None
# Create AxonaData separately for each recording area
for area in channel_map.keys():
# Load spike data
tetrode_nrs = hfunct.get_tetrode_nrs(channel_map[area]['list'])
print('Loading spikes for tetrodes nr: ' + ', '.join(map(str, tetrode_nrs)))
if spike_name == 'first_available':
spike_data = get_first_available_spike_data(OpenEphysDataPath, tetrode_nrs,
use_idx_keep=True, use_badChan=True,
clustering_name=clustering_name)
else:
spike_data = NWBio.load_spikes(OpenEphysDataPath, tetrode_nrs=tetrode_nrs,
spike_name=spike_name, use_idx_keep=True,
use_badChan=True, clustering_name=clustering_name)
# Load eeg data
if eegChans is None:
eegData = None
else:
eegChansInArea_Bool = [x in channel_map[area]['list'] for x in eegChans]
if any(eegChansInArea_Bool):
eegChansInArea = [x for (x,y) in zip(eegChans, eegChansInArea_Bool) if y]
print('Loading LFP data for channels: ' + ', '.join(map(str, eegChansInArea)))
eegData = load_eegData(OpenEphysDataPath, eegChansInArea)
else:
eegData = None
createAxonaData(AxonaDataPath, spike_data, data_time_edges, posdata=posdata,
experiment_info=experiment_info, axona_file_name=area,
eegData=eegData, pixels_per_metre=pixels_per_metre,
show_output=show_output)
def __init__(self, fpath, fps=30, method=None, threshold_multiplier=0.75):
# Parse input
self._fpath = NWBio.get_filename(fpath)
if not NWBio.check_if_open_ephys_nwb_file(self._fpath):
raise ValueError('Path {} does not lead to a recognised recording file.'.format(self._fpath))
self._timestep = 1 / float(fps)
settings = NWBio.load_settings(self._fpath)
self._method = settings['CameraSettings']['General']['tracking_mode'] if method is None else method
if not (self._method in self.supported_methods):
raise ValueError('Method {} not found in support methods {}'.format(self._method, self.supported_methods))
self._threshold_multiplier = threshold_multiplier
# Get list of cameras_ids
self._camera_ids = tuple(map(str, settings['CameraSettings']['CameraSpecific'].keys()))
if len(self._camera_ids) == 0:
raise Exception('No cameras specified in CameraSettings.')
# Get settings for processing camera images
resolution_setting = settings['CameraSettings']['General']['resolution_option']
self._arena_size = settings['General']['arena_size']
self._led_separation = settings['CameraSettings']['General']['LED_separation']
self._camera_transfer_radius = settings['CameraSettings']['General']['camera_transfer_radius']
self._smoothing_size = {'high': 4, 'low': 2}[resolution_setting]
self._calibration_matrix = {}
self._camera_position = {}
for camera_id in self._camera_ids:
camera_id_settings = settings['CameraSettings']['CameraSpecific'][camera_id]
self._calibration_matrix[camera_id] = \
camera_id_settings['CalibrationData'][resolution_setting]['calibrationTmatrix']
self._camera_position[camera_id] = camera_id_settings['position_xy']
# Initialise video feed
self._video = {camera_id: RecordingCameraVideo(self._fpath, camera_id) for camera_id in self._camera_ids}
self._align_videos()
# Initialise tracking process
self._current_timestamp = min([self._video[camera_id].current_timestamp
for camera_id in self._camera_ids])
self._final_timestamp = min([self._video[camera_id].final_timestamp
for camera_id in self._camera_ids])
self._timestamps = []
self._tracking_data = []
self._process_started = False
self._process_finished = False
def save_settings(self, fpath):
NWBio.save_settings(fpath, self.task_settings, path='/TaskSettings/')
def load_settings(self, fpath):
self._task_settings = NWBio.load_settings(fpath, path='/TaskSettings/')
def createAxonaData_for_multiple_NWBfiles(OpenEphysDataPaths, AxonaDataPath,
spike_name='first_available', channel_map=None,
eegChans=None, pixels_per_metre=None,
show_output=False, clustering_name=None):
# Get experiment info
if len(OpenEphysDataPaths) > 1:
print('Using experiment_info from first recording only.')
experiment_info = getExperimentInfo(OpenEphysDataPaths[0])
# Get channel_map for this dataset
if channel_map is None:
if len(OpenEphysDataPaths) > 1:
print('Using channel_map from first recording only.')
if NWBio.check_if_settings_available(OpenEphysDataPaths[0],'/General/channel_map/'):
channel_map = NWBio.load_settings(OpenEphysDataPaths[0],'/General/channel_map/')
else:
raise ValueError('Channel map could not be generated. Enter channels to process.')
# Compute start and end times of each segment of the recording
data_time_edges = []
for OpenEphysDataPath in OpenEphysDataPaths:
data_time_edges.append(NWBio.get_processed_tracking_data_timestamp_edges(OpenEphysDataPath))
recording_edges = []
recording_duration = 0
for dte in data_time_edges:
end_of_this_recording = recording_duration + (dte[1] - dte[0])
recording_edges.append([recording_duration, end_of_this_recording])
recording_duration = end_of_this_recording
combined_data_time_edges = [recording_edges[0][0], recording_edges[-1][1]]
# Get position data for these recordings
print('Loading position data.')
posdata = []
for OpenEphysDataPath in OpenEphysDataPaths:
if NWBio.check_if_processed_position_data_available(OpenEphysDataPath):
posdata.append(NWBio.load_processed_tracking_data(OpenEphysDataPath))
else:
posdata.append(None)
if any([x is None for x in posdata]):
posdata = None
else:
posdata = concatenate_posdata_across_recordings(posdata, data_time_edges,
recording_edges)
# Create AxonaData separately for each recording area
for area in channel_map.keys():
# Load spike data
tetrode_nrs = hfunct.get_tetrode_nrs(channel_map[area]['list'])
print('Loading spikes for tetrodes nr: ' + ', '.join(map(str, tetrode_nrs)))
spike_data = []
for OpenEphysDataPath in OpenEphysDataPaths:
if spike_name == 'first_available':
spike_data.append(get_first_available_spike_data(OpenEphysDataPath, tetrode_nrs,
use_idx_keep=True, use_badChan=True))
else:
print([hfunct.time_string(), 'DEBUG: loading spikes of tet ', tetrode_nrs, ' from ', OpenEphysDataPath])
spike_data.append(NWBio.load_spikes(OpenEphysDataPath, tetrode_nrs=tetrode_nrs,
spike_name=spike_name, use_idx_keep=True,
use_badChan=True, clustering_name=clustering_name))
spike_data = concatenate_spike_data_across_recordings(spike_data, data_time_edges,
recording_edges)
# Load eeg data
if eegChans is None:
eegData = None
else:
eegChansInArea_Bool = [x in channel_map[area]['list'] for x in eegChans]
if any(eegChansInArea_Bool):
eegChansInArea = [x for (x,y) in zip(eegChans, eegChansInArea_Bool) if y]
print('Loading LFP data for channels: ' + ', '.join(map(str, eegChansInArea)))
eegData = []
for OpenEphysDataPath in OpenEphysDataPaths:
print([hfunct.time_string(), 'DEBUG: loading eegData for ', OpenEphysDataPath])
eegData.append(load_eegData(OpenEphysDataPath, eegChansInArea))
print([hfunct.time_string(), 'DEBUG: concatenating eeg data'])
eegData = concatenate_eegData_across_recordings(eegData, data_time_edges,
recording_edges)
else:
eegData = None
createAxonaData(AxonaDataPath, spike_data, combined_data_time_edges,
posdata=posdata, experiment_info=experiment_info,
axona_file_name=area, eegData=eegData,
pixels_per_metre=pixels_per_metre, show_output=show_output)
with open(os.path.join(AxonaDataPath, 'recording_edges'), 'w') as file:
for edges, OpenEphysDataPath in zip(recording_edges, OpenEphysDataPaths):
file.write(str(edges) + ' path: ' + OpenEphysDataPath + '\n')
def iteratively_combine_multicamera_data_for_recording(
CameraSettings, arena_size, posdatas, OE_GC_times, verbose=False):
"""Return ProcessedPos
Combines raw tracking data from multiple cameras into a single ProcessedPos array.
If a single camera data is provided, this will be converted into same format.
"""
cameraIDs = sorted(CameraSettings['CameraSpecific'].keys())
# Load position data for all cameras
for i, posdata in enumerate(posdatas):
if isinstance(posdata, dict):
# This is conditional because for early recordings tracking data was immediately stored in Open Ephys time.
# Remove datapoints where all position data is None
idx_None = np.all(np.isnan(posdata['OnlineTrackerData']), 1)
posdata['OnlineTrackerData'] = np.delete(posdata['OnlineTrackerData'],
np.where(idx_None)[0], axis=0)
posdata['OnlineTrackerData_timestamps'] = np.delete(posdata['OnlineTrackerData_timestamps'],
np.where(idx_None)[0], axis=0)
# Compute position data timestamps in OpenEphys time
RPi_frame_in_OE_times = NWBio.estimate_open_ephys_timestamps_from_other_timestamps(
OE_GC_times,
posdata['GlobalClock_timestamps'],
posdata['OnlineTrackerData_timestamps'],
other_times_divider=10 ** 6
)
# Combine timestamps and position data into a single array
posdata = np.concatenate((RPi_frame_in_OE_times.astype(np.float64)[:, None], posdata['OnlineTrackerData']), axis=1)
posdatas[i] = posdata
if len(posdatas) > 1:
# If data from multiple cameras available, combine it
PosDataFramesPerSecond = 30.0
# Find first and last timepoint for position data
first_timepoints = []
last_timepoints = []
for posdata in posdatas:
first_timepoints.append(posdata[0, 0])
last_timepoints.append(posdata[-1, 0])
# Combine position data step-wise from first to last timepoint at PosDataFramesPerSecond
# At each timepoint the closest matchin datapoints will be taken from different cameras
timepoints = np.arange(np.array(first_timepoints).min(), np.array(last_timepoints).max(), 1.0 / PosDataFramesPerSecond)
combPosData = [None]
listNaNs = []
if verbose:
print('Combining iteratively camera data for each position timepoint')
for npoint in (tqdm(range(len(timepoints))) if verbose else range(len(timepoints))):
# Find closest matchin timepoint from all RPis
idx_tp = np.zeros(4, dtype=np.int32)
for nRPi in range(len(posdatas)):
idx_tp[nRPi] = np.argmin(np.abs(timepoints[npoint] - posdatas[nRPi][:,0]))
# Convert posdatas for use in combineCamerasData function
cameraPos = []
for nRPi in range(len(posdatas)):
cameraPos.append(posdatas[nRPi][idx_tp[nRPi], 1:5])
tmp_comb_data = combineCamerasData(cameraPos, combPosData[-1], cameraIDs, CameraSettings, arena_size)
combPosData.append(tmp_comb_data)
if tmp_comb_data is None:
listNaNs.append(npoint)
# Remove the extra element from combPosData
del combPosData[0]
# Remove all None elements
for nanElement in listNaNs[::-1]:
del combPosData[nanElement]
timepoints = np.delete(timepoints, listNaNs)
# Combine timepoints and position data
ProcessedPos = np.concatenate((np.expand_dims(np.array(timepoints), axis=1), np.array(combPosData)), axis=1)
# Print info about None elements
if len(listNaNs) > 0:
print('Total of ' + str(len(listNaNs) * (1.0 / PosDataFramesPerSecond)) + ' seconds of position data was lost')
if listNaNs[0] == 0:
print('This was in the beginning and ' + str(np.sum(np.diff(np.array(listNaNs)) > 1)) + ' other epochs.')
else:
print('This was not in the beginning, but in ' + str(np.sum(np.diff(np.array(listNaNs)) > 1) + 1) + ' other epochs.')
else:
# In case of single camera being used, just use data from that camera
ProcessedPos = posdatas[0]
ProcessedPos = remove_tracking_data_outside_boundaries(ProcessedPos, arena_size, max_error=10)
ProcessedPos = ProcessedPos.astype(np.float64)
return ProcessedPos