def make(self, nwb_content: NWBFile):
position = Position(name='position')
cameras_ids = get_cameras_ids(self.dataset_names, self.metadata)
meters_per_pixels = get_meters_per_pixels(cameras_ids, self.metadata)
pos_online_paths = []
for dataset in self.datasets:
for pos_file in dataset.get_all_data_from_dataset('pos'):
if pos_file.endswith('.pos_online.dat'):
pos_online_paths.append(
os.path.join(dataset.get_data_path_from_dataset('pos'),
pos_file))
first_timestamps = []
for series_id, (conversion, pos_online_path) in enumerate(
zip(meters_per_pixels, pos_online_paths)):
position_df = self.get_position_with_corrected_timestamps(
pos_online_path)
position.create_spatial_series(
name=f'series_{series_id}',
description=', '.join(position_df.columns.tolist()),
data=np.asarray(position_df),
conversion=conversion,
reference_frame='Upper left corner of video frame',
timestamps=np.asarray(position_df.index),
)
first_timestamps.append(position_df.index[0])
# check if timestamps are in order
first_timestamps = np.asarray(first_timestamps)
assert np.all(first_timestamps[:-1] < first_timestamps[1:])
logger.info('Position: Injecting into Processing Module')
nwb_content.processing['behavior'].add(position)
def add_behavior(nwbfile, expt):
bd = expt.behaviorData(imageSync=True)
fs = 1 / expt.frame_period()
behavior_module = nwbfile.create_processing_module(
name='Behavior', description='Data relevant to behavior')
# Add Normalized Position
pos = Position(name='Normalized Position')
pos.create_spatial_series(name='Normalized Position',
rate=fs,
data=bd['treadmillPosition'][:, np.newaxis],
reference_frame='0 is belt start',
conversion=0.001 * bd['trackLength'])
behavior_module.add_container(pos)
# Add Licking
licking = BehavioralTimeSeries(name='Licking')
licking.create_timeseries(
name='Licking',
data=bd['licking'],
rate=fs,
unit='na',
description='1 if mouse licked during this imaging frame')
behavior_module.add_container(licking)
# Add Water Reward Delivery
water = BehavioralTimeSeries(name='Water')
water.create_timeseries(
name='Water',
data=bd['water'],
rate=fs,
unit='na',
description='1 if water was delivered during this imaging frame')
behavior_module.add_container(water)
# Add Lap Times
laps = BehavioralEvents(name='Lap Starts')
# TODO probably not best to have laps as data and timestamps here
laps.create_timeseries(name='Lap Starts',
data=bd['lap'],
timestamps=bd['lap'],
description='Frames at which laps began',
unit='na')
behavior_module.add_container(laps)
def create(position: Position, series_id: int, fl_position: FlPosition):
validate_parameters_not_none(__name__, fl_position.column_labels, fl_position.position_data,
fl_position.conversion)
position.create_spatial_series(
name='series_' + str(series_id),
description=fl_position.column_labels,
data=fl_position.position_data,
conversion=fl_position.conversion,
reference_frame='Upper left corner of video frame',
timestamps=fl_position.timestamps,
)
# :py:class:`~pynwb.behavior.SpatialSeries` objects. :py:class:`~pynwb.behavior.SpatialSeries` is a subtype of
# :py:class:`~pynwb.base.TimeSeries` that represents the spatial position of an animal over time. By putting
# your position data into a :py:class:`~pynwb.behavior.Position` container, downstream users and
# tools know where to look to retrieve position data. For a comprehensive list of available data interfaces, see the
# :ref:`overview page <modules_overview>`. Here is how to create a :py:class:`~pynwb.behavior.Position` object
# named '`Position'` [#]_.
from pynwb.behavior import Position
position = Position()
####################
# You can add objects to a data interface as a method of the data interface:
position.create_spatial_series(name='position1',
data=np.linspace(0, 1, 20),
rate=50.,
reference_frame='starting gate')
####################
# or you can add pre-existing objects:
from pynwb.behavior import SpatialSeries
spatial_series = SpatialSeries(name='position2',
data=np.linspace(0, 1, 20),
rate=50.,
reference_frame='starting gate')
position.add_spatial_series(spatial_series)
####################
def conversion_function(source_paths,
f_nwb,
metadata,
add_raw=False,
add_processed=True,
add_behavior=True,
plot_rois=False):
"""
Copy data stored in a set of .npz files to a single NWB file.
Parameters
----------
source_paths : dict
Dictionary with paths to source files/directories. e.g.:
{'raw_data': {'type': 'file', 'path': ''},
'raw_info': {'type': 'file', 'path': ''}
'processed_data': {'type': 'file', 'path': ''},
'sparse_matrix': {'type': 'file', 'path': ''},
'ref_image',: {'type': 'file', 'path': ''}}
f_nwb : str
Path to output NWB file, e.g. 'my_file.nwb'.
metadata : dict
Metadata dictionary
add_raw : bool
Whether to convert raw data or not.
add_processed : bool
Whether to convert processed data or not.
add_behavior : bool
Whether to convert behavior data or not.
plot_rois : bool
Plot ROIs
"""
# Source files
file_raw = None
file_info = None
file_processed = None
file_sparse_matrix = None
file_reference_image = None
for k, v in source_paths.items():
if source_paths[k]['path'] != '':
fname = source_paths[k]['path']
if k == 'raw_data':
file_raw = h5py.File(fname, 'r')
if k == 'raw_info':
file_info = scipy.io.loadmat(fname,
struct_as_record=False,
squeeze_me=True)
if k == 'processed_data':
file_processed = np.load(fname)
if k == 'sparse_matrix':
file_sparse_matrix = np.load(fname)
if k == 'ref_image':
file_reference_image = np.load(fname)
# Initialize a NWB object
nwb = NWBFile(**metadata['NWBFile'])
# Create and add device
device = Device(name=metadata['Ophys']['Device'][0]['name'])
nwb.add_device(device)
# Creates one Imaging Plane for each channel
fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0])
for meta_ip in metadata['Ophys']['ImagingPlane']:
# Optical channel
opt_ch = OpticalChannel(
name=meta_ip['optical_channel'][0]['name'],
description=meta_ip['optical_channel'][0]['description'],
emission_lambda=meta_ip['optical_channel'][0]['emission_lambda'])
nwb.create_imaging_plane(
name=meta_ip['name'],
optical_channel=opt_ch,
description=meta_ip['description'],
device=device,
excitation_lambda=meta_ip['excitation_lambda'],
imaging_rate=fs,
indicator=meta_ip['indicator'],
location=meta_ip['location'],
)
# Raw optical data
if add_raw:
print('Adding raw data...')
for meta_tps in metadata['Ophys']['TwoPhotonSeries']:
if meta_tps['name'][-1] == 'R':
raw_data = file_raw['R']
else:
raw_data = file_raw['Y']
def data_gen(data):
xl, yl, zl, tl = data.shape
chunk = 0
while chunk < tl:
val = data[:, :, :, chunk]
chunk += 1
print('adding data chunk: ', chunk)
yield val
xl, yl, zl, tl = raw_data.shape
tps_data = DataChunkIterator(data=data_gen(data=raw_data),
iter_axis=0,
maxshape=(tl, xl, yl, zl))
# Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard
#raw_data = np.moveaxis(raw_data, -1, 0)
tps = TwoPhotonSeries(
name=meta_tps['name'],
imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']],
data=tps_data,
rate=file_info['info'].daq.scanRate)
nwb.add_acquisition(tps)
# Processed data
if add_processed:
print('Adding processed data...')
ophys_module = ProcessingModule(
name='Ophys',
description='contains optical physiology processed data.',
)
nwb.add_processing_module(ophys_module)
# Create Image Segmentation compartment
img_seg = ImageSegmentation(
name=metadata['Ophys']['ImageSegmentation']['name'])
ophys_module.add(img_seg)
# Create plane segmentation and add ROIs
meta_ps = metadata['Ophys']['ImageSegmentation'][
'plane_segmentations'][0]
ps = img_seg.create_plane_segmentation(
name=meta_ps['name'],
description=meta_ps['description'],
imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']],
)
# Add ROIs
indices = file_sparse_matrix['indices']
indptr = file_sparse_matrix['indptr']
dims = np.squeeze(file_processed['dims'])
for start, stop in zip(indptr, indptr[1:]):
voxel_mask = make_voxel_mask(indices[start:stop], dims)
ps.add_roi(voxel_mask=voxel_mask)
# Visualize 3D voxel masks
if plot_rois:
plot_rois_function(plane_segmentation=ps, indptr=indptr)
# DFF measures
dff = DfOverF(name=metadata['Ophys']['DfOverF']['name'])
ophys_module.add(dff)
# create ROI regions
n_cells = file_processed['dFF'].shape[0]
roi_region = ps.create_roi_table_region(description='RoiTableRegion',
region=list(range(n_cells)))
# create ROI response series
dff_data = file_processed['dFF']
tt = file_processed['time'].ravel()
meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0]
meta_rrs['data'] = dff_data.T
meta_rrs['rois'] = roi_region
meta_rrs['timestamps'] = tt
dff.create_roi_response_series(**meta_rrs)
# Creates GrayscaleVolume containers and add a reference image
grayscale_volume = GrayscaleVolume(
name=metadata['Ophys']['GrayscaleVolume']['name'],
data=file_reference_image['im'])
ophys_module.add(grayscale_volume)
# Behavior data
if add_behavior:
print('Adding behavior data...')
# Ball motion
behavior_mod = nwb.create_processing_module(
name='Behavior',
description='holds processed behavior data',
)
meta_ts = metadata['Behavior']['TimeSeries'][0]
meta_ts['data'] = file_processed['ball'].ravel()
tt = file_processed['time'].ravel()
meta_ts['timestamps'] = tt
behavior_ts = TimeSeries(**meta_ts)
behavior_mod.add(behavior_ts)
# Re-arranges spatial data of body-points positions tracking
pos = file_processed['dlc']
n_points = 8
pos_reshaped = pos.reshape(
(-1, n_points, 3)) # dims=(nSamples,n_points,3)
# Creates a Position object and add one SpatialSeries for each body-point position
position = Position()
for i in range(n_points):
position.create_spatial_series(
name='SpatialSeries_' + str(i),
data=pos_reshaped[:, i, :],
timestamps=tt,
reference_frame=
'Description defining what the zero-position is.',
conversion=np.nan)
behavior_mod.add(position)
# Trial times
trialFlag = file_processed['trialFlag'].ravel()
trial_inds = np.hstack(
(0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1))
trial_times = tt[trial_inds]
for start, stop in zip(trial_times, trial_times[1:]):
nwb.add_trial(start_time=start, stop_time=stop)
# Saves to NWB file
with NWBHDF5IO(f_nwb, mode='w') as io:
io.write(nwb)
print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def convert(
input_file,
session_start_time,
subject_date_of_birth,
subject_id='I5',
subject_description='naive',
subject_genotype='wild-type',
subject_sex='M',
subject_weight='11.6g',
subject_species='Mus musculus',
subject_brain_region='Medial Entorhinal Cortex',
surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible',
session_id='npI5_0417_baseline_1',
experimenter='Kei Masuda',
experiment_description='Virtual Hallway Task',
institution='Stanford University School of Medicine',
lab_name='Giocomo Lab'):
"""
Read in the .mat file specified by input_file and convert to .nwb format.
Parameters
----------
input_file : np.ndarray (..., n_channels, n_time)
the .mat file to be converted
subject_id : string
the unique subject ID number for the subject of the experiment
subject_date_of_birth : datetime ISO 8601
the date and time the subject was born
subject_description : string
important information specific to this subject that differentiates it from other members of it's species
subject_genotype : string
the genetic strain of this species.
subject_sex : string
Male or Female
subject_weight :
the weight of the subject around the time of the experiment
subject_species : string
the name of the species of the subject
subject_brain_region : basestring
the name of the brain region where the electrode probe is recording from
surgery : str
information about the subject's surgery to implant electrodes
session_id: string
human-readable ID# for the experiment session that has a one-to-one relationship with a recording session
session_start_time : datetime
date and time that the experiment started
experimenter : string
who ran the experiment, first and last name
experiment_description : string
what task was being run during the session
institution : string
what institution was the experiment performed in
lab_name : string
the lab where the experiment was performed
Returns
-------
nwbfile : NWBFile
The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5
"""
# input matlab data
matfile = hdf5storage.loadmat(input_file)
# output path for nwb data
def replace_last(source_string, replace_what, replace_with):
head, _sep, tail = source_string.rpartition(replace_what)
return head + replace_with + tail
outpath = replace_last(input_file, '.mat', '.nwb')
create_date = datetime.today()
timezone_cali = pytz.timezone('US/Pacific')
create_date_tz = timezone_cali.localize(create_date)
# if loading data from config.yaml, convert string dates into datetime
if isinstance(session_start_time, str):
session_start_time = datetime.strptime(session_start_time,
'%B %d, %Y %I:%M%p')
session_start_time = timezone_cali.localize(session_start_time)
if isinstance(subject_date_of_birth, str):
subject_date_of_birth = datetime.strptime(subject_date_of_birth,
'%B %d, %Y %I:%M%p')
subject_date_of_birth = timezone_cali.localize(subject_date_of_birth)
# create unique identifier for this experimental session
uuid_identifier = uuid.uuid1()
# Create NWB file
nwbfile = NWBFile(
session_description=experiment_description, # required
identifier=uuid_identifier.hex, # required
session_id=session_id,
experiment_description=experiment_description,
experimenter=experimenter,
surgery=surgery,
institution=institution,
lab=lab_name,
session_start_time=session_start_time, # required
file_create_date=create_date_tz) # optional
# add information about the subject of the experiment
experiment_subject = Subject(subject_id=subject_id,
species=subject_species,
description=subject_description,
genotype=subject_genotype,
date_of_birth=subject_date_of_birth,
weight=subject_weight,
sex=subject_sex)
nwbfile.subject = experiment_subject
# adding constants via LabMetaData container
# constants
sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0])
n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0])
dat_path = matfile['sp'][0]['dat_path'][0][0][0]
offset = int(matfile['sp'][0]['offset'][0][0][0])
data_dtype = matfile['sp'][0]['dtype'][0][0][0]
hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0])
vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0]
# container
lab_metadata = LabMetaData_ext(name='LabMetaData',
acquisition_sampling_rate=sample_rate,
number_of_electrodes=n_channels_dat,
file_path=dat_path,
bytes_to_skip=offset,
raw_data_dtype=data_dtype,
high_pass_filtered=hp_filtered,
movie_start_time=vr_session_offset)
nwbfile.add_lab_meta_data(lab_metadata)
# Adding trial information
nwbfile.add_trial_column(
'trial_contrast',
'visual contrast of the maze through which the mouse is running')
trial = np.ravel(matfile['trial'])
trial_nums = np.unique(trial)
position_time = np.ravel(matfile['post'])
# matlab trial numbers start at 1. To correctly index trial_contract vector,
# subtracting 1 from 'num' so index starts at 0
for num in trial_nums:
trial_times = position_time[trial == num]
nwbfile.add_trial(start_time=trial_times[0],
stop_time=trial_times[-1],
trial_contrast=matfile['trial_contrast'][num - 1][0])
# Add mouse position inside:
position = Position()
position_virtual = np.ravel(matfile['posx'])
# position inside the virtual environment
sampling_rate = 1 / (position_time[1] - position_time[0])
position.create_spatial_series(
name='Position',
data=position_virtual,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='The start of the trial, which begins at the start '
'of the virtual hallway.',
conversion=0.01,
description='Subject position in the virtual hallway.',
comments='The values should be >0 and <400cm. Values greater than '
'400cm mean that the mouse briefly exited the maze.',
)
# physical position on the mouse wheel
physical_posx = position_virtual
trial_gain = np.ravel(matfile['trial_gain'])
for num in trial_nums:
physical_posx[trial ==
num] = physical_posx[trial == num] / trial_gain[num - 1]
position.create_spatial_series(
name='PhysicalPosition',
data=physical_posx,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='Location on wheel re-referenced to zero '
'at the start of each trial.',
conversion=0.01,
description='Physical location on the wheel measured '
'since the beginning of the trial.',
comments='Physical location found by dividing the '
'virtual position by the "trial_gain"')
nwbfile.add_acquisition(position)
# Add timing of lick events, as well as mouse's virtual position during lick event
lick_events = BehavioralEvents()
lick_events.create_timeseries(
'LickEvents',
data=np.ravel(matfile['lickx']),
timestamps=np.ravel(matfile['lickt']),
unit='centimeter',
description='Subject position in virtual hallway during the lick.')
nwbfile.add_acquisition(lick_events)
# Add information on the visual stimulus that was shown to the subject
# Assumed rate=60 [Hz]. Update if necessary
# Update external_file to link to Unity environment file
visualization = ImageSeries(
name='ImageSeries',
unit='seconds',
format='external',
external_file=list(['https://unity.com/VR-and-AR-corner']),
starting_time=vr_session_offset,
starting_frame=[[0]],
rate=float(60),
description='virtual Unity environment that the mouse navigates through'
)
nwbfile.add_stimulus(visualization)
# Add the recording device, a neuropixel probe
recording_device = nwbfile.create_device(name='neuropixel_probes')
electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded'
electrode_group_name = 'probe1'
electrode_group = nwbfile.create_electrode_group(
electrode_group_name,
description=electrode_group_description,
location=subject_brain_region,
device=recording_device)
# Add information about each electrode
xcoords = np.ravel(matfile['sp'][0]['xcoords'][0])
ycoords = np.ravel(matfile['sp'][0]['ycoords'][0])
data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0]
if data_filtered_flag:
filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered'
else:
filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered'
num_recording_electrodes = xcoords.shape[0]
recording_electrodes = range(0, num_recording_electrodes)
# create electrode columns for the x,y location on the neuropixel probe
# the standard x,y,z locations are reserved for Allen Brain Atlas location
nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe')
nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe')
for idx in recording_electrodes:
nwbfile.add_electrode(id=idx,
x=np.nan,
y=np.nan,
z=np.nan,
rel_x=float(xcoords[idx]),
rel_y=float(ycoords[idx]),
imp=np.nan,
location='medial entorhinal cortex',
filtering=filter_desc,
group=electrode_group)
# Add information about each unit, termed 'cluster' in giocomo data
# create new columns in unit table
nwbfile.add_unit_column(
'quality',
'labels given to clusters during manual sorting in phy (1=MUA, '
'2=Good, 3=Unsorted)')
# cluster information
cluster_ids = matfile['sp'][0]['cids'][0][0]
cluster_quality = matfile['sp'][0]['cgs'][0][0]
# spikes in time
spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike
spike_cluster = np.ravel(
matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time
for i, cluster_id in enumerate(cluster_ids):
unit_spike_times = spike_times[spike_cluster == cluster_id]
waveforms = matfile['sp'][0]['temps'][0][cluster_id]
nwbfile.add_unit(id=int(cluster_id),
spike_times=unit_spike_times,
quality=cluster_quality[i],
waveform_mean=waveforms,
electrode_group=electrode_group)
# Trying to add another Units table to hold the results of the automatic spike sorting
# create TemplateUnits units table
template_units = Units(
name='TemplateUnits',
description='units assigned during automatic spike sorting')
template_units.add_column(
'tempScalingAmps',
'scaling amplitude applied to the template when extracting spike',
index=True)
# information on extracted spike templates
spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0])
spike_template_ids = np.unique(spike_templates)
# template scaling amplitudes
temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0])
for i, spike_template_id in enumerate(spike_template_ids):
template_spike_times = spike_times[spike_templates ==
spike_template_id]
temp_scaling_amps_per_template = temp_scaling_amps[spike_templates ==
spike_template_id]
template_units.add_unit(id=int(spike_template_id),
spike_times=template_spike_times,
electrode_group=electrode_group,
tempScalingAmps=temp_scaling_amps_per_template)
# create ecephys processing module
spike_template_module = nwbfile.create_processing_module(
name='ecephys',
description='units assigned during automatic spike sorting')
# add template_units table to processing module
spike_template_module.add(template_units)
print(nwbfile)
print('converted to NWB:N')
print('saving ...')
with NWBHDF5IO(outpath, 'w') as io:
io.write(nwbfile)
print('saved', outpath)
def add_behavior(nwbfile, behavior_file, metadata_behavior, t0):
# The timestamps for behavior data, coming from .mat files, are in milliseconds
# and should be transformed to seconds with / 1000
print("adding behavior...")
# process raw behavior
behavior_data = loadmat(behavior_file)
behavior_module = nwbfile.create_processing_module(
name='behavior', description='preprocessed behavioral data')
# Player Position
meta_pos = metadata_behavior['Position']
pos = Position(name=meta_pos['name'])
all_pos = np.atleast_1d([[], [], []]).T
all_tme = []
for iepoch in range(len(behavior_data["behavior"])):
if 'posdat' in behavior_data['behavior'][iepoch]:
epoch_data = behavior_data["behavior"][iepoch]
all_pos = np.r_[all_pos, epoch_data['posdat']]
all_tme = np.r_[all_tme, epoch_data['tme']]
# Metadata for SpatialSeries stored in Position
pos.create_spatial_series(
name=meta_pos['spatial_series'][0]['name'],
data=all_pos,
reference_frame=meta_pos['spatial_series'][0]['reference_frame'],
timestamps=all_tme / 1000. - t0,
conversion=np.nan)
behavior_module.add(pos)
# nlx eye movements
nlxeye = EyeTracking(name='EyeTracking')
# metadata for SpatialSeries stored in EyeTracking
meta_et = metadata_behavior['EyeTracking']
tt = np.array(behavior_data["nlxtme"]) / 1000. - t0
nlxeye.create_spatial_series(
name=meta_et['spatial_series'][0]['name'],
data=np.array(behavior_data["nlxeye"]).T,
reference_frame=meta_et['spatial_series'][0]['reference_frame'],
rate=len(tt) / (tt[-1] - tt[0]),
starting_time=tt[0],
#timestamps=np.array(behavior_data["nlxtme"]) / 1000. - t0,
conversion=np.nan)
behavior_module.add(nlxeye)
# trial columns
nwbfile.add_trial_column(
name='environment',
description='trial environment (calibration if calibration trial)')
# nwbfile.add_trial_column(name='trial_vars', description='trial variables - different between calibration and task')
# event key dictionary
event_dict = {
"new_trial": 1000,
"right_trial": 1001,
"left_trial": 1002,
"reward_on": 1,
"reward_off": 0,
"end_trial": 100,
"end_presentation": 101,
"successful_trial": 200
}
# process behavior here
for iepoch in range(len(behavior_data["behavior"])):
epoch_data = behavior_data["behavior"][iepoch]
if iepoch < 2:
process_behavior_calibration(nwbfile, iepoch + 1, epoch_data, t0)
else:
banana_flag = 1
process_behavior(nwbfile, iepoch + 1, epoch_data, banana_flag,
event_dict, t0)
def conversion_function(source_paths,
f_nwb,
metadata,
add_spikeglx=False,
add_processed=False):
"""
Copy data stored in a set of .npz files to a single NWB file.
Parameters
----------
source_paths : dict
Dictionary with paths to source files/directories. e.g.:
{'spikeglx data': {'type': 'file', 'path': ''}
'processed data': {'type': 'file', 'path': ''}}
f_nwb : str
Path to output NWB file, e.g. 'my_file.nwb'.
metadata : dict
Dictionary containing metadata
add_spikeglx: bool
add_processed: bool
"""
# Source files
npx_file_path = None
mat_file_path = None
for k, v in source_paths.items():
if source_paths[k]['path'] != '':
if k == 'spikeglx data':
npx_file_path = source_paths[k]['path']
if k == 'processed data':
mat_file_path = source_paths[k]['path']
# Remove lab_meta_data from metadata, it will be added later
metadata0 = copy.deepcopy(metadata)
metadata0['NWBFile'].pop('lab_meta_data', None)
# Create nwb
nwbfile = pynwb.NWBFile(**metadata0['NWBFile'])
# If adding processed data
if add_processed:
# Source matlab data
matfile = hdf5storage.loadmat(mat_file_path)
# Adding trial information
nwbfile.add_trial_column(
name='trial_contrast',
description=
'visual contrast of the maze through which the mouse is running')
trial = np.ravel(matfile['trial'])
trial_nums = np.unique(trial)
position_time = np.ravel(matfile['post'])
# matlab trial numbers start at 1. To correctly index trial_contract vector,
# subtracting 1 from 'num' so index starts at 0
for num in trial_nums:
trial_times = position_time[trial == num]
nwbfile.add_trial(start_time=trial_times[0],
stop_time=trial_times[-1],
trial_contrast=matfile['trial_contrast'][num -
1][0])
# create behavior processing module
behavior = nwbfile.create_processing_module(
name='behavior', description='behavior processing module')
# Add mouse position
position = Position(name=metadata['Behavior']['Position']['name'])
meta_pos_names = [
sps['name']
for sps in metadata['Behavior']['Position']['spatial_series']
]
# Position inside the virtual environment
pos_vir_meta_ind = meta_pos_names.index('VirtualPosition')
meta_vir = metadata['Behavior']['Position']['spatial_series'][
pos_vir_meta_ind]
position_virtual = np.ravel(matfile['posx'])
sampling_rate = 1 / (position_time[1] - position_time[0])
position.create_spatial_series(
name=meta_vir['name'],
data=position_virtual,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame=meta_vir['reference_frame'],
conversion=meta_vir['conversion'],
description=meta_vir['description'],
comments=meta_vir['comments'])
# Physical position on the mouse wheel
pos_phys_meta_ind = meta_pos_names.index('PhysicalPosition')
meta_phys = metadata['Behavior']['Position']['spatial_series'][
pos_phys_meta_ind]
physical_posx = position_virtual
trial_gain = np.ravel(matfile['trial_gain'])
for num in trial_nums:
physical_posx[trial == num] = physical_posx[
trial == num] / trial_gain[num - 1]
position.create_spatial_series(
name=meta_phys['name'],
data=physical_posx,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame=meta_phys['reference_frame'],
conversion=meta_phys['conversion'],
description=meta_phys['description'],
comments=meta_phys['comments'])
behavior.add(position)
# Add timing of lick events, as well as mouse's virtual position during lick event
lick_events = BehavioralEvents(
name=metadata['Behavior']['BehavioralEvents']['name'])
meta_ts = metadata['Behavior']['BehavioralEvents']['time_series']
meta_ts['data'] = np.ravel(matfile['lickx'])
meta_ts['timestamps'] = np.ravel(matfile['lickt'])
lick_events.create_timeseries(**meta_ts)
behavior.add(lick_events)
# Add the recording device, a neuropixel probe
recording_device = nwbfile.create_device(
name=metadata['Ecephys']['Device'][0]['name'])
# Add ElectrodeGroup
electrode_group = nwbfile.create_electrode_group(
name=metadata['Ecephys']['ElectrodeGroup'][0]['name'],
description=metadata['Ecephys']['ElectrodeGroup'][0]
['description'],
location=metadata['Ecephys']['ElectrodeGroup'][0]['location'],
device=recording_device)
# Add information about each electrode
xcoords = np.ravel(matfile['sp'][0]['xcoords'][0])
ycoords = np.ravel(matfile['sp'][0]['ycoords'][0])
data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0]
if metadata['NWBFile']['lab_meta_data']['high_pass_filtered']:
filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered'
else:
filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered'
num_recording_electrodes = xcoords.shape[0]
recording_electrodes = range(0, num_recording_electrodes)
# create electrode columns for the x,y location on the neuropixel probe
# the standard x,y,z locations are reserved for Allen Brain Atlas location
nwbfile.add_electrode_column('relativex',
'electrode x-location on the probe')
nwbfile.add_electrode_column('relativey',
'electrode y-location on the probe')
for idx in recording_electrodes:
nwbfile.add_electrode(id=idx,
x=np.nan,
y=np.nan,
z=np.nan,
relativex=float(xcoords[idx]),
relativey=float(ycoords[idx]),
imp=np.nan,
location='medial entorhinal cortex',
filtering=filter_desc,
group=electrode_group)
# Add information about each unit, termed 'cluster' in giocomo data
# create new columns in unit table
nwbfile.add_unit_column(
name='quality',
description='labels given to clusters during manual sorting in phy '
'(1=MUA, 2=Good, 3=Unsorted)')
# cluster information
cluster_ids = matfile['sp'][0]['cids'][0][0]
cluster_quality = matfile['sp'][0]['cgs'][0][0]
# spikes in time
spike_times = np.ravel(
matfile['sp'][0]['st'][0]) # the time of each spike
spike_cluster = np.ravel(
matfile['sp'][0]['clu']
[0]) # the cluster_id that spiked at that time
for i, cluster_id in enumerate(cluster_ids):
unit_spike_times = spike_times[spike_cluster == cluster_id]
waveforms = matfile['sp'][0]['temps'][0][cluster_id]
nwbfile.add_unit(id=int(cluster_id),
spike_times=unit_spike_times,
quality=cluster_quality[i],
waveform_mean=waveforms,
electrode_group=electrode_group)
# Trying to add another Units table to hold the results of the automatic spike sorting
# create TemplateUnits units table
template_units = Units(
name='TemplateUnits',
description='units assigned during automatic spike sorting')
template_units.add_column(
name='tempScalingAmps',
description=
'scaling amplitude applied to the template when extracting spike',
index=True)
# information on extracted spike templates
spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0])
spike_template_ids = np.unique(spike_templates)
# template scaling amplitudes
temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0])
for i, spike_template_id in enumerate(spike_template_ids):
template_spike_times = spike_times[spike_templates ==
spike_template_id]
temp_scaling_amps_per_template = temp_scaling_amps[
spike_templates == spike_template_id]
template_units.add_unit(
id=int(spike_template_id),
spike_times=template_spike_times,
electrode_group=electrode_group,
tempScalingAmps=temp_scaling_amps_per_template)
# create ecephys processing module
spike_template_module = nwbfile.create_processing_module(
name='ecephys',
description='units assigned during automatic spike sorting')
# add template_units table to processing module
spike_template_module.add(template_units)
# Add other fields
# Add lab_meta_data
if 'lab_meta_data' in metadata['NWBFile']:
lab_metadata = LabMetaData_ext(
name=metadata['NWBFile']['lab_meta_data']['name'],
acquisition_sampling_rate=metadata['NWBFile']['lab_meta_data']
['acquisition_sampling_rate'],
number_of_electrodes=metadata['NWBFile']['lab_meta_data']
['number_of_electrodes'],
file_path=metadata['NWBFile']['lab_meta_data']['file_path'],
bytes_to_skip=metadata['NWBFile']['lab_meta_data']
['bytes_to_skip'],
raw_data_dtype=metadata['NWBFile']['lab_meta_data']
['raw_data_dtype'],
high_pass_filtered=metadata['NWBFile']['lab_meta_data']
['high_pass_filtered'],
movie_start_time=metadata['NWBFile']['lab_meta_data']
['movie_start_time'],
subject_brain_region=metadata['NWBFile']['lab_meta_data']
['subject_brain_region'])
nwbfile.add_lab_meta_data(lab_metadata)
# add information about the subject of the experiment
if 'Subject' in metadata:
experiment_subject = Subject(
subject_id=metadata['Subject']['subject_id'],
species=metadata['Subject']['species'],
description=metadata['Subject']['description'],
genotype=metadata['Subject']['genotype'],
date_of_birth=metadata['Subject']['date_of_birth'],
weight=metadata['Subject']['weight'],
sex=metadata['Subject']['sex'])
nwbfile.subject = experiment_subject
# If adding SpikeGLX data
if add_spikeglx:
# Create extractor for SpikeGLX data
extractor = Spikeglx2NWB(nwbfile=nwbfile,
metadata=metadata0,
npx_file=npx_file_path)
# Add acquisition data
extractor.add_acquisition(es_name='ElectricalSeries',
metadata=metadata['Ecephys'])
# Run spike sorting method
#extractor.run_spike_sorting()
# Save content to NWB file
extractor.save(to_path=f_nwb)
else:
# Write to nwb file
with pynwb.NWBHDF5IO(f_nwb, 'w') as io:
io.write(nwbfile)
print(nwbfile)
# Check file was saved and inform on screen
print('File saved at:')
print(f_nwb)
print('Size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
description=
'which side of the assay the mouse chose (correct or incorrect)'
)
for trial in range(0, len(concentration_level)):
nwbfile.add_trial(start_time=trial_start[trial],
stop_time=trial_end[trial],
level=concentration_level[trial],
side=stimulus_side[trial],
chosen=chosen_side[trial])
if frame_data_exists == True:
tracking = Position(
) #create a position container for tracking data
tracking.create_spatial_series(name='nose x-position',
data=nose_x,
timestamps=frame_msec,
reference_frame='session start')
tracking.create_spatial_series(name='nose y-position',
data=nose_y,
timestamps=frame_msec,
reference_frame='session start')
tracking.create_spatial_series(name='head x-position',
data=head_x,
timestamps=frame_msec,
reference_frame='session start')
tracking.create_spatial_series(name='head y-position',
data=head_y,
timestamps=frame_msec,
reference_frame='session start')
tracking.create_spatial_series(name='body x-position',
data=body_x,
