[0].keys() if tag not in acquisition.TrialStimInfo.primary_key
}
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
photostim_tag = (acquisition.TrialStimInfo
& trial).fetch(as_dict=True)
trial_tag_value = {
**trial,
**photostim_tag[0]
} if len(photostim_tag) == 1 else {
**trial,
**photostim_tag_default
}
# rename 'trial_id' to 'id'
trial_tag_value['id'] = trial_tag_value['trial_id']
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
nwbfile.add_trial(**trial_tag_value)
# =============== Write NWB 2.0 file ===============
save_path = os.path.join('data', 'NWB 2.0')
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(save_path):
os.makedirs(save_path)
with NWBHDF5IO(os.path.join(save_path, save_file_name), mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
#session: 'TWH103_111918'
NOID = 144
#Get all Spike Times
allSpikeTimes = demoHelper.getSpikeTimesAllCells(...)
for cluster in allSpikeTimes.keys():
# Add spike times to Units Table
nwb.add_unit(id=int(cluster), spike_times=allSpikeTimes[cluster][0])
#Add Trial(s) information (Learning)
events_learn_stim_on = [4.218311e+09, 4.222039e+09, ...]
events_learn_stim_off = [4.219308e+09, 4.223034e+09, ...]
...
for i in range(len(events_learn_stim_on)):
nwb.add_trial(stim_on=events_learn_stim_on[i],
stim_off=events_learn_stim_off[i], ...)
#Add Trial(s) information (Recognition)
events_recog_stim_on = [5.222291e+09, 5.229686e+09, ...]
events_recog_stim_off = [5.223292e+09, 5.230683e+09, ...]
...
for i in range(len(events_recog_stim_on)):
nwb.add_trial(stim_on=events_recog_stim_on[i],
stim_off=events_recog_stim_off[i], ...)
io = NWBHDF5IO('TWH103_111918.nwb', mode='w')
io.write(nwb)
io.close()
def run_network(model, env, run_trial, num_trial=1000, file=None):
"""Run trained networks for analysis on trial-based tasks.
Args:
model: model of arbitrary format, must provide a run_one_trial function
that works with it
env: neurogym environment
run_trial: function handle for running model for one trial,
takes (model, env) as inputs and
returns (model, env, activity, trial_info), where activity has
shape (N_time, N_unit)
num_trial: int, number of trials to run
file: str or None, file name to save
Returns:
activity: a list of activity matrices, each matrix has shape (
N_time, N_neuron)
info: pandas dataframe, each row is information of a trial
config: dict of network, training configurations
"""
env.reset(no_step=True)
# Make NWB file
nwbfile = NWBFile(
session_description=str(env), # required
identifier='NWB_default', # required
session_start_time=datetime.datetime.now(), # required
file_create_date=datetime.datetime.now())
info = pd.DataFrame()
spike_times = defaultdict(list)
start_time = 0.
for i in range(num_trial):
model, env, hidden, trial_info = run_trial(model, env)
# Log trial info
for key, val in env.start_t.items():
# NWB time default unit is second, ngym default is ms
trial_info['start_' + key] = val / 1000. + start_time
for key, val in env.end_t.items():
trial_info['end_' + key] = val / 1000. + start_time
info = info.append(trial_info, ignore_index=True)
# Store results to NWB file
if i == 0:
for key in trial_info.keys():
nwbfile.add_trial_column(name=key, description=key)
stop_time = start_time + hidden.shape[0] * env.dt / 1000.
# Generate simulated spikes from rates
scale_rate = 10.
for j in range(hidden.shape[-1]):
spikes = sample_spikes(hidden[:, j] * scale_rate,
dt=env.dt / 1000.) + start_time
spike_times[j].append(spikes)
nwbfile.add_trial(start_time=start_time,
stop_time=stop_time,
**trial_info)
start_time = stop_time # Assuming continous trials
try:
print('Average performance', np.mean(info['correct']))
except:
pass
for j in range(hidden.shape[-1]): # For each neuron
nwbfile.add_unit(id=j, spike_times=np.concatenate(spike_times[j]))
# TODO: Check why the file.units['spike_times'] is weird
if file is None:
file = str(get_modelpath(envid) / (envid + '.nwb'))
with pynwb.NWBHDF5IO(file, 'w') as io:
io.write(nwbfile)
start_time = np.full(outcomes.shape, np.nan)
stop_time = np.full(outcomes.shape, np.nan)
else:
alldata_frameTimes = postmat['alldata_frameTimes'] # timestamps of each trial for all trials
start_time = [t[0] for t in alldata_frameTimes]
stop_time = [t[-1] for t in alldata_frameTimes]
# - now insert each trial into trial table
for k in range(outcomes.size):
nwbfile.add_trial(
start_time=start_time[k],
stop_time=stop_time[k],
trial_type=('High-rate' if postmat['stimrate'][k] >= 16 else 'Low-rate'),
trial_pulse_rate=postmat['stimrate'][k],
trial_response=trial_response_dict[outcomes[k]],
trial_is_good=(outcomes[k] >= 0),
init_tone=init_tone[k],
stim_onset=postmat['timeStimOnsetAll'][k],
stim_offset=postmat['timeSingleStimOffset'][k],
go_tone=postmat['timeCommitCL_CR_Gotone'][k],
first_commit=postmat['time1stSideTry'][k],
second_commit=second_commit_times[k])
# ------ Image Segmentation processing module ------
img_seg_mod = nwbfile.create_processing_module(
'Image-Segmentation', 'Plane segmentation and ROI identification')
img_segmentation = nwb_ophys.ImageSegmentation(name='img_seg')
img_seg_mod.add_data_interface(img_segmentation)
plane_segmentation = nwb_ophys.PlaneSegmentation(
name='pln_seg',
description='description here',
def nwb_copy_file(old_file, new_file, cp_objs={}):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path
String such as '/path/to/old_file.nwb'.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
"""
manager = get_manager()
# Open original signal file
with NWBHDF5IO(old_file, 'r', manager=manager,
load_namespaces=True) as io1:
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(session_description=str(nwb_old.session_description),
identifier='',
session_start_time=datetime.now(tzlocal()))
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[
aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name)
)
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux])
)
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering',
'group', 'group_name']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [str(elem) for elem in nwb_old.electrodes[
var].data[:]]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(name=str(var),
description=
str(nwb_old.electrodes[
var].description),
data=var_data)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags',
'timeseries']
[new_vars.remove(var) for var in default_vars if
var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(name=var,
description=nwb_old.epochs[
var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(
start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(name=var,
description=nwb_old.trials[
var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
if cp_objs['ecephys'] is True:
interfaces = nwb_old.processing[
'ecephys'].data_interfaces.keys()
else: # list of items
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.'
)
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
if 'acquisition' in cp_objs:
if cp_objs['acquisition'] is True:
all_acq_names = list(nwb_old.acquisition.keys())
else: # list of items
all_acq_names = cp_objs['acquisition']
for acq_name in all_acq_names:
obj_old = nwb_old.acquisition[acq_name]
obj = copy_obj(obj_old, nwb_old, nwb_new)
if obj is not None:
nwb_new.add_acquisition(obj)
# Subject ---------------------------------------------------------
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
subject_id=nwb_old.subject.subject_id,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
# Write new file with copied fields
io2.write(nwb_new, link_data=False)
def conversion_function(source_paths,
f_nwb,
metadata,
add_raw=False,
add_processed=True,
add_behavior=True,
plot_rois=False):
"""
Copy data stored in a set of .npz files to a single NWB file.
Parameters
----------
source_paths : dict
Dictionary with paths to source files/directories. e.g.:
{'raw_data': {'type': 'file', 'path': ''},
'raw_info': {'type': 'file', 'path': ''}
'processed_data': {'type': 'file', 'path': ''},
'sparse_matrix': {'type': 'file', 'path': ''},
'ref_image',: {'type': 'file', 'path': ''}}
f_nwb : str
Path to output NWB file, e.g. 'my_file.nwb'.
metadata : dict
Metadata dictionary
add_raw : bool
Whether to convert raw data or not.
add_processed : bool
Whether to convert processed data or not.
add_behavior : bool
Whether to convert behavior data or not.
plot_rois : bool
Plot ROIs
"""
# Source files
file_raw = None
file_info = None
file_processed = None
file_sparse_matrix = None
file_reference_image = None
for k, v in source_paths.items():
if source_paths[k]['path'] != '':
fname = source_paths[k]['path']
if k == 'raw_data':
file_raw = h5py.File(fname, 'r')
if k == 'raw_info':
file_info = scipy.io.loadmat(fname,
struct_as_record=False,
squeeze_me=True)
if k == 'processed_data':
file_processed = np.load(fname)
if k == 'sparse_matrix':
file_sparse_matrix = np.load(fname)
if k == 'ref_image':
file_reference_image = np.load(fname)
# Initialize a NWB object
nwb = NWBFile(**metadata['NWBFile'])
# Create and add device
device = Device(name=metadata['Ophys']['Device'][0]['name'])
nwb.add_device(device)
# Creates one Imaging Plane for each channel
fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0])
for meta_ip in metadata['Ophys']['ImagingPlane']:
# Optical channel
opt_ch = OpticalChannel(
name=meta_ip['optical_channel'][0]['name'],
description=meta_ip['optical_channel'][0]['description'],
emission_lambda=meta_ip['optical_channel'][0]['emission_lambda'])
nwb.create_imaging_plane(
name=meta_ip['name'],
optical_channel=opt_ch,
description=meta_ip['description'],
device=device,
excitation_lambda=meta_ip['excitation_lambda'],
imaging_rate=fs,
indicator=meta_ip['indicator'],
location=meta_ip['location'],
)
# Raw optical data
if add_raw:
print('Adding raw data...')
for meta_tps in metadata['Ophys']['TwoPhotonSeries']:
if meta_tps['name'][-1] == 'R':
raw_data = file_raw['R']
else:
raw_data = file_raw['Y']
def data_gen(data):
xl, yl, zl, tl = data.shape
chunk = 0
while chunk < tl:
val = data[:, :, :, chunk]
chunk += 1
print('adding data chunk: ', chunk)
yield val
xl, yl, zl, tl = raw_data.shape
tps_data = DataChunkIterator(data=data_gen(data=raw_data),
iter_axis=0,
maxshape=(tl, xl, yl, zl))
# Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard
#raw_data = np.moveaxis(raw_data, -1, 0)
tps = TwoPhotonSeries(
name=meta_tps['name'],
imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']],
data=tps_data,
rate=file_info['info'].daq.scanRate)
nwb.add_acquisition(tps)
# Processed data
if add_processed:
print('Adding processed data...')
ophys_module = ProcessingModule(
name='Ophys',
description='contains optical physiology processed data.',
)
nwb.add_processing_module(ophys_module)
# Create Image Segmentation compartment
img_seg = ImageSegmentation(
name=metadata['Ophys']['ImageSegmentation']['name'])
ophys_module.add(img_seg)
# Create plane segmentation and add ROIs
meta_ps = metadata['Ophys']['ImageSegmentation'][
'plane_segmentations'][0]
ps = img_seg.create_plane_segmentation(
name=meta_ps['name'],
description=meta_ps['description'],
imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']],
)
# Add ROIs
indices = file_sparse_matrix['indices']
indptr = file_sparse_matrix['indptr']
dims = np.squeeze(file_processed['dims'])
for start, stop in zip(indptr, indptr[1:]):
voxel_mask = make_voxel_mask(indices[start:stop], dims)
ps.add_roi(voxel_mask=voxel_mask)
# Visualize 3D voxel masks
if plot_rois:
plot_rois_function(plane_segmentation=ps, indptr=indptr)
# DFF measures
dff = DfOverF(name=metadata['Ophys']['DfOverF']['name'])
ophys_module.add(dff)
# create ROI regions
n_cells = file_processed['dFF'].shape[0]
roi_region = ps.create_roi_table_region(description='RoiTableRegion',
region=list(range(n_cells)))
# create ROI response series
dff_data = file_processed['dFF']
tt = file_processed['time'].ravel()
meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0]
meta_rrs['data'] = dff_data.T
meta_rrs['rois'] = roi_region
meta_rrs['timestamps'] = tt
dff.create_roi_response_series(**meta_rrs)
# Creates GrayscaleVolume containers and add a reference image
grayscale_volume = GrayscaleVolume(
name=metadata['Ophys']['GrayscaleVolume']['name'],
data=file_reference_image['im'])
ophys_module.add(grayscale_volume)
# Behavior data
if add_behavior:
print('Adding behavior data...')
# Ball motion
behavior_mod = nwb.create_processing_module(
name='Behavior',
description='holds processed behavior data',
)
meta_ts = metadata['Behavior']['TimeSeries'][0]
meta_ts['data'] = file_processed['ball'].ravel()
tt = file_processed['time'].ravel()
meta_ts['timestamps'] = tt
behavior_ts = TimeSeries(**meta_ts)
behavior_mod.add(behavior_ts)
# Re-arranges spatial data of body-points positions tracking
pos = file_processed['dlc']
n_points = 8
pos_reshaped = pos.reshape(
(-1, n_points, 3)) # dims=(nSamples,n_points,3)
# Creates a Position object and add one SpatialSeries for each body-point position
position = Position()
for i in range(n_points):
position.create_spatial_series(
name='SpatialSeries_' + str(i),
data=pos_reshaped[:, i, :],
timestamps=tt,
reference_frame=
'Description defining what the zero-position is.',
conversion=np.nan)
behavior_mod.add(position)
# Trial times
trialFlag = file_processed['trialFlag'].ravel()
trial_inds = np.hstack(
(0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1))
trial_times = tt[trial_inds]
for start, stop in zip(trial_times, trial_times[1:]):
nwb.add_trial(start_time=start, stop_time=stop)
# Saves to NWB file
with NWBHDF5IO(f_nwb, mode='w') as io:
io.write(nwb)
print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
mat_file_path = self.source_data["mat_file_path"]
mat_file = loadmat(mat_file_path)
trial_info = mat_file["SessionNP"]
nwbfile.add_trial_column(
name="reward_time",
description="Time when subject began consuming reward.")
nwbfile.add_trial_column(
name="left_or_right",
description="Time when subject began consuming reward.")
l_r_dict = {1: "Right", 2: "Left"}
for trial in trial_info:
nwbfile.add_trial(start_time=trial[0],
stop_time=trial[1],
reward_time=trial[2],
left_or_right=l_r_dict[int(trial[3])])
# Position
pos_info = mat_file["whlrl"]
pos_data = [pos_info[:, 0:1], pos_info[:, 2:3]]
starting_time = 0.0
rate = 20000 / 512 # from CRCNS info
conversion = np.nan # whl are arbitrary units
pos_obj = Position(name="Position")
for j in range(2):
spatial_series_object = SpatialSeries(
name=f"SpatialSeries{j+1}",
description=
"(x,y) coordinates tracking subject movement through the maze.",
data=H5DataIO(pos_data[j], compression="gzip"),
reference_frame="unknown",
conversion=conversion,
starting_time=starting_time,
rate=rate,
resolution=np.nan,
)
pos_obj.add_spatial_series(spatial_series_object)
get_module(nwbfile=nwbfile,
name="behavior",
description="Contains processed behavioral data."
).add_data_interface(pos_obj)
linearized_pos = mat_file["whlrld"][:, 6]
lin_pos_obj = Position(name="LinearizedPosition")
lin_spatial_series_object = SpatialSeries(
name="LinearizedTimeSeries",
description=
("Linearized position, with '1' defined as start position (the position at the time of last nose-poking "
"in the trial), and d=2 being the end position (position at the tiome just before reward consumption). "
"d=0 means subject is not performing working memory trials."),
data=H5DataIO(linearized_pos, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
starting_time=starting_time,
rate=rate,
resolution=np.nan,
)
lin_pos_obj.add_spatial_series(lin_spatial_series_object)
get_module(nwbfile=nwbfile,
name="behavior").add_data_interface(lin_pos_obj)
data=sweepDAC,
electrode=elec,
unit='mV',
rate=10e4,
gain=0.00,
starting_time=0.0,
bias_current=np.nan,
bridge_balance=np.nan,
capacitance_compensation=np.nan)
nwbFile.add_trial(start_time=float(start),
stop_time=float(end),
Stimulus=stimulus,
Acquisition=acquisition,
CellNo=metadata['Cell No.'],
RMP=metadata['RMP'],
Layer=metadata['Layer'],
Tau=metadata['Tau'],
Gain=metadata['Gain'],
DC=metadata['DC'],
Date=metadata['Session Date'])
start += 1
end += 1
output = sys.argv[2]
io = NWBHDF5IO(join(
output,
str(sessionDate) + ' - ' + metadata['Cell No.'] + '.nwb'),
mode='w')
io.write(nwbFile)
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversion for this data interface.
Reads labview experiment behavioral data and adds it to nwbfile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
print("Converting Labview data...")
# Get list of trial summary files
dir_behavior_labview = self.source_data['dir_behavior_labview']
all_files = os.listdir(dir_behavior_labview)
trials_files = [f for f in all_files if '_sum.txt' in f]
trials_files.sort()
# Get session_start_time from first file timestamps
fpath = os.path.join(dir_behavior_labview, trials_files[0])
colnames = [
'Trial', 'StartT', 'EndT', 'Result', 'InitT', 'SpecificResults',
'ProbLeft', 'OptoDur', 'LRew', 'RRew', 'InterT', 'LTrial',
'ReactionTime', 'OptoCond', 'OptoTrial'
]
df_0 = pd.read_csv(fpath, sep='\t', index_col=False, names=colnames)
t0 = df_0['StartT'][0] # initial time in Labview seconds
# Add trials
print("Converting Labview trials data...")
if nwbfile.trials is not None:
print(
'Trials already exist in current nwb file. Labview behavior trials not added.'
)
else:
# Make dataframe
frames = []
for f in trials_files:
fpath = os.path.join(dir_behavior_labview, f)
frames.append(
pd.read_csv(fpath,
sep='\t',
index_col=False,
names=colnames))
df_trials_summary = pd.concat(frames)
nwbfile.add_trial_column(
name='results',
description=
"0 means sucess (rewarded trial), 1 means licks during intitial "
"period, which leads to a failed trial. 2 means early lick failure. 3 means "
"wrong lick or no response.")
nwbfile.add_trial_column(
name='init_t', description="duration of initial delay period.")
nwbfile.add_trial_column(
name='specific_results',
description=
"Possible outcomes classified based on raw data & meta file (_tr.m)."
)
nwbfile.add_trial_column(
name='prob_left',
description=
"probability for left trials in order to keep the number of "
"left and right trials balanced within the session. ")
nwbfile.add_trial_column(
name='opto_dur',
description="the duration of optical stimulation.")
nwbfile.add_trial_column(
name='l_rew_n',
description="counting the number of left rewards.")
nwbfile.add_trial_column(
name='r_rew_n',
description="counting the number of rightrewards.")
nwbfile.add_trial_column(name='inter_t',
description="inter-trial delay period.")
nwbfile.add_trial_column(
name='l_trial',
description=
"trial type (which side the air-puff is applied). 1 means "
"left-trial, 0 means right-trial")
nwbfile.add_trial_column(
name='reaction_time',
description=
"if it is a successful trial or wrong lick during response "
"period trial: ReactionTime = time between the first decision "
"lick and the beginning of the response period. If it is a failed "
"trial due to early licks: reaction time = the duration of "
"the air-puff period (in other words, when the animal licks "
"during the sample period).")
nwbfile.add_trial_column(
name='opto_cond',
description="0: no opto. 1: opto is on during sample period. "
"2: opto is on half way through the sample period (0.5s) "
"and 0.5 during the response period. 3. opto is on during "
"the response period.")
nwbfile.add_trial_column(
name='opto_trial',
description="1: opto trials. 0: Non-opto trials.")
for index, row in df_trials_summary.iterrows():
nwbfile.add_trial(
start_time=row['StartT'] - t0,
stop_time=row['EndT'] - t0,
results=int(row['Result']),
init_t=row['InitT'],
specific_results=int(row['SpecificResults']),
prob_left=row['ProbLeft'],
opto_dur=row['OptoDur'],
l_rew_n=int(row['LRew']),
r_rew_n=int(row['RRew']),
inter_t=row['InterT'],
l_trial=int(row['LTrial']),
reaction_time=int(row['ReactionTime']),
opto_cond=int(row['OptoCond']),
opto_trial=int(row['OptoTrial']),
)
# Get list of files: continuous data
continuous_files = [f.replace('_sum', '') for f in trials_files]
# Adds continuous behavioral data
frames = []
for f in continuous_files:
fpath_lick = os.path.join(dir_behavior_labview, f)
frames.append(pd.read_csv(fpath_lick, sep='\t', index_col=False))
df_continuous = pd.concat(frames)
# Behavioral data
print("Converting Labview behavior data...")
l1_ts = TimeSeries(name="left_lick",
data=df_continuous['Lick 1'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
l2_ts = TimeSeries(name="right_lick",
data=df_continuous['Lick 2'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
nwbfile.add_acquisition(l1_ts)
nwbfile.add_acquisition(l2_ts)
# Optogenetics stimulation data
print("Converting Labview optogenetics data...")
ogen_device = nwbfile.create_device(
name=metadata['Ogen']['Device']['name'],
description=metadata['Ogen']['Device']['description'])
meta_ogen_site = metadata['Ogen']['OptogeneticStimulusSite']
ogen_stim_site = OptogeneticStimulusSite(
name=meta_ogen_site['name'],
device=ogen_device,
description=meta_ogen_site['description'],
excitation_lambda=float(meta_ogen_site['excitation_lambda']),
location=meta_ogen_site['location'])
nwbfile.add_ogen_site(ogen_stim_site)
meta_ogen_series = metadata['Ogen']['OptogeneticSeries']
ogen_series = OptogeneticSeries(
name=meta_ogen_series['name'],
data=df_continuous['Opto'].to_numpy(),
site=ogen_stim_site,
timestamps=df_continuous['Time'].to_numpy() - t0,
description=meta_ogen_series['description'],
)
nwbfile.add_stimulus(ogen_series)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
identifier = '_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
str(this_session['session_id'])
])
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier=identifier,
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
sex=subj['sex'],
species=subj['species'])
# =============== Extracellular ====================
probe_insertion = ((extracellular.ProbeInsertion
& session_key).fetch1() if extracellular.ProbeInsertion
& session_key else None)
if probe_insertion:
probe = nwbfile.create_device(name=probe_insertion['probe_name'])
electrode_group = nwbfile.create_electrode_group(name='; '.join([
f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}'
]),
description='N/A',
device=probe,
location='; '.join([
f'{k}: {str(v)}'
for k, v in
(reference.
BrainLocation
& probe_insertion
).fetch1().items(
)
]))
for chn in (reference.Probe.Channel
& probe_insertion).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['channel_id'],
group=electrode_group,
filtering=hardware_filter,
imp=np.nan,
x=np.nan, # not available from data
y=np.nan, # not available from data
z=np.nan, # not available from data
location=electrode_group.location)
# --- unit spike times ---
nwbfile.add_unit_column(name='depth',
description='depth this unit (um)')
nwbfile.add_unit_column(
name='quality',
description=
'quality of the spike sorted unit (e.g. excellent, good, poor, fair, etc.)'
)
nwbfile.add_unit_column(
name='cell_type', description='cell type (e.g. PTlower, PTupper)')
for unit in (extracellular.UnitSpikeTimes
& probe_insertion).fetch(as_dict=True):
# make an electrode table region (which electrode(s) is this unit coming from)
unit_chn = unit['channel_id'] if isinstance(
unit['channel_id'], np.ndarray) else [unit['channel_id']]
nwbfile.add_unit(id=unit['unit_id'],
electrodes=np.where(
np.in1d(np.array(nwbfile.electrodes.id.data),
unit_chn))[0],
depth=unit['unit_depth'],
quality=unit['unit_quality'],
cell_type=unit['unit_cell_type'],
spike_times=unit['spike_times'])
# =============== Behavior ====================
behavior_data = ((behavior.LickTimes
& session_key).fetch1() if behavior.LickTimes
& session_key else None)
if behavior_data:
behav_acq = pynwb.behavior.BehavioralEvents(name='lick_times')
nwbfile.add_acquisition(behav_acq)
[behavior_data.pop(k) for k in behavior.LickTimes.primary_key]
for b_k, b_v in behavior_data.items():
behav_acq.create_timeseries(name=b_k,
unit='a.u.',
conversion=1.0,
data=np.full_like(b_v, 1).astype(bool),
timestamps=b_v)
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
# adjust trial event times to be relative to session's start time
q_trial_event = (acquisition.TrialSet.EventTime *
acquisition.TrialSet.Trial.proj('start_time')).proj(
event_time='event_time + start_time')
if acquisition.TrialSet & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str(acquisition.TrialSet.Trial.heading)).groups()[-1].strip()
} for tag in acquisition.TrialSet.Trial.heading.names
if tag not in acquisition.TrialSet.Trial.primary_key +
['start_time', 'stop_time']]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(q_trial_event & trial & [{
'trial_event': e
} for e in trial_events]).fetch('trial_event', 'event_time')))
trial_tag_value = {
**trial,
**events, 'stop_time': np.nan
} # No stop_time available for this dataset
trial_tag_value['id'] = trial_tag_value[
'trial_id'] # rename 'trial_id' to 'id'
# convert None to np.nan since nwb fields does not take None
for k, v in trial_tag_value.items():
trial_tag_value[k] = v if v is not None else np.nan
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def convert_data(self,
nwbfile: NWBFile,
metadata_dict: dict,
stub_test: bool = False,
include_spike_waveforms: bool = False):
session_path = self.input_args['folder_path']
# TODO: check/enforce format?
task_types = metadata_dict['task_types']
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
sleep_state_fpath = os.path.join(
session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(
os.path.isfile(
os.path.join(
session_path, '{}__{}.mat'.format(session_id,
task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path,
session_id + '__' + label + '.mat')
if os.path.isfile(file):
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown',
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, 'behavior',
'contains processed behavioral data').add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0],
stop_time=window[1],
label=label + '_' + str(i))
trialdata_path = os.path.join(session_path,
session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base,
'EightMazeRunInfo.mat')
trialdatainfo = [
x[0]
for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]
]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[
nwbfile.add_trial_column(x, 'description')
for x in features[4:] + ['condition']
]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6])
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states',
description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({
'start_time': row[0],
'stop_time': row[1],
'label': name
})
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x['start_time'])
]
check_module(nwbfile, 'behavior',
'contains behavioral data').add_data_interface(table)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=False):
this_session = (experiment.Session & session_key).fetch1()
print(f'Exporting to NWB 2.0 for session: {this_session}...')
# ===============================================================================
# ============================== META INFORMATION ===============================
# ===============================================================================
sess_desc = session_description_mapper[(
experiment.ProjectSession & session_key).fetch1('project_name')]
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(
identifier='_'.join([
'ANM' + str(this_session['subject_id']),
this_session['session_date'].strftime('%Y-%m-%d'),
str(this_session['session'])
]),
session_description='',
session_start_time=datetime.combine(this_session['session_date'],
zero_zero_time),
file_create_date=datetime.now(tzlocal()),
experimenter=this_session['username'],
institution=institution,
experiment_description=sess_desc['experiment_description'],
related_publications=sess_desc['related_publications'],
keywords=sess_desc['keywords'])
# -- subject
subj = (lab.Subject & session_key).aggr(
lab.Subject.Strain, ...,
strains='GROUP_CONCAT(animal_strain)').fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=str(this_session['subject_id']),
description=
f'source: {subj["animal_source"]}; strains: {subj["strains"]}',
genotype=' x '.join((lab.Subject.GeneModification
& subj).fetch('gene_modification')),
sex=subj['sex'],
species=subj['species'],
date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time)
if subj['date_of_birth'] else None)
# -- virus
nwbfile.virus = json.dumps([{
k: str(v)
for k, v in virus_injection.items() if k not in subj
} for virus_injection in virus.VirusInjection * virus.Virus & session_key])
# ===============================================================================
# ======================== EXTRACELLULAR & CLUSTERING ===========================
# ===============================================================================
"""
In the event of multiple probe recording (i.e. multiple probe insertions), the clustering results
(and the associated units) are associated with the corresponding probe.
Each probe insertion is associated with one ElectrodeConfiguration (which may define multiple electrode groups)
"""
dj_insert_location = ephys.ProbeInsertion.InsertionLocation.aggr(
ephys.ProbeInsertion.RecordableBrainRegion.proj(
brain_region='CONCAT(hemisphere, " ", brain_area)'), ...,
brain_regions='GROUP_CONCAT(brain_region)')
for probe_insertion in ephys.ProbeInsertion & session_key:
electrode_config = (lab.ElectrodeConfig & probe_insertion).fetch1()
electrode_groups = {}
for electrode_group in lab.ElectrodeConfig.ElectrodeGroup & electrode_config:
electrode_groups[electrode_group[
'electrode_group']] = nwbfile.create_electrode_group(
name=electrode_config['electrode_config_name'] + '_g' +
str(electrode_group['electrode_group']),
description='N/A',
device=nwbfile.create_device(
name=electrode_config['probe']),
location=json.dumps({
k: str(v)
for k, v in (dj_insert_location
& session_key).fetch1().items()
if k not in dj_insert_location.primary_key
}))
for chn in (lab.ElectrodeConfig.Electrode * lab.Probe.Electrode
& electrode_config).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['electrode'],
group=electrode_groups[chn['electrode_group']],
filtering=hardware_filter,
imp=-1.,
x=chn['x_coord'] if chn['x_coord'] else np.nan,
y=chn['y_coord'] if chn['y_coord'] else np.nan,
z=chn['z_coord'] if chn['z_coord'] else np.nan,
location=electrode_groups[chn['electrode_group']].location)
# --- unit spike times ---
nwbfile.add_unit_column(
name='sampling_rate',
description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='quality',
description='unit quality from clustering')
nwbfile.add_unit_column(
name='posx',
description=
'estimated x position of the unit relative to probe (0,0) (um)')
nwbfile.add_unit_column(
name='posy',
description=
'estimated y position of the unit relative to probe (0,0) (um)')
nwbfile.add_unit_column(
name='cell_type',
description='cell type (e.g. fast spiking or pyramidal)')
for unit_key in (ephys.Unit * ephys.UnitCellType
& probe_insertion).fetch('KEY'):
unit = (ephys.Unit * ephys.UnitCellType & probe_insertion
& unit_key).proj(..., '-spike_times').fetch1()
if ephys.TrialSpikes & unit_key:
obs_intervals = np.array(
list(
zip(*(ephys.TrialSpikes * experiment.SessionTrial
& unit_key).fetch('start_time',
'stop_time')))).astype(float)
tr_spike_times, tr_start, tr_go = (
ephys.TrialSpikes * experiment.SessionTrial *
(experiment.TrialEvent & 'trial_event_type = "go"')
& unit_key).fetch('spike_times', 'start_time',
'trial_event_time')
spike_times = np.hstack([
spks + float(t_go) + float(t_start) for spks, t_go, t_start
in zip(tr_spike_times, tr_start, tr_go)
])
else: # the case of unavailable `TrialSpikes`
spike_times = (ephys.Unit & unit_key).fetch1('spike_times')
obs_intervals = np.array(
list(
zip(*(experiment.SessionTrial & unit_key).fetch(
'start_time', 'stop_time')))).astype(float)
obs_intervals = [
interval for interval in obs_intervals
if np.logical_and(spike_times >= interval[0],
spike_times <= interval[-1]).any()
]
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(
id=unit['unit'],
electrodes=np.where(
np.array(nwbfile.electrodes.id.data) ==
unit['electrode'])[0],
electrode_group=electrode_groups[unit['electrode_group']],
obs_intervals=obs_intervals,
sampling_rate=ecephys_fs,
quality=unit['unit_quality'],
posx=unit['unit_posx'],
posy=unit['unit_posy'],
cell_type=unit['cell_type'],
spike_times=spike_times,
waveform_mean=np.mean(unit['waveform'], axis=0),
waveform_sd=np.std(unit['waveform'], axis=0))
# ===============================================================================
# ============================= BEHAVIOR TRACKING ===============================
# ===============================================================================
if tracking.LickTrace * experiment.SessionTrial & session_key:
# re-concatenating trialized tracking traces
lick_traces, time_vecs, trial_starts = (
tracking.LickTrace * experiment.SessionTrial & session_key).fetch(
'lick_trace', 'lick_trace_timestamps', 'start_time')
behav_acq = pynwb.behavior.BehavioralTimeSeries(
name='BehavioralTimeSeries')
nwbfile.add_acquisition(behav_acq)
behav_acq.create_timeseries(
name='lick_trace',
unit='a.u.',
conversion=1.0,
data=np.hstack(lick_traces),
description=
"Time-series of the animal's tongue movement when licking",
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
# ===============================================================================
# ============================= PHOTO-STIMULATION ===============================
# ===============================================================================
stim_sites = {}
for photostim in experiment.Photostim * experiment.PhotostimBrainRegion * lab.PhotostimDevice & session_key:
stim_device = (nwbfile.get_device(photostim['photostim_device'])
if photostim['photostim_device'] in nwbfile.devices else
nwbfile.create_device(
name=photostim['photostim_device']))
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name=photostim['stim_laterality'] + ' ' +
photostim['stim_brain_area'],
device=stim_device,
excitation_lambda=float(photostim['excitation_wavelength']),
location=json.dumps([{
k: v
for k, v in stim_locs.items()
if k not in experiment.Photostim.primary_key
} for stim_locs in (experiment.Photostim.PhotostimLocation.proj(
..., '-brain_area')
& photostim).fetch(as_dict=True)],
default=str),
description='')
nwbfile.add_ogen_site(stim_site)
stim_sites[photostim['photo_stim']] = stim_site
# re-concatenating trialized photostim traces
dj_photostim = (experiment.PhotostimTrace * experiment.SessionTrial *
experiment.PhotostimEvent * experiment.Photostim
& session_key)
for photo_stim, stim_site in stim_sites.items():
if dj_photostim & {'photo_stim': photo_stim}:
aom_input_trace, laser_power, time_vecs, trial_starts = (
dj_photostim & {
'photo_stim': photo_stim
}).fetch('aom_input_trace', 'laser_power',
'photostim_timestamps', 'start_time')
aom_series = pynwb.ogen.OptogeneticSeries(
name=stim_site.name + '_aom_input_trace',
site=stim_site,
conversion=1e-3,
data=np.hstack(aom_input_trace),
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
laser_series = pynwb.ogen.OptogeneticSeries(
name=stim_site.name + '_laser_power',
site=stim_site,
conversion=1e-3,
data=np.hstack(laser_power),
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
nwbfile.add_stimulus(aom_series)
nwbfile.add_stimulus(laser_series)
# ===============================================================================
# =============================== BEHAVIOR TRIALS ===============================
# ===============================================================================
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time'
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
dj_trial = experiment.SessionTrial * experiment.BehaviorTrial
skip_adding_columns = experiment.Session.primary_key + [
'trial_uid', 'trial'
]
if experiment.SessionTrial & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{
'name':
tag,
'description':
re.sub('\s+:|\s+', ' ',
re.search(f'(?<={tag})(.*)',
str(dj_trial.heading)).group()).strip()
} for tag in dj_trial.heading.names if tag not in skip_adding_columns +
['start_time', 'stop_time']]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (dj_trial & session_key).fetch(as_dict=True):
trial['start_time'] = float(trial['start_time'])
trial['stop_time'] = float(
trial['stop_time']) if trial['stop_time'] else np.nan
trial['id'] = trial['trial'] # rename 'trial_id' to 'id'
[trial.pop(k) for k in skip_adding_columns]
nwbfile.add_trial(**trial)
# ===============================================================================
# =============================== BEHAVIOR TRIAL EVENTS ==========================
# ===============================================================================
behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents')
nwbfile.add_acquisition(behav_event)
for trial_event_type in (experiment.TrialEventType & experiment.TrialEvent
& session_key).fetch('trial_event_type'):
event_times, trial_starts = (
experiment.TrialEvent * experiment.SessionTrial
& session_key & {
'trial_event_type': trial_event_type
}).fetch('trial_event_time', 'start_time')
if len(event_times) > 0:
event_times = np.hstack(
event_times.astype(float) + trial_starts.astype(float))
behav_event.create_timeseries(name=trial_event_type,
unit='a.u.',
conversion=1.0,
data=np.full_like(event_times, 1),
timestamps=event_times)
photostim_event_time, trial_starts, photo_stim, power, duration = (
experiment.PhotostimEvent * experiment.SessionTrial
& session_key).fetch('photostim_event_time', 'start_time',
'photo_stim', 'power', 'duration')
if len(photostim_event_time) > 0:
behav_event.create_timeseries(
name='photostim_start_time',
unit='a.u.',
conversion=1.0,
data=power,
timestamps=photostim_event_time.astype(float) +
trial_starts.astype(float),
control=photo_stim.astype('uint8'),
control_description=stim_sites)
behav_event.create_timeseries(
name='photostim_stop_time',
unit='a.u.',
conversion=1.0,
data=np.full_like(photostim_event_time, 0),
timestamps=photostim_event_time.astype(float) +
duration.astype(float) + trial_starts.astype(float),
control=photo_stim.astype('uint8'),
control_description=stim_sites)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
class NWBConverter:
"""
Common conversion code factored out so it can be used by multiple conversion projects
"""
def __init__(self, metadata, nwbfile=None, source_paths=None):
"""
Parameters
----------
metadata: dict
nwbfile: pynwb.NWBFile
source_paths: dict
"""
self.metadata = metadata
self.source_paths = source_paths
# create self.nwbfile object
if nwbfile is None:
self.create_nwbfile(metadata['NWBFile'])
else:
self.nwbfile = nwbfile
# add subject information
if 'Subject' in metadata:
self.create_subject(metadata['Subject'])
# add devices
self.devices = dict()
for domain in ('Icephys', 'Ecephys', 'Ophys'):
if domain in metadata and 'Device' in metadata[domain]:
self.devices.update(
self.create_devices(metadata[domain]['Device']))
if 'Ecephys' in metadata:
if 'ElectrodeGroup' in metadata['Ecephys']:
self.create_electrode_groups(metadata['Ecephys'])
if 'Icephys' in metadata:
if 'Electrode' in metadata['Icephys']:
self.ic_elecs = self.create_icephys_elecs(
metadata['Icephys']['Electrode'])
def create_nwbfile(self, metadata_nwbfile):
"""
This method is called at __init__.
This method can be overridden by child classes if necessary.
Creates self.nwbfile object.
Parameters
----------
metadata_nwbfile: dict
"""
nwbfile_args = dict(identifier=str(uuid.uuid4()), )
nwbfile_args.update(**metadata_nwbfile)
self.nwbfile = NWBFile(**nwbfile_args)
def create_subject(self, metadata_subject):
"""
This method is called at __init__.
This method can be overridden by child classes if necessary.
Adds information about Subject to self.nwbfile.
Parameters
----------
metadata_subject: dict
"""
self.nwbfile.subject = Subject(**metadata_subject)
def create_devices(self, metadata_device) -> Dict:
"""
This method is called at __init__.
Use metadata to create Device object(s) in the NWBFile
Parameters
----------
metadata_device: list or dict
Returns
-------
dict
"""
if isinstance(metadata_device, list):
devices = dict()
[
devices.update(self.create_devices(idevice_meta))
for idevice_meta in metadata_device
]
return devices
else:
if 'tag' in metadata_device:
key = metadata_device['tag']
else:
key = metadata_device['name']
return {key: self.nwbfile.create_device(**metadata_device)}
def create_electrode_groups(self, metadata_ecephys):
"""
This method is called at __init__.
Use metadata to create ElectrodeGroup object(s) in the NWBFile
Parameters
----------
metadata_ecephys : dict
Dict with key:value pairs for defining the Ecephys group from where this
ElectrodeGroup belongs. This should contain keys for required groups
such as 'Device', 'ElectrodeGroup', etc.
"""
for metadata_elec_group in metadata_ecephys['ElectrodeGroup']:
eg_name = metadata_elec_group['name']
# Tests if ElectrodeGroup already exists
aux = [i.name == eg_name for i in self.nwbfile.children]
if any(aux):
print(eg_name + ' already exists in current NWBFile.')
else:
device_name = metadata_elec_group['device']
if device_name in self.nwbfile.devices:
device = self.nwbfile.devices[device_name]
else:
print('Device ', device_name, ' for ElectrodeGroup ',
eg_name, ' does not exist.')
print('Make sure ', device_name,
' is defined in metadata.')
eg_description = metadata_elec_group['description']
eg_location = metadata_elec_group['location']
self.nwbfile.create_electrode_group(name=eg_name,
location=eg_location,
device=device,
description=eg_description)
def create_electrodes_ecephys(self):
"""
This method should be overridden by child classes if necessary.
Create electrodes in the NWBFile.
"""
pass
def create_icephys_elecs(self, elec_meta) -> Dict:
"""
Use metadata to generate intracellular electrode object(s) in the NWBFile
Parameters
----------
elec_meta: list or dict
Returns
-------
list
"""
if isinstance(elec_meta, list):
elecs = dict()
[
elecs.update(self.create_icephys_elecs(**ielec_meta))
for ielec_meta in elec_meta
]
return elecs
else:
if len(self.devices) == 1:
device = list(self.devices.values())[0]
elif elec_meta['device'] in self.devices:
device = self.devices[elec_meta['device']]
else:
raise ValueError(
'device not found for icephys electrode {}'.format(
elec_meta['name']))
if 'tag' in elec_meta:
key = elec_meta['tag']
else:
key = elec_meta['name']
return {
key: self.nwbfile.create_ic_electrode(device=device,
**elec_meta)
}
def create_trials_from_df(self, df):
"""
This method should not be overridden.
Creates a trials table in self.nwbfile from a Pandas DataFrame.
Parameters
----------
df: Pandas DataFrame
"""
# Tests if trials table already exists
if self.nwbfile.trials is not None:
print("Trials table already exist in current nwb file.\n"
"Use 'add_trials_columns_from_df' to include new columns.\n"
"Use 'add_trials_from_df' to include new trials.")
pass
# Tests if required column names are present in df
if 'start_time' not in df.columns:
print("Required column 'start_time' not present in DataFrame.")
pass
if 'stop_time' not in df.columns:
print("Required column 'stop_time' not present in DataFrame.")
pass
# Creates new columns
for colname in df.columns:
if colname not in ['start_time', 'stop_time']:
# Indexed columns should be of type 'object' in the dataframe
if df[colname].dtype == 'object':
self.nwbfile.add_trial_column(name=colname,
description='no description',
index=True)
else:
self.nwbfile.add_trial_column(name=colname,
description='no description')
# Populates trials table from df values
for index, row in df.iterrows():
self.nwbfile.add_trial(**dict(row))
def add_trials_from_df(self, df):
"""
This method should not be overridden.
Adds trials from a Pandas DataFrame to existing trials table in self.nwbfile.
Parameters
----------
df: Pandas DataFrame
"""
# Tests for mismatch between trials table columns and dataframe columns
A = set(self.nwbfile.trials.colnames)
B = set(df.columns)
if len(A - B) > 0:
print("Missing columns in DataFrame: ", A - B)
pass
if len(B - A) > 0:
print("NWBFile trials table does not contain: ", B - A)
pass
# Adds trials from df values
for index, row in df.iterrows():
self.nwbfile.add_trial(**dict(row))
def add_trials_columns_from_df(self, df):
"""
This method should not be overridden.
Adds trials columns from a Pandas DataFrame to existing trials table in self.nwbfile.
Parameters
----------
df: Pandas DataFrame
"""
# Tests if dataframe columns already exist in nwbfile trials table
A = set(self.nwbfile.trials.colnames)
B = set(df.columns)
intersection = A.intersection(B)
if len(intersection) > 0:
print("These columns already exist in nwbfile trials: ",
intersection)
pass
# Adds trials columns with data from df values
for (colname, coldata) in df.iteritems():
# Indexed columns should be of type 'object' in the dataframe
if df[colname].dtype == 'object':
index = True
else:
index = False
self.nwbfile.add_trial_column(name=colname,
description='no description',
data=coldata,
index=index)
def save(self, to_path, read_check=True):
"""
This method should not be overridden.
Saves object self.nwbfile.
Parameters
----------
to_path: str
read_check: bool
If True, try to read the file after writing
"""
print('Saving to file, please wait...')
with NWBHDF5IO(to_path, 'w') as io:
io.write(self.nwbfile)
print('File successfully saved at: ', str(to_path))
if read_check:
with NWBHDF5IO(to_path, 'r') as io:
io.read()
print('Read check: OK')
def check_module(self, name, description=None):
"""
Check if processing module exists. If not, create it. Then return module
Parameters
----------
name: str
description: str | None (optional)
Returns
-------
pynwb.module
"""
if name in self.nwbfile.processing:
return self.nwbfile.processing[name]
else:
if description is None:
description = name
return self.nwbfile.create_processing_module(name, description)
def add_sortingextractor(self, sortingextractor: SortingExtractor):
self.nwbfile = NwbSortingExtractor.read_sorting(
sortingextractor, self.nwbfile)
def add_recordingextractor(self, recordingextractor: RecordingExtractor):
self.nwbfile = NwbRecordingExtractor.read_recording(
recordingextractor, self.nwbfile)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
identifier = '_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']
])
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier=identifier,
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join(
(subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell
& session_key).fetch1() if len(intracellular.Cell
& session_key) == 1 else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([
f'{k}: {str(v)}'
for k, v in dict((reference.BrainLocation & cell).fetch1(),
depth=cell['cell_depth']).items()
]))
# acquisition - membrane potential
mp, mp_wo_spike, mp_start_time, mp_fs = (
intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_wo_spike',
'membrane_potential_start_time',
'membrane_potential_sampling_rate')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
starting_time=mp_start_time,
rate=mp_fs))
# acquisition - current injection
if (intracellular.CurrentInjection & cell):
current_injection, ci_start_time, ci_fs = (
intracellular.CurrentInjection & cell).fetch1(
'current_injection', 'current_injection_start_time',
'current_injection_sampling_rate')
nwbfile.add_stimulus(
pynwb.icephys.CurrentClampStimulusSeries(
name='CurrentClampStimulus',
electrode=ic_electrode,
conversion=1e-9,
gain=1.0,
data=current_injection,
starting_time=ci_start_time,
rate=ci_fs))
# analysis - membrane potential without spike
mp_rmv_spike = nwbfile.create_processing_module(
name='icephys', description='Spike removal')
mp_rmv_spike.add_data_interface(
pynwb.icephys.PatchClampSeries(name='icephys',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp_wo_spike,
starting_time=mp_start_time,
rate=mp_fs))
# =============== Extracellular ====================
probe_insertion = ((extracellular.ProbeInsertion
& session_key).fetch1() if extracellular.ProbeInsertion
& session_key else None)
if probe_insertion:
probe = nwbfile.create_device(name=probe_insertion['probe_name'])
electrode_group = nwbfile.create_electrode_group(name='; '.join([
f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}'
]),
description='N/A',
device=probe,
location='; '.join([
f'{k}: {str(v)}'
for k, v in
(reference.
BrainLocation
& probe_insertion
).fetch1().items(
)
]))
for chn in (reference.Probe.Channel
& probe_insertion).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['channel_id'],
group=electrode_group,
filtering=hardware_filter,
imp=-1.,
x=0.0, # not available from data
y=0.0, # not available from data
z=0.0, # not available from data
location=electrode_group.location)
# --- unit spike times ---
nwbfile.add_unit_column(
name='sampling_rate',
description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='depth',
description='depth this unit (mm)')
nwbfile.add_unit_column(name='spike_width',
description='spike width of this unit (ms)')
nwbfile.add_unit_column(
name='cell_type',
description='cell type (e.g. wide width, narrow width spiking)')
for unit in (extracellular.UnitSpikeTimes
& probe_insertion).fetch(as_dict=True):
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(
id=unit['unit_id'],
electrodes=(unit['channel_id'] if isinstance(
unit['channel_id'], np.ndarray) else [unit['channel_id']]),
depth=unit['unit_depth'],
sampling_rate=ecephys_fs,
spike_width=unit['unit_spike_width'],
cell_type=unit['unit_cell_type'],
spike_times=unit['spike_times'],
waveform_mean=unit['spike_waveform'])
# =============== Behavior ====================
# Note: for this study, raw behavioral data were not available, only trialized data were provided
# here, we reconstruct raw behavioral data by concatenation
trial_seg_setting = (analysis.TrialSegmentationSetting
& 'trial_seg_setting=0').fetch1()
seg_behav_query = (
behavior.TrialSegmentedLickTrace * acquisition.TrialSet.Trial *
(analysis.RealignedEvent.RealignedEventTime
& 'trial_event="trial_start"')
& session_key & trial_seg_setting)
if seg_behav_query:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='lick_times')
nwbfile.add_acquisition(behav_acq)
seg_behav = pd.DataFrame(
seg_behav_query.fetch('start_time', 'realigned_event_time',
'segmented_lick_left_on',
'segmented_lick_left_off',
'segmented_lick_right_on',
'segmented_lick_right_off')).T
seg_behav.columns = [
'start_time', 'realigned_event_time', 'segmented_lick_left_on',
'segmented_lick_left_off', 'segmented_lick_right_on',
'segmented_lick_right_off'
]
for behav_name in [
'lick_left_on', 'lick_left_off', 'lick_right_on',
'lick_right_off'
]:
lick_times = np.hstack(r['segmented_' + behav_name] -
r.realigned_event_time + r.start_time
for _, r in seg_behav.iterrows())
behav_acq.create_timeseries(name=behav_name,
unit='a.u.',
conversion=1.0,
data=np.full_like(lick_times, 1),
timestamps=lick_times)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if stimulation.PhotoStimulation
& session_key else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(
(stimulation.PhotoStimProtocol
& photostim).fetch1('photo_stim_excitation_lambda')),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimProtocol
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if acquisition.TrialSet & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any)
trial_columns = [
{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimParam
).heading)).groups()[-1].strip()
} for tag in (acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimParam).heading.names
if tag not in (acquisition.TrialSet.Trial
& stimulation.TrialPhotoStimParam).primary_key +
['start_time', 'stop_time']
]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d + ' - (s) relative to trial start time'
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
photostim_tag_default = {
tag: ''
for tag in stimulation.TrialPhotoStimParam.heading.names
if tag not in stimulation.TrialPhotoStimParam.primary_key
}
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
trial_tag_value = ({
**trial,
**events,
**(stimulation.TrialPhotoStimParam & trial).fetch1()
} if (stimulation.TrialPhotoStimParam & trial) else {
**trial,
**events,
**photostim_tag_default
})
trial_tag_value['id'] = trial_tag_value[
'trial_id'] # rename 'trial_id' to 'id'
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# convert None to np.nan since nwb fields does not take None
for k, v in trial_tag_value.items():
trial_tag_value[k] = v if v is not None else np.nan
trial_tag_value['delay_duration'] = float(
trial_tag_value['delay_duration']) # convert Decimal to float
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier='_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']
]),
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join(
(subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell
& session_key).fetch1() if len(intracellular.Cell
& session_key) == 1 else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([
f'{k}: {str(v)}'
for k, v in dict((reference.BrainLocation & cell).fetch1(),
depth=cell['recording_depth']).items()
]))
# acquisition - membrane potential
mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
timestamps=mp_timestamps))
# acquisition - spike train
spk, spk_timestamps = (intracellular.SpikeTrain & cell).fetch1(
'spike_train', 'spike_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='SpikeTrain',
electrode=ic_electrode,
unit='a.u.',
conversion=1e1,
gain=1.0,
data=spk,
timestamps=spk_timestamps))
# =============== Behavior ====================
behavior_data = ((behavior.Behavior & session_key).fetch1()
if len(behavior.Behavior & session_key) == 1 else None)
if behavior_data:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='behavior')
nwbfile.add_acquisition(behav_acq)
[behavior_data.pop(k) for k in behavior.Behavior.primary_key]
timestamps = behavior_data.pop('behavior_timestamps')
# get behavior data description from the comments of table definition
behavior_descriptions = {
attr:
re.search(f'(?<={attr})(.*)#(.*)',
str(behavior.Behavior.heading)).groups()[-1].strip()
for attr in behavior_data
}
for b_k, b_v in behavior_data.items():
behav_acq.create_timeseries(name=b_k,
description=behavior_descriptions[b_k],
unit='a.u.',
conversion=1.0,
data=b_v,
timestamps=timestamps)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if len(stimulation.PhotoStimulation
& session_key) == 1 else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(
(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_excitation_lambda')),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if len((acquisition.TrialSet & session_key).fetch()) == 1:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any)
trial_columns = [{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimInfo
).heading)).groups()[-1].strip()
} for tag in acquisition.TrialSet.Trial.heading.names
if tag not in acquisition.TrialSet.Trial.primary_key +
['start_time', 'stop_time']]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
# shift event times to be relative to session_start (currently relative to trial_start)
events = {k: v + trial['start_time'] for k, v in events.items()}
trial_tag_value = {**trial, **events}
# rename 'trial_id' to 'id'
trial_tag_value['id'] = trial_tag_value['trial_id']
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def run_conversion(self,
nwbfile: NWBFile,
metadata: dict,
stub_test: bool = False):
session_path = Path(self.source_data["folder_path"])
task_types = [
dict(name="OpenFieldPosition_ExtraLarge"),
dict(name="OpenFieldPosition_New_Curtain", conversion=0.46),
dict(name="OpenFieldPosition_New", conversion=0.46),
dict(name="OpenFieldPosition_Old_Curtain", conversion=0.46),
dict(name="OpenFieldPosition_Old", conversion=0.46),
dict(name="OpenFieldPosition_Oldlast", conversion=0.46),
dict(name="EightMazePosition", conversion=0.65 / 2),
]
subject_path = session_path.parent
session_id = session_path.stem
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
sleep_state_fpath = session_path / f"{session_id}--StatePeriod.mat"
exist_pos_data = any([
(session_path / "{session_id}__{task_type['name']}.mat").is_file()
for task_type in task_types
])
if exist_pos_data:
nwbfile.add_epoch_column("label", "Name of epoch.")
# Epoch intervals
for task_type in task_types:
label = task_type["name"]
file = session_path / f"{session_id}__{label}.mat"
if file.is_file():
pos_obj = Position(name=f"{label}_position")
matin = loadmat(file)
tt = matin["twhl_norm"][:, 0]
exp_times = find_discontinuities(tt)
if "conversion" in task_type:
conversion = task_type["conversion"]
else:
conversion = np.nan
for pos_type in ("twhl_norm", "twhl_linearized"):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=f"{label}_{pos_type}_spatial_series",
data=H5DataIO(pos_data_norm, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression="gzip"),
)
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, "behavior",
"Contains processed behavioral data.").add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(
start_time=window[0],
stop_time=window[1],
tags=f"{label}_{str(i)}",
)
# Trial intervals
trialdata_path = session_path / f"{session_id}__EightMazeRun.mat"
if trialdata_path.is_file():
trials_data = loadmat(trialdata_path)["EightMazeRun"]
trialdatainfo_path = subject_path / "EightMazeRunInfo.mat"
trialdatainfo = [
x[0]
for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]
]
features = trialdatainfo[:7]
features[:2] = (
"start_time",
"stop_time",
)
[
nwbfile.add_trial_column(x, "description")
for x in features[4:] + ["condition"]
]
for trial_data in trials_data:
if trial_data[3]:
cond = "run_left"
else:
cond = "run_right"
nwbfile.add_trial(
start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6],
)
# SLeep states
if sleep_state_fpath.is_file():
matin = loadmat(sleep_state_fpath)["StatePeriod"]
table = TimeIntervals(name="states",
description="sleep states of animal")
table.add_column(name="label", description="sleep state")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(start_time=row[0], stop_time=row[1], label=name))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"Contains behavioral data.").add_data_interface(table)
def convert(
input_file,
session_start_time,
subject_date_of_birth,
subject_id='I5',
subject_description='naive',
subject_genotype='wild-type',
subject_sex='M',
subject_weight='11.6g',
subject_species='Mus musculus',
subject_brain_region='Medial Entorhinal Cortex',
surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible',
session_id='npI5_0417_baseline_1',
experimenter='Kei Masuda',
experiment_description='Virtual Hallway Task',
institution='Stanford University School of Medicine',
lab_name='Giocomo Lab'):
"""
Read in the .mat file specified by input_file and convert to .nwb format.
Parameters
----------
input_file : np.ndarray (..., n_channels, n_time)
the .mat file to be converted
subject_id : string
the unique subject ID number for the subject of the experiment
subject_date_of_birth : datetime ISO 8601
the date and time the subject was born
subject_description : string
important information specific to this subject that differentiates it from other members of it's species
subject_genotype : string
the genetic strain of this species.
subject_sex : string
Male or Female
subject_weight :
the weight of the subject around the time of the experiment
subject_species : string
the name of the species of the subject
subject_brain_region : basestring
the name of the brain region where the electrode probe is recording from
surgery : str
information about the subject's surgery to implant electrodes
session_id: string
human-readable ID# for the experiment session that has a one-to-one relationship with a recording session
session_start_time : datetime
date and time that the experiment started
experimenter : string
who ran the experiment, first and last name
experiment_description : string
what task was being run during the session
institution : string
what institution was the experiment performed in
lab_name : string
the lab where the experiment was performed
Returns
-------
nwbfile : NWBFile
The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5
"""
# input matlab data
matfile = hdf5storage.loadmat(input_file)
# output path for nwb data
def replace_last(source_string, replace_what, replace_with):
head, _sep, tail = source_string.rpartition(replace_what)
return head + replace_with + tail
outpath = replace_last(input_file, '.mat', '.nwb')
create_date = datetime.today()
timezone_cali = pytz.timezone('US/Pacific')
create_date_tz = timezone_cali.localize(create_date)
# if loading data from config.yaml, convert string dates into datetime
if isinstance(session_start_time, str):
session_start_time = datetime.strptime(session_start_time,
'%B %d, %Y %I:%M%p')
session_start_time = timezone_cali.localize(session_start_time)
if isinstance(subject_date_of_birth, str):
subject_date_of_birth = datetime.strptime(subject_date_of_birth,
'%B %d, %Y %I:%M%p')
subject_date_of_birth = timezone_cali.localize(subject_date_of_birth)
# create unique identifier for this experimental session
uuid_identifier = uuid.uuid1()
# Create NWB file
nwbfile = NWBFile(
session_description=experiment_description, # required
identifier=uuid_identifier.hex, # required
session_id=session_id,
experiment_description=experiment_description,
experimenter=experimenter,
surgery=surgery,
institution=institution,
lab=lab_name,
session_start_time=session_start_time, # required
file_create_date=create_date_tz) # optional
# add information about the subject of the experiment
experiment_subject = Subject(subject_id=subject_id,
species=subject_species,
description=subject_description,
genotype=subject_genotype,
date_of_birth=subject_date_of_birth,
weight=subject_weight,
sex=subject_sex)
nwbfile.subject = experiment_subject
# adding constants via LabMetaData container
# constants
sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0])
n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0])
dat_path = matfile['sp'][0]['dat_path'][0][0][0]
offset = int(matfile['sp'][0]['offset'][0][0][0])
data_dtype = matfile['sp'][0]['dtype'][0][0][0]
hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0])
vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0]
# container
lab_metadata = LabMetaData_ext(name='LabMetaData',
acquisition_sampling_rate=sample_rate,
number_of_electrodes=n_channels_dat,
file_path=dat_path,
bytes_to_skip=offset,
raw_data_dtype=data_dtype,
high_pass_filtered=hp_filtered,
movie_start_time=vr_session_offset)
nwbfile.add_lab_meta_data(lab_metadata)
# Adding trial information
nwbfile.add_trial_column(
'trial_contrast',
'visual contrast of the maze through which the mouse is running')
trial = np.ravel(matfile['trial'])
trial_nums = np.unique(trial)
position_time = np.ravel(matfile['post'])
# matlab trial numbers start at 1. To correctly index trial_contract vector,
# subtracting 1 from 'num' so index starts at 0
for num in trial_nums:
trial_times = position_time[trial == num]
nwbfile.add_trial(start_time=trial_times[0],
stop_time=trial_times[-1],
trial_contrast=matfile['trial_contrast'][num - 1][0])
# Add mouse position inside:
position = Position()
position_virtual = np.ravel(matfile['posx'])
# position inside the virtual environment
sampling_rate = 1 / (position_time[1] - position_time[0])
position.create_spatial_series(
name='Position',
data=position_virtual,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='The start of the trial, which begins at the start '
'of the virtual hallway.',
conversion=0.01,
description='Subject position in the virtual hallway.',
comments='The values should be >0 and <400cm. Values greater than '
'400cm mean that the mouse briefly exited the maze.',
)
# physical position on the mouse wheel
physical_posx = position_virtual
trial_gain = np.ravel(matfile['trial_gain'])
for num in trial_nums:
physical_posx[trial ==
num] = physical_posx[trial == num] / trial_gain[num - 1]
position.create_spatial_series(
name='PhysicalPosition',
data=physical_posx,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='Location on wheel re-referenced to zero '
'at the start of each trial.',
conversion=0.01,
description='Physical location on the wheel measured '
'since the beginning of the trial.',
comments='Physical location found by dividing the '
'virtual position by the "trial_gain"')
nwbfile.add_acquisition(position)
# Add timing of lick events, as well as mouse's virtual position during lick event
lick_events = BehavioralEvents()
lick_events.create_timeseries(
'LickEvents',
data=np.ravel(matfile['lickx']),
timestamps=np.ravel(matfile['lickt']),
unit='centimeter',
description='Subject position in virtual hallway during the lick.')
nwbfile.add_acquisition(lick_events)
# Add information on the visual stimulus that was shown to the subject
# Assumed rate=60 [Hz]. Update if necessary
# Update external_file to link to Unity environment file
visualization = ImageSeries(
name='ImageSeries',
unit='seconds',
format='external',
external_file=list(['https://unity.com/VR-and-AR-corner']),
starting_time=vr_session_offset,
starting_frame=[[0]],
rate=float(60),
description='virtual Unity environment that the mouse navigates through'
)
nwbfile.add_stimulus(visualization)
# Add the recording device, a neuropixel probe
recording_device = nwbfile.create_device(name='neuropixel_probes')
electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded'
electrode_group_name = 'probe1'
electrode_group = nwbfile.create_electrode_group(
electrode_group_name,
description=electrode_group_description,
location=subject_brain_region,
device=recording_device)
# Add information about each electrode
xcoords = np.ravel(matfile['sp'][0]['xcoords'][0])
ycoords = np.ravel(matfile['sp'][0]['ycoords'][0])
data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0]
if data_filtered_flag:
filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered'
else:
filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered'
num_recording_electrodes = xcoords.shape[0]
recording_electrodes = range(0, num_recording_electrodes)
# create electrode columns for the x,y location on the neuropixel probe
# the standard x,y,z locations are reserved for Allen Brain Atlas location
nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe')
nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe')
for idx in recording_electrodes:
nwbfile.add_electrode(id=idx,
x=np.nan,
y=np.nan,
z=np.nan,
rel_x=float(xcoords[idx]),
rel_y=float(ycoords[idx]),
imp=np.nan,
location='medial entorhinal cortex',
filtering=filter_desc,
group=electrode_group)
# Add information about each unit, termed 'cluster' in giocomo data
# create new columns in unit table
nwbfile.add_unit_column(
'quality',
'labels given to clusters during manual sorting in phy (1=MUA, '
'2=Good, 3=Unsorted)')
# cluster information
cluster_ids = matfile['sp'][0]['cids'][0][0]
cluster_quality = matfile['sp'][0]['cgs'][0][0]
# spikes in time
spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike
spike_cluster = np.ravel(
matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time
for i, cluster_id in enumerate(cluster_ids):
unit_spike_times = spike_times[spike_cluster == cluster_id]
waveforms = matfile['sp'][0]['temps'][0][cluster_id]
nwbfile.add_unit(id=int(cluster_id),
spike_times=unit_spike_times,
quality=cluster_quality[i],
waveform_mean=waveforms,
electrode_group=electrode_group)
# Trying to add another Units table to hold the results of the automatic spike sorting
# create TemplateUnits units table
template_units = Units(
name='TemplateUnits',
description='units assigned during automatic spike sorting')
template_units.add_column(
'tempScalingAmps',
'scaling amplitude applied to the template when extracting spike',
index=True)
# information on extracted spike templates
spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0])
spike_template_ids = np.unique(spike_templates)
# template scaling amplitudes
temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0])
for i, spike_template_id in enumerate(spike_template_ids):
template_spike_times = spike_times[spike_templates ==
spike_template_id]
temp_scaling_amps_per_template = temp_scaling_amps[spike_templates ==
spike_template_id]
template_units.add_unit(id=int(spike_template_id),
spike_times=template_spike_times,
electrode_group=electrode_group,
tempScalingAmps=temp_scaling_amps_per_template)
# create ecephys processing module
spike_template_module = nwbfile.create_processing_module(
name='ecephys',
description='units assigned during automatic spike sorting')
# add template_units table to processing module
spike_template_module.add(template_units)
print(nwbfile)
print('converted to NWB:N')
print('saving ...')
with NWBHDF5IO(outpath, 'w') as io:
io.write(nwbfile)
print('saved', outpath)
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversion for this data interface.
Reads treadmill experiment behavioral data from csv files and adds it to nwbfile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
# Detect relevant files: trials summary, treadmill data and nose data
dir_behavior_treadmill = self.source_data['dir_behavior_treadmill']
trials_file = [
f for f in Path(dir_behavior_treadmill).glob('*_tr.csv')
if '~lock' not in f.name
][0]
treadmill_file = trials_file.name.split('_tr')[0] + '.csv'
nose_file = trials_file.name.split('_tr')[0] + '_mk.csv'
trials_file = os.path.join(dir_behavior_treadmill, trials_file)
treadmill_file = os.path.join(dir_behavior_treadmill, treadmill_file)
nose_file = os.path.join(dir_behavior_treadmill, nose_file)
# Add trials
if nwbfile.trials is not None:
print(
'Trials already exist in current nwb file. Treadmill behavior trials not added.'
)
else:
df_trials_summary = pd.read_csv(trials_file)
nwbfile.add_trial_column(name='fail', description='no description')
nwbfile.add_trial_column(name='reward_given',
description='no description')
nwbfile.add_trial_column(name='total_rewards',
description='no description')
nwbfile.add_trial_column(name='init_dur',
description='no description')
nwbfile.add_trial_column(name='light_dur',
description='no description')
nwbfile.add_trial_column(name='motor_dur',
description='no description')
nwbfile.add_trial_column(name='post_motor',
description='no description')
nwbfile.add_trial_column(name='speed',
description='no description')
nwbfile.add_trial_column(name='speed_mode',
description='no description')
nwbfile.add_trial_column(name='amplitude',
description='no description')
nwbfile.add_trial_column(name='period',
description='no description')
nwbfile.add_trial_column(name='deviation',
description='no description')
t_offset = df_trials_summary.loc[0]['Start Time']
for index, row in df_trials_summary.iterrows():
nwbfile.add_trial(
start_time=row['Start Time'] - t_offset,
stop_time=row['End Time'] - t_offset,
fail=row['Fail'],
reward_given=row['Reward Given'],
total_rewards=row['Total Rewards'],
init_dur=row['Init Dur'],
light_dur=row['Light Dur'],
motor_dur=row['Motor Dur'],
post_motor=row['Post Motor'],
speed=row['Speed'],
speed_mode=row['Speed Mode'],
amplitude=row['Amplitude'],
period=row['Period'],
deviation=row['+/- Deviation'],
)
# Treadmill continuous data
df_treadmill = pd.read_csv(treadmill_file, index_col=False)
# Nose position continuous data
df_nose = pd.read_csv(nose_file, index_col=False)
# All behavioral data
df_all = pd.concat([df_treadmill, df_nose], axis=1, sort=False)
meta_behavioral_ts = metadata['Behavior']
t_offset = df_treadmill.loc[0]['Time']
for meta in meta_behavioral_ts.values():
ts = TimeSeries(name=meta['name'],
data=df_all[meta['name']].to_numpy(),
timestamps=df_all['Time'].to_numpy() - t_offset,
description=meta['description'])
nwbfile.add_acquisition(ts)
class NWBFileTest(unittest.TestCase):
def setUp(self):
self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create = [
datetime(2017, 5, 1, 12, tzinfo=tzlocal()),
datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()),
datetime(2017, 5, 2, 14, tzinfo=tzutc())
]
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile(
'a test session description for a test NWBFile',
'FILE123',
self.start,
file_create_date=self.create,
timestamps_reference_time=self.ref_time,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
notes='my notes',
pharmacology='drugs',
protocol='protocol',
related_publications='my pubs',
slices='my slices',
surgery='surgery',
virus='a virus',
source_script='noscript',
source_script_file_name='nofilename',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
keywords=('these', 'are', 'keywords'))
def test_constructor(self):
self.assertEqual(self.nwbfile.session_description,
'a test session description for a test NWBFile')
self.assertEqual(self.nwbfile.identifier, 'FILE123')
self.assertEqual(self.nwbfile.session_start_time, self.start)
self.assertEqual(self.nwbfile.file_create_date, self.create)
self.assertEqual(self.nwbfile.lab, 'a test lab')
self.assertEqual(self.nwbfile.experimenter, 'A test experimenter')
self.assertEqual(self.nwbfile.institution, 'a test institution')
self.assertEqual(self.nwbfile.experiment_description,
'a test experiment description')
self.assertEqual(self.nwbfile.session_id, 'test1')
self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes')
self.assertEqual(self.nwbfile.data_collection,
'test data collection notes')
self.assertEqual(self.nwbfile.source_script, 'noscript')
self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename')
self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords'))
self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time)
def test_create_electrode_group(self):
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
d = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d)
self.assertEqual(elecgrp.description, desc)
self.assertEqual(elecgrp.location, loc)
self.assertIs(elecgrp.device, d)
def test_create_electrode_group_invalid_index(self):
"""
Test the case where the user creates an electrode table region with
indexes that are out of range of the amount of electrodes added.
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i)
with self.assertRaises(IndexError):
nwbfile.create_electrode_table_region(list(range(6)), 'test')
def test_access_group_after_io(self):
"""
Motivated by #739
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i + 1)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
os.remove("electrodes_mwe.nwb")
def test_epoch_tags(self):
tags1 = ['t1', 't2']
tags2 = ['t3', 't4']
tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float)
ts = TimeSeries("test_ts",
list(range(len(tstamps))),
'unit',
timestamps=tstamps)
expected_tags = tags1 + tags2
self.nwbfile.add_epoch(0.0, 1.0, tags1, ts)
self.nwbfile.add_epoch(0.0, 1.0, tags2, ts)
tags = self.nwbfile.epoch_tags
six.assertCountEqual(self, expected_tags, tags)
def test_add_acquisition(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.acquisition), 1)
def test_add_stimulus(self):
self.nwbfile.add_stimulus(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus), 1)
def test_add_stimulus_template(self):
self.nwbfile.add_stimulus_template(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus_template), 1)
def test_add_analysis(self):
self.nwbfile.add_analysis(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.analysis), 1)
def test_add_acquisition_check_dups(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
with self.assertRaises(ValueError):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
def test_get_acquisition_empty(self):
with self.assertRaisesRegex(ValueError,
"acquisition of NWBFile 'root' is empty"):
self.nwbfile.get_acquisition()
def test_get_acquisition_multiple_elements(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.nwbfile.add_acquisition(
TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name"
with self.assertRaisesRegex(ValueError, msg):
self.nwbfile.get_acquisition()
def test_add_acquisition_invalid_name(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "'TEST_TS' not found in acquisition of NWBFile 'root'"
with self.assertRaisesRegex(KeyError, msg):
self.nwbfile.get_acquisition("TEST_TS")
def test_set_electrode_table(self):
table = ElectrodeTable() # noqa: F405
dev1 = self.nwbfile.create_device('dev1') # noqa: F405
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-2.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-3.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-4.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
self.nwbfile.set_electrode_table(table)
self.assertIs(self.nwbfile.electrodes, table)
self.assertIs(table.parent, self.nwbfile)
def test_add_unit_column(self):
self.nwbfile.add_unit_column('unit_type', 'the type of unit')
self.assertEqual(self.nwbfile.units.colnames, ('unit_type', ))
def test_add_unit(self):
self.nwbfile.add_unit(id=1)
self.assertEqual(len(self.nwbfile.units), 1)
self.nwbfile.add_unit(id=2)
self.nwbfile.add_unit(id=3)
self.assertEqual(len(self.nwbfile.units), 3)
def test_add_trial_column(self):
self.nwbfile.add_trial_column('trial_type', 'the type of trial')
self.assertEqual(self.nwbfile.trials.colnames,
('start_time', 'stop_time', 'trial_type'))
def test_add_trial(self):
self.nwbfile.add_trial(start_time=10.0, stop_time=20.0)
self.assertEqual(len(self.nwbfile.trials), 1)
self.nwbfile.add_trial(start_time=30.0, stop_time=40.0)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.assertEqual(len(self.nwbfile.trials), 3)
def test_add_invalid_times_column(self):
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.assertEqual(self.nwbfile.invalid_times.colnames,
('start_time', 'stop_time', 'comments'))
def test_add_invalid_time_interval(self):
self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0)
self.assertEqual(len(self.nwbfile.invalid_times), 1)
self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0)
self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5)
self.assertEqual(len(self.nwbfile.invalid_times), 3)
def test_add_invalid_time_w_ts(self):
ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na')
self.nwbfile.add_invalid_time_interval(start_time=18.0,
stop_time=20.6,
timeseries=ts,
tags=('hi', 'there'))
def test_add_electrode(self):
dev1 = self.nwbfile.create_device('dev1') # noqa: F405
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
self.nwbfile.add_electrode(1.0,
2.0,
3.0,
-1.0,
'CA1',
'none',
group=group,
id=1)
self.assertEqual(self.nwbfile.electrodes[0][0], 1)
self.assertEqual(self.nwbfile.electrodes[0][1], 1.0)
self.assertEqual(self.nwbfile.electrodes[0][2], 2.0)
self.assertEqual(self.nwbfile.electrodes[0][3], 3.0)
self.assertEqual(self.nwbfile.electrodes[0][4], -1.0)
self.assertEqual(self.nwbfile.electrodes[0][5], 'CA1')
self.assertEqual(self.nwbfile.electrodes[0][6], 'none')
self.assertEqual(self.nwbfile.electrodes[0][7], group)
def test_all_children(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
device = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device)
children = self.nwbfile.all_children()
self.assertIn(ts1, children)
self.assertIn(ts2, children)
self.assertIn(device, children)
self.assertIn(elecgrp, children)
def test_fail_if_source_script_file_name_without_source_script(self):
with self.assertRaises(ValueError):
# <-- source_script_file_name without source_script is not allowed
NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
source_script=None,
source_script_file_name='nofilename')
def test_get_neurodata_type(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
p1 = ts1.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p1, self.nwbfile)
p2 = ts2.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p2, self.nwbfile)
def test_print_units(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
self.assertEqual(str(self.nwbfile.units), """
units <class 'pynwb.misc.Units'>
Fields:
colnames: ('spike_times',)
columns: (
spike_times_index <class 'pynwb.core.VectorIndex'>
Fields:
target: spike_times <class 'pynwb.core.VectorData'>
,
spike_times <class 'pynwb.core.VectorData'>
Fields:
description: the spike times for each unit
)
description: Autogenerated by NWBFile
id: id <class 'pynwb.core.ElementIdentifiers'>
""") # noqa: W291
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None, include_spike_waveforms=True, stub=True, cache_spec=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id, age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b)
print(lfp_channel)
custom_column = [{'name': 'theta_reference',
'description': 'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel}]
ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks)
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp',
description='lfp signal for all shank electrodes')
for name, channel in special_electrode_dict.items():
ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data',
data=all_channels_data[:, channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
nshanks = min((max_shanks, len(ns.get_shank_channels(session_path))))
for shankn in np.arange(nshanks, dtype=int) + 1:
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
decomp_series = DecompositionSeries(name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data, rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase', unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown', conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
custom_unit_columns = [
{
'name': 'cell_type',
'description': 'name of cell type',
'data': celltype_names},
{
'name': 'global_id',
'description': 'global id for cell for entire experiment',
'data': df_unit_features['unitID'].values},
{
'name': 'max_electrode',
'description': 'electrode that has the maximum amplitude of the waveform',
'data': get_max_electrodes(nwbfile, session_path),
'table': nwbfile.electrodes
}]
ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks)
trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond,
error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states', description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({'start_time': row[0], 'stop_time': row[1], 'label': name})
[table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])]
check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile, cache_spec=cache_spec)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')
# and :py:meth:`~pynwb.file.NWBFile.add_trial_column`. Together, these methods maintains a
# table-like structure that can define arbitrary columns without having to go through the
# extension process.
#
# By default, NWBFile only requires trial start time and trial end time. Additional columns
# can be added using :py:meth:`~pynwb.file.NWBFile.add_trial_column`. This method takes a name
# for the column and a description of what the column stores. You do not need to supply data
# type, as this will inferred.
# Once all columns have been added, trial data can be populated using :py:meth:`~pynwb.file.NWBFile.add_trial`.
#
# Lets add an additional column and some trial data.
nwbfile.add_trial_column(name='stim',
description='the visual stimuli during the trial')
nwbfile.add_trial(start_time=0.0, stop_time=2.0, stim='person')
nwbfile.add_trial(start_time=3.0, stop_time=5.0, stim='ocean')
nwbfile.add_trial(start_time=6.0, stop_time=8.0, stim='desert')
####################
# Tabular data such as trials can be converted to a `pandas.DataFrame`.
print(nwbfile.trials.to_dataframe())
####################
# ::
#
# start_time stop_time stim
# id
# 0 0.0 2.0 person
# 1 3.0 5.0 ocean
def convert_from_old_neo(old_file, bird_name, electrode_df):
root_dir, fname = os.path.split(old_file)
rec_date = bird_info[bird_name]['recording_date']
rec_start = bird_info[bird_name]['recording_start']
rec_end = bird_info[bird_name]['recording_end']
rec_datetime = date_parser.parse('{} {}'.format(rec_date, rec_start))
# create an electrode array, electrode groups, and electrode table for the electrodes
i = electrode_df.bird == bird_name
edf = electrode_df[i]
for block, gdf in edf.groupby(['block']):
lfp_series = dict()
spike_series = dict()
print('*************** Processing block {}'.format(block))
if bird_name == 'WhiWhi4522M' and block == 'Site1':
continue
# get the LFP and spike data for the block
hf = h5py.File(old_file, 'r')
block_data = read_old_neo(hf, block)
hf.close()
recording_name = '{}_{}_{}'.format(bird_name, block,
block_data['protocol'])
nwb_file = os.path.join(root_dir, '{}.nwb'.format(recording_name))
session_desc = """ A single recording session, roughly one hour long, at a single depth, in a series of
recordings from the auditory "cortex" of zebra finch {}. Block {}, stimulus protocol
{}.
""".format(bird_name, block, block_data['protocol'])
exp_desc = """ A randomly interleaved mixture of Zebra finch vocalizations, songs, and modulation limited
noise played to a Zebra finch under urethane anaesthesia in an acute experiment. Two 16 microwire
electrode arrays were implanted, one in each hemisphere. Experiments designed and performed by
Julie Elie, vocal repertoire recorded by Julie Elie. Data converted to NWB by Mike Schachter. For
a full description of methods, please consult and also cite the following publications:
Elie, J. E., & Theunissen, F. E. (2015). Meaning in the avian auditory cortex: neural
representation of communication calls. European Journal of Neuroscience, 41(5), 546-567.
Elie, J. E., & Theunissen, F. E. (2016). The vocal repertoire of the domesticated zebra finch:
a data-driven approach to decipher the information-bearing acoustic features of communication
signals. Animal cognition, 19(2), 285-315.
"""
nf = NWBFile(recording_name,
session_desc,
bird_name,
rec_datetime,
experimenter='Julie Elie',
lab='Theunissen Lab',
institution='UC Berkeley',
experiment_description=exp_desc,
session_id=bird_name)
# create the electrodes and electrode tables
for hemi, ggdf in gdf.groupby(['hemisphere']):
electrode_array_name = '16 electrode microwire array on {} hemisphere'.format(
hemi)
electrode_array = nf.create_device(name=electrode_array_name,
source='')
# create an electrode group
egrp_desc = """ The (x,y) locations of the electrodes refer to the distance from region midline and distance
from L2A, respectively, in mm.
"""
electrode_group = nf.create_electrode_group(
hemi,
source=electrode_array_name,
description=egrp_desc,
location='{} Hemisphere, Field L, CM, NCM'.format(hemi),
device=electrode_array)
# add electrodes to electrode group
for row_idx, row in ggdf.iterrows():
electrode_number = row['electrode'] - 1
dist_l2a = row['dist_l2a']
dist_midline = row['dist_midline']
region = row['region']
if bird_name == 'GreBlu9508M':
dist_l2a *= 4 # correct for the error in the original data
nf.add_electrode(electrode_number,
x=dist_midline,
y=dist_l2a,
z=0.0,
imp=0.0,
location=region,
filtering='none',
description='Row {}, Column {}'.format(
row['row'], row['col']),
group=electrode_group)
# create an electrode table region
electrode_numbers = list(
np.array(sorted(ggdf.electrode.unique())) - 1)
print('electrode_numbers=', electrode_numbers)
etable = nf.create_electrode_table_region(
electrode_numbers,
'All electrodes in array for hemisphere {} with LFP'.format(
hemi))
lfp_data = np.array([
block_data['electrodes'][e + 1]['lfp']
for e in electrode_numbers
])
sr = block_data['sample_rate']
t = np.arange(lfp_data.shape[1]) / sr
# add the raw LFP
lfp_series_name = 'Multi-electrode LFP on {} hemisphere from block {}'.format(
hemi, block)
lfp = ElectricalSeries(
lfp_series_name,
electrode_array_name,
lfp_data,
etable,
timestamps=t,
resolution=1e-12,
comments='',
description='Low-passed LFP recorded from microwire array')
lfp_series[hemi] = lfp
# add the spikes and their waveforms
for row_idx, row in ggdf.iterrows():
# electrode_number ranges from 1-32, same as the keys in block_data['electrodes']
electrode_number = row['electrode']
for k, unit_data in enumerate(
block_data['electrodes'][electrode_number]['units']):
spike_times = unit_data['spike_times']
waveforms = unit_data['waveforms']
unit_name = unit_data['name']
e_num = unit_data['electrode']
print('{} electrode_number={}({}), e_num={}, k={}'.format(
unit_name, electrode_number, electrode_number - 1,
e_num, k))
assert e_num == electrode_number
if unit_name.startswith('RF Sort'):
xarr = unit_name.split(' ')
unit_num = xarr[-1]
sort_type = 'Spike-sorted unit {}'.format(unit_num)
elif unit_name.startswith('Chan'):
if 'Code0' not in unit_name:
print(
'Skipping multi-unit channel with non-zero sort code'
)
continue
sort_type = 'Multi-unit'
else:
raise Exception(
'Unknown unit name: {}'.format(unit_name))
full_unit_name = '{} on block {} and electrode {}'.format(
sort_type, block, e_num - 1)
spikes = SpikeEventSeries(
full_unit_name,
lfp_series_name,
waveforms,
spike_times,
etable,
resolution=1e-12,
conversion=1e6,
comments='',
description='',
)
print(
'\tAdding spikes acquisition: {} ({}), waveforms.shape={}'
.format(full_unit_name, sort_type,
str(waveforms.shape)))
spike_series[(hemi, electrode_number, unit_name)] = spikes
all_series = list()
all_series.extend(lfp_series.values())
all_series.extend(spike_series.values())
print('len(all_series)=', len(all_series))
nf.add_acquisition(all_series)
# create trials for each stim presentation
nf.add_trial_column('stim_id',
'The ID of the sound played during the trial.')
nf.add_trial_column(
'trial', 'The number of times this stimulus has been presented.')
for stim_epoch_data in block_data['epochs']:
stim_id = stim_epoch_data['stim_id']
trial_num = stim_epoch_data['trial']
stime = stim_epoch_data['start']
etime = stim_epoch_data['end']
nf.add_trial({
'start': stime,
'end': etime,
'stim_id': stim_id,
'trial': trial_num
})
print('Writing to {}'.format(nwb_file))
with NWBHDF5IO(nwb_file, mode='w') as io:
io.write(nf)
del nf
class NWBFileTest(TestCase):
def setUp(self):
self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create = [datetime(2017, 5, 1, 12, tzinfo=tzlocal()),
datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()),
datetime(2017, 5, 2, 14, tzinfo=tzutc())]
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
file_create_date=self.create,
timestamps_reference_time=self.ref_time,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
notes='my notes',
pharmacology='drugs',
protocol='protocol',
related_publications='my pubs',
slices='my slices',
surgery='surgery',
virus='a virus',
source_script='noscript',
source_script_file_name='nofilename',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
keywords=('these', 'are', 'keywords'))
def test_constructor(self):
self.assertEqual(self.nwbfile.session_description, 'a test session description for a test NWBFile')
self.assertEqual(self.nwbfile.identifier, 'FILE123')
self.assertEqual(self.nwbfile.session_start_time, self.start)
self.assertEqual(self.nwbfile.file_create_date, self.create)
self.assertEqual(self.nwbfile.lab, 'a test lab')
self.assertEqual(self.nwbfile.experimenter, ('A test experimenter',))
self.assertEqual(self.nwbfile.institution, 'a test institution')
self.assertEqual(self.nwbfile.experiment_description, 'a test experiment description')
self.assertEqual(self.nwbfile.session_id, 'test1')
self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes')
self.assertEqual(self.nwbfile.data_collection, 'test data collection notes')
self.assertEqual(self.nwbfile.related_publications, ('my pubs',))
self.assertEqual(self.nwbfile.source_script, 'noscript')
self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename')
self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords'))
self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time)
def test_create_electrode_group(self):
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
d = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d)
self.assertEqual(elecgrp.description, desc)
self.assertEqual(elecgrp.location, loc)
self.assertIs(elecgrp.device, d)
def test_create_custom_intervals(self):
df_words = pd.DataFrame({'start_time': [.1, 2.], 'stop_time': [.8, 2.3],
'label': ['hello', 'there']})
words = TimeIntervals.from_dataframe(df_words, name='words')
self.nwbfile.add_time_intervals(words)
self.assertEqual(self.nwbfile.intervals['words'], words)
def test_create_electrode_group_invalid_index(self):
"""
Test the case where the user creates an electrode table region with
indexes that are out of range of the amount of electrodes added.
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a')
for i in range(4):
nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i)
with self.assertRaises(IndexError):
nwbfile.create_electrode_table_region(list(range(6)), 'test')
def test_access_group_after_io(self):
"""
Motivated by #739
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a')
nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
for i in range(4):
nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i + 1)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
remove_test_file("electrodes_mwe.nwb")
def test_access_processing(self):
self.nwbfile.create_processing_module('test_mod', 'test_description')
# test deprecate .modules
with self.assertWarnsWith(DeprecationWarning, 'replaced by NWBFile.processing'):
modules = self.nwbfile.modules['test_mod']
self.assertIs(self.nwbfile.processing['test_mod'], modules)
def test_epoch_tags(self):
tags1 = ['t1', 't2']
tags2 = ['t3', 't4']
tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float)
ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps)
expected_tags = tags1 + tags2
self.nwbfile.add_epoch(0.0, 1.0, tags1, ts)
self.nwbfile.add_epoch(0.0, 1.0, tags2, ts)
tags = self.nwbfile.epoch_tags
self.assertEqual(set(expected_tags), set(tags))
def test_add_acquisition(self):
self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.acquisition), 1)
def test_add_stimulus(self):
self.nwbfile.add_stimulus(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus), 1)
def test_add_stimulus_template(self):
self.nwbfile.add_stimulus_template(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus_template), 1)
def test_add_analysis(self):
self.nwbfile.add_analysis(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.analysis), 1)
def test_add_acquisition_check_dups(self):
self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
with self.assertRaises(ValueError):
self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
def test_get_acquisition_empty(self):
with self.assertRaisesWith(ValueError, "acquisition of NWBFile 'root' is empty"):
self.nwbfile.get_acquisition()
def test_get_acquisition_multiple_elements(self):
self.nwbfile.add_acquisition(TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.nwbfile.add_acquisition(TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name"
with self.assertRaisesWith(ValueError, msg):
self.nwbfile.get_acquisition()
def test_add_acquisition_invalid_name(self):
self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "\"'TEST_TS' not found in acquisition of NWBFile 'root'\""
with self.assertRaisesWith(KeyError, msg):
self.nwbfile.get_acquisition("TEST_TS")
def test_set_electrode_table(self):
table = ElectrodeTable()
dev1 = self.nwbfile.create_device('dev1')
group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1)
table.add_row(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group,
group_name='tetrode1')
table.add_row(x=1.0, y=2.0, z=3.0, imp=-2.0, location='CA1', filtering='none', group=group,
group_name='tetrode1')
table.add_row(x=1.0, y=2.0, z=3.0, imp=-3.0, location='CA1', filtering='none', group=group,
group_name='tetrode1')
table.add_row(x=1.0, y=2.0, z=3.0, imp=-4.0, location='CA1', filtering='none', group=group,
group_name='tetrode1')
self.nwbfile.set_electrode_table(table)
self.assertIs(self.nwbfile.electrodes, table)
self.assertIs(table.parent, self.nwbfile)
def test_add_unit_column(self):
self.nwbfile.add_unit_column('unit_type', 'the type of unit')
self.assertEqual(self.nwbfile.units.colnames, ('unit_type',))
def test_add_unit(self):
self.nwbfile.add_unit(id=1)
self.assertEqual(len(self.nwbfile.units), 1)
self.nwbfile.add_unit(id=2)
self.nwbfile.add_unit(id=3)
self.assertEqual(len(self.nwbfile.units), 3)
def test_add_trial_column(self):
self.nwbfile.add_trial_column('trial_type', 'the type of trial')
self.assertEqual(self.nwbfile.trials.colnames, ('start_time', 'stop_time', 'trial_type'))
def test_add_trial(self):
self.nwbfile.add_trial(start_time=10.0, stop_time=20.0)
self.assertEqual(len(self.nwbfile.trials), 1)
self.nwbfile.add_trial(start_time=30.0, stop_time=40.0)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.assertEqual(len(self.nwbfile.trials), 3)
def test_add_invalid_times_column(self):
self.nwbfile.add_invalid_times_column('comments', 'description of reason for omitting time')
self.assertEqual(self.nwbfile.invalid_times.colnames, ('start_time', 'stop_time', 'comments'))
def test_add_invalid_time_interval(self):
self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0)
self.assertEqual(len(self.nwbfile.invalid_times), 1)
self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0)
self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5)
self.assertEqual(len(self.nwbfile.invalid_times), 3)
def test_add_invalid_time_w_ts(self):
ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na')
self.nwbfile.add_invalid_time_interval(start_time=18.0, stop_time=20.6,
timeseries=ts, tags=('hi', 'there'))
def test_add_electrode(self):
dev1 = self.nwbfile.create_device('dev1')
group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1)
self.nwbfile.add_electrode(1.0, 2.0, 3.0, -1.0, 'CA1', 'none', group=group, id=1)
elec = self.nwbfile.electrodes[0]
self.assertEqual(elec.index[0], 1)
self.assertEqual(elec.iloc[0]['x'], 1.0)
self.assertEqual(elec.iloc[0]['y'], 2.0)
self.assertEqual(elec.iloc[0]['z'], 3.0)
self.assertEqual(elec.iloc[0]['location'], 'CA1')
self.assertEqual(elec.iloc[0]['filtering'], 'none')
self.assertEqual(elec.iloc[0]['group'], group)
def test_all_children(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
device = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device)
children = self.nwbfile.all_children()
self.assertIn(ts1, children)
self.assertIn(ts2, children)
self.assertIn(device, children)
self.assertIn(elecgrp, children)
def test_fail_if_source_script_file_name_without_source_script(self):
with self.assertRaises(ValueError):
# <-- source_script_file_name without source_script is not allowed
NWBFile('a test session description for a test NWBFile', 'FILE123', self.start,
source_script=None,
source_script_file_name='nofilename')
def test_get_neurodata_type(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
p1 = ts1.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p1, self.nwbfile)
p2 = ts2.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p2, self.nwbfile)
def test_print_units(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
expected = """units pynwb.misc.Units at 0x%d
Fields:
colnames: ['spike_times']
columns: (
spike_times_index <class 'hdmf.common.table.VectorIndex'>,
spike_times <class 'hdmf.common.table.VectorData'>
)
description: Autogenerated by NWBFile
id: id <class 'hdmf.common.table.ElementIdentifiers'>
waveform_unit: volts
"""
expected = expected % id(self.nwbfile.units)
self.assertEqual(str(self.nwbfile.units), expected)
def test_copy(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
device = self.nwbfile.create_device('a')
elecgrp = self.nwbfile.create_electrode_group('a', 'b', device=device, location='a')
self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0)
self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'b', 'b', elecgrp)
elec_region = self.nwbfile.create_electrode_table_region([1], 'name')
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = ElectricalSeries('test_ts2', [0, 1, 2, 3, 4, 5],
electrodes=elec_region, timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.nwbfile.add_invalid_times_column('comments', 'description of reason for omitting time')
self.nwbfile.create_processing_module('test_mod', 'test_description')
self.nwbfile.create_time_intervals('custom_interval', 'a custom time interval')
self.nwbfile.intervals['custom_interval'].add_interval(start_time=10., stop_time=20.)
newfile = self.nwbfile.copy()
# test dictionaries
self.assertIs(self.nwbfile.devices['a'], newfile.devices['a'])
self.assertIs(self.nwbfile.acquisition['test_ts1'], newfile.acquisition['test_ts1'])
self.assertIs(self.nwbfile.acquisition['test_ts2'], newfile.acquisition['test_ts2'])
self.assertIs(self.nwbfile.processing['test_mod'], newfile.processing['test_mod'])
# test dynamic tables
self.assertIsNot(self.nwbfile.electrodes, newfile.electrodes)
self.assertIs(self.nwbfile.electrodes['x'], newfile.electrodes['x'])
self.assertIsNot(self.nwbfile.units, newfile.units)
self.assertIs(self.nwbfile.units['spike_times'], newfile.units['spike_times'])
self.assertIsNot(self.nwbfile.trials, newfile.trials)
self.assertIsNot(self.nwbfile.trials.parent, newfile.trials.parent)
self.assertIs(self.nwbfile.trials.id, newfile.trials.id)
self.assertIs(self.nwbfile.trials['start_time'], newfile.trials['start_time'])
self.assertIs(self.nwbfile.trials['stop_time'], newfile.trials['stop_time'])
self.assertIsNot(self.nwbfile.invalid_times, newfile.invalid_times)
self.assertTupleEqual(self.nwbfile.invalid_times.colnames, newfile.invalid_times.colnames)
self.assertIsNot(self.nwbfile.intervals['custom_interval'], newfile.intervals['custom_interval'])
self.assertTupleEqual(self.nwbfile.intervals['custom_interval'].colnames,
newfile.intervals['custom_interval'].colnames)
self.assertIs(self.nwbfile.intervals['custom_interval']['start_time'],
newfile.intervals['custom_interval']['start_time'])
self.assertIs(self.nwbfile.intervals['custom_interval']['stop_time'],
newfile.intervals['custom_interval']['stop_time'])
def test_multi_experimenters(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
experimenter=('experimenter1', 'experimenter2'))
self.assertTupleEqual(self.nwbfile.experimenter, ('experimenter1', 'experimenter2'))
def test_multi_publications(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
related_publications=('pub1', 'pub2'))
self.assertTupleEqual(self.nwbfile.related_publications, ('pub1', 'pub2'))
def convert_data(
self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False
):
session_path = self.input_args["folder_path"]
# TODO: check/enforce format?
task_types = metadata_dict.get("task_types", [])
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
exist_pos_data = any(
os.path.isfile(os.path.join(session_path, "{}__{}.mat".format(session_id, task_type["name"])))
for task_type in task_types
)
if exist_pos_data:
nwbfile.add_epoch_column("label", "name of epoch")
for task_type in task_types:
label = task_type["name"]
file = os.path.join(session_path, session_id + "__" + label + ".mat")
if os.path.isfile(file):
pos_obj = Position(name=label + "_position")
matin = loadmat(file)
tt = matin["twhl_norm"][:, 0]
exp_times = find_discontinuities(tt)
if "conversion" in task_type:
conversion = task_type["conversion"]
else:
conversion = np.nan
for pos_type in ("twhl_norm", "twhl_linearized"):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + "_{}_spatial_series".format(pos_type),
data=H5DataIO(pos_data_norm, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression="gzip"),
)
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, "behavior", "contains processed behavioral data").add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + "_" + str(i))
trialdata_path = os.path.join(session_path, session_id + "__EightMazeRun.mat")
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)["EightMazeRun"]
trialdatainfo_path = os.path.join(fpath_base, "EightMazeRunInfo.mat")
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]]
features = trialdatainfo[:7]
features[:2] = (
"start_time",
"stop_time",
)
[nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"]]
for trial_data in trials_data:
if trial_data[3]:
cond = "run_left"
else:
cond = "run_right"
nwbfile.add_trial(
start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6],
)
sleep_state_fpath = os.path.join(session_path, "{}.SleepState.states.mat".format(session_id))
# label renaming specific to Watson
state_label_names = {"WAKEstate": "Awake", "NREMstate": "Non-REM", "REMstate": "REM"}
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states", description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({"start_time": row[0], "stop_time": row[1], "label": state_label_names[name]})
[table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"])]
check_module(nwbfile, "behavior", "contains behavioral data").add_data_interface(table)
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversionfor this data interface.
Reads optophysiology raw data from .rsd files and adds it to nwbfile.
XXXXXXX_A.rsd - Raw data from donor
XXXXXXX_B.rsd - Raw data from acceptor
XXXXXXXXX.rsh - Header data
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
dir_cortical_imaging = self.source_data['dir_cortical_imaging']
ophys_list = [i.name for i in Path(dir_cortical_imaging).glob('*.rsh')]
if len(ophys_list) > 0:
fname_prefix = ophys_list[0].split('-')[0]
else:
raise OSError(f"No .rsd file found in directory: {dir_cortical_imaging}.\n"
"Did you choose the correct path for source data?")
def data_gen(channel, trial):
"""channel = 'A' or 'B'"""
# Read trial-specific metadata file .rsh
trial_meta = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{trial}_{channel}.rsh")
file_rsm, files_raw, acquisition_date, sample_rate, n_frames = self.read_trial_meta(trial_meta=trial_meta)
# Iterates over all files within the same trial
for fn, fraw in enumerate(files_raw):
print('adding channel ' + channel + ', trial: ', trial, ': ', 100 * fn / len(files_raw), '%')
fpath = os.path.join(dir_cortical_imaging, fraw)
# Open file as a byte array
with open(fpath, "rb") as f:
byte = f.read(1000000000)
# Data as word array: 'h' signed, 'H' unsigned
words = np.array(struct.unpack('h' * (len(byte) // 2), byte))
# Iterates over frames within the same file (n_frames, 100, 100)
n_frames = int(len(words) / 12800)
words_reshaped = words.reshape(12800, n_frames, order='F')
frames = np.zeros((n_frames, 100, 100))
excess_frames = np.zeros((n_frames, 20, 100))
for ifr in range(n_frames):
iframe = -words_reshaped[:, ifr].reshape(128, 100, order='F').astype('int16')
frames[ifr, :, :] = iframe[20:120, :]
excess_frames[ifr, :, :] = iframe[0:20, :]
yield iframe[20:120, :]
# # Analog signals are taken from excess data variable
# analog_1 = np.squeeze(np.squeeze(excess_frames[:, 12, 0:80:4]).reshape(20*256, 1))
# analog_2 = np.squeeze(np.squeeze(excess_frames[:, 14, 0:80:4]).reshape(20*256, 1))
# stim_trg = np.squeeze(np.squeeze(excess_frames[:, 8, 0:80:4]).reshape(20*256, 1))
# Get session_start_time from first header file
all_files = os.listdir(dir_cortical_imaging)
all_headers = [f for f in all_files if ('.rsh' in f) and ('_A' not in f) and ('_B' not in f)]
all_headers.sort()
_, _, acquisition_date, _, _ = self.read_trial_meta(trial_meta=Path(dir_cortical_imaging) / all_headers[0])
session_start_time = datetime.strptime(acquisition_date, '%Y/%m/%d %H:%M:%S')
session_start_time_tzaware = pytz.timezone('EST').localize(session_start_time)
if session_start_time_tzaware != nwbfile.session_start_time:
print("Session start time in current nwbfile does not match the start time from rsd files.")
print("Ophys data conversion aborted.")
return
# Create and add device
device = Device(name=metadata['Ophys']['Device']['name'])
nwbfile.add_device(device)
# Get FRETSeries metadata
meta_donor = metadata['Ophys']['FRET']['donor'][0]
meta_acceptor = metadata['Ophys']['FRET']['acceptor'][0]
# OpticalChannels
opt_ch_donor = OpticalChannel(
name=meta_donor['optical_channel'][0]['name'],
description=meta_donor['optical_channel'][0]['description'],
emission_lambda=float(meta_donor['optical_channel'][0]['emission_lambda'])
)
opt_ch_acceptor = OpticalChannel(
name=meta_acceptor['optical_channel'][0]['name'],
description=meta_acceptor['optical_channel'][0]['description'],
emission_lambda=float(meta_acceptor['optical_channel'][0]['emission_lambda'])
)
# Add trials intervals values only if no trials data exists in nwbfile
if nwbfile.trials is not None:
add_trials = False
print('Trials already exist in current nwb file. Ophys trials intervals not added.')
else:
add_trials = True
# Iterate over trials, creates a FRET group per trial
trials_numbers = [f.split('-')[1].replace('.rsh', '') for f in all_headers]
for tr in trials_numbers:
# Read trial-specific metadata file .rsh
trial_meta_A = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_A.rsh")
trial_meta_B = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_B.rsh")
file_rsm_A, files_raw_A, acquisition_date_A, sample_rate_A, n_frames_A = self.read_trial_meta(trial_meta=trial_meta_A)
file_rsm_B, files_raw_B, acquisition_date_B, sample_rate_B, n_frames_B = self.read_trial_meta(trial_meta=trial_meta_B)
absolute_start_time = datetime.strptime(acquisition_date_A, '%Y/%m/%d %H:%M:%S')
relative_start_time = float((absolute_start_time - nwbfile.session_start_time.replace(tzinfo=None)).seconds)
# Checks if Acceptor and Donor channels have the same basic parameters
assert acquisition_date_A == acquisition_date_B, \
"Acquisition date of channels do not match. Trial=" + str(tr)
assert sample_rate_A == sample_rate_B, \
"Sample rate of channels do not match. Trial=" + str(tr)
assert n_frames_A == n_frames_B, \
"Number of frames of channels do not match. Trial=" + str(tr)
assert relative_start_time >= 0., \
"Starting time is negative. Trial=" + str(tr)
# Create iterator
data_donor = DataChunkIterator(
data=data_gen(channel='A', trial=tr),
iter_axis=0,
buffer_size=10000,
maxshape=(None, 100, 100)
)
data_acceptor = DataChunkIterator(
data=data_gen(channel='B', trial=tr),
iter_axis=0,
buffer_size=10000,
maxshape=(None, 100, 100)
)
# FRETSeries
frets_donor = FRETSeries(
name='donor',
fluorophore=meta_donor['fluorophore'],
optical_channel=opt_ch_donor,
device=device,
description=meta_donor['description'],
data=data_donor,
starting_time=relative_start_time,
rate=sample_rate_A,
unit=meta_donor['unit'],
)
frets_acceptor = FRETSeries(
name='acceptor',
fluorophore=meta_acceptor['fluorophore'],
optical_channel=opt_ch_acceptor,
device=device,
description=meta_acceptor['description'],
data=data_acceptor,
starting_time=relative_start_time,
rate=sample_rate_B,
unit=meta_acceptor['unit']
)
# Add FRET to acquisition
meta_fret = metadata['Ophys']['FRET']
fret = FRET(
name=meta_fret['name'] + '_' + str(tr),
excitation_lambda=float(meta_fret['excitation_lambda']),
donor=frets_donor,
acceptor=frets_acceptor
)
nwbfile.add_acquisition(fret)
# Add trial
if add_trials:
tr_stop = relative_start_time + n_frames_A / sample_rate_A
nwbfile.add_trial(
start_time=relative_start_time,
stop_time=tr_stop
)
class ShowPSTHTestCase(unittest.TestCase):
def setUp(self):
start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal())
create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
self.nwbfile = NWBFile(session_description='NWBFile for PSTH',
identifier='NWB123',
session_start_time=start_time,
file_create_date=create_date)
self.nwbfile.add_unit_column('location',
'the anatomical location of this unit')
self.nwbfile.add_unit_column(
'quality', 'the quality for the inference of this unit')
self.nwbfile.add_unit(id=1,
spike_times=[2.2, 3.0, 4.5],
obs_intervals=[[1, 10]],
location='CA1',
quality=0.95)
self.nwbfile.add_unit(id=2,
spike_times=[2.2, 3.0, 25.0, 26.0],
obs_intervals=[[1, 10], [20, 30]],
location='CA3',
quality=0.85)
self.nwbfile.add_unit(id=3,
spike_times=[1.2, 2.3, 3.3, 4.5],
obs_intervals=[[1, 10], [20, 30]],
location='CA1',
quality=0.90)
self.nwbfile.add_trial_column(
name='stim', description='the visual stimuli during the trial')
self.nwbfile.add_trial(start_time=0.0, stop_time=2.0, stim='person')
self.nwbfile.add_trial(start_time=3.0, stop_time=5.0, stim='ocean')
self.nwbfile.add_trial(start_time=6.0, stop_time=8.0, stim='desert')
def test_get_min_spike_time(self):
assert (get_min_spike_time(self.nwbfile.units) == 1.2)
def test_align_by_trials(self):
ComparetoAT = [
np.array([2.2, 3.0, 25.0, 26.0]),
np.array([-0.8, 0., 22., 23.]),
np.array([-3.8, -3., 19., 20.])
]
AT = align_by_trials(self.nwbfile.units,
index=1,
before=20.,
after=30.)
np.testing.assert_allclose(AT, ComparetoAT, rtol=1e-02)
def test_align_by_time_intervals_Nonetrials_select(self):
time_intervals = TimeIntervals(name='Test Time Interval')
time_intervals.add_interval(start_time=21.0, stop_time=28.0)
time_intervals.add_interval(start_time=22.0, stop_time=26.0)
time_intervals.add_interval(start_time=22.0, stop_time=28.4)
ATI = align_by_time_intervals(self.nwbfile.units,
index=1,
intervals=time_intervals,
stop_label=None,
before=20.,
after=30.)
ComparedtoATI = [
np.array([-18.8, -18., 4., 5.]),
np.array([-19.8, -19., 3., 4.]),
np.array([-19.8, -19., 3., 4.])
]
np.testing.assert_array_equal(ATI, ComparedtoATI)
def test_align_by_time_intervals(self):
time_intervals = TimeIntervals(name='Test Time Interval')
time_intervals.add_interval(start_time=21.0, stop_time=28.0)
time_intervals.add_interval(start_time=22.0, stop_time=26.0)
time_intervals.add_interval(start_time=22.0, stop_time=28.4)
ATI = align_by_time_intervals(self.nwbfile.units,
index=1,
intervals=time_intervals,
stop_label=None,
before=20.,
after=30.,
rows_select=[0, 1])
ComparedtoATI = [
np.array([-18.8, -18., 4., 5.]),
np.array([-19.8, -19., 3., 4.])
]
np.testing.assert_array_equal(ATI, ComparedtoATI)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=False):
this_session = (experiment.Session & session_key).fetch1()
print(f'Exporting to NWB 2.0 for session: {this_session}...')
# ===============================================================================
# ============================== META INFORMATION ===============================
# ===============================================================================
# -- NWB file - a NWB2.0 file for each session
file_name = '_'.join([
this_session['subject_nickname'],
this_session['session_date'].strftime('%Y-%m-%d'),
str(this_session['session'])
])
nwbfile = NWBFile(
identifier=file_name,
related_publications='http://dx.doi.org/10.1016/j.neuron.2017.05.005',
experiment_description='Two-photon experiment recorded in {}'.format(
this_session['brain_location_name']),
session_description='Imaging session',
session_start_time=datetime.combine(this_session['session_date'],
zero_zero_time),
file_create_date=datetime.now(tzlocal()),
experimenter=this_session['username'],
institution=institution,
keywords=[
'motor planning', 'anterior lateral cortex', 'medial motor cortex',
'ALM', 'MM', 'Two-photon imaging'
])
# -- subject
subj = (lab.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_nickname'],
genotype=' x '.join((lab.Subject.GeneModification
& subj).fetch('gene_modification')),
sex=subj['sex'],
species=subj['species'],
date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time)
if subj['date_of_birth'] else None)
# -- virus
nwbfile.virus = json.dumps([{
k: str(v)
for k, v in virus_injection.items() if k not in subj
} for virus_injection in virus.VirusInjection * virus.Virus & session_key])
# ===============================================================================
# ======================== IMAGING & SEGMENTATION ===============================
# ===============================================================================
scan = (imaging.Scan & session_key).fetch1()
# ---- Structural Images ----------
images = pynwb.base.Images('images')
if isinstance(scan['image_beads'], collections.Sequence):
gcamp = pynwb.image.GrayscaleImage('GCaMP at 940nm',
scan['image_gcamp'])
images.add_image(gcamp)
if isinstance(scan['image_ctb'], collections.Sequence):
ctb = pynwb.image.RGBImage('CTB-647 IT', scan['image_ctb'])
images.add_image(ctb)
if isinstance(scan['image_beads'], collections.Sequence):
beads = pynwb.image.GrayscaleImage('Beads PT', scan['image_beads'])
images.add_image(beads)
nwbfile.add_acquisition(images)
imaging_plane = nwbfile.create_imaging_plane(
name='Imaging plane',
optical_channel=pynwb.ophys.OpticalChannel(name='green',
description='green channel',
emission_lambda=500.),
description=
'Imaging session for PT and IT neurons during audio delay task',
device=nwbfile.create_device(
name='two-photon microscope with Thorlabs resonant galvo scannner'
),
excitation_lambda=940.,
imaging_rate=300.,
indicator='GCaMP6s',
location='ALM',
conversion=1e-6,
unit='micrometers')
# ---- Frame Time information -----
frame_time = pynwb.image.TimeSeries(name='Frame Time',
data=list(
range(0, len(scan['frame_time']))),
unit='a.u',
timestamps=scan['frame_time'])
nwbfile.add_acquisition(frame_time)
# ----- Segementation information -----
# link the imaging segmentation to the nwb file
ophys = nwbfile.create_processing_module('Ophys',
'Processing result of imaging')
img_seg = pynwb.ophys.ImageSegmentation()
ophys.add_data_interface(img_seg)
pln_seg = pynwb.ophys.PlaneSegmentation(name='Plane Segmentation',
description='plane segmentation',
imaging_plane=imaging_plane)
img_seg.add_plane_segmentation([pln_seg])
# insert ROI mask
rois = (imaging.Scan.Roi & session_key).fetch(as_dict=True)
for k, v in dict(roi_id='roi id',
cell_type='PT, IT, or unknown',
roi_trace='Trace on this session of this roi',
neuropil_trace='Trace on this session of the neurophil',
included='whether to include this roi into later analyses'
).items():
pln_seg.add_column(name=k, description=v)
for roi in rois:
mask = np.zeros([512, 512])
mask[np.unravel_index(roi['roi_pixel_list'] - 1, mask.shape, 'F')] = 1
pln_seg.add_roi(roi_id=roi['roi_idx'],
image_mask=mask,
cell_type=roi['cell_type'],
roi_trace=roi['roi_trace'],
neuropil_trace=roi['roi_trace'],
included=roi['inc'])
# ===============================================================================
# =============================== BEHAVIOR TRIALS ===============================
# ===============================================================================
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time'
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
dj_trial = experiment.SessionTrial * experiment.BehaviorTrial
skip_adding_columns = experiment.Session.primary_key + ['trial_uid']
if experiment.SessionTrial & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{
'name':
tag,
'description':
re.sub('\s+:|\s+', ' ',
re.search(f'(?<={tag})(.*)',
str(dj_trial.heading)).group()).strip()
} for tag in dj_trial.heading.names if tag not in skip_adding_columns +
['start_time', 'stop_time']]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (dj_trial & session_key).fetch(as_dict=True):
trial['start_time'] = float(trial['start_time'])
trial['stop_time'] = float(
trial['stop_time']) if trial['stop_time'] else 5.0
[trial.pop(k) for k in skip_adding_columns]
nwbfile.add_trial(**trial)
# ===============================================================================
# =============================== BEHAVIOR TRIAL EVENTS ==========================
# ===============================================================================
behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents')
nwbfile.add_acquisition(behav_event)
for trial_event_type in (experiment.TrialEventType & experiment.TrialEvent
& session_key).fetch('trial_event_type'):
event_times, trial_starts = (
experiment.TrialEvent * experiment.SessionTrial
& session_key & {
'trial_event_type': trial_event_type
}).fetch('trial_event_time', 'start_time')
if len(event_times) > 0:
event_times = np.hstack(
event_times.astype(float) + trial_starts.astype(float))
behav_event.create_timeseries(name=trial_event_type,
unit='a.u.',
conversion=1.0,
data=np.full_like(event_times, 1),
timestamps=event_times)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def run_conversion(
self,
nwbfile: NWBFile,
metadata: dict,
stub_test: bool = False,
):
session_path = Path(self.source_data["folder_path"])
session_id = session_path.name
# Stimuli
[
nwbfile.add_stimulus(x) for x in get_events(
session_path=session_path,
suffixes=[".lrw.evt", ".puf.evt", ".rip.evt", ".rrw.evt"])
]
# Epochs
df = pd.read_csv(session_path / f"{session_id}.cat.evt",
sep=" ",
names=("time", "begin_or_end", "of", "epoch_name"))
epoch_starts = []
for j in range(int(len(df) / 2)):
epoch_starts.append(df["time"][2 * j])
nwbfile.add_epoch(start_time=epoch_starts[j],
stop_time=df["time"][2 * j + 1],
tags=[df["epoch_name"][2 * j][18:]])
# Trials
trialdata_path = session_path / f"{session_id}-TrackRunTimes.mat"
if trialdata_path.is_file():
trials_data = loadmat(trialdata_path)["trackruntimes"]
for trial_data in trials_data:
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1])
# .whl position
whl_files = []
for whl_file in whl_files:
add_position_data(nwbfile=nwbfile,
session_path=session_path,
whl_file_path=whl_file,
starting_time=epoch_starts[j])
# States
sleep_state_fpath = session_path / f"{session_id}.SleepState.states.mat"
# label renaming
state_label_names = dict(WAKEstate="Awake",
NREMstate="Non-REM",
REMstate="REM")
if sleep_state_fpath.is_file():
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states",
description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(start_time=row[0],
stop_time=row[1],
label=state_label_names[name]))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"Contains behavioral data.").add(table)
module_cellular.add_container(cci_obj)
trialdata_path = os.path.join(fpath, fname + '__EightMazeRun.mat')
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath, fname + '__EightMazeRunInfo.mat')
trialdatainfo = [
x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]
]
features = trialdatainfo[:7]
features[:2] = 'start', 'end'
[nwbfile.add_trial_column(x, 'description') for x in features]
for trial_data in trials_data:
nwbfile.add_trial({lab: dat for lab, dat in zip(features, trial_data[:7])})
mono_syn_fpath = os.path.join(fpath, fname + '-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
exc_obj = CatCellInfo('excitatory_connections',
'YutaMouse41-150903-MonoSynConvClick.mat',
values=[],
cell_index=exc[:, 0] - 1,
indices=exc[:, 1] - 1)
module_cellular.add_container(exc_obj)
inh_obj = CatCellInfo('inhibitory_connections',
'YutaMouse41-150903-MonoSynConvClick.mat',
def chang2nwb(blockpath,
outpath=None,
session_start_time=None,
session_description=None,
identifier=None,
anin4=False,
ecog_format='auto',
external_subject=True,
include_pitch=False,
include_intensity=False,
speakers=True,
mic=False,
mini=False,
hilb=False,
verbose=False,
imaging_path=None,
parse_transcript=False,
include_cortical_surfaces=True,
include_electrodes=True,
include_ekg=True,
subject_image_list=None,
rest_period=None,
load_warped=False,
**kwargs):
"""
Parameters
----------
blockpath: str
outpath: None | str
if None, output = [blockpath]/[blockname].nwb
session_start_time: datetime.datetime
default: datetime(1900, 1, 1)
session_description: str
default: blockname
identifier: str
default: blockname
anin4: False | str
Whether or not to convert ANIN4. ANIN4 is used as an extra channel for
things like button presses, and is usually unused. If a string is
supplied, that is used as the name of the timeseries.
ecog_format: str
({'htk'}, 'mat', 'raw')
external_subject: bool (optional)
True: (default) cortical mesh is saved in an external file and a link is
provided to that file. This is useful if you have multiple sessions for a single subject.
False: cortical mesh is saved normally
include_pitch: bool (optional)
add pitch data. Default: False
include_intensity: bool (optional)
add intensity data. Default: False
speakers: bool (optional)
Default: False
mic: bool (optional)
default: False
mini: only save data stub. Used for testing
hilb: bool
include Hilbert Transform data. Default: False
verbose: bool (optional)
imaging_path: str (optional)
None: use IMAGING_DIR
'local': use subject_dir/Imaging/
else: use supplied string
parse_transcript: str (optional)
include_cortical_surfaces: bool (optional)
include_electrodes: bool (optional)
include_ekg: bool (optional)
subject_image_list: list (optional)
List of paths of images to include
rest_period: None | array-like
kwargs: dict
passed to pynwb.NWBFile
Returns
-------
"""
behav_module = None
basepath, blockname = os.path.split(blockpath)
subject_id = get_subject_id(blockname)
if identifier is None:
identifier = blockname
if session_description is None:
session_description = blockname
if outpath is None:
outpath = blockpath + '.nwb'
out_base_path = os.path.split(outpath)[0]
if session_start_time is None:
session_start_time = datetime(1900, 1, 1).astimezone(timezone('UTC'))
if imaging_path is None:
subj_imaging_path = path.join(IMAGING_PATH, subject_id)
elif imaging_path == 'local':
subj_imaging_path = path.join(basepath, 'imaging')
else:
subj_imaging_path = os.path.join(imaging_path, subject_id)
# file paths
bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat')
ecog_path = path.join(blockpath, 'RawHTK')
ecog400_path = path.join(blockpath, 'ecog400', 'ecog.mat')
elec_metadata_file = path.join(subj_imaging_path, 'elecs',
'TDT_elecs_all.mat')
mesh_path = path.join(subj_imaging_path, 'Meshes')
pial_files = glob.glob(path.join(mesh_path, '*pial.mat'))
# Create the NWB file object
nwbfile = NWBFile(session_description,
identifier,
session_start_time,
datetime.now().astimezone(),
session_id=identifier,
institution='University of California, San Francisco',
lab='Chang Lab',
**kwargs)
nwbfile.add_electrode_column('bad', 'electrode identified as too noisy')
bad_elecs_inds = get_bad_elecs(blockpath)
if include_electrodes:
add_electrodes(nwbfile,
elec_metadata_file,
bad_elecs_inds,
load_warped=load_warped)
else:
device = nwbfile.create_device('256Grid')
electrode_group = nwbfile.create_electrode_group(
name='256Grid electrodes',
description='auto_group',
location='location',
device=device)
for elec_counter in range(256):
bad = elec_counter in bad_elecs_inds
nwbfile.add_electrode(id=elec_counter + 1,
x=np.nan,
y=np.nan,
z=np.nan,
imp=np.nan,
location=' ',
filtering='none',
group=electrode_group,
bad=bad)
ecog_elecs = list(range(len(nwbfile.electrodes)))
ecog_elecs_region = nwbfile.create_electrode_table_region(
ecog_elecs, 'ECoG electrodes on brain')
# Read electrophysiology data from HTK files and add them to NWB file
if ecog_format == 'auto':
ecog_rate, data, ecog_path = auto_ecog(blockpath,
ecog_elecs,
verbose=False)
elif ecog_format == 'htk':
if verbose:
print('reading htk acquisition...', flush=True)
ecog_rate, data = readhtks(ecog_path, ecog_elecs)
data = data.squeeze()
if verbose:
print('done', flush=True)
elif ecog_format == 'mat':
with File(ecog400_path, 'r') as f:
data = f['ecogDS']['data'][:, ecog_elecs]
ecog_rate = f['ecogDS']['sampFreq'][:].ravel()[0]
ecog_path = ecog400_path
elif ecog_format == 'raw':
ecog_path = os.path.join(tdt_data_path, subject_id, blockname,
'raw.mat')
ecog_rate, data = load_wavs(ecog_path)
else:
raise ValueError('unrecognized argument: ecog_format')
ts_desc = "all Wav data"
if mini:
data = data[:2000]
ecog_ts = ElectricalSeries(name='ElectricalSeries',
data=H5DataIO(data, compression='gzip'),
electrodes=ecog_elecs_region,
rate=ecog_rate,
description=ts_desc,
conversion=0.001)
nwbfile.add_acquisition(ecog_ts)
if include_ekg:
ekg_elecs = find_ekg_elecs(elec_metadata_file)
if len(ekg_elecs):
add_ekg(nwbfile, ecog_path, ekg_elecs)
if mic:
# Add microphone recording from room
fs, data = get_analog(blockpath, 1)
nwbfile.add_acquisition(
TimeSeries('microphone',
data,
'audio unit',
rate=fs,
description="audio recording from microphone in room"))
if speakers:
fs, data = get_analog(blockpath, 2)
# Add audio stimulus 1
nwbfile.add_stimulus(
TimeSeries('speaker 1',
data,
'NA',
rate=fs,
description="audio stimulus 1"))
# Add audio stimulus 2
fs, data = get_analog(blockpath, 3)
if fs is not None:
nwbfile.add_stimulus(
TimeSeries('speaker 2',
data,
'NA',
rate=fs,
description='the second stimulus source'))
if anin4:
fs, data = get_analog(blockpath, 4)
nwbfile.add_acquisition(
TimeSeries(anin4,
data,
'aux unit',
rate=fs,
description="aux analog recording"))
# Add bad time segments
if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size:
bad_time = sio.loadmat(bad_time_file)['badTimeSegments']
for row in bad_time:
nwbfile.add_invalid_time_interval(start_time=row[0],
stop_time=row[1],
tags=('ECoG artifact', ),
timeseries=ecog_ts)
if rest_period is not None:
nwbfile.add_epoch_column(name='label', description='label')
nwbfile.add_epoch(start_time=rest_period[0],
stop_time=rest_period[1],
label='rest_period')
if hilb:
block_hilb_path = os.path.join(hilb_dir, subject_id, blockname,
blockname + '_AA.h5')
file = File(block_hilb_path, 'r')
data = transpose_iter(
file['X']) # transposes data during iterative write
filter_center = file['filter_center'][:]
filter_sigma = file['filter_sigma'][:]
data = H5DataIO(DataChunkIterator(tqdm(data,
desc='writing hilbert data'),
buffer_size=400 * 20),
compression='gzip')
decomp_series = DecompositionSeries(
name='LFPDecompositionSeries',
description='Gaussian band Hilbert transform',
data=data,
rate=400.,
source_timeseries=ecog_ts,
metric='amplitude')
for band_mean, band_stdev in zip(filter_center, filter_sigma):
decomp_series.add_band(band_mean=band_mean, band_stdev=band_stdev)
hilb_mod = nwbfile.create_processing_module(
name='ecephys', description='holds hilbert analysis results')
hilb_mod.add_container(decomp_series)
if include_cortical_surfaces:
subject = ECoGSubject(subject_id=subject_id)
subject.cortical_surfaces = create_cortical_surfaces(
pial_files, subject_id)
else:
subject = Subject(subject_id=subject_id, species='H**o sapiens')
if subject_image_list is not None:
subject = add_images_to_subject(subject, subject_image_list)
if external_subject:
subj_fpath = path.join(out_base_path, subject_id + '.nwb')
if not os.path.isfile(subj_fpath):
subj_nwbfile = NWBFile(session_description=subject_id,
identifier=subject_id,
subject=subject,
session_start_time=datetime(
1900, 1, 1).astimezone(timezone('UTC')))
with NWBHDF5IO(subj_fpath, manager=manager, mode='w') as subj_io:
subj_io.write(subj_nwbfile)
subj_read_io = NWBHDF5IO(subj_fpath, manager=manager, mode='r')
subj_nwbfile = subj_read_io.read()
subject = subj_nwbfile.subject
nwbfile.subject = subject
if parse_transcript:
if parse_transcript == 'CV':
parseout = parse(blockpath, blockname)
df = make_df(parseout, 0, subject_id, align_pos=1)
nwbfile.add_trial_column('cv_transition_time',
'time of CV transition in seconds')
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
nwbfile.add_trial_column('condition', 'syllable spoken')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
cv_transition_time=row['align'],
speak=row['mode'] == 'speak',
condition=row['label'])
elif parse_transcript == 'singing':
parseout = parse(blockpath, blockname)
df = make_df(parseout, 0, subject_id, align_pos=0)
if not len(df):
df = pd.DataFrame(parseout)
df['mode'] = 'speak'
df = df.loc[df['label'].astype('bool'), :] # handle empty labels
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
nwbfile.add_trial_column('condition', 'syllable spoken')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
speak=row['mode'] == 'speak',
condition=row['label'])
elif parse_transcript == 'emphasis':
parseout = parse(blockpath, blockname)
try:
df = make_df(parseout, 0, subject_id, align_pos=0)
except:
df = pd.DataFrame(parseout)
if not len(df):
df = pd.DataFrame(parseout)
df = df.loc[df['label'].astype('bool'), :] # handle empty labels
nwbfile.add_trial_column('condition', 'word emphasized')
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
speak=True,
condition=row['label'])
elif parse_transcript == 'MOCHA':
nwbfile = create_transcription(nwbfile, transcript_path, blockname)
# behavior
if include_pitch:
if behav_module is None:
behav_module = nwbfile.create_processing_module(
'behavior', 'processing about behavior')
if os.path.isfile(
os.path.join(blockpath, 'pitch_' + blockname + '.mat')):
fs, data = load_pitch(blockpath)
pitch_ts = TimeSeries(
data=data,
rate=fs,
unit='Hz',
name='pitch',
description=
'Pitch as extracted from Praat. NaNs mark unvoiced regions.')
behav_module.add_container(
BehavioralTimeSeries(name='pitch', time_series=pitch_ts))
else:
print('No pitch file for ' + blockname)
if include_intensity:
if behav_module is None:
behav_module = nwbfile.create_processing_module(
'behavior', 'processing about behavior')
if os.path.isfile(
os.path.join(blockpath, 'intensity_' + blockname + '.mat')):
fs, data = load_pitch(blockpath)
intensity_ts = TimeSeries(
data=data,
rate=fs,
unit='dB',
name='intensity',
description='Intensity of speech in dB extracted from Praat.')
behav_module.add_container(
BehavioralTimeSeries(name='intensity',
time_series=intensity_ts))
else:
print('No intensity file for ' + blockname)
# Export the NWB file
with NWBHDF5IO(outpath, manager=manager, mode='w') as io:
io.write(nwbfile)
if external_subject:
subj_read_io.close()
if hilb:
file.close()
# read check
with NWBHDF5IO(outpath, manager=manager, mode='r') as io:
io.read()
