[0].keys() if tag not in acquisition.TrialStimInfo.primary_key } # Add entry to the trial-table for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True): photostim_tag = (acquisition.TrialStimInfo & trial).fetch(as_dict=True) trial_tag_value = { **trial, **photostim_tag[0] } if len(photostim_tag) == 1 else { **trial, **photostim_tag_default } # rename 'trial_id' to 'id' trial_tag_value['id'] = trial_tag_value['trial_id'] [ trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key ] nwbfile.add_trial(**trial_tag_value) # =============== Write NWB 2.0 file =============== save_path = os.path.join('data', 'NWB 2.0') save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(save_path): os.makedirs(save_path) with NWBHDF5IO(os.path.join(save_path, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}')
#session: 'TWH103_111918' NOID = 144 #Get all Spike Times allSpikeTimes = demoHelper.getSpikeTimesAllCells(...) for cluster in allSpikeTimes.keys(): # Add spike times to Units Table nwb.add_unit(id=int(cluster), spike_times=allSpikeTimes[cluster][0]) #Add Trial(s) information (Learning) events_learn_stim_on = [4.218311e+09, 4.222039e+09, ...] events_learn_stim_off = [4.219308e+09, 4.223034e+09, ...] ... for i in range(len(events_learn_stim_on)): nwb.add_trial(stim_on=events_learn_stim_on[i], stim_off=events_learn_stim_off[i], ...) #Add Trial(s) information (Recognition) events_recog_stim_on = [5.222291e+09, 5.229686e+09, ...] events_recog_stim_off = [5.223292e+09, 5.230683e+09, ...] ... for i in range(len(events_recog_stim_on)): nwb.add_trial(stim_on=events_recog_stim_on[i], stim_off=events_recog_stim_off[i], ...) io = NWBHDF5IO('TWH103_111918.nwb', mode='w') io.write(nwb) io.close()
def run_network(model, env, run_trial, num_trial=1000, file=None): """Run trained networks for analysis on trial-based tasks. Args: model: model of arbitrary format, must provide a run_one_trial function that works with it env: neurogym environment run_trial: function handle for running model for one trial, takes (model, env) as inputs and returns (model, env, activity, trial_info), where activity has shape (N_time, N_unit) num_trial: int, number of trials to run file: str or None, file name to save Returns: activity: a list of activity matrices, each matrix has shape ( N_time, N_neuron) info: pandas dataframe, each row is information of a trial config: dict of network, training configurations """ env.reset(no_step=True) # Make NWB file nwbfile = NWBFile( session_description=str(env), # required identifier='NWB_default', # required session_start_time=datetime.datetime.now(), # required file_create_date=datetime.datetime.now()) info = pd.DataFrame() spike_times = defaultdict(list) start_time = 0. for i in range(num_trial): model, env, hidden, trial_info = run_trial(model, env) # Log trial info for key, val in env.start_t.items(): # NWB time default unit is second, ngym default is ms trial_info['start_' + key] = val / 1000. + start_time for key, val in env.end_t.items(): trial_info['end_' + key] = val / 1000. + start_time info = info.append(trial_info, ignore_index=True) # Store results to NWB file if i == 0: for key in trial_info.keys(): nwbfile.add_trial_column(name=key, description=key) stop_time = start_time + hidden.shape[0] * env.dt / 1000. # Generate simulated spikes from rates scale_rate = 10. for j in range(hidden.shape[-1]): spikes = sample_spikes(hidden[:, j] * scale_rate, dt=env.dt / 1000.) + start_time spike_times[j].append(spikes) nwbfile.add_trial(start_time=start_time, stop_time=stop_time, **trial_info) start_time = stop_time # Assuming continous trials try: print('Average performance', np.mean(info['correct'])) except: pass for j in range(hidden.shape[-1]): # For each neuron nwbfile.add_unit(id=j, spike_times=np.concatenate(spike_times[j])) # TODO: Check why the file.units['spike_times'] is weird if file is None: file = str(get_modelpath(envid) / (envid + '.nwb')) with pynwb.NWBHDF5IO(file, 'w') as io: io.write(nwbfile)
start_time = np.full(outcomes.shape, np.nan) stop_time = np.full(outcomes.shape, np.nan) else: alldata_frameTimes = postmat['alldata_frameTimes'] # timestamps of each trial for all trials start_time = [t[0] for t in alldata_frameTimes] stop_time = [t[-1] for t in alldata_frameTimes] # - now insert each trial into trial table for k in range(outcomes.size): nwbfile.add_trial( start_time=start_time[k], stop_time=stop_time[k], trial_type=('High-rate' if postmat['stimrate'][k] >= 16 else 'Low-rate'), trial_pulse_rate=postmat['stimrate'][k], trial_response=trial_response_dict[outcomes[k]], trial_is_good=(outcomes[k] >= 0), init_tone=init_tone[k], stim_onset=postmat['timeStimOnsetAll'][k], stim_offset=postmat['timeSingleStimOffset'][k], go_tone=postmat['timeCommitCL_CR_Gotone'][k], first_commit=postmat['time1stSideTry'][k], second_commit=second_commit_times[k]) # ------ Image Segmentation processing module ------ img_seg_mod = nwbfile.create_processing_module( 'Image-Segmentation', 'Plane segmentation and ROI identification') img_segmentation = nwb_ophys.ImageSegmentation(name='img_seg') img_seg_mod.add_data_interface(img_segmentation) plane_segmentation = nwb_ophys.PlaneSegmentation( name='pln_seg', description='description here',
def nwb_copy_file(old_file, new_file, cp_objs={}): """ Copy fields defined in 'obj', from existing NWB file to new NWB file. Parameters ---------- old_file : str, path String such as '/path/to/old_file.nwb'. new_file : str, path String such as '/path/to/new_file.nwb'. cp_objs : dict Name:Value pairs (Group:Children) listing the groups and respective children from the current NWB file to be copied. Children can be: - Boolean, indicating an attribute (e.g. for institution, lab) - List of strings, containing several children names Example: {'institution':True, 'lab':True, 'acquisition':['microphone'], 'ecephys':['LFP','DecompositionSeries']} """ manager = get_manager() # Open original signal file with NWBHDF5IO(old_file, 'r', manager=manager, load_namespaces=True) as io1: nwb_old = io1.read() # Creates new file nwb_new = NWBFile(session_description=str(nwb_old.session_description), identifier='', session_start_time=datetime.now(tzlocal())) with NWBHDF5IO(new_file, mode='w', manager=manager, load_namespaces=False) as io2: # Institution name ------------------------------------------------ if 'institution' in cp_objs: nwb_new.institution = str(nwb_old.institution) # Lab name -------------------------------------------------------- if 'lab' in cp_objs: nwb_new.lab = str(nwb_old.lab) # Session id ------------------------------------------------------ if 'session' in cp_objs: nwb_new.session_id = nwb_old.session_id # Devices --------------------------------------------------------- if 'devices' in cp_objs: for aux in list(nwb_old.devices.keys()): dev = Device(nwb_old.devices[aux].name) nwb_new.add_device(dev) # Electrode groups ------------------------------------------------ if 'electrode_groups' in cp_objs: for aux in list(nwb_old.electrode_groups.keys()): nwb_new.create_electrode_group( name=str(nwb_old.electrode_groups[aux].name), description=str(nwb_old.electrode_groups[ aux].description), location=str(nwb_old.electrode_groups[aux].location), device=nwb_new.get_device( nwb_old.electrode_groups[aux].device.name) ) # Electrodes ------------------------------------------------------ if 'electrodes' in cp_objs: nElec = len(nwb_old.electrodes['x'].data[:]) for aux in np.arange(nElec): nwb_new.add_electrode( x=nwb_old.electrodes['x'][aux], y=nwb_old.electrodes['y'][aux], z=nwb_old.electrodes['z'][aux], imp=nwb_old.electrodes['imp'][aux], location=str(nwb_old.electrodes['location'][aux]), filtering=str(nwb_old.electrodes['filtering'][aux]), group=nwb_new.get_electrode_group( nwb_old.electrodes['group'][aux].name), group_name=str(nwb_old.electrodes['group_name'][aux]) ) # if there are custom variables new_vars = list(nwb_old.electrodes.colnames) default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering', 'group', 'group_name'] [new_vars.remove(var) for var in default_vars] for var in new_vars: if var == 'label': var_data = [str(elem) for elem in nwb_old.electrodes[ var].data[:]] else: var_data = np.array(nwb_old.electrodes[var].data[:]) nwb_new.add_electrode_column(name=str(var), description= str(nwb_old.electrodes[ var].description), data=var_data) # Epochs ---------------------------------------------------------- if 'epochs' in cp_objs: nEpochs = len(nwb_old.epochs['start_time'].data[:]) for i in np.arange(nEpochs): nwb_new.add_epoch( start_time=nwb_old.epochs['start_time'].data[i], stop_time=nwb_old.epochs['stop_time'].data[i]) # if there are custom variables new_vars = list(nwb_old.epochs.colnames) default_vars = ['start_time', 'stop_time', 'tags', 'timeseries'] [new_vars.remove(var) for var in default_vars if var in new_vars] for var in new_vars: nwb_new.add_epoch_column(name=var, description=nwb_old.epochs[ var].description, data=nwb_old.epochs[var].data[:]) # Invalid times --------------------------------------------------- if 'invalid_times' in cp_objs: nInvalid = len(nwb_old.invalid_times['start_time'][:]) for aux in np.arange(nInvalid): nwb_new.add_invalid_time_interval( start_time=nwb_old.invalid_times['start_time'][aux], stop_time=nwb_old.invalid_times['stop_time'][aux]) # Trials ---------------------------------------------------------- if 'trials' in cp_objs: nTrials = len(nwb_old.trials['start_time']) for aux in np.arange(nTrials): nwb_new.add_trial( start_time=nwb_old.trials['start_time'][aux], stop_time=nwb_old.trials['stop_time'][aux]) # if there are custom variables new_vars = list(nwb_old.trials.colnames) default_vars = ['start_time', 'stop_time'] [new_vars.remove(var) for var in default_vars] for var in new_vars: nwb_new.add_trial_column(name=var, description=nwb_old.trials[ var].description, data=nwb_old.trials[var].data[:]) # Intervals ------------------------------------------------------- if 'intervals' in cp_objs: all_objs_names = list(nwb_old.intervals.keys()) for obj_name in all_objs_names: obj_old = nwb_old.intervals[obj_name] # create and add TimeIntervals obj = TimeIntervals(name=obj_old.name, description=obj_old.description) nInt = len(obj_old['start_time']) for ind in np.arange(nInt): obj.add_interval(start_time=obj_old['start_time'][ind], stop_time=obj_old['stop_time'][ind]) # Add to file nwb_new.add_time_intervals(obj) # Stimulus -------------------------------------------------------- if 'stimulus' in cp_objs: all_objs_names = list(nwb_old.stimulus.keys()) for obj_name in all_objs_names: obj_old = nwb_old.stimulus[obj_name] obj = TimeSeries(name=obj_old.name, description=obj_old.description, data=obj_old.data[:], rate=obj_old.rate, resolution=obj_old.resolution, conversion=obj_old.conversion, starting_time=obj_old.starting_time, unit=obj_old.unit) nwb_new.add_stimulus(obj) # Processing modules ---------------------------------------------- if 'ecephys' in cp_objs: if cp_objs['ecephys'] is True: interfaces = nwb_old.processing[ 'ecephys'].data_interfaces.keys() else: # list of items interfaces = [ nwb_old.processing['ecephys'].data_interfaces[key] for key in cp_objs['ecephys'] ] # Add ecephys module to NWB file ecephys_module = ProcessingModule( name='ecephys', description='Extracellular electrophysiology data.' ) nwb_new.add_processing_module(ecephys_module) for interface_old in interfaces: obj = copy_obj(interface_old, nwb_old, nwb_new) if obj is not None: ecephys_module.add_data_interface(obj) # Acquisition ----------------------------------------------------- if 'acquisition' in cp_objs: if cp_objs['acquisition'] is True: all_acq_names = list(nwb_old.acquisition.keys()) else: # list of items all_acq_names = cp_objs['acquisition'] for acq_name in all_acq_names: obj_old = nwb_old.acquisition[acq_name] obj = copy_obj(obj_old, nwb_old, nwb_new) if obj is not None: nwb_new.add_acquisition(obj) # Subject --------------------------------------------------------- if 'subject' in cp_objs: try: cortical_surfaces = CorticalSurfaces() surfaces = nwb_old.subject.cortical_surfaces.surfaces for sfc in list(surfaces.keys()): cortical_surfaces.create_surface( name=surfaces[sfc].name, faces=surfaces[sfc].faces, vertices=surfaces[sfc].vertices) nwb_new.subject = ECoGSubject( cortical_surfaces=cortical_surfaces, subject_id=nwb_old.subject.subject_id, age=nwb_old.subject.age, description=nwb_old.subject.description, genotype=nwb_old.subject.genotype, sex=nwb_old.subject.sex, species=nwb_old.subject.species, weight=nwb_old.subject.weight, date_of_birth=nwb_old.subject.date_of_birth) except: nwb_new.subject = Subject(age=nwb_old.subject.age, description=nwb_old.subject.description, genotype=nwb_old.subject.genotype, sex=nwb_old.subject.sex, species=nwb_old.subject.species, subject_id=nwb_old.subject.subject_id, weight=nwb_old.subject.weight, date_of_birth=nwb_old.subject.date_of_birth) # Write new file with copied fields io2.write(nwb_new, link_data=False)
def conversion_function(source_paths, f_nwb, metadata, add_raw=False, add_processed=True, add_behavior=True, plot_rois=False): """ Copy data stored in a set of .npz files to a single NWB file. Parameters ---------- source_paths : dict Dictionary with paths to source files/directories. e.g.: {'raw_data': {'type': 'file', 'path': ''}, 'raw_info': {'type': 'file', 'path': ''} 'processed_data': {'type': 'file', 'path': ''}, 'sparse_matrix': {'type': 'file', 'path': ''}, 'ref_image',: {'type': 'file', 'path': ''}} f_nwb : str Path to output NWB file, e.g. 'my_file.nwb'. metadata : dict Metadata dictionary add_raw : bool Whether to convert raw data or not. add_processed : bool Whether to convert processed data or not. add_behavior : bool Whether to convert behavior data or not. plot_rois : bool Plot ROIs """ # Source files file_raw = None file_info = None file_processed = None file_sparse_matrix = None file_reference_image = None for k, v in source_paths.items(): if source_paths[k]['path'] != '': fname = source_paths[k]['path'] if k == 'raw_data': file_raw = h5py.File(fname, 'r') if k == 'raw_info': file_info = scipy.io.loadmat(fname, struct_as_record=False, squeeze_me=True) if k == 'processed_data': file_processed = np.load(fname) if k == 'sparse_matrix': file_sparse_matrix = np.load(fname) if k == 'ref_image': file_reference_image = np.load(fname) # Initialize a NWB object nwb = NWBFile(**metadata['NWBFile']) # Create and add device device = Device(name=metadata['Ophys']['Device'][0]['name']) nwb.add_device(device) # Creates one Imaging Plane for each channel fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0]) for meta_ip in metadata['Ophys']['ImagingPlane']: # Optical channel opt_ch = OpticalChannel( name=meta_ip['optical_channel'][0]['name'], description=meta_ip['optical_channel'][0]['description'], emission_lambda=meta_ip['optical_channel'][0]['emission_lambda']) nwb.create_imaging_plane( name=meta_ip['name'], optical_channel=opt_ch, description=meta_ip['description'], device=device, excitation_lambda=meta_ip['excitation_lambda'], imaging_rate=fs, indicator=meta_ip['indicator'], location=meta_ip['location'], ) # Raw optical data if add_raw: print('Adding raw data...') for meta_tps in metadata['Ophys']['TwoPhotonSeries']: if meta_tps['name'][-1] == 'R': raw_data = file_raw['R'] else: raw_data = file_raw['Y'] def data_gen(data): xl, yl, zl, tl = data.shape chunk = 0 while chunk < tl: val = data[:, :, :, chunk] chunk += 1 print('adding data chunk: ', chunk) yield val xl, yl, zl, tl = raw_data.shape tps_data = DataChunkIterator(data=data_gen(data=raw_data), iter_axis=0, maxshape=(tl, xl, yl, zl)) # Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard #raw_data = np.moveaxis(raw_data, -1, 0) tps = TwoPhotonSeries( name=meta_tps['name'], imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']], data=tps_data, rate=file_info['info'].daq.scanRate) nwb.add_acquisition(tps) # Processed data if add_processed: print('Adding processed data...') ophys_module = ProcessingModule( name='Ophys', description='contains optical physiology processed data.', ) nwb.add_processing_module(ophys_module) # Create Image Segmentation compartment img_seg = ImageSegmentation( name=metadata['Ophys']['ImageSegmentation']['name']) ophys_module.add(img_seg) # Create plane segmentation and add ROIs meta_ps = metadata['Ophys']['ImageSegmentation'][ 'plane_segmentations'][0] ps = img_seg.create_plane_segmentation( name=meta_ps['name'], description=meta_ps['description'], imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']], ) # Add ROIs indices = file_sparse_matrix['indices'] indptr = file_sparse_matrix['indptr'] dims = np.squeeze(file_processed['dims']) for start, stop in zip(indptr, indptr[1:]): voxel_mask = make_voxel_mask(indices[start:stop], dims) ps.add_roi(voxel_mask=voxel_mask) # Visualize 3D voxel masks if plot_rois: plot_rois_function(plane_segmentation=ps, indptr=indptr) # DFF measures dff = DfOverF(name=metadata['Ophys']['DfOverF']['name']) ophys_module.add(dff) # create ROI regions n_cells = file_processed['dFF'].shape[0] roi_region = ps.create_roi_table_region(description='RoiTableRegion', region=list(range(n_cells))) # create ROI response series dff_data = file_processed['dFF'] tt = file_processed['time'].ravel() meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0] meta_rrs['data'] = dff_data.T meta_rrs['rois'] = roi_region meta_rrs['timestamps'] = tt dff.create_roi_response_series(**meta_rrs) # Creates GrayscaleVolume containers and add a reference image grayscale_volume = GrayscaleVolume( name=metadata['Ophys']['GrayscaleVolume']['name'], data=file_reference_image['im']) ophys_module.add(grayscale_volume) # Behavior data if add_behavior: print('Adding behavior data...') # Ball motion behavior_mod = nwb.create_processing_module( name='Behavior', description='holds processed behavior data', ) meta_ts = metadata['Behavior']['TimeSeries'][0] meta_ts['data'] = file_processed['ball'].ravel() tt = file_processed['time'].ravel() meta_ts['timestamps'] = tt behavior_ts = TimeSeries(**meta_ts) behavior_mod.add(behavior_ts) # Re-arranges spatial data of body-points positions tracking pos = file_processed['dlc'] n_points = 8 pos_reshaped = pos.reshape( (-1, n_points, 3)) # dims=(nSamples,n_points,3) # Creates a Position object and add one SpatialSeries for each body-point position position = Position() for i in range(n_points): position.create_spatial_series( name='SpatialSeries_' + str(i), data=pos_reshaped[:, i, :], timestamps=tt, reference_frame= 'Description defining what the zero-position is.', conversion=np.nan) behavior_mod.add(position) # Trial times trialFlag = file_processed['trialFlag'].ravel() trial_inds = np.hstack( (0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1)) trial_times = tt[trial_inds] for start, stop in zip(trial_times, trial_times[1:]): nwb.add_trial(start_time=start, stop_time=stop) # Saves to NWB file with NWBHDF5IO(f_nwb, mode='w') as io: io.write(nwb) print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def run_conversion(self, nwbfile: NWBFile, metadata: dict): mat_file_path = self.source_data["mat_file_path"] mat_file = loadmat(mat_file_path) trial_info = mat_file["SessionNP"] nwbfile.add_trial_column( name="reward_time", description="Time when subject began consuming reward.") nwbfile.add_trial_column( name="left_or_right", description="Time when subject began consuming reward.") l_r_dict = {1: "Right", 2: "Left"} for trial in trial_info: nwbfile.add_trial(start_time=trial[0], stop_time=trial[1], reward_time=trial[2], left_or_right=l_r_dict[int(trial[3])]) # Position pos_info = mat_file["whlrl"] pos_data = [pos_info[:, 0:1], pos_info[:, 2:3]] starting_time = 0.0 rate = 20000 / 512 # from CRCNS info conversion = np.nan # whl are arbitrary units pos_obj = Position(name="Position") for j in range(2): spatial_series_object = SpatialSeries( name=f"SpatialSeries{j+1}", description= "(x,y) coordinates tracking subject movement through the maze.", data=H5DataIO(pos_data[j], compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) pos_obj.add_spatial_series(spatial_series_object) get_module(nwbfile=nwbfile, name="behavior", description="Contains processed behavioral data." ).add_data_interface(pos_obj) linearized_pos = mat_file["whlrld"][:, 6] lin_pos_obj = Position(name="LinearizedPosition") lin_spatial_series_object = SpatialSeries( name="LinearizedTimeSeries", description= ("Linearized position, with '1' defined as start position (the position at the time of last nose-poking " "in the trial), and d=2 being the end position (position at the tiome just before reward consumption). " "d=0 means subject is not performing working memory trials."), data=H5DataIO(linearized_pos, compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) lin_pos_obj.add_spatial_series(lin_spatial_series_object) get_module(nwbfile=nwbfile, name="behavior").add_data_interface(lin_pos_obj)
data=sweepDAC, electrode=elec, unit='mV', rate=10e4, gain=0.00, starting_time=0.0, bias_current=np.nan, bridge_balance=np.nan, capacitance_compensation=np.nan) nwbFile.add_trial(start_time=float(start), stop_time=float(end), Stimulus=stimulus, Acquisition=acquisition, CellNo=metadata['Cell No.'], RMP=metadata['RMP'], Layer=metadata['Layer'], Tau=metadata['Tau'], Gain=metadata['Gain'], DC=metadata['DC'], Date=metadata['Session Date']) start += 1 end += 1 output = sys.argv[2] io = NWBHDF5IO(join( output, str(sessionDate) + ' - ' + metadata['Cell No.'] + '.nwb'), mode='w') io.write(nwbFile)
def run_conversion(self, nwbfile: NWBFile, metadata: dict): """ Run conversion for this data interface. Reads labview experiment behavioral data and adds it to nwbfile. Parameters ---------- nwbfile : NWBFile metadata : dict """ print("Converting Labview data...") # Get list of trial summary files dir_behavior_labview = self.source_data['dir_behavior_labview'] all_files = os.listdir(dir_behavior_labview) trials_files = [f for f in all_files if '_sum.txt' in f] trials_files.sort() # Get session_start_time from first file timestamps fpath = os.path.join(dir_behavior_labview, trials_files[0]) colnames = [ 'Trial', 'StartT', 'EndT', 'Result', 'InitT', 'SpecificResults', 'ProbLeft', 'OptoDur', 'LRew', 'RRew', 'InterT', 'LTrial', 'ReactionTime', 'OptoCond', 'OptoTrial' ] df_0 = pd.read_csv(fpath, sep='\t', index_col=False, names=colnames) t0 = df_0['StartT'][0] # initial time in Labview seconds # Add trials print("Converting Labview trials data...") if nwbfile.trials is not None: print( 'Trials already exist in current nwb file. Labview behavior trials not added.' ) else: # Make dataframe frames = [] for f in trials_files: fpath = os.path.join(dir_behavior_labview, f) frames.append( pd.read_csv(fpath, sep='\t', index_col=False, names=colnames)) df_trials_summary = pd.concat(frames) nwbfile.add_trial_column( name='results', description= "0 means sucess (rewarded trial), 1 means licks during intitial " "period, which leads to a failed trial. 2 means early lick failure. 3 means " "wrong lick or no response.") nwbfile.add_trial_column( name='init_t', description="duration of initial delay period.") nwbfile.add_trial_column( name='specific_results', description= "Possible outcomes classified based on raw data & meta file (_tr.m)." ) nwbfile.add_trial_column( name='prob_left', description= "probability for left trials in order to keep the number of " "left and right trials balanced within the session. ") nwbfile.add_trial_column( name='opto_dur', description="the duration of optical stimulation.") nwbfile.add_trial_column( name='l_rew_n', description="counting the number of left rewards.") nwbfile.add_trial_column( name='r_rew_n', description="counting the number of rightrewards.") nwbfile.add_trial_column(name='inter_t', description="inter-trial delay period.") nwbfile.add_trial_column( name='l_trial', description= "trial type (which side the air-puff is applied). 1 means " "left-trial, 0 means right-trial") nwbfile.add_trial_column( name='reaction_time', description= "if it is a successful trial or wrong lick during response " "period trial: ReactionTime = time between the first decision " "lick and the beginning of the response period. If it is a failed " "trial due to early licks: reaction time = the duration of " "the air-puff period (in other words, when the animal licks " "during the sample period).") nwbfile.add_trial_column( name='opto_cond', description="0: no opto. 1: opto is on during sample period. " "2: opto is on half way through the sample period (0.5s) " "and 0.5 during the response period. 3. opto is on during " "the response period.") nwbfile.add_trial_column( name='opto_trial', description="1: opto trials. 0: Non-opto trials.") for index, row in df_trials_summary.iterrows(): nwbfile.add_trial( start_time=row['StartT'] - t0, stop_time=row['EndT'] - t0, results=int(row['Result']), init_t=row['InitT'], specific_results=int(row['SpecificResults']), prob_left=row['ProbLeft'], opto_dur=row['OptoDur'], l_rew_n=int(row['LRew']), r_rew_n=int(row['RRew']), inter_t=row['InterT'], l_trial=int(row['LTrial']), reaction_time=int(row['ReactionTime']), opto_cond=int(row['OptoCond']), opto_trial=int(row['OptoTrial']), ) # Get list of files: continuous data continuous_files = [f.replace('_sum', '') for f in trials_files] # Adds continuous behavioral data frames = [] for f in continuous_files: fpath_lick = os.path.join(dir_behavior_labview, f) frames.append(pd.read_csv(fpath_lick, sep='\t', index_col=False)) df_continuous = pd.concat(frames) # Behavioral data print("Converting Labview behavior data...") l1_ts = TimeSeries(name="left_lick", data=df_continuous['Lick 1'].to_numpy(), timestamps=df_continuous['Time'].to_numpy() - t0, description="no description") l2_ts = TimeSeries(name="right_lick", data=df_continuous['Lick 2'].to_numpy(), timestamps=df_continuous['Time'].to_numpy() - t0, description="no description") nwbfile.add_acquisition(l1_ts) nwbfile.add_acquisition(l2_ts) # Optogenetics stimulation data print("Converting Labview optogenetics data...") ogen_device = nwbfile.create_device( name=metadata['Ogen']['Device']['name'], description=metadata['Ogen']['Device']['description']) meta_ogen_site = metadata['Ogen']['OptogeneticStimulusSite'] ogen_stim_site = OptogeneticStimulusSite( name=meta_ogen_site['name'], device=ogen_device, description=meta_ogen_site['description'], excitation_lambda=float(meta_ogen_site['excitation_lambda']), location=meta_ogen_site['location']) nwbfile.add_ogen_site(ogen_stim_site) meta_ogen_series = metadata['Ogen']['OptogeneticSeries'] ogen_series = OptogeneticSeries( name=meta_ogen_series['name'], data=df_continuous['Opto'].to_numpy(), site=ogen_stim_site, timestamps=df_continuous['Time'].to_numpy() - t0, description=meta_ogen_series['description'], ) nwbfile.add_stimulus(ogen_series)
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=True): this_session = (acquisition.Session & session_key).fetch1() identifier = '_'.join([ this_session['subject_id'], this_session['session_time'].strftime('%Y-%m-%d'), str(this_session['session_id']) ]) # =============== General ==================== # -- NWB file - a NWB2.0 file for each session nwbfile = NWBFile(session_description=this_session['session_note'], identifier=identifier, session_start_time=this_session['session_time'], file_create_date=datetime.now(tzlocal()), experimenter='; '.join( (acquisition.Session.Experimenter & session_key).fetch('experimenter')), institution=institution, experiment_description=experiment_description, related_publications=related_publications, keywords=keywords) # -- subject subj = (subject.Subject & session_key).fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=this_session['subject_id'], description=subj['subject_description'], sex=subj['sex'], species=subj['species']) # =============== Extracellular ==================== probe_insertion = ((extracellular.ProbeInsertion & session_key).fetch1() if extracellular.ProbeInsertion & session_key else None) if probe_insertion: probe = nwbfile.create_device(name=probe_insertion['probe_name']) electrode_group = nwbfile.create_electrode_group(name='; '.join([ f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}' ]), description='N/A', device=probe, location='; '.join([ f'{k}: {str(v)}' for k, v in (reference. BrainLocation & probe_insertion ).fetch1().items( ) ])) for chn in (reference.Probe.Channel & probe_insertion).fetch(as_dict=True): nwbfile.add_electrode( id=chn['channel_id'], group=electrode_group, filtering=hardware_filter, imp=np.nan, x=np.nan, # not available from data y=np.nan, # not available from data z=np.nan, # not available from data location=electrode_group.location) # --- unit spike times --- nwbfile.add_unit_column(name='depth', description='depth this unit (um)') nwbfile.add_unit_column( name='quality', description= 'quality of the spike sorted unit (e.g. excellent, good, poor, fair, etc.)' ) nwbfile.add_unit_column( name='cell_type', description='cell type (e.g. PTlower, PTupper)') for unit in (extracellular.UnitSpikeTimes & probe_insertion).fetch(as_dict=True): # make an electrode table region (which electrode(s) is this unit coming from) unit_chn = unit['channel_id'] if isinstance( unit['channel_id'], np.ndarray) else [unit['channel_id']] nwbfile.add_unit(id=unit['unit_id'], electrodes=np.where( np.in1d(np.array(nwbfile.electrodes.id.data), unit_chn))[0], depth=unit['unit_depth'], quality=unit['unit_quality'], cell_type=unit['unit_cell_type'], spike_times=unit['spike_times']) # =============== Behavior ==================== behavior_data = ((behavior.LickTimes & session_key).fetch1() if behavior.LickTimes & session_key else None) if behavior_data: behav_acq = pynwb.behavior.BehavioralEvents(name='lick_times') nwbfile.add_acquisition(behav_acq) [behavior_data.pop(k) for k in behavior.LickTimes.primary_key] for b_k, b_v in behavior_data.items(): behav_acq.create_timeseries(name=b_k, unit='a.u.', conversion=1.0, data=np.full_like(b_v, 1).astype(bool), timestamps=b_v) # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: # 'id', 'start_time' and 'stop_time'. # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) # adjust trial event times to be relative to session's start time q_trial_event = (acquisition.TrialSet.EventTime * acquisition.TrialSet.Trial.proj('start_time')).proj( event_time='event_time + start_time') if acquisition.TrialSet & session_key: # Get trial descriptors from TrialSet.Trial and TrialStimInfo trial_columns = [{ 'name': tag.replace('trial_', ''), 'description': re.search( f'(?<={tag})(.*)#(.*)', str(acquisition.TrialSet.Trial.heading)).groups()[-1].strip() } for tag in acquisition.TrialSet.Trial.heading.names if tag not in acquisition.TrialSet.Trial.primary_key + ['start_time', 'stop_time']] # Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time # also add `_time` suffix to all events trial_events = set(((acquisition.TrialSet.EventTime & session_key) - [{ 'trial_event': 'trial_start' }, { 'trial_event': 'trial_stop' }]).fetch('trial_event')) event_names = [{ 'name': e + '_time', 'description': d } for e, d in zip(*(reference.ExperimentalEvent & [{ 'event': k } for k in trial_events]).fetch('event', 'description'))] # Add new table columns to nwb trial-table for trial-label for c in trial_columns + event_names: nwbfile.add_trial_column(**c) # Add entry to the trial-table for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True): events = dict( zip(*(q_trial_event & trial & [{ 'trial_event': e } for e in trial_events]).fetch('trial_event', 'event_time'))) trial_tag_value = { **trial, **events, 'stop_time': np.nan } # No stop_time available for this dataset trial_tag_value['id'] = trial_tag_value[ 'trial_id'] # rename 'trial_id' to 'id' # convert None to np.nan since nwb fields does not take None for k, v in trial_tag_value.items(): trial_tag_value[k] = v if v is not None else np.nan [ trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key ] # Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix events = {k + '_time': trial_tag_value.pop(k) for k in events} trial_attrs = { k.replace('trial_', ''): trial_tag_value.pop(k) for k in [n for n in trial_tag_value if n.startswith('trial_')] } nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
def convert_data(self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False): session_path = self.input_args['folder_path'] # TODO: check/enforce format? task_types = metadata_dict['task_types'] subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] [nwbfile.add_stimulus(x) for x in get_events(session_path)] sleep_state_fpath = os.path.join( session_path, '{}--StatePeriod.mat'.format(session_id)) exist_pos_data = any( os.path.isfile( os.path.join( session_path, '{}__{}.mat'.format(session_id, task_type['name']))) for task_type in task_types) if exist_pos_data: nwbfile.add_epoch_column('label', 'name of epoch') for task_type in task_types: label = task_type['name'] file = os.path.join(session_path, session_id + '__' + label + '.mat') if os.path.isfile(file): pos_obj = Position(name=label + '_position') matin = loadmat(file) tt = matin['twhl_norm'][:, 0] exp_times = find_discontinuities(tt) if 'conversion' in task_type: conversion = task_type['conversion'] else: conversion = np.nan for pos_type in ('twhl_norm', 'twhl_linearized'): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + '_{}_spatial_series'.format(pos_type), data=H5DataIO(pos_data_norm, compression='gzip'), reference_frame='unknown', conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression='gzip')) pos_obj.add_spatial_series(spatial_series_object) check_module( nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface( pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + '_' + str(i)) trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat') if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat') trialdatainfo = [ x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0] ] features = trialdatainfo[:7] features[:2] = 'start_time', 'stop_time', [ nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition'] ] for trial_data in trials_data: if trial_data[3]: cond = 'run_left' else: cond = 'run_right' nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6]) if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)['StatePeriod'] table = TimeIntervals(name='states', description='sleep states of animal') table.add_column(name='label', description='sleep state') data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({ 'start_time': row[0], 'stop_time': row[1], 'label': name }) [ table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time']) ] check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=False): this_session = (experiment.Session & session_key).fetch1() print(f'Exporting to NWB 2.0 for session: {this_session}...') # =============================================================================== # ============================== META INFORMATION =============================== # =============================================================================== sess_desc = session_description_mapper[( experiment.ProjectSession & session_key).fetch1('project_name')] # -- NWB file - a NWB2.0 file for each session nwbfile = NWBFile( identifier='_'.join([ 'ANM' + str(this_session['subject_id']), this_session['session_date'].strftime('%Y-%m-%d'), str(this_session['session']) ]), session_description='', session_start_time=datetime.combine(this_session['session_date'], zero_zero_time), file_create_date=datetime.now(tzlocal()), experimenter=this_session['username'], institution=institution, experiment_description=sess_desc['experiment_description'], related_publications=sess_desc['related_publications'], keywords=sess_desc['keywords']) # -- subject subj = (lab.Subject & session_key).aggr( lab.Subject.Strain, ..., strains='GROUP_CONCAT(animal_strain)').fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=str(this_session['subject_id']), description= f'source: {subj["animal_source"]}; strains: {subj["strains"]}', genotype=' x '.join((lab.Subject.GeneModification & subj).fetch('gene_modification')), sex=subj['sex'], species=subj['species'], date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time) if subj['date_of_birth'] else None) # -- virus nwbfile.virus = json.dumps([{ k: str(v) for k, v in virus_injection.items() if k not in subj } for virus_injection in virus.VirusInjection * virus.Virus & session_key]) # =============================================================================== # ======================== EXTRACELLULAR & CLUSTERING =========================== # =============================================================================== """ In the event of multiple probe recording (i.e. multiple probe insertions), the clustering results (and the associated units) are associated with the corresponding probe. Each probe insertion is associated with one ElectrodeConfiguration (which may define multiple electrode groups) """ dj_insert_location = ephys.ProbeInsertion.InsertionLocation.aggr( ephys.ProbeInsertion.RecordableBrainRegion.proj( brain_region='CONCAT(hemisphere, " ", brain_area)'), ..., brain_regions='GROUP_CONCAT(brain_region)') for probe_insertion in ephys.ProbeInsertion & session_key: electrode_config = (lab.ElectrodeConfig & probe_insertion).fetch1() electrode_groups = {} for electrode_group in lab.ElectrodeConfig.ElectrodeGroup & electrode_config: electrode_groups[electrode_group[ 'electrode_group']] = nwbfile.create_electrode_group( name=electrode_config['electrode_config_name'] + '_g' + str(electrode_group['electrode_group']), description='N/A', device=nwbfile.create_device( name=electrode_config['probe']), location=json.dumps({ k: str(v) for k, v in (dj_insert_location & session_key).fetch1().items() if k not in dj_insert_location.primary_key })) for chn in (lab.ElectrodeConfig.Electrode * lab.Probe.Electrode & electrode_config).fetch(as_dict=True): nwbfile.add_electrode( id=chn['electrode'], group=electrode_groups[chn['electrode_group']], filtering=hardware_filter, imp=-1., x=chn['x_coord'] if chn['x_coord'] else np.nan, y=chn['y_coord'] if chn['y_coord'] else np.nan, z=chn['z_coord'] if chn['z_coord'] else np.nan, location=electrode_groups[chn['electrode_group']].location) # --- unit spike times --- nwbfile.add_unit_column( name='sampling_rate', description='Sampling rate of the raw voltage traces (Hz)') nwbfile.add_unit_column(name='quality', description='unit quality from clustering') nwbfile.add_unit_column( name='posx', description= 'estimated x position of the unit relative to probe (0,0) (um)') nwbfile.add_unit_column( name='posy', description= 'estimated y position of the unit relative to probe (0,0) (um)') nwbfile.add_unit_column( name='cell_type', description='cell type (e.g. fast spiking or pyramidal)') for unit_key in (ephys.Unit * ephys.UnitCellType & probe_insertion).fetch('KEY'): unit = (ephys.Unit * ephys.UnitCellType & probe_insertion & unit_key).proj(..., '-spike_times').fetch1() if ephys.TrialSpikes & unit_key: obs_intervals = np.array( list( zip(*(ephys.TrialSpikes * experiment.SessionTrial & unit_key).fetch('start_time', 'stop_time')))).astype(float) tr_spike_times, tr_start, tr_go = ( ephys.TrialSpikes * experiment.SessionTrial * (experiment.TrialEvent & 'trial_event_type = "go"') & unit_key).fetch('spike_times', 'start_time', 'trial_event_time') spike_times = np.hstack([ spks + float(t_go) + float(t_start) for spks, t_go, t_start in zip(tr_spike_times, tr_start, tr_go) ]) else: # the case of unavailable `TrialSpikes` spike_times = (ephys.Unit & unit_key).fetch1('spike_times') obs_intervals = np.array( list( zip(*(experiment.SessionTrial & unit_key).fetch( 'start_time', 'stop_time')))).astype(float) obs_intervals = [ interval for interval in obs_intervals if np.logical_and(spike_times >= interval[0], spike_times <= interval[-1]).any() ] # make an electrode table region (which electrode(s) is this unit coming from) nwbfile.add_unit( id=unit['unit'], electrodes=np.where( np.array(nwbfile.electrodes.id.data) == unit['electrode'])[0], electrode_group=electrode_groups[unit['electrode_group']], obs_intervals=obs_intervals, sampling_rate=ecephys_fs, quality=unit['unit_quality'], posx=unit['unit_posx'], posy=unit['unit_posy'], cell_type=unit['cell_type'], spike_times=spike_times, waveform_mean=np.mean(unit['waveform'], axis=0), waveform_sd=np.std(unit['waveform'], axis=0)) # =============================================================================== # ============================= BEHAVIOR TRACKING =============================== # =============================================================================== if tracking.LickTrace * experiment.SessionTrial & session_key: # re-concatenating trialized tracking traces lick_traces, time_vecs, trial_starts = ( tracking.LickTrace * experiment.SessionTrial & session_key).fetch( 'lick_trace', 'lick_trace_timestamps', 'start_time') behav_acq = pynwb.behavior.BehavioralTimeSeries( name='BehavioralTimeSeries') nwbfile.add_acquisition(behav_acq) behav_acq.create_timeseries( name='lick_trace', unit='a.u.', conversion=1.0, data=np.hstack(lick_traces), description= "Time-series of the animal's tongue movement when licking", timestamps=np.hstack(time_vecs + trial_starts.astype(float))) # =============================================================================== # ============================= PHOTO-STIMULATION =============================== # =============================================================================== stim_sites = {} for photostim in experiment.Photostim * experiment.PhotostimBrainRegion * lab.PhotostimDevice & session_key: stim_device = (nwbfile.get_device(photostim['photostim_device']) if photostim['photostim_device'] in nwbfile.devices else nwbfile.create_device( name=photostim['photostim_device'])) stim_site = pynwb.ogen.OptogeneticStimulusSite( name=photostim['stim_laterality'] + ' ' + photostim['stim_brain_area'], device=stim_device, excitation_lambda=float(photostim['excitation_wavelength']), location=json.dumps([{ k: v for k, v in stim_locs.items() if k not in experiment.Photostim.primary_key } for stim_locs in (experiment.Photostim.PhotostimLocation.proj( ..., '-brain_area') & photostim).fetch(as_dict=True)], default=str), description='') nwbfile.add_ogen_site(stim_site) stim_sites[photostim['photo_stim']] = stim_site # re-concatenating trialized photostim traces dj_photostim = (experiment.PhotostimTrace * experiment.SessionTrial * experiment.PhotostimEvent * experiment.Photostim & session_key) for photo_stim, stim_site in stim_sites.items(): if dj_photostim & {'photo_stim': photo_stim}: aom_input_trace, laser_power, time_vecs, trial_starts = ( dj_photostim & { 'photo_stim': photo_stim }).fetch('aom_input_trace', 'laser_power', 'photostim_timestamps', 'start_time') aom_series = pynwb.ogen.OptogeneticSeries( name=stim_site.name + '_aom_input_trace', site=stim_site, conversion=1e-3, data=np.hstack(aom_input_trace), timestamps=np.hstack(time_vecs + trial_starts.astype(float))) laser_series = pynwb.ogen.OptogeneticSeries( name=stim_site.name + '_laser_power', site=stim_site, conversion=1e-3, data=np.hstack(laser_power), timestamps=np.hstack(time_vecs + trial_starts.astype(float))) nwbfile.add_stimulus(aom_series) nwbfile.add_stimulus(laser_series) # =============================================================================== # =============================== BEHAVIOR TRIALS =============================== # =============================================================================== # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time' # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) dj_trial = experiment.SessionTrial * experiment.BehaviorTrial skip_adding_columns = experiment.Session.primary_key + [ 'trial_uid', 'trial' ] if experiment.SessionTrial & session_key: # Get trial descriptors from TrialSet.Trial and TrialStimInfo trial_columns = [{ 'name': tag, 'description': re.sub('\s+:|\s+', ' ', re.search(f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip() } for tag in dj_trial.heading.names if tag not in skip_adding_columns + ['start_time', 'stop_time']] # Add new table columns to nwb trial-table for trial-label for c in trial_columns: nwbfile.add_trial_column(**c) # Add entry to the trial-table for trial in (dj_trial & session_key).fetch(as_dict=True): trial['start_time'] = float(trial['start_time']) trial['stop_time'] = float( trial['stop_time']) if trial['stop_time'] else np.nan trial['id'] = trial['trial'] # rename 'trial_id' to 'id' [trial.pop(k) for k in skip_adding_columns] nwbfile.add_trial(**trial) # =============================================================================== # =============================== BEHAVIOR TRIAL EVENTS ========================== # =============================================================================== behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') nwbfile.add_acquisition(behav_event) for trial_event_type in (experiment.TrialEventType & experiment.TrialEvent & session_key).fetch('trial_event_type'): event_times, trial_starts = ( experiment.TrialEvent * experiment.SessionTrial & session_key & { 'trial_event_type': trial_event_type }).fetch('trial_event_time', 'start_time') if len(event_times) > 0: event_times = np.hstack( event_times.astype(float) + trial_starts.astype(float)) behav_event.create_timeseries(name=trial_event_type, unit='a.u.', conversion=1.0, data=np.full_like(event_times, 1), timestamps=event_times) photostim_event_time, trial_starts, photo_stim, power, duration = ( experiment.PhotostimEvent * experiment.SessionTrial & session_key).fetch('photostim_event_time', 'start_time', 'photo_stim', 'power', 'duration') if len(photostim_event_time) > 0: behav_event.create_timeseries( name='photostim_start_time', unit='a.u.', conversion=1.0, data=power, timestamps=photostim_event_time.astype(float) + trial_starts.astype(float), control=photo_stim.astype('uint8'), control_description=stim_sites) behav_event.create_timeseries( name='photostim_stop_time', unit='a.u.', conversion=1.0, data=np.full_like(photostim_event_time, 0), timestamps=photostim_event_time.astype(float) + duration.astype(float) + trial_starts.astype(float), control=photo_stim.astype('uint8'), control_description=stim_sites) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
class NWBConverter: """ Common conversion code factored out so it can be used by multiple conversion projects """ def __init__(self, metadata, nwbfile=None, source_paths=None): """ Parameters ---------- metadata: dict nwbfile: pynwb.NWBFile source_paths: dict """ self.metadata = metadata self.source_paths = source_paths # create self.nwbfile object if nwbfile is None: self.create_nwbfile(metadata['NWBFile']) else: self.nwbfile = nwbfile # add subject information if 'Subject' in metadata: self.create_subject(metadata['Subject']) # add devices self.devices = dict() for domain in ('Icephys', 'Ecephys', 'Ophys'): if domain in metadata and 'Device' in metadata[domain]: self.devices.update( self.create_devices(metadata[domain]['Device'])) if 'Ecephys' in metadata: if 'ElectrodeGroup' in metadata['Ecephys']: self.create_electrode_groups(metadata['Ecephys']) if 'Icephys' in metadata: if 'Electrode' in metadata['Icephys']: self.ic_elecs = self.create_icephys_elecs( metadata['Icephys']['Electrode']) def create_nwbfile(self, metadata_nwbfile): """ This method is called at __init__. This method can be overridden by child classes if necessary. Creates self.nwbfile object. Parameters ---------- metadata_nwbfile: dict """ nwbfile_args = dict(identifier=str(uuid.uuid4()), ) nwbfile_args.update(**metadata_nwbfile) self.nwbfile = NWBFile(**nwbfile_args) def create_subject(self, metadata_subject): """ This method is called at __init__. This method can be overridden by child classes if necessary. Adds information about Subject to self.nwbfile. Parameters ---------- metadata_subject: dict """ self.nwbfile.subject = Subject(**metadata_subject) def create_devices(self, metadata_device) -> Dict: """ This method is called at __init__. Use metadata to create Device object(s) in the NWBFile Parameters ---------- metadata_device: list or dict Returns ------- dict """ if isinstance(metadata_device, list): devices = dict() [ devices.update(self.create_devices(idevice_meta)) for idevice_meta in metadata_device ] return devices else: if 'tag' in metadata_device: key = metadata_device['tag'] else: key = metadata_device['name'] return {key: self.nwbfile.create_device(**metadata_device)} def create_electrode_groups(self, metadata_ecephys): """ This method is called at __init__. Use metadata to create ElectrodeGroup object(s) in the NWBFile Parameters ---------- metadata_ecephys : dict Dict with key:value pairs for defining the Ecephys group from where this ElectrodeGroup belongs. This should contain keys for required groups such as 'Device', 'ElectrodeGroup', etc. """ for metadata_elec_group in metadata_ecephys['ElectrodeGroup']: eg_name = metadata_elec_group['name'] # Tests if ElectrodeGroup already exists aux = [i.name == eg_name for i in self.nwbfile.children] if any(aux): print(eg_name + ' already exists in current NWBFile.') else: device_name = metadata_elec_group['device'] if device_name in self.nwbfile.devices: device = self.nwbfile.devices[device_name] else: print('Device ', device_name, ' for ElectrodeGroup ', eg_name, ' does not exist.') print('Make sure ', device_name, ' is defined in metadata.') eg_description = metadata_elec_group['description'] eg_location = metadata_elec_group['location'] self.nwbfile.create_electrode_group(name=eg_name, location=eg_location, device=device, description=eg_description) def create_electrodes_ecephys(self): """ This method should be overridden by child classes if necessary. Create electrodes in the NWBFile. """ pass def create_icephys_elecs(self, elec_meta) -> Dict: """ Use metadata to generate intracellular electrode object(s) in the NWBFile Parameters ---------- elec_meta: list or dict Returns ------- list """ if isinstance(elec_meta, list): elecs = dict() [ elecs.update(self.create_icephys_elecs(**ielec_meta)) for ielec_meta in elec_meta ] return elecs else: if len(self.devices) == 1: device = list(self.devices.values())[0] elif elec_meta['device'] in self.devices: device = self.devices[elec_meta['device']] else: raise ValueError( 'device not found for icephys electrode {}'.format( elec_meta['name'])) if 'tag' in elec_meta: key = elec_meta['tag'] else: key = elec_meta['name'] return { key: self.nwbfile.create_ic_electrode(device=device, **elec_meta) } def create_trials_from_df(self, df): """ This method should not be overridden. Creates a trials table in self.nwbfile from a Pandas DataFrame. Parameters ---------- df: Pandas DataFrame """ # Tests if trials table already exists if self.nwbfile.trials is not None: print("Trials table already exist in current nwb file.\n" "Use 'add_trials_columns_from_df' to include new columns.\n" "Use 'add_trials_from_df' to include new trials.") pass # Tests if required column names are present in df if 'start_time' not in df.columns: print("Required column 'start_time' not present in DataFrame.") pass if 'stop_time' not in df.columns: print("Required column 'stop_time' not present in DataFrame.") pass # Creates new columns for colname in df.columns: if colname not in ['start_time', 'stop_time']: # Indexed columns should be of type 'object' in the dataframe if df[colname].dtype == 'object': self.nwbfile.add_trial_column(name=colname, description='no description', index=True) else: self.nwbfile.add_trial_column(name=colname, description='no description') # Populates trials table from df values for index, row in df.iterrows(): self.nwbfile.add_trial(**dict(row)) def add_trials_from_df(self, df): """ This method should not be overridden. Adds trials from a Pandas DataFrame to existing trials table in self.nwbfile. Parameters ---------- df: Pandas DataFrame """ # Tests for mismatch between trials table columns and dataframe columns A = set(self.nwbfile.trials.colnames) B = set(df.columns) if len(A - B) > 0: print("Missing columns in DataFrame: ", A - B) pass if len(B - A) > 0: print("NWBFile trials table does not contain: ", B - A) pass # Adds trials from df values for index, row in df.iterrows(): self.nwbfile.add_trial(**dict(row)) def add_trials_columns_from_df(self, df): """ This method should not be overridden. Adds trials columns from a Pandas DataFrame to existing trials table in self.nwbfile. Parameters ---------- df: Pandas DataFrame """ # Tests if dataframe columns already exist in nwbfile trials table A = set(self.nwbfile.trials.colnames) B = set(df.columns) intersection = A.intersection(B) if len(intersection) > 0: print("These columns already exist in nwbfile trials: ", intersection) pass # Adds trials columns with data from df values for (colname, coldata) in df.iteritems(): # Indexed columns should be of type 'object' in the dataframe if df[colname].dtype == 'object': index = True else: index = False self.nwbfile.add_trial_column(name=colname, description='no description', data=coldata, index=index) def save(self, to_path, read_check=True): """ This method should not be overridden. Saves object self.nwbfile. Parameters ---------- to_path: str read_check: bool If True, try to read the file after writing """ print('Saving to file, please wait...') with NWBHDF5IO(to_path, 'w') as io: io.write(self.nwbfile) print('File successfully saved at: ', str(to_path)) if read_check: with NWBHDF5IO(to_path, 'r') as io: io.read() print('Read check: OK') def check_module(self, name, description=None): """ Check if processing module exists. If not, create it. Then return module Parameters ---------- name: str description: str | None (optional) Returns ------- pynwb.module """ if name in self.nwbfile.processing: return self.nwbfile.processing[name] else: if description is None: description = name return self.nwbfile.create_processing_module(name, description) def add_sortingextractor(self, sortingextractor: SortingExtractor): self.nwbfile = NwbSortingExtractor.read_sorting( sortingextractor, self.nwbfile) def add_recordingextractor(self, recordingextractor: RecordingExtractor): self.nwbfile = NwbRecordingExtractor.read_recording( recordingextractor, self.nwbfile)
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=True): this_session = (acquisition.Session & session_key).fetch1() identifier = '_'.join([ this_session['subject_id'], this_session['session_time'].strftime('%Y-%m-%d'), this_session['session_id'] ]) # =============== General ==================== # -- NWB file - a NWB2.0 file for each session nwbfile = NWBFile(session_description=this_session['session_note'], identifier=identifier, session_start_time=this_session['session_time'], file_create_date=datetime.now(tzlocal()), experimenter='; '.join( (acquisition.Session.Experimenter & session_key).fetch('experimenter')), institution=institution, experiment_description=experiment_description, related_publications=related_publications, keywords=keywords) # -- subject subj = (subject.Subject & session_key).fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=this_session['subject_id'], description=subj['subject_description'], genotype=' x '.join( (subject.Subject.Allele & session_key).fetch('allele')), sex=subj['sex'], species=subj['species']) # =============== Intracellular ==================== cell = ((intracellular.Cell & session_key).fetch1() if len(intracellular.Cell & session_key) == 1 else None) if cell: # metadata whole_cell_device = nwbfile.create_device(name=cell['device_name']) ic_electrode = nwbfile.create_ic_electrode( name=cell['cell_id'], device=whole_cell_device, description='N/A', filtering='N/A', location='; '.join([ f'{k}: {str(v)}' for k, v in dict((reference.BrainLocation & cell).fetch1(), depth=cell['cell_depth']).items() ])) # acquisition - membrane potential mp, mp_wo_spike, mp_start_time, mp_fs = ( intracellular.MembranePotential & cell).fetch1( 'membrane_potential', 'membrane_potential_wo_spike', 'membrane_potential_start_time', 'membrane_potential_sampling_rate') nwbfile.add_acquisition( pynwb.icephys.PatchClampSeries(name='PatchClampSeries', electrode=ic_electrode, unit='mV', conversion=1e-3, gain=1.0, data=mp, starting_time=mp_start_time, rate=mp_fs)) # acquisition - current injection if (intracellular.CurrentInjection & cell): current_injection, ci_start_time, ci_fs = ( intracellular.CurrentInjection & cell).fetch1( 'current_injection', 'current_injection_start_time', 'current_injection_sampling_rate') nwbfile.add_stimulus( pynwb.icephys.CurrentClampStimulusSeries( name='CurrentClampStimulus', electrode=ic_electrode, conversion=1e-9, gain=1.0, data=current_injection, starting_time=ci_start_time, rate=ci_fs)) # analysis - membrane potential without spike mp_rmv_spike = nwbfile.create_processing_module( name='icephys', description='Spike removal') mp_rmv_spike.add_data_interface( pynwb.icephys.PatchClampSeries(name='icephys', electrode=ic_electrode, unit='mV', conversion=1e-3, gain=1.0, data=mp_wo_spike, starting_time=mp_start_time, rate=mp_fs)) # =============== Extracellular ==================== probe_insertion = ((extracellular.ProbeInsertion & session_key).fetch1() if extracellular.ProbeInsertion & session_key else None) if probe_insertion: probe = nwbfile.create_device(name=probe_insertion['probe_name']) electrode_group = nwbfile.create_electrode_group(name='; '.join([ f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}' ]), description='N/A', device=probe, location='; '.join([ f'{k}: {str(v)}' for k, v in (reference. BrainLocation & probe_insertion ).fetch1().items( ) ])) for chn in (reference.Probe.Channel & probe_insertion).fetch(as_dict=True): nwbfile.add_electrode( id=chn['channel_id'], group=electrode_group, filtering=hardware_filter, imp=-1., x=0.0, # not available from data y=0.0, # not available from data z=0.0, # not available from data location=electrode_group.location) # --- unit spike times --- nwbfile.add_unit_column( name='sampling_rate', description='Sampling rate of the raw voltage traces (Hz)') nwbfile.add_unit_column(name='depth', description='depth this unit (mm)') nwbfile.add_unit_column(name='spike_width', description='spike width of this unit (ms)') nwbfile.add_unit_column( name='cell_type', description='cell type (e.g. wide width, narrow width spiking)') for unit in (extracellular.UnitSpikeTimes & probe_insertion).fetch(as_dict=True): # make an electrode table region (which electrode(s) is this unit coming from) nwbfile.add_unit( id=unit['unit_id'], electrodes=(unit['channel_id'] if isinstance( unit['channel_id'], np.ndarray) else [unit['channel_id']]), depth=unit['unit_depth'], sampling_rate=ecephys_fs, spike_width=unit['unit_spike_width'], cell_type=unit['unit_cell_type'], spike_times=unit['spike_times'], waveform_mean=unit['spike_waveform']) # =============== Behavior ==================== # Note: for this study, raw behavioral data were not available, only trialized data were provided # here, we reconstruct raw behavioral data by concatenation trial_seg_setting = (analysis.TrialSegmentationSetting & 'trial_seg_setting=0').fetch1() seg_behav_query = ( behavior.TrialSegmentedLickTrace * acquisition.TrialSet.Trial * (analysis.RealignedEvent.RealignedEventTime & 'trial_event="trial_start"') & session_key & trial_seg_setting) if seg_behav_query: behav_acq = pynwb.behavior.BehavioralTimeSeries(name='lick_times') nwbfile.add_acquisition(behav_acq) seg_behav = pd.DataFrame( seg_behav_query.fetch('start_time', 'realigned_event_time', 'segmented_lick_left_on', 'segmented_lick_left_off', 'segmented_lick_right_on', 'segmented_lick_right_off')).T seg_behav.columns = [ 'start_time', 'realigned_event_time', 'segmented_lick_left_on', 'segmented_lick_left_off', 'segmented_lick_right_on', 'segmented_lick_right_off' ] for behav_name in [ 'lick_left_on', 'lick_left_off', 'lick_right_on', 'lick_right_off' ]: lick_times = np.hstack(r['segmented_' + behav_name] - r.realigned_event_time + r.start_time for _, r in seg_behav.iterrows()) behav_acq.create_timeseries(name=behav_name, unit='a.u.', conversion=1.0, data=np.full_like(lick_times, 1), timestamps=lick_times) # =============== Photostimulation ==================== photostim = ((stimulation.PhotoStimulation & session_key).fetch1() if stimulation.PhotoStimulation & session_key else None) if photostim: photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1() stim_device = nwbfile.create_device( name=photostim_device['device_name']) stim_site = pynwb.ogen.OptogeneticStimulusSite( name='-'.join([photostim['hemisphere'], photostim['brain_region']]), device=stim_device, excitation_lambda=float( (stimulation.PhotoStimProtocol & photostim).fetch1('photo_stim_excitation_lambda')), location='; '.join([ f'{k}: {str(v)}' for k, v in (reference.ActionLocation & photostim).fetch1().items() ]), description=(stimulation.PhotoStimProtocol & photostim).fetch1('photo_stim_notes')) nwbfile.add_ogen_site(stim_site) if photostim['photostim_timeseries'] is not None: nwbfile.add_stimulus( pynwb.ogen.OptogeneticSeries( name='_'.join([ 'photostim_on', photostim['photostim_datetime'].strftime( '%Y-%m-%d_%H-%M-%S') ]), site=stim_site, resolution=0.0, conversion=1e-3, data=photostim['photostim_timeseries'], starting_time=photostim['photostim_start_time'], rate=photostim['photostim_sampling_rate'])) # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: # 'id', 'start_time' and 'stop_time'. # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) if acquisition.TrialSet & session_key: # Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any) trial_columns = [ { 'name': tag.replace('trial_', ''), 'description': re.search( f'(?<={tag})(.*)#(.*)', str((acquisition.TrialSet.Trial * stimulation.TrialPhotoStimParam ).heading)).groups()[-1].strip() } for tag in (acquisition.TrialSet.Trial * stimulation.TrialPhotoStimParam).heading.names if tag not in (acquisition.TrialSet.Trial & stimulation.TrialPhotoStimParam).primary_key + ['start_time', 'stop_time'] ] # Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time # also add `_time` suffix to all events trial_events = set(((acquisition.TrialSet.EventTime & session_key) - [{ 'trial_event': 'trial_start' }, { 'trial_event': 'trial_stop' }]).fetch('trial_event')) event_names = [{ 'name': e + '_time', 'description': d + ' - (s) relative to trial start time' } for e, d in zip(*(reference.ExperimentalEvent & [{ 'event': k } for k in trial_events]).fetch('event', 'description'))] # Add new table columns to nwb trial-table for trial-label for c in trial_columns + event_names: nwbfile.add_trial_column(**c) photostim_tag_default = { tag: '' for tag in stimulation.TrialPhotoStimParam.heading.names if tag not in stimulation.TrialPhotoStimParam.primary_key } # Add entry to the trial-table for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True): events = dict( zip(*(acquisition.TrialSet.EventTime & trial & [{ 'trial_event': e } for e in trial_events] ).fetch('trial_event', 'event_time'))) trial_tag_value = ({ **trial, **events, **(stimulation.TrialPhotoStimParam & trial).fetch1() } if (stimulation.TrialPhotoStimParam & trial) else { **trial, **events, **photostim_tag_default }) trial_tag_value['id'] = trial_tag_value[ 'trial_id'] # rename 'trial_id' to 'id' [ trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key ] # convert None to np.nan since nwb fields does not take None for k, v in trial_tag_value.items(): trial_tag_value[k] = v if v is not None else np.nan trial_tag_value['delay_duration'] = float( trial_tag_value['delay_duration']) # convert Decimal to float # Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix events = {k + '_time': trial_tag_value.pop(k) for k in events} trial_attrs = { k.replace('trial_', ''): trial_tag_value.pop(k) for k in [n for n in trial_tag_value if n.startswith('trial_')] } nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=True): this_session = (acquisition.Session & session_key).fetch1() # =============== General ==================== # -- NWB file - a NWB2.0 file for each session nwbfile = NWBFile(session_description=this_session['session_note'], identifier='_'.join([ this_session['subject_id'], this_session['session_time'].strftime('%Y-%m-%d'), this_session['session_id'] ]), session_start_time=this_session['session_time'], file_create_date=datetime.now(tzlocal()), experimenter='; '.join( (acquisition.Session.Experimenter & session_key).fetch('experimenter')), institution=institution, experiment_description=experiment_description, related_publications=related_publications, keywords=keywords) # -- subject subj = (subject.Subject & session_key).fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=this_session['subject_id'], description=subj['subject_description'], genotype=' x '.join( (subject.Subject.Allele & session_key).fetch('allele')), sex=subj['sex'], species=subj['species']) # =============== Intracellular ==================== cell = ((intracellular.Cell & session_key).fetch1() if len(intracellular.Cell & session_key) == 1 else None) if cell: # metadata whole_cell_device = nwbfile.create_device(name=cell['device_name']) ic_electrode = nwbfile.create_ic_electrode( name=cell['cell_id'], device=whole_cell_device, description='N/A', filtering='N/A', location='; '.join([ f'{k}: {str(v)}' for k, v in dict((reference.BrainLocation & cell).fetch1(), depth=cell['recording_depth']).items() ])) # acquisition - membrane potential mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1( 'membrane_potential', 'membrane_potential_timestamps') nwbfile.add_acquisition( pynwb.icephys.PatchClampSeries(name='PatchClampSeries', electrode=ic_electrode, unit='mV', conversion=1e-3, gain=1.0, data=mp, timestamps=mp_timestamps)) # acquisition - spike train spk, spk_timestamps = (intracellular.SpikeTrain & cell).fetch1( 'spike_train', 'spike_timestamps') nwbfile.add_acquisition( pynwb.icephys.PatchClampSeries(name='SpikeTrain', electrode=ic_electrode, unit='a.u.', conversion=1e1, gain=1.0, data=spk, timestamps=spk_timestamps)) # =============== Behavior ==================== behavior_data = ((behavior.Behavior & session_key).fetch1() if len(behavior.Behavior & session_key) == 1 else None) if behavior_data: behav_acq = pynwb.behavior.BehavioralTimeSeries(name='behavior') nwbfile.add_acquisition(behav_acq) [behavior_data.pop(k) for k in behavior.Behavior.primary_key] timestamps = behavior_data.pop('behavior_timestamps') # get behavior data description from the comments of table definition behavior_descriptions = { attr: re.search(f'(?<={attr})(.*)#(.*)', str(behavior.Behavior.heading)).groups()[-1].strip() for attr in behavior_data } for b_k, b_v in behavior_data.items(): behav_acq.create_timeseries(name=b_k, description=behavior_descriptions[b_k], unit='a.u.', conversion=1.0, data=b_v, timestamps=timestamps) # =============== Photostimulation ==================== photostim = ((stimulation.PhotoStimulation & session_key).fetch1() if len(stimulation.PhotoStimulation & session_key) == 1 else None) if photostim: photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1() stim_device = nwbfile.create_device( name=photostim_device['device_name']) stim_site = pynwb.ogen.OptogeneticStimulusSite( name='-'.join([photostim['hemisphere'], photostim['brain_region']]), device=stim_device, excitation_lambda=float( (stimulation.PhotoStimulationProtocol & photostim).fetch1('photo_stim_excitation_lambda')), location='; '.join([ f'{k}: {str(v)}' for k, v in (reference.ActionLocation & photostim).fetch1().items() ]), description=(stimulation.PhotoStimulationProtocol & photostim).fetch1('photo_stim_notes')) nwbfile.add_ogen_site(stim_site) if photostim['photostim_timeseries'] is not None: nwbfile.add_stimulus( pynwb.ogen.OptogeneticSeries( name='_'.join([ 'photostim_on', photostim['photostim_datetime'].strftime( '%Y-%m-%d_%H-%M-%S') ]), site=stim_site, resolution=0.0, conversion=1e-3, data=photostim['photostim_timeseries'], starting_time=photostim['photostim_start_time'], rate=photostim['photostim_sampling_rate'])) # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: # 'id', 'start_time' and 'stop_time'. # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) if len((acquisition.TrialSet & session_key).fetch()) == 1: # Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any) trial_columns = [{ 'name': tag.replace('trial_', ''), 'description': re.search( f'(?<={tag})(.*)#(.*)', str((acquisition.TrialSet.Trial * stimulation.TrialPhotoStimInfo ).heading)).groups()[-1].strip() } for tag in acquisition.TrialSet.Trial.heading.names if tag not in acquisition.TrialSet.Trial.primary_key + ['start_time', 'stop_time']] # Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time # also add `_time` suffix to all events trial_events = set(((acquisition.TrialSet.EventTime & session_key) - [{ 'trial_event': 'trial_start' }, { 'trial_event': 'trial_stop' }]).fetch('trial_event')) event_names = [{ 'name': e + '_time', 'description': d } for e, d in zip(*(reference.ExperimentalEvent & [{ 'event': k } for k in trial_events]).fetch('event', 'description'))] # Add new table columns to nwb trial-table for trial-label for c in trial_columns + event_names: nwbfile.add_trial_column(**c) # Add entry to the trial-table for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True): events = dict( zip(*(acquisition.TrialSet.EventTime & trial & [{ 'trial_event': e } for e in trial_events] ).fetch('trial_event', 'event_time'))) # shift event times to be relative to session_start (currently relative to trial_start) events = {k: v + trial['start_time'] for k, v in events.items()} trial_tag_value = {**trial, **events} # rename 'trial_id' to 'id' trial_tag_value['id'] = trial_tag_value['trial_id'] [ trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key ] # Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix events = {k + '_time': trial_tag_value.pop(k) for k in events} trial_attrs = { k.replace('trial_', ''): trial_tag_value.pop(k) for k in [n for n in trial_tag_value if n.startswith('trial_')] } nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
def run_conversion(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False): session_path = Path(self.source_data["folder_path"]) task_types = [ dict(name="OpenFieldPosition_ExtraLarge"), dict(name="OpenFieldPosition_New_Curtain", conversion=0.46), dict(name="OpenFieldPosition_New", conversion=0.46), dict(name="OpenFieldPosition_Old_Curtain", conversion=0.46), dict(name="OpenFieldPosition_Old", conversion=0.46), dict(name="OpenFieldPosition_Oldlast", conversion=0.46), dict(name="EightMazePosition", conversion=0.65 / 2), ] subject_path = session_path.parent session_id = session_path.stem [nwbfile.add_stimulus(x) for x in get_events(session_path)] sleep_state_fpath = session_path / f"{session_id}--StatePeriod.mat" exist_pos_data = any([ (session_path / "{session_id}__{task_type['name']}.mat").is_file() for task_type in task_types ]) if exist_pos_data: nwbfile.add_epoch_column("label", "Name of epoch.") # Epoch intervals for task_type in task_types: label = task_type["name"] file = session_path / f"{session_id}__{label}.mat" if file.is_file(): pos_obj = Position(name=f"{label}_position") matin = loadmat(file) tt = matin["twhl_norm"][:, 0] exp_times = find_discontinuities(tt) if "conversion" in task_type: conversion = task_type["conversion"] else: conversion = np.nan for pos_type in ("twhl_norm", "twhl_linearized"): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=f"{label}_{pos_type}_spatial_series", data=H5DataIO(pos_data_norm, compression="gzip"), reference_frame="unknown", conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression="gzip"), ) pos_obj.add_spatial_series(spatial_series_object) check_module( nwbfile, "behavior", "Contains processed behavioral data.").add_data_interface( pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch( start_time=window[0], stop_time=window[1], tags=f"{label}_{str(i)}", ) # Trial intervals trialdata_path = session_path / f"{session_id}__EightMazeRun.mat" if trialdata_path.is_file(): trials_data = loadmat(trialdata_path)["EightMazeRun"] trialdatainfo_path = subject_path / "EightMazeRunInfo.mat" trialdatainfo = [ x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0] ] features = trialdatainfo[:7] features[:2] = ( "start_time", "stop_time", ) [ nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"] ] for trial_data in trials_data: if trial_data[3]: cond = "run_left" else: cond = "run_right" nwbfile.add_trial( start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6], ) # SLeep states if sleep_state_fpath.is_file(): matin = loadmat(sleep_state_fpath)["StatePeriod"] table = TimeIntervals(name="states", description="sleep states of animal") table.add_column(name="label", description="sleep state") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append( dict(start_time=row[0], stop_time=row[1], label=name)) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] check_module(nwbfile, "behavior", "Contains behavioral data.").add_data_interface(table)
def convert( input_file, session_start_time, subject_date_of_birth, subject_id='I5', subject_description='naive', subject_genotype='wild-type', subject_sex='M', subject_weight='11.6g', subject_species='Mus musculus', subject_brain_region='Medial Entorhinal Cortex', surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible', session_id='npI5_0417_baseline_1', experimenter='Kei Masuda', experiment_description='Virtual Hallway Task', institution='Stanford University School of Medicine', lab_name='Giocomo Lab'): """ Read in the .mat file specified by input_file and convert to .nwb format. Parameters ---------- input_file : np.ndarray (..., n_channels, n_time) the .mat file to be converted subject_id : string the unique subject ID number for the subject of the experiment subject_date_of_birth : datetime ISO 8601 the date and time the subject was born subject_description : string important information specific to this subject that differentiates it from other members of it's species subject_genotype : string the genetic strain of this species. subject_sex : string Male or Female subject_weight : the weight of the subject around the time of the experiment subject_species : string the name of the species of the subject subject_brain_region : basestring the name of the brain region where the electrode probe is recording from surgery : str information about the subject's surgery to implant electrodes session_id: string human-readable ID# for the experiment session that has a one-to-one relationship with a recording session session_start_time : datetime date and time that the experiment started experimenter : string who ran the experiment, first and last name experiment_description : string what task was being run during the session institution : string what institution was the experiment performed in lab_name : string the lab where the experiment was performed Returns ------- nwbfile : NWBFile The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5 """ # input matlab data matfile = hdf5storage.loadmat(input_file) # output path for nwb data def replace_last(source_string, replace_what, replace_with): head, _sep, tail = source_string.rpartition(replace_what) return head + replace_with + tail outpath = replace_last(input_file, '.mat', '.nwb') create_date = datetime.today() timezone_cali = pytz.timezone('US/Pacific') create_date_tz = timezone_cali.localize(create_date) # if loading data from config.yaml, convert string dates into datetime if isinstance(session_start_time, str): session_start_time = datetime.strptime(session_start_time, '%B %d, %Y %I:%M%p') session_start_time = timezone_cali.localize(session_start_time) if isinstance(subject_date_of_birth, str): subject_date_of_birth = datetime.strptime(subject_date_of_birth, '%B %d, %Y %I:%M%p') subject_date_of_birth = timezone_cali.localize(subject_date_of_birth) # create unique identifier for this experimental session uuid_identifier = uuid.uuid1() # Create NWB file nwbfile = NWBFile( session_description=experiment_description, # required identifier=uuid_identifier.hex, # required session_id=session_id, experiment_description=experiment_description, experimenter=experimenter, surgery=surgery, institution=institution, lab=lab_name, session_start_time=session_start_time, # required file_create_date=create_date_tz) # optional # add information about the subject of the experiment experiment_subject = Subject(subject_id=subject_id, species=subject_species, description=subject_description, genotype=subject_genotype, date_of_birth=subject_date_of_birth, weight=subject_weight, sex=subject_sex) nwbfile.subject = experiment_subject # adding constants via LabMetaData container # constants sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0]) n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0]) dat_path = matfile['sp'][0]['dat_path'][0][0][0] offset = int(matfile['sp'][0]['offset'][0][0][0]) data_dtype = matfile['sp'][0]['dtype'][0][0][0] hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0]) vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0] # container lab_metadata = LabMetaData_ext(name='LabMetaData', acquisition_sampling_rate=sample_rate, number_of_electrodes=n_channels_dat, file_path=dat_path, bytes_to_skip=offset, raw_data_dtype=data_dtype, high_pass_filtered=hp_filtered, movie_start_time=vr_session_offset) nwbfile.add_lab_meta_data(lab_metadata) # Adding trial information nwbfile.add_trial_column( 'trial_contrast', 'visual contrast of the maze through which the mouse is running') trial = np.ravel(matfile['trial']) trial_nums = np.unique(trial) position_time = np.ravel(matfile['post']) # matlab trial numbers start at 1. To correctly index trial_contract vector, # subtracting 1 from 'num' so index starts at 0 for num in trial_nums: trial_times = position_time[trial == num] nwbfile.add_trial(start_time=trial_times[0], stop_time=trial_times[-1], trial_contrast=matfile['trial_contrast'][num - 1][0]) # Add mouse position inside: position = Position() position_virtual = np.ravel(matfile['posx']) # position inside the virtual environment sampling_rate = 1 / (position_time[1] - position_time[0]) position.create_spatial_series( name='Position', data=position_virtual, starting_time=position_time[0], rate=sampling_rate, reference_frame='The start of the trial, which begins at the start ' 'of the virtual hallway.', conversion=0.01, description='Subject position in the virtual hallway.', comments='The values should be >0 and <400cm. Values greater than ' '400cm mean that the mouse briefly exited the maze.', ) # physical position on the mouse wheel physical_posx = position_virtual trial_gain = np.ravel(matfile['trial_gain']) for num in trial_nums: physical_posx[trial == num] = physical_posx[trial == num] / trial_gain[num - 1] position.create_spatial_series( name='PhysicalPosition', data=physical_posx, starting_time=position_time[0], rate=sampling_rate, reference_frame='Location on wheel re-referenced to zero ' 'at the start of each trial.', conversion=0.01, description='Physical location on the wheel measured ' 'since the beginning of the trial.', comments='Physical location found by dividing the ' 'virtual position by the "trial_gain"') nwbfile.add_acquisition(position) # Add timing of lick events, as well as mouse's virtual position during lick event lick_events = BehavioralEvents() lick_events.create_timeseries( 'LickEvents', data=np.ravel(matfile['lickx']), timestamps=np.ravel(matfile['lickt']), unit='centimeter', description='Subject position in virtual hallway during the lick.') nwbfile.add_acquisition(lick_events) # Add information on the visual stimulus that was shown to the subject # Assumed rate=60 [Hz]. Update if necessary # Update external_file to link to Unity environment file visualization = ImageSeries( name='ImageSeries', unit='seconds', format='external', external_file=list(['https://unity.com/VR-and-AR-corner']), starting_time=vr_session_offset, starting_frame=[[0]], rate=float(60), description='virtual Unity environment that the mouse navigates through' ) nwbfile.add_stimulus(visualization) # Add the recording device, a neuropixel probe recording_device = nwbfile.create_device(name='neuropixel_probes') electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded' electrode_group_name = 'probe1' electrode_group = nwbfile.create_electrode_group( electrode_group_name, description=electrode_group_description, location=subject_brain_region, device=recording_device) # Add information about each electrode xcoords = np.ravel(matfile['sp'][0]['xcoords'][0]) ycoords = np.ravel(matfile['sp'][0]['ycoords'][0]) data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0] if data_filtered_flag: filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered' else: filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered' num_recording_electrodes = xcoords.shape[0] recording_electrodes = range(0, num_recording_electrodes) # create electrode columns for the x,y location on the neuropixel probe # the standard x,y,z locations are reserved for Allen Brain Atlas location nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe') nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe') for idx in recording_electrodes: nwbfile.add_electrode(id=idx, x=np.nan, y=np.nan, z=np.nan, rel_x=float(xcoords[idx]), rel_y=float(ycoords[idx]), imp=np.nan, location='medial entorhinal cortex', filtering=filter_desc, group=electrode_group) # Add information about each unit, termed 'cluster' in giocomo data # create new columns in unit table nwbfile.add_unit_column( 'quality', 'labels given to clusters during manual sorting in phy (1=MUA, ' '2=Good, 3=Unsorted)') # cluster information cluster_ids = matfile['sp'][0]['cids'][0][0] cluster_quality = matfile['sp'][0]['cgs'][0][0] # spikes in time spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike spike_cluster = np.ravel( matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time for i, cluster_id in enumerate(cluster_ids): unit_spike_times = spike_times[spike_cluster == cluster_id] waveforms = matfile['sp'][0]['temps'][0][cluster_id] nwbfile.add_unit(id=int(cluster_id), spike_times=unit_spike_times, quality=cluster_quality[i], waveform_mean=waveforms, electrode_group=electrode_group) # Trying to add another Units table to hold the results of the automatic spike sorting # create TemplateUnits units table template_units = Units( name='TemplateUnits', description='units assigned during automatic spike sorting') template_units.add_column( 'tempScalingAmps', 'scaling amplitude applied to the template when extracting spike', index=True) # information on extracted spike templates spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0]) spike_template_ids = np.unique(spike_templates) # template scaling amplitudes temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0]) for i, spike_template_id in enumerate(spike_template_ids): template_spike_times = spike_times[spike_templates == spike_template_id] temp_scaling_amps_per_template = temp_scaling_amps[spike_templates == spike_template_id] template_units.add_unit(id=int(spike_template_id), spike_times=template_spike_times, electrode_group=electrode_group, tempScalingAmps=temp_scaling_amps_per_template) # create ecephys processing module spike_template_module = nwbfile.create_processing_module( name='ecephys', description='units assigned during automatic spike sorting') # add template_units table to processing module spike_template_module.add(template_units) print(nwbfile) print('converted to NWB:N') print('saving ...') with NWBHDF5IO(outpath, 'w') as io: io.write(nwbfile) print('saved', outpath)
def run_conversion(self, nwbfile: NWBFile, metadata: dict): """ Run conversion for this data interface. Reads treadmill experiment behavioral data from csv files and adds it to nwbfile. Parameters ---------- nwbfile : NWBFile metadata : dict """ # Detect relevant files: trials summary, treadmill data and nose data dir_behavior_treadmill = self.source_data['dir_behavior_treadmill'] trials_file = [ f for f in Path(dir_behavior_treadmill).glob('*_tr.csv') if '~lock' not in f.name ][0] treadmill_file = trials_file.name.split('_tr')[0] + '.csv' nose_file = trials_file.name.split('_tr')[0] + '_mk.csv' trials_file = os.path.join(dir_behavior_treadmill, trials_file) treadmill_file = os.path.join(dir_behavior_treadmill, treadmill_file) nose_file = os.path.join(dir_behavior_treadmill, nose_file) # Add trials if nwbfile.trials is not None: print( 'Trials already exist in current nwb file. Treadmill behavior trials not added.' ) else: df_trials_summary = pd.read_csv(trials_file) nwbfile.add_trial_column(name='fail', description='no description') nwbfile.add_trial_column(name='reward_given', description='no description') nwbfile.add_trial_column(name='total_rewards', description='no description') nwbfile.add_trial_column(name='init_dur', description='no description') nwbfile.add_trial_column(name='light_dur', description='no description') nwbfile.add_trial_column(name='motor_dur', description='no description') nwbfile.add_trial_column(name='post_motor', description='no description') nwbfile.add_trial_column(name='speed', description='no description') nwbfile.add_trial_column(name='speed_mode', description='no description') nwbfile.add_trial_column(name='amplitude', description='no description') nwbfile.add_trial_column(name='period', description='no description') nwbfile.add_trial_column(name='deviation', description='no description') t_offset = df_trials_summary.loc[0]['Start Time'] for index, row in df_trials_summary.iterrows(): nwbfile.add_trial( start_time=row['Start Time'] - t_offset, stop_time=row['End Time'] - t_offset, fail=row['Fail'], reward_given=row['Reward Given'], total_rewards=row['Total Rewards'], init_dur=row['Init Dur'], light_dur=row['Light Dur'], motor_dur=row['Motor Dur'], post_motor=row['Post Motor'], speed=row['Speed'], speed_mode=row['Speed Mode'], amplitude=row['Amplitude'], period=row['Period'], deviation=row['+/- Deviation'], ) # Treadmill continuous data df_treadmill = pd.read_csv(treadmill_file, index_col=False) # Nose position continuous data df_nose = pd.read_csv(nose_file, index_col=False) # All behavioral data df_all = pd.concat([df_treadmill, df_nose], axis=1, sort=False) meta_behavioral_ts = metadata['Behavior'] t_offset = df_treadmill.loc[0]['Time'] for meta in meta_behavioral_ts.values(): ts = TimeSeries(name=meta['name'], data=df_all[meta['name']].to_numpy(), timestamps=df_all['Time'].to_numpy() - t_offset, description=meta['description']) nwbfile.add_acquisition(ts)
class NWBFileTest(unittest.TestCase): def setUp(self): self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()) self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) self.create = [ datetime(2017, 5, 1, 12, tzinfo=tzlocal()), datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()), datetime(2017, 5, 2, 14, tzinfo=tzutc()) ] self.path = 'nwbfile_test.h5' self.nwbfile = NWBFile( 'a test session description for a test NWBFile', 'FILE123', self.start, file_create_date=self.create, timestamps_reference_time=self.ref_time, experimenter='A test experimenter', lab='a test lab', institution='a test institution', experiment_description='a test experiment description', session_id='test1', notes='my notes', pharmacology='drugs', protocol='protocol', related_publications='my pubs', slices='my slices', surgery='surgery', virus='a virus', source_script='noscript', source_script_file_name='nofilename', stimulus_notes='test stimulus notes', data_collection='test data collection notes', keywords=('these', 'are', 'keywords')) def test_constructor(self): self.assertEqual(self.nwbfile.session_description, 'a test session description for a test NWBFile') self.assertEqual(self.nwbfile.identifier, 'FILE123') self.assertEqual(self.nwbfile.session_start_time, self.start) self.assertEqual(self.nwbfile.file_create_date, self.create) self.assertEqual(self.nwbfile.lab, 'a test lab') self.assertEqual(self.nwbfile.experimenter, 'A test experimenter') self.assertEqual(self.nwbfile.institution, 'a test institution') self.assertEqual(self.nwbfile.experiment_description, 'a test experiment description') self.assertEqual(self.nwbfile.session_id, 'test1') self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes') self.assertEqual(self.nwbfile.data_collection, 'test data collection notes') self.assertEqual(self.nwbfile.source_script, 'noscript') self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename') self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords')) self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time) def test_create_electrode_group(self): name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' d = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d) self.assertEqual(elecgrp.description, desc) self.assertEqual(elecgrp.location, loc) self.assertIs(elecgrp.device, d) def test_create_electrode_group_invalid_index(self): """ Test the case where the user creates an electrode table region with indexes that are out of range of the amount of electrodes added. """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i) with self.assertRaises(IndexError): nwbfile.create_electrode_table_region(list(range(6)), 'test') def test_access_group_after_io(self): """ Motivated by #739 """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i + 1) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) os.remove("electrodes_mwe.nwb") def test_epoch_tags(self): tags1 = ['t1', 't2'] tags2 = ['t3', 't4'] tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float) ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) expected_tags = tags1 + tags2 self.nwbfile.add_epoch(0.0, 1.0, tags1, ts) self.nwbfile.add_epoch(0.0, 1.0, tags2, ts) tags = self.nwbfile.epoch_tags six.assertCountEqual(self, expected_tags, tags) def test_add_acquisition(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.acquisition), 1) def test_add_stimulus(self): self.nwbfile.add_stimulus( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus), 1) def test_add_stimulus_template(self): self.nwbfile.add_stimulus_template( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus_template), 1) def test_add_analysis(self): self.nwbfile.add_analysis( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.analysis), 1) def test_add_acquisition_check_dups(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) with self.assertRaises(ValueError): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) def test_get_acquisition_empty(self): with self.assertRaisesRegex(ValueError, "acquisition of NWBFile 'root' is empty"): self.nwbfile.get_acquisition() def test_get_acquisition_multiple_elements(self): self.nwbfile.add_acquisition( TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.nwbfile.add_acquisition( TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name" with self.assertRaisesRegex(ValueError, msg): self.nwbfile.get_acquisition() def test_add_acquisition_invalid_name(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "'TEST_TS' not found in acquisition of NWBFile 'root'" with self.assertRaisesRegex(KeyError, msg): self.nwbfile.get_acquisition("TEST_TS") def test_set_electrode_table(self): table = ElectrodeTable() # noqa: F405 dev1 = self.nwbfile.create_device('dev1') # noqa: F405 group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1) table.add_row(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-2.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-3.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-4.0, location='CA1', filtering='none', group=group, group_name='tetrode1') self.nwbfile.set_electrode_table(table) self.assertIs(self.nwbfile.electrodes, table) self.assertIs(table.parent, self.nwbfile) def test_add_unit_column(self): self.nwbfile.add_unit_column('unit_type', 'the type of unit') self.assertEqual(self.nwbfile.units.colnames, ('unit_type', )) def test_add_unit(self): self.nwbfile.add_unit(id=1) self.assertEqual(len(self.nwbfile.units), 1) self.nwbfile.add_unit(id=2) self.nwbfile.add_unit(id=3) self.assertEqual(len(self.nwbfile.units), 3) def test_add_trial_column(self): self.nwbfile.add_trial_column('trial_type', 'the type of trial') self.assertEqual(self.nwbfile.trials.colnames, ('start_time', 'stop_time', 'trial_type')) def test_add_trial(self): self.nwbfile.add_trial(start_time=10.0, stop_time=20.0) self.assertEqual(len(self.nwbfile.trials), 1) self.nwbfile.add_trial(start_time=30.0, stop_time=40.0) self.nwbfile.add_trial(start_time=50.0, stop_time=70.0) self.assertEqual(len(self.nwbfile.trials), 3) def test_add_invalid_times_column(self): self.nwbfile.add_invalid_times_column( 'comments', 'description of reason for omitting time') self.assertEqual(self.nwbfile.invalid_times.colnames, ('start_time', 'stop_time', 'comments')) def test_add_invalid_time_interval(self): self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0) self.assertEqual(len(self.nwbfile.invalid_times), 1) self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0) self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5) self.assertEqual(len(self.nwbfile.invalid_times), 3) def test_add_invalid_time_w_ts(self): ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na') self.nwbfile.add_invalid_time_interval(start_time=18.0, stop_time=20.6, timeseries=ts, tags=('hi', 'there')) def test_add_electrode(self): dev1 = self.nwbfile.create_device('dev1') # noqa: F405 group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1) self.nwbfile.add_electrode(1.0, 2.0, 3.0, -1.0, 'CA1', 'none', group=group, id=1) self.assertEqual(self.nwbfile.electrodes[0][0], 1) self.assertEqual(self.nwbfile.electrodes[0][1], 1.0) self.assertEqual(self.nwbfile.electrodes[0][2], 2.0) self.assertEqual(self.nwbfile.electrodes[0][3], 3.0) self.assertEqual(self.nwbfile.electrodes[0][4], -1.0) self.assertEqual(self.nwbfile.electrodes[0][5], 'CA1') self.assertEqual(self.nwbfile.electrodes[0][6], 'none') self.assertEqual(self.nwbfile.electrodes[0][7], group) def test_all_children(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' device = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device) children = self.nwbfile.all_children() self.assertIn(ts1, children) self.assertIn(ts2, children) self.assertIn(device, children) self.assertIn(elecgrp, children) def test_fail_if_source_script_file_name_without_source_script(self): with self.assertRaises(ValueError): # <-- source_script_file_name without source_script is not allowed NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, source_script=None, source_script_file_name='nofilename') def test_get_neurodata_type(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) p1 = ts1.get_ancestor(neurodata_type='NWBFile') self.assertIs(p1, self.nwbfile) p2 = ts2.get_ancestor(neurodata_type='NWBFile') self.assertIs(p2, self.nwbfile) def test_print_units(self): self.nwbfile.add_unit(spike_times=[1., 2., 3.]) self.assertEqual(str(self.nwbfile.units), """ units <class 'pynwb.misc.Units'> Fields: colnames: ('spike_times',) columns: ( spike_times_index <class 'pynwb.core.VectorIndex'> Fields: target: spike_times <class 'pynwb.core.VectorData'> , spike_times <class 'pynwb.core.VectorData'> Fields: description: the spike times for each unit ) description: Autogenerated by NWBFile id: id <class 'pynwb.core.ElementIdentifiers'> """) # noqa: W291
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903', subject_xls=None, include_spike_waveforms=True, stub=True, cache_spec=True): subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] identifier = session_id mouse_number = session_id[9:11] if '-' in session_id: subject_id, date_text = session_id.split('-') b = False else: subject_id, date_text = session_id.split('b') b = True if subject_xls is None: subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx') else: if not subject_xls[-4:] == 'xlsx': subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx') session_start_time = dateparse(date_text, yearfirst=True) df = pd.read_excel(subject_xls) subject_data = {} for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']: names = df.iloc[:, 0] if key in names.values: subject_data[key] = df.iloc[np.argmax(names == key), 1] if isinstance(subject_data['DOB'], datetime): age = session_start_time - subject_data['DOB'] else: age = None subject = Subject(subject_id=subject_id, age=str(age), genotype=subject_data['genotype'], species='mouse') nwbfile = NWBFile(session_description='mouse in open exploration and theta maze', identifier=identifier, session_start_time=session_start_time.astimezone(), file_create_date=datetime.now().astimezone(), experimenter='Yuta Senzai', session_id=session_id, institution='NYU', lab='Buzsaki', subject=subject, related_publications='DOI:10.1016/j.neuron.2016.12.011') print('reading and writing raw position data...', end='', flush=True) ns.add_position_data(nwbfile, session_path) shank_channels = ns.get_shank_channels(session_path)[:8] all_shank_channels = np.concatenate(shank_channels) print('setting up electrodes...', end='', flush=True) hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv') lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b) print(lfp_channel) custom_column = [{'name': 'theta_reference', 'description': 'this electrode was used to calculate LFP canonical bands', 'data': all_shank_channels == lfp_channel}] ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks) print('reading LFPs...', end='', flush=True) lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub) lfp_data = all_channels_data[:, all_shank_channels] print('writing LFPs...', flush=True) # lfp_data[:int(len(lfp_data)/4)] lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp', description='lfp signal for all shank electrodes') for name, channel in special_electrode_dict.items(): ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data', data=all_channels_data[:, channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan) nwbfile.add_acquisition(ts) # compute filtered LFP print('filtering LFP...', end='', flush=True) all_lfp_phases = [] for passband in ('theta', 'gamma'): lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband) lfp_phase, _ = hilbert_lfp(lfp_fft) all_lfp_phases.append(lfp_phase[:, np.newaxis]) data = np.dstack(all_lfp_phases) print('done.', flush=True) if include_spike_waveforms: print('writing waveforms...', end='', flush=True) nshanks = min((max_shanks, len(ns.get_shank_channels(session_path)))) for shankn in np.arange(nshanks, dtype=int) + 1: ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub) print('done.', flush=True) decomp_series = DecompositionSeries(name='LFPDecompositionSeries', description='Theta and Gamma phase for reference LFP', data=data, rate=lfp_fs, source_timeseries=lfp_ts, metric='phase', unit='radians') decomp_series.add_band(band_name='theta', band_limits=(4, 10)) decomp_series.add_band(band_name='gamma', band_limits=(30, 80)) check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series) [nwbfile.add_stimulus(x) for x in ns.get_events(session_path)] # create epochs corresponding to experiments/environments for the mouse sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id)) exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name']))) for task_type in task_types) if exist_pos_data: nwbfile.add_epoch_column('label', 'name of epoch') for task_type in task_types: label = task_type['name'] file = os.path.join(session_path, session_id + '__' + label + '.mat') if os.path.isfile(file): print('loading position for ' + label + '...', end='', flush=True) pos_obj = Position(name=label + '_position') matin = loadmat(file) tt = matin['twhl_norm'][:, 0] exp_times = find_discontinuities(tt) if 'conversion' in task_type: conversion = task_type['conversion'] else: conversion = np.nan for pos_type in ('twhl_norm', 'twhl_linearized'): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + '_{}_spatial_series'.format(pos_type), data=H5DataIO(pos_data_norm, compression='gzip'), reference_frame='unknown', conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression='gzip')) pos_obj.add_spatial_series(spatial_series_object) check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + '_' + str(i)) print('done.') # there are occasional mismatches between the matlab struct and the neuroscope files # regions: 3: 'CA3', 4: 'DG' df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path) celltype_names = [] for celltype_id, region_id in zip(df_unit_features['fineCellType'].values, df_unit_features['region'].values): if celltype_id == 1: if region_id == 3: celltype_names.append('pyramidal cell') elif region_id == 4: celltype_names.append('granule cell') else: raise Exception('unknown type') elif not np.isfinite(celltype_id): celltype_names.append('missing') else: celltype_names.append(celltype_dict[celltype_id]) custom_unit_columns = [ { 'name': 'cell_type', 'description': 'name of cell type', 'data': celltype_names}, { 'name': 'global_id', 'description': 'global id for cell for entire experiment', 'data': df_unit_features['unitID'].values}, { 'name': 'max_electrode', 'description': 'electrode that has the maximum amplitude of the waveform', 'data': get_max_electrodes(nwbfile, session_path), 'table': nwbfile.electrodes }] ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks) trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat') if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat') trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]] features = trialdatainfo[:7] features[:2] = 'start_time', 'stop_time', [nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']] for trial_data in trials_data: if trial_data[3]: cond = 'run_left' else: cond = 'run_right' nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6]) """ mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat') matin = loadmat(mono_syn_fpath) exc = matin['FinalExcMonoSynID'] inh = matin['FinalInhMonoSynID'] #exc_obj = CatCellInfo(name='excitatory_connections', # indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1) #module_cellular.add_container(exc_obj) #inh_obj = CatCellInfo(name='inhibitory_connections', # indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1) #module_cellular.add_container(inh_obj) """ if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)['StatePeriod'] table = TimeIntervals(name='states', description='sleep states of animal') table.add_column(name='label', description='sleep state') data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({'start_time': row[0], 'stop_time': row[1], 'label': name}) [table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])] check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table) if stub: out_fname = session_path + '_stub.nwb' else: out_fname = session_path + '.nwb' print('writing NWB file...', end='', flush=True) with NWBHDF5IO(out_fname, mode='w') as io: io.write(nwbfile, cache_spec=cache_spec) print('done.') print('testing read...', end='', flush=True) # test read with NWBHDF5IO(out_fname, mode='r') as io: io.read() print('done.')
# and :py:meth:`~pynwb.file.NWBFile.add_trial_column`. Together, these methods maintains a # table-like structure that can define arbitrary columns without having to go through the # extension process. # # By default, NWBFile only requires trial start time and trial end time. Additional columns # can be added using :py:meth:`~pynwb.file.NWBFile.add_trial_column`. This method takes a name # for the column and a description of what the column stores. You do not need to supply data # type, as this will inferred. # Once all columns have been added, trial data can be populated using :py:meth:`~pynwb.file.NWBFile.add_trial`. # # Lets add an additional column and some trial data. nwbfile.add_trial_column(name='stim', description='the visual stimuli during the trial') nwbfile.add_trial(start_time=0.0, stop_time=2.0, stim='person') nwbfile.add_trial(start_time=3.0, stop_time=5.0, stim='ocean') nwbfile.add_trial(start_time=6.0, stop_time=8.0, stim='desert') #################### # Tabular data such as trials can be converted to a `pandas.DataFrame`. print(nwbfile.trials.to_dataframe()) #################### # :: # # start_time stop_time stim # id # 0 0.0 2.0 person # 1 3.0 5.0 ocean
def convert_from_old_neo(old_file, bird_name, electrode_df): root_dir, fname = os.path.split(old_file) rec_date = bird_info[bird_name]['recording_date'] rec_start = bird_info[bird_name]['recording_start'] rec_end = bird_info[bird_name]['recording_end'] rec_datetime = date_parser.parse('{} {}'.format(rec_date, rec_start)) # create an electrode array, electrode groups, and electrode table for the electrodes i = electrode_df.bird == bird_name edf = electrode_df[i] for block, gdf in edf.groupby(['block']): lfp_series = dict() spike_series = dict() print('*************** Processing block {}'.format(block)) if bird_name == 'WhiWhi4522M' and block == 'Site1': continue # get the LFP and spike data for the block hf = h5py.File(old_file, 'r') block_data = read_old_neo(hf, block) hf.close() recording_name = '{}_{}_{}'.format(bird_name, block, block_data['protocol']) nwb_file = os.path.join(root_dir, '{}.nwb'.format(recording_name)) session_desc = """ A single recording session, roughly one hour long, at a single depth, in a series of recordings from the auditory "cortex" of zebra finch {}. Block {}, stimulus protocol {}. """.format(bird_name, block, block_data['protocol']) exp_desc = """ A randomly interleaved mixture of Zebra finch vocalizations, songs, and modulation limited noise played to a Zebra finch under urethane anaesthesia in an acute experiment. Two 16 microwire electrode arrays were implanted, one in each hemisphere. Experiments designed and performed by Julie Elie, vocal repertoire recorded by Julie Elie. Data converted to NWB by Mike Schachter. For a full description of methods, please consult and also cite the following publications: Elie, J. E., & Theunissen, F. E. (2015). Meaning in the avian auditory cortex: neural representation of communication calls. European Journal of Neuroscience, 41(5), 546-567. Elie, J. E., & Theunissen, F. E. (2016). The vocal repertoire of the domesticated zebra finch: a data-driven approach to decipher the information-bearing acoustic features of communication signals. Animal cognition, 19(2), 285-315. """ nf = NWBFile(recording_name, session_desc, bird_name, rec_datetime, experimenter='Julie Elie', lab='Theunissen Lab', institution='UC Berkeley', experiment_description=exp_desc, session_id=bird_name) # create the electrodes and electrode tables for hemi, ggdf in gdf.groupby(['hemisphere']): electrode_array_name = '16 electrode microwire array on {} hemisphere'.format( hemi) electrode_array = nf.create_device(name=electrode_array_name, source='') # create an electrode group egrp_desc = """ The (x,y) locations of the electrodes refer to the distance from region midline and distance from L2A, respectively, in mm. """ electrode_group = nf.create_electrode_group( hemi, source=electrode_array_name, description=egrp_desc, location='{} Hemisphere, Field L, CM, NCM'.format(hemi), device=electrode_array) # add electrodes to electrode group for row_idx, row in ggdf.iterrows(): electrode_number = row['electrode'] - 1 dist_l2a = row['dist_l2a'] dist_midline = row['dist_midline'] region = row['region'] if bird_name == 'GreBlu9508M': dist_l2a *= 4 # correct for the error in the original data nf.add_electrode(electrode_number, x=dist_midline, y=dist_l2a, z=0.0, imp=0.0, location=region, filtering='none', description='Row {}, Column {}'.format( row['row'], row['col']), group=electrode_group) # create an electrode table region electrode_numbers = list( np.array(sorted(ggdf.electrode.unique())) - 1) print('electrode_numbers=', electrode_numbers) etable = nf.create_electrode_table_region( electrode_numbers, 'All electrodes in array for hemisphere {} with LFP'.format( hemi)) lfp_data = np.array([ block_data['electrodes'][e + 1]['lfp'] for e in electrode_numbers ]) sr = block_data['sample_rate'] t = np.arange(lfp_data.shape[1]) / sr # add the raw LFP lfp_series_name = 'Multi-electrode LFP on {} hemisphere from block {}'.format( hemi, block) lfp = ElectricalSeries( lfp_series_name, electrode_array_name, lfp_data, etable, timestamps=t, resolution=1e-12, comments='', description='Low-passed LFP recorded from microwire array') lfp_series[hemi] = lfp # add the spikes and their waveforms for row_idx, row in ggdf.iterrows(): # electrode_number ranges from 1-32, same as the keys in block_data['electrodes'] electrode_number = row['electrode'] for k, unit_data in enumerate( block_data['electrodes'][electrode_number]['units']): spike_times = unit_data['spike_times'] waveforms = unit_data['waveforms'] unit_name = unit_data['name'] e_num = unit_data['electrode'] print('{} electrode_number={}({}), e_num={}, k={}'.format( unit_name, electrode_number, electrode_number - 1, e_num, k)) assert e_num == electrode_number if unit_name.startswith('RF Sort'): xarr = unit_name.split(' ') unit_num = xarr[-1] sort_type = 'Spike-sorted unit {}'.format(unit_num) elif unit_name.startswith('Chan'): if 'Code0' not in unit_name: print( 'Skipping multi-unit channel with non-zero sort code' ) continue sort_type = 'Multi-unit' else: raise Exception( 'Unknown unit name: {}'.format(unit_name)) full_unit_name = '{} on block {} and electrode {}'.format( sort_type, block, e_num - 1) spikes = SpikeEventSeries( full_unit_name, lfp_series_name, waveforms, spike_times, etable, resolution=1e-12, conversion=1e6, comments='', description='', ) print( '\tAdding spikes acquisition: {} ({}), waveforms.shape={}' .format(full_unit_name, sort_type, str(waveforms.shape))) spike_series[(hemi, electrode_number, unit_name)] = spikes all_series = list() all_series.extend(lfp_series.values()) all_series.extend(spike_series.values()) print('len(all_series)=', len(all_series)) nf.add_acquisition(all_series) # create trials for each stim presentation nf.add_trial_column('stim_id', 'The ID of the sound played during the trial.') nf.add_trial_column( 'trial', 'The number of times this stimulus has been presented.') for stim_epoch_data in block_data['epochs']: stim_id = stim_epoch_data['stim_id'] trial_num = stim_epoch_data['trial'] stime = stim_epoch_data['start'] etime = stim_epoch_data['end'] nf.add_trial({ 'start': stime, 'end': etime, 'stim_id': stim_id, 'trial': trial_num }) print('Writing to {}'.format(nwb_file)) with NWBHDF5IO(nwb_file, mode='w') as io: io.write(nf) del nf
class NWBFileTest(TestCase): def setUp(self): self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()) self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) self.create = [datetime(2017, 5, 1, 12, tzinfo=tzlocal()), datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()), datetime(2017, 5, 2, 14, tzinfo=tzutc())] self.path = 'nwbfile_test.h5' self.nwbfile = NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, file_create_date=self.create, timestamps_reference_time=self.ref_time, experimenter='A test experimenter', lab='a test lab', institution='a test institution', experiment_description='a test experiment description', session_id='test1', notes='my notes', pharmacology='drugs', protocol='protocol', related_publications='my pubs', slices='my slices', surgery='surgery', virus='a virus', source_script='noscript', source_script_file_name='nofilename', stimulus_notes='test stimulus notes', data_collection='test data collection notes', keywords=('these', 'are', 'keywords')) def test_constructor(self): self.assertEqual(self.nwbfile.session_description, 'a test session description for a test NWBFile') self.assertEqual(self.nwbfile.identifier, 'FILE123') self.assertEqual(self.nwbfile.session_start_time, self.start) self.assertEqual(self.nwbfile.file_create_date, self.create) self.assertEqual(self.nwbfile.lab, 'a test lab') self.assertEqual(self.nwbfile.experimenter, ('A test experimenter',)) self.assertEqual(self.nwbfile.institution, 'a test institution') self.assertEqual(self.nwbfile.experiment_description, 'a test experiment description') self.assertEqual(self.nwbfile.session_id, 'test1') self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes') self.assertEqual(self.nwbfile.data_collection, 'test data collection notes') self.assertEqual(self.nwbfile.related_publications, ('my pubs',)) self.assertEqual(self.nwbfile.source_script, 'noscript') self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename') self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords')) self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time) def test_create_electrode_group(self): name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' d = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d) self.assertEqual(elecgrp.description, desc) self.assertEqual(elecgrp.location, loc) self.assertIs(elecgrp.device, d) def test_create_custom_intervals(self): df_words = pd.DataFrame({'start_time': [.1, 2.], 'stop_time': [.8, 2.3], 'label': ['hello', 'there']}) words = TimeIntervals.from_dataframe(df_words, name='words') self.nwbfile.add_time_intervals(words) self.assertEqual(self.nwbfile.intervals['words'], words) def test_create_electrode_group_invalid_index(self): """ Test the case where the user creates an electrode table region with indexes that are out of range of the amount of electrodes added. """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i) with self.assertRaises(IndexError): nwbfile.create_electrode_table_region(list(range(6)), 'test') def test_access_group_after_io(self): """ Motivated by #739 """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i + 1) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) remove_test_file("electrodes_mwe.nwb") def test_access_processing(self): self.nwbfile.create_processing_module('test_mod', 'test_description') # test deprecate .modules with self.assertWarnsWith(DeprecationWarning, 'replaced by NWBFile.processing'): modules = self.nwbfile.modules['test_mod'] self.assertIs(self.nwbfile.processing['test_mod'], modules) def test_epoch_tags(self): tags1 = ['t1', 't2'] tags2 = ['t3', 't4'] tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float) ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) expected_tags = tags1 + tags2 self.nwbfile.add_epoch(0.0, 1.0, tags1, ts) self.nwbfile.add_epoch(0.0, 1.0, tags2, ts) tags = self.nwbfile.epoch_tags self.assertEqual(set(expected_tags), set(tags)) def test_add_acquisition(self): self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.acquisition), 1) def test_add_stimulus(self): self.nwbfile.add_stimulus(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus), 1) def test_add_stimulus_template(self): self.nwbfile.add_stimulus_template(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus_template), 1) def test_add_analysis(self): self.nwbfile.add_analysis(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.analysis), 1) def test_add_acquisition_check_dups(self): self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) with self.assertRaises(ValueError): self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) def test_get_acquisition_empty(self): with self.assertRaisesWith(ValueError, "acquisition of NWBFile 'root' is empty"): self.nwbfile.get_acquisition() def test_get_acquisition_multiple_elements(self): self.nwbfile.add_acquisition(TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.nwbfile.add_acquisition(TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name" with self.assertRaisesWith(ValueError, msg): self.nwbfile.get_acquisition() def test_add_acquisition_invalid_name(self): self.nwbfile.add_acquisition(TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "\"'TEST_TS' not found in acquisition of NWBFile 'root'\"" with self.assertRaisesWith(KeyError, msg): self.nwbfile.get_acquisition("TEST_TS") def test_set_electrode_table(self): table = ElectrodeTable() dev1 = self.nwbfile.create_device('dev1') group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1) table.add_row(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-2.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-3.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-4.0, location='CA1', filtering='none', group=group, group_name='tetrode1') self.nwbfile.set_electrode_table(table) self.assertIs(self.nwbfile.electrodes, table) self.assertIs(table.parent, self.nwbfile) def test_add_unit_column(self): self.nwbfile.add_unit_column('unit_type', 'the type of unit') self.assertEqual(self.nwbfile.units.colnames, ('unit_type',)) def test_add_unit(self): self.nwbfile.add_unit(id=1) self.assertEqual(len(self.nwbfile.units), 1) self.nwbfile.add_unit(id=2) self.nwbfile.add_unit(id=3) self.assertEqual(len(self.nwbfile.units), 3) def test_add_trial_column(self): self.nwbfile.add_trial_column('trial_type', 'the type of trial') self.assertEqual(self.nwbfile.trials.colnames, ('start_time', 'stop_time', 'trial_type')) def test_add_trial(self): self.nwbfile.add_trial(start_time=10.0, stop_time=20.0) self.assertEqual(len(self.nwbfile.trials), 1) self.nwbfile.add_trial(start_time=30.0, stop_time=40.0) self.nwbfile.add_trial(start_time=50.0, stop_time=70.0) self.assertEqual(len(self.nwbfile.trials), 3) def test_add_invalid_times_column(self): self.nwbfile.add_invalid_times_column('comments', 'description of reason for omitting time') self.assertEqual(self.nwbfile.invalid_times.colnames, ('start_time', 'stop_time', 'comments')) def test_add_invalid_time_interval(self): self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0) self.assertEqual(len(self.nwbfile.invalid_times), 1) self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0) self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5) self.assertEqual(len(self.nwbfile.invalid_times), 3) def test_add_invalid_time_w_ts(self): ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na') self.nwbfile.add_invalid_time_interval(start_time=18.0, stop_time=20.6, timeseries=ts, tags=('hi', 'there')) def test_add_electrode(self): dev1 = self.nwbfile.create_device('dev1') group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1) self.nwbfile.add_electrode(1.0, 2.0, 3.0, -1.0, 'CA1', 'none', group=group, id=1) elec = self.nwbfile.electrodes[0] self.assertEqual(elec.index[0], 1) self.assertEqual(elec.iloc[0]['x'], 1.0) self.assertEqual(elec.iloc[0]['y'], 2.0) self.assertEqual(elec.iloc[0]['z'], 3.0) self.assertEqual(elec.iloc[0]['location'], 'CA1') self.assertEqual(elec.iloc[0]['filtering'], 'none') self.assertEqual(elec.iloc[0]['group'], group) def test_all_children(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' device = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device) children = self.nwbfile.all_children() self.assertIn(ts1, children) self.assertIn(ts2, children) self.assertIn(device, children) self.assertIn(elecgrp, children) def test_fail_if_source_script_file_name_without_source_script(self): with self.assertRaises(ValueError): # <-- source_script_file_name without source_script is not allowed NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, source_script=None, source_script_file_name='nofilename') def test_get_neurodata_type(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) p1 = ts1.get_ancestor(neurodata_type='NWBFile') self.assertIs(p1, self.nwbfile) p2 = ts2.get_ancestor(neurodata_type='NWBFile') self.assertIs(p2, self.nwbfile) def test_print_units(self): self.nwbfile.add_unit(spike_times=[1., 2., 3.]) expected = """units pynwb.misc.Units at 0x%d Fields: colnames: ['spike_times'] columns: ( spike_times_index <class 'hdmf.common.table.VectorIndex'>, spike_times <class 'hdmf.common.table.VectorData'> ) description: Autogenerated by NWBFile id: id <class 'hdmf.common.table.ElementIdentifiers'> waveform_unit: volts """ expected = expected % id(self.nwbfile.units) self.assertEqual(str(self.nwbfile.units), expected) def test_copy(self): self.nwbfile.add_unit(spike_times=[1., 2., 3.]) device = self.nwbfile.create_device('a') elecgrp = self.nwbfile.create_electrode_group('a', 'b', device=device, location='a') self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0) self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'b', 'b', elecgrp) elec_region = self.nwbfile.create_electrode_table_region([1], 'name') ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = ElectricalSeries('test_ts2', [0, 1, 2, 3, 4, 5], electrodes=elec_region, timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) self.nwbfile.add_trial(start_time=50.0, stop_time=70.0) self.nwbfile.add_invalid_times_column('comments', 'description of reason for omitting time') self.nwbfile.create_processing_module('test_mod', 'test_description') self.nwbfile.create_time_intervals('custom_interval', 'a custom time interval') self.nwbfile.intervals['custom_interval'].add_interval(start_time=10., stop_time=20.) newfile = self.nwbfile.copy() # test dictionaries self.assertIs(self.nwbfile.devices['a'], newfile.devices['a']) self.assertIs(self.nwbfile.acquisition['test_ts1'], newfile.acquisition['test_ts1']) self.assertIs(self.nwbfile.acquisition['test_ts2'], newfile.acquisition['test_ts2']) self.assertIs(self.nwbfile.processing['test_mod'], newfile.processing['test_mod']) # test dynamic tables self.assertIsNot(self.nwbfile.electrodes, newfile.electrodes) self.assertIs(self.nwbfile.electrodes['x'], newfile.electrodes['x']) self.assertIsNot(self.nwbfile.units, newfile.units) self.assertIs(self.nwbfile.units['spike_times'], newfile.units['spike_times']) self.assertIsNot(self.nwbfile.trials, newfile.trials) self.assertIsNot(self.nwbfile.trials.parent, newfile.trials.parent) self.assertIs(self.nwbfile.trials.id, newfile.trials.id) self.assertIs(self.nwbfile.trials['start_time'], newfile.trials['start_time']) self.assertIs(self.nwbfile.trials['stop_time'], newfile.trials['stop_time']) self.assertIsNot(self.nwbfile.invalid_times, newfile.invalid_times) self.assertTupleEqual(self.nwbfile.invalid_times.colnames, newfile.invalid_times.colnames) self.assertIsNot(self.nwbfile.intervals['custom_interval'], newfile.intervals['custom_interval']) self.assertTupleEqual(self.nwbfile.intervals['custom_interval'].colnames, newfile.intervals['custom_interval'].colnames) self.assertIs(self.nwbfile.intervals['custom_interval']['start_time'], newfile.intervals['custom_interval']['start_time']) self.assertIs(self.nwbfile.intervals['custom_interval']['stop_time'], newfile.intervals['custom_interval']['stop_time']) def test_multi_experimenters(self): self.nwbfile = NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, experimenter=('experimenter1', 'experimenter2')) self.assertTupleEqual(self.nwbfile.experimenter, ('experimenter1', 'experimenter2')) def test_multi_publications(self): self.nwbfile = NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, related_publications=('pub1', 'pub2')) self.assertTupleEqual(self.nwbfile.related_publications, ('pub1', 'pub2'))
def convert_data( self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False ): session_path = self.input_args["folder_path"] # TODO: check/enforce format? task_types = metadata_dict.get("task_types", []) subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] [nwbfile.add_stimulus(x) for x in get_events(session_path)] exist_pos_data = any( os.path.isfile(os.path.join(session_path, "{}__{}.mat".format(session_id, task_type["name"]))) for task_type in task_types ) if exist_pos_data: nwbfile.add_epoch_column("label", "name of epoch") for task_type in task_types: label = task_type["name"] file = os.path.join(session_path, session_id + "__" + label + ".mat") if os.path.isfile(file): pos_obj = Position(name=label + "_position") matin = loadmat(file) tt = matin["twhl_norm"][:, 0] exp_times = find_discontinuities(tt) if "conversion" in task_type: conversion = task_type["conversion"] else: conversion = np.nan for pos_type in ("twhl_norm", "twhl_linearized"): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + "_{}_spatial_series".format(pos_type), data=H5DataIO(pos_data_norm, compression="gzip"), reference_frame="unknown", conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression="gzip"), ) pos_obj.add_spatial_series(spatial_series_object) check_module(nwbfile, "behavior", "contains processed behavioral data").add_data_interface(pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + "_" + str(i)) trialdata_path = os.path.join(session_path, session_id + "__EightMazeRun.mat") if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)["EightMazeRun"] trialdatainfo_path = os.path.join(fpath_base, "EightMazeRunInfo.mat") trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]] features = trialdatainfo[:7] features[:2] = ( "start_time", "stop_time", ) [nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"]] for trial_data in trials_data: if trial_data[3]: cond = "run_left" else: cond = "run_right" nwbfile.add_trial( start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6], ) sleep_state_fpath = os.path.join(session_path, "{}.SleepState.states.mat".format(session_id)) # label renaming specific to Watson state_label_names = {"WAKEstate": "Awake", "NREMstate": "Non-REM", "REMstate": "REM"} if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0] table = TimeIntervals(name="states", description="Sleep states of animal.") table.add_column(name="label", description="Sleep state.") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({"start_time": row[0], "stop_time": row[1], "label": state_label_names[name]}) [table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"])] check_module(nwbfile, "behavior", "contains behavioral data").add_data_interface(table)
def run_conversion(self, nwbfile: NWBFile, metadata: dict): """ Run conversionfor this data interface. Reads optophysiology raw data from .rsd files and adds it to nwbfile. XXXXXXX_A.rsd - Raw data from donor XXXXXXX_B.rsd - Raw data from acceptor XXXXXXXXX.rsh - Header data Parameters ---------- nwbfile : NWBFile metadata : dict """ dir_cortical_imaging = self.source_data['dir_cortical_imaging'] ophys_list = [i.name for i in Path(dir_cortical_imaging).glob('*.rsh')] if len(ophys_list) > 0: fname_prefix = ophys_list[0].split('-')[0] else: raise OSError(f"No .rsd file found in directory: {dir_cortical_imaging}.\n" "Did you choose the correct path for source data?") def data_gen(channel, trial): """channel = 'A' or 'B'""" # Read trial-specific metadata file .rsh trial_meta = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{trial}_{channel}.rsh") file_rsm, files_raw, acquisition_date, sample_rate, n_frames = self.read_trial_meta(trial_meta=trial_meta) # Iterates over all files within the same trial for fn, fraw in enumerate(files_raw): print('adding channel ' + channel + ', trial: ', trial, ': ', 100 * fn / len(files_raw), '%') fpath = os.path.join(dir_cortical_imaging, fraw) # Open file as a byte array with open(fpath, "rb") as f: byte = f.read(1000000000) # Data as word array: 'h' signed, 'H' unsigned words = np.array(struct.unpack('h' * (len(byte) // 2), byte)) # Iterates over frames within the same file (n_frames, 100, 100) n_frames = int(len(words) / 12800) words_reshaped = words.reshape(12800, n_frames, order='F') frames = np.zeros((n_frames, 100, 100)) excess_frames = np.zeros((n_frames, 20, 100)) for ifr in range(n_frames): iframe = -words_reshaped[:, ifr].reshape(128, 100, order='F').astype('int16') frames[ifr, :, :] = iframe[20:120, :] excess_frames[ifr, :, :] = iframe[0:20, :] yield iframe[20:120, :] # # Analog signals are taken from excess data variable # analog_1 = np.squeeze(np.squeeze(excess_frames[:, 12, 0:80:4]).reshape(20*256, 1)) # analog_2 = np.squeeze(np.squeeze(excess_frames[:, 14, 0:80:4]).reshape(20*256, 1)) # stim_trg = np.squeeze(np.squeeze(excess_frames[:, 8, 0:80:4]).reshape(20*256, 1)) # Get session_start_time from first header file all_files = os.listdir(dir_cortical_imaging) all_headers = [f for f in all_files if ('.rsh' in f) and ('_A' not in f) and ('_B' not in f)] all_headers.sort() _, _, acquisition_date, _, _ = self.read_trial_meta(trial_meta=Path(dir_cortical_imaging) / all_headers[0]) session_start_time = datetime.strptime(acquisition_date, '%Y/%m/%d %H:%M:%S') session_start_time_tzaware = pytz.timezone('EST').localize(session_start_time) if session_start_time_tzaware != nwbfile.session_start_time: print("Session start time in current nwbfile does not match the start time from rsd files.") print("Ophys data conversion aborted.") return # Create and add device device = Device(name=metadata['Ophys']['Device']['name']) nwbfile.add_device(device) # Get FRETSeries metadata meta_donor = metadata['Ophys']['FRET']['donor'][0] meta_acceptor = metadata['Ophys']['FRET']['acceptor'][0] # OpticalChannels opt_ch_donor = OpticalChannel( name=meta_donor['optical_channel'][0]['name'], description=meta_donor['optical_channel'][0]['description'], emission_lambda=float(meta_donor['optical_channel'][0]['emission_lambda']) ) opt_ch_acceptor = OpticalChannel( name=meta_acceptor['optical_channel'][0]['name'], description=meta_acceptor['optical_channel'][0]['description'], emission_lambda=float(meta_acceptor['optical_channel'][0]['emission_lambda']) ) # Add trials intervals values only if no trials data exists in nwbfile if nwbfile.trials is not None: add_trials = False print('Trials already exist in current nwb file. Ophys trials intervals not added.') else: add_trials = True # Iterate over trials, creates a FRET group per trial trials_numbers = [f.split('-')[1].replace('.rsh', '') for f in all_headers] for tr in trials_numbers: # Read trial-specific metadata file .rsh trial_meta_A = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_A.rsh") trial_meta_B = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_B.rsh") file_rsm_A, files_raw_A, acquisition_date_A, sample_rate_A, n_frames_A = self.read_trial_meta(trial_meta=trial_meta_A) file_rsm_B, files_raw_B, acquisition_date_B, sample_rate_B, n_frames_B = self.read_trial_meta(trial_meta=trial_meta_B) absolute_start_time = datetime.strptime(acquisition_date_A, '%Y/%m/%d %H:%M:%S') relative_start_time = float((absolute_start_time - nwbfile.session_start_time.replace(tzinfo=None)).seconds) # Checks if Acceptor and Donor channels have the same basic parameters assert acquisition_date_A == acquisition_date_B, \ "Acquisition date of channels do not match. Trial=" + str(tr) assert sample_rate_A == sample_rate_B, \ "Sample rate of channels do not match. Trial=" + str(tr) assert n_frames_A == n_frames_B, \ "Number of frames of channels do not match. Trial=" + str(tr) assert relative_start_time >= 0., \ "Starting time is negative. Trial=" + str(tr) # Create iterator data_donor = DataChunkIterator( data=data_gen(channel='A', trial=tr), iter_axis=0, buffer_size=10000, maxshape=(None, 100, 100) ) data_acceptor = DataChunkIterator( data=data_gen(channel='B', trial=tr), iter_axis=0, buffer_size=10000, maxshape=(None, 100, 100) ) # FRETSeries frets_donor = FRETSeries( name='donor', fluorophore=meta_donor['fluorophore'], optical_channel=opt_ch_donor, device=device, description=meta_donor['description'], data=data_donor, starting_time=relative_start_time, rate=sample_rate_A, unit=meta_donor['unit'], ) frets_acceptor = FRETSeries( name='acceptor', fluorophore=meta_acceptor['fluorophore'], optical_channel=opt_ch_acceptor, device=device, description=meta_acceptor['description'], data=data_acceptor, starting_time=relative_start_time, rate=sample_rate_B, unit=meta_acceptor['unit'] ) # Add FRET to acquisition meta_fret = metadata['Ophys']['FRET'] fret = FRET( name=meta_fret['name'] + '_' + str(tr), excitation_lambda=float(meta_fret['excitation_lambda']), donor=frets_donor, acceptor=frets_acceptor ) nwbfile.add_acquisition(fret) # Add trial if add_trials: tr_stop = relative_start_time + n_frames_A / sample_rate_A nwbfile.add_trial( start_time=relative_start_time, stop_time=tr_stop )
class ShowPSTHTestCase(unittest.TestCase): def setUp(self): start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal()) create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) self.nwbfile = NWBFile(session_description='NWBFile for PSTH', identifier='NWB123', session_start_time=start_time, file_create_date=create_date) self.nwbfile.add_unit_column('location', 'the anatomical location of this unit') self.nwbfile.add_unit_column( 'quality', 'the quality for the inference of this unit') self.nwbfile.add_unit(id=1, spike_times=[2.2, 3.0, 4.5], obs_intervals=[[1, 10]], location='CA1', quality=0.95) self.nwbfile.add_unit(id=2, spike_times=[2.2, 3.0, 25.0, 26.0], obs_intervals=[[1, 10], [20, 30]], location='CA3', quality=0.85) self.nwbfile.add_unit(id=3, spike_times=[1.2, 2.3, 3.3, 4.5], obs_intervals=[[1, 10], [20, 30]], location='CA1', quality=0.90) self.nwbfile.add_trial_column( name='stim', description='the visual stimuli during the trial') self.nwbfile.add_trial(start_time=0.0, stop_time=2.0, stim='person') self.nwbfile.add_trial(start_time=3.0, stop_time=5.0, stim='ocean') self.nwbfile.add_trial(start_time=6.0, stop_time=8.0, stim='desert') def test_get_min_spike_time(self): assert (get_min_spike_time(self.nwbfile.units) == 1.2) def test_align_by_trials(self): ComparetoAT = [ np.array([2.2, 3.0, 25.0, 26.0]), np.array([-0.8, 0., 22., 23.]), np.array([-3.8, -3., 19., 20.]) ] AT = align_by_trials(self.nwbfile.units, index=1, before=20., after=30.) np.testing.assert_allclose(AT, ComparetoAT, rtol=1e-02) def test_align_by_time_intervals_Nonetrials_select(self): time_intervals = TimeIntervals(name='Test Time Interval') time_intervals.add_interval(start_time=21.0, stop_time=28.0) time_intervals.add_interval(start_time=22.0, stop_time=26.0) time_intervals.add_interval(start_time=22.0, stop_time=28.4) ATI = align_by_time_intervals(self.nwbfile.units, index=1, intervals=time_intervals, stop_label=None, before=20., after=30.) ComparedtoATI = [ np.array([-18.8, -18., 4., 5.]), np.array([-19.8, -19., 3., 4.]), np.array([-19.8, -19., 3., 4.]) ] np.testing.assert_array_equal(ATI, ComparedtoATI) def test_align_by_time_intervals(self): time_intervals = TimeIntervals(name='Test Time Interval') time_intervals.add_interval(start_time=21.0, stop_time=28.0) time_intervals.add_interval(start_time=22.0, stop_time=26.0) time_intervals.add_interval(start_time=22.0, stop_time=28.4) ATI = align_by_time_intervals(self.nwbfile.units, index=1, intervals=time_intervals, stop_label=None, before=20., after=30., rows_select=[0, 1]) ComparedtoATI = [ np.array([-18.8, -18., 4., 5.]), np.array([-19.8, -19., 3., 4.]) ] np.testing.assert_array_equal(ATI, ComparedtoATI)
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=False): this_session = (experiment.Session & session_key).fetch1() print(f'Exporting to NWB 2.0 for session: {this_session}...') # =============================================================================== # ============================== META INFORMATION =============================== # =============================================================================== # -- NWB file - a NWB2.0 file for each session file_name = '_'.join([ this_session['subject_nickname'], this_session['session_date'].strftime('%Y-%m-%d'), str(this_session['session']) ]) nwbfile = NWBFile( identifier=file_name, related_publications='http://dx.doi.org/10.1016/j.neuron.2017.05.005', experiment_description='Two-photon experiment recorded in {}'.format( this_session['brain_location_name']), session_description='Imaging session', session_start_time=datetime.combine(this_session['session_date'], zero_zero_time), file_create_date=datetime.now(tzlocal()), experimenter=this_session['username'], institution=institution, keywords=[ 'motor planning', 'anterior lateral cortex', 'medial motor cortex', 'ALM', 'MM', 'Two-photon imaging' ]) # -- subject subj = (lab.Subject & session_key).fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=this_session['subject_nickname'], genotype=' x '.join((lab.Subject.GeneModification & subj).fetch('gene_modification')), sex=subj['sex'], species=subj['species'], date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time) if subj['date_of_birth'] else None) # -- virus nwbfile.virus = json.dumps([{ k: str(v) for k, v in virus_injection.items() if k not in subj } for virus_injection in virus.VirusInjection * virus.Virus & session_key]) # =============================================================================== # ======================== IMAGING & SEGMENTATION =============================== # =============================================================================== scan = (imaging.Scan & session_key).fetch1() # ---- Structural Images ---------- images = pynwb.base.Images('images') if isinstance(scan['image_beads'], collections.Sequence): gcamp = pynwb.image.GrayscaleImage('GCaMP at 940nm', scan['image_gcamp']) images.add_image(gcamp) if isinstance(scan['image_ctb'], collections.Sequence): ctb = pynwb.image.RGBImage('CTB-647 IT', scan['image_ctb']) images.add_image(ctb) if isinstance(scan['image_beads'], collections.Sequence): beads = pynwb.image.GrayscaleImage('Beads PT', scan['image_beads']) images.add_image(beads) nwbfile.add_acquisition(images) imaging_plane = nwbfile.create_imaging_plane( name='Imaging plane', optical_channel=pynwb.ophys.OpticalChannel(name='green', description='green channel', emission_lambda=500.), description= 'Imaging session for PT and IT neurons during audio delay task', device=nwbfile.create_device( name='two-photon microscope with Thorlabs resonant galvo scannner' ), excitation_lambda=940., imaging_rate=300., indicator='GCaMP6s', location='ALM', conversion=1e-6, unit='micrometers') # ---- Frame Time information ----- frame_time = pynwb.image.TimeSeries(name='Frame Time', data=list( range(0, len(scan['frame_time']))), unit='a.u', timestamps=scan['frame_time']) nwbfile.add_acquisition(frame_time) # ----- Segementation information ----- # link the imaging segmentation to the nwb file ophys = nwbfile.create_processing_module('Ophys', 'Processing result of imaging') img_seg = pynwb.ophys.ImageSegmentation() ophys.add_data_interface(img_seg) pln_seg = pynwb.ophys.PlaneSegmentation(name='Plane Segmentation', description='plane segmentation', imaging_plane=imaging_plane) img_seg.add_plane_segmentation([pln_seg]) # insert ROI mask rois = (imaging.Scan.Roi & session_key).fetch(as_dict=True) for k, v in dict(roi_id='roi id', cell_type='PT, IT, or unknown', roi_trace='Trace on this session of this roi', neuropil_trace='Trace on this session of the neurophil', included='whether to include this roi into later analyses' ).items(): pln_seg.add_column(name=k, description=v) for roi in rois: mask = np.zeros([512, 512]) mask[np.unravel_index(roi['roi_pixel_list'] - 1, mask.shape, 'F')] = 1 pln_seg.add_roi(roi_id=roi['roi_idx'], image_mask=mask, cell_type=roi['cell_type'], roi_trace=roi['roi_trace'], neuropil_trace=roi['roi_trace'], included=roi['inc']) # =============================================================================== # =============================== BEHAVIOR TRIALS =============================== # =============================================================================== # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time' # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) dj_trial = experiment.SessionTrial * experiment.BehaviorTrial skip_adding_columns = experiment.Session.primary_key + ['trial_uid'] if experiment.SessionTrial & session_key: # Get trial descriptors from TrialSet.Trial and TrialStimInfo trial_columns = [{ 'name': tag, 'description': re.sub('\s+:|\s+', ' ', re.search(f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip() } for tag in dj_trial.heading.names if tag not in skip_adding_columns + ['start_time', 'stop_time']] # Add new table columns to nwb trial-table for trial-label for c in trial_columns: nwbfile.add_trial_column(**c) # Add entry to the trial-table for trial in (dj_trial & session_key).fetch(as_dict=True): trial['start_time'] = float(trial['start_time']) trial['stop_time'] = float( trial['stop_time']) if trial['stop_time'] else 5.0 [trial.pop(k) for k in skip_adding_columns] nwbfile.add_trial(**trial) # =============================================================================== # =============================== BEHAVIOR TRIAL EVENTS ========================== # =============================================================================== behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') nwbfile.add_acquisition(behav_event) for trial_event_type in (experiment.TrialEventType & experiment.TrialEvent & session_key).fetch('trial_event_type'): event_times, trial_starts = ( experiment.TrialEvent * experiment.SessionTrial & session_key & { 'trial_event_type': trial_event_type }).fetch('trial_event_time', 'start_time') if len(event_times) > 0: event_times = np.hstack( event_times.astype(float) + trial_starts.astype(float)) behav_event.create_timeseries(name=trial_event_type, unit='a.u.', conversion=1.0, data=np.full_like(event_times, 1), timestamps=event_times) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
def run_conversion( self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False, ): session_path = Path(self.source_data["folder_path"]) session_id = session_path.name # Stimuli [ nwbfile.add_stimulus(x) for x in get_events( session_path=session_path, suffixes=[".lrw.evt", ".puf.evt", ".rip.evt", ".rrw.evt"]) ] # Epochs df = pd.read_csv(session_path / f"{session_id}.cat.evt", sep=" ", names=("time", "begin_or_end", "of", "epoch_name")) epoch_starts = [] for j in range(int(len(df) / 2)): epoch_starts.append(df["time"][2 * j]) nwbfile.add_epoch(start_time=epoch_starts[j], stop_time=df["time"][2 * j + 1], tags=[df["epoch_name"][2 * j][18:]]) # Trials trialdata_path = session_path / f"{session_id}-TrackRunTimes.mat" if trialdata_path.is_file(): trials_data = loadmat(trialdata_path)["trackruntimes"] for trial_data in trials_data: nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1]) # .whl position whl_files = [] for whl_file in whl_files: add_position_data(nwbfile=nwbfile, session_path=session_path, whl_file_path=whl_file, starting_time=epoch_starts[j]) # States sleep_state_fpath = session_path / f"{session_id}.SleepState.states.mat" # label renaming state_label_names = dict(WAKEstate="Awake", NREMstate="Non-REM", REMstate="REM") if sleep_state_fpath.is_file(): matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0] table = TimeIntervals(name="states", description="Sleep states of animal.") table.add_column(name="label", description="Sleep state.") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append( dict(start_time=row[0], stop_time=row[1], label=state_label_names[name])) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] check_module(nwbfile, "behavior", "Contains behavioral data.").add(table)
module_cellular.add_container(cci_obj) trialdata_path = os.path.join(fpath, fname + '__EightMazeRun.mat') trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath, fname + '__EightMazeRunInfo.mat') trialdatainfo = [ x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0] ] features = trialdatainfo[:7] features[:2] = 'start', 'end' [nwbfile.add_trial_column(x, 'description') for x in features] for trial_data in trials_data: nwbfile.add_trial({lab: dat for lab, dat in zip(features, trial_data[:7])}) mono_syn_fpath = os.path.join(fpath, fname + '-MonoSynConvClick.mat') matin = loadmat(mono_syn_fpath) exc = matin['FinalExcMonoSynID'] inh = matin['FinalInhMonoSynID'] exc_obj = CatCellInfo('excitatory_connections', 'YutaMouse41-150903-MonoSynConvClick.mat', values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1) module_cellular.add_container(exc_obj) inh_obj = CatCellInfo('inhibitory_connections', 'YutaMouse41-150903-MonoSynConvClick.mat',
def chang2nwb(blockpath, outpath=None, session_start_time=None, session_description=None, identifier=None, anin4=False, ecog_format='auto', external_subject=True, include_pitch=False, include_intensity=False, speakers=True, mic=False, mini=False, hilb=False, verbose=False, imaging_path=None, parse_transcript=False, include_cortical_surfaces=True, include_electrodes=True, include_ekg=True, subject_image_list=None, rest_period=None, load_warped=False, **kwargs): """ Parameters ---------- blockpath: str outpath: None | str if None, output = [blockpath]/[blockname].nwb session_start_time: datetime.datetime default: datetime(1900, 1, 1) session_description: str default: blockname identifier: str default: blockname anin4: False | str Whether or not to convert ANIN4. ANIN4 is used as an extra channel for things like button presses, and is usually unused. If a string is supplied, that is used as the name of the timeseries. ecog_format: str ({'htk'}, 'mat', 'raw') external_subject: bool (optional) True: (default) cortical mesh is saved in an external file and a link is provided to that file. This is useful if you have multiple sessions for a single subject. False: cortical mesh is saved normally include_pitch: bool (optional) add pitch data. Default: False include_intensity: bool (optional) add intensity data. Default: False speakers: bool (optional) Default: False mic: bool (optional) default: False mini: only save data stub. Used for testing hilb: bool include Hilbert Transform data. Default: False verbose: bool (optional) imaging_path: str (optional) None: use IMAGING_DIR 'local': use subject_dir/Imaging/ else: use supplied string parse_transcript: str (optional) include_cortical_surfaces: bool (optional) include_electrodes: bool (optional) include_ekg: bool (optional) subject_image_list: list (optional) List of paths of images to include rest_period: None | array-like kwargs: dict passed to pynwb.NWBFile Returns ------- """ behav_module = None basepath, blockname = os.path.split(blockpath) subject_id = get_subject_id(blockname) if identifier is None: identifier = blockname if session_description is None: session_description = blockname if outpath is None: outpath = blockpath + '.nwb' out_base_path = os.path.split(outpath)[0] if session_start_time is None: session_start_time = datetime(1900, 1, 1).astimezone(timezone('UTC')) if imaging_path is None: subj_imaging_path = path.join(IMAGING_PATH, subject_id) elif imaging_path == 'local': subj_imaging_path = path.join(basepath, 'imaging') else: subj_imaging_path = os.path.join(imaging_path, subject_id) # file paths bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat') ecog_path = path.join(blockpath, 'RawHTK') ecog400_path = path.join(blockpath, 'ecog400', 'ecog.mat') elec_metadata_file = path.join(subj_imaging_path, 'elecs', 'TDT_elecs_all.mat') mesh_path = path.join(subj_imaging_path, 'Meshes') pial_files = glob.glob(path.join(mesh_path, '*pial.mat')) # Create the NWB file object nwbfile = NWBFile(session_description, identifier, session_start_time, datetime.now().astimezone(), session_id=identifier, institution='University of California, San Francisco', lab='Chang Lab', **kwargs) nwbfile.add_electrode_column('bad', 'electrode identified as too noisy') bad_elecs_inds = get_bad_elecs(blockpath) if include_electrodes: add_electrodes(nwbfile, elec_metadata_file, bad_elecs_inds, load_warped=load_warped) else: device = nwbfile.create_device('256Grid') electrode_group = nwbfile.create_electrode_group( name='256Grid electrodes', description='auto_group', location='location', device=device) for elec_counter in range(256): bad = elec_counter in bad_elecs_inds nwbfile.add_electrode(id=elec_counter + 1, x=np.nan, y=np.nan, z=np.nan, imp=np.nan, location=' ', filtering='none', group=electrode_group, bad=bad) ecog_elecs = list(range(len(nwbfile.electrodes))) ecog_elecs_region = nwbfile.create_electrode_table_region( ecog_elecs, 'ECoG electrodes on brain') # Read electrophysiology data from HTK files and add them to NWB file if ecog_format == 'auto': ecog_rate, data, ecog_path = auto_ecog(blockpath, ecog_elecs, verbose=False) elif ecog_format == 'htk': if verbose: print('reading htk acquisition...', flush=True) ecog_rate, data = readhtks(ecog_path, ecog_elecs) data = data.squeeze() if verbose: print('done', flush=True) elif ecog_format == 'mat': with File(ecog400_path, 'r') as f: data = f['ecogDS']['data'][:, ecog_elecs] ecog_rate = f['ecogDS']['sampFreq'][:].ravel()[0] ecog_path = ecog400_path elif ecog_format == 'raw': ecog_path = os.path.join(tdt_data_path, subject_id, blockname, 'raw.mat') ecog_rate, data = load_wavs(ecog_path) else: raise ValueError('unrecognized argument: ecog_format') ts_desc = "all Wav data" if mini: data = data[:2000] ecog_ts = ElectricalSeries(name='ElectricalSeries', data=H5DataIO(data, compression='gzip'), electrodes=ecog_elecs_region, rate=ecog_rate, description=ts_desc, conversion=0.001) nwbfile.add_acquisition(ecog_ts) if include_ekg: ekg_elecs = find_ekg_elecs(elec_metadata_file) if len(ekg_elecs): add_ekg(nwbfile, ecog_path, ekg_elecs) if mic: # Add microphone recording from room fs, data = get_analog(blockpath, 1) nwbfile.add_acquisition( TimeSeries('microphone', data, 'audio unit', rate=fs, description="audio recording from microphone in room")) if speakers: fs, data = get_analog(blockpath, 2) # Add audio stimulus 1 nwbfile.add_stimulus( TimeSeries('speaker 1', data, 'NA', rate=fs, description="audio stimulus 1")) # Add audio stimulus 2 fs, data = get_analog(blockpath, 3) if fs is not None: nwbfile.add_stimulus( TimeSeries('speaker 2', data, 'NA', rate=fs, description='the second stimulus source')) if anin4: fs, data = get_analog(blockpath, 4) nwbfile.add_acquisition( TimeSeries(anin4, data, 'aux unit', rate=fs, description="aux analog recording")) # Add bad time segments if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size: bad_time = sio.loadmat(bad_time_file)['badTimeSegments'] for row in bad_time: nwbfile.add_invalid_time_interval(start_time=row[0], stop_time=row[1], tags=('ECoG artifact', ), timeseries=ecog_ts) if rest_period is not None: nwbfile.add_epoch_column(name='label', description='label') nwbfile.add_epoch(start_time=rest_period[0], stop_time=rest_period[1], label='rest_period') if hilb: block_hilb_path = os.path.join(hilb_dir, subject_id, blockname, blockname + '_AA.h5') file = File(block_hilb_path, 'r') data = transpose_iter( file['X']) # transposes data during iterative write filter_center = file['filter_center'][:] filter_sigma = file['filter_sigma'][:] data = H5DataIO(DataChunkIterator(tqdm(data, desc='writing hilbert data'), buffer_size=400 * 20), compression='gzip') decomp_series = DecompositionSeries( name='LFPDecompositionSeries', description='Gaussian band Hilbert transform', data=data, rate=400., source_timeseries=ecog_ts, metric='amplitude') for band_mean, band_stdev in zip(filter_center, filter_sigma): decomp_series.add_band(band_mean=band_mean, band_stdev=band_stdev) hilb_mod = nwbfile.create_processing_module( name='ecephys', description='holds hilbert analysis results') hilb_mod.add_container(decomp_series) if include_cortical_surfaces: subject = ECoGSubject(subject_id=subject_id) subject.cortical_surfaces = create_cortical_surfaces( pial_files, subject_id) else: subject = Subject(subject_id=subject_id, species='H**o sapiens') if subject_image_list is not None: subject = add_images_to_subject(subject, subject_image_list) if external_subject: subj_fpath = path.join(out_base_path, subject_id + '.nwb') if not os.path.isfile(subj_fpath): subj_nwbfile = NWBFile(session_description=subject_id, identifier=subject_id, subject=subject, session_start_time=datetime( 1900, 1, 1).astimezone(timezone('UTC'))) with NWBHDF5IO(subj_fpath, manager=manager, mode='w') as subj_io: subj_io.write(subj_nwbfile) subj_read_io = NWBHDF5IO(subj_fpath, manager=manager, mode='r') subj_nwbfile = subj_read_io.read() subject = subj_nwbfile.subject nwbfile.subject = subject if parse_transcript: if parse_transcript == 'CV': parseout = parse(blockpath, blockname) df = make_df(parseout, 0, subject_id, align_pos=1) nwbfile.add_trial_column('cv_transition_time', 'time of CV transition in seconds') nwbfile.add_trial_column( 'speak', 'if True, subject is speaking. If False, subject is listening') nwbfile.add_trial_column('condition', 'syllable spoken') for _, row in df.iterrows(): nwbfile.add_trial(start_time=row['start'], stop_time=row['stop'], cv_transition_time=row['align'], speak=row['mode'] == 'speak', condition=row['label']) elif parse_transcript == 'singing': parseout = parse(blockpath, blockname) df = make_df(parseout, 0, subject_id, align_pos=0) if not len(df): df = pd.DataFrame(parseout) df['mode'] = 'speak' df = df.loc[df['label'].astype('bool'), :] # handle empty labels nwbfile.add_trial_column( 'speak', 'if True, subject is speaking. If False, subject is listening') nwbfile.add_trial_column('condition', 'syllable spoken') for _, row in df.iterrows(): nwbfile.add_trial(start_time=row['start'], stop_time=row['stop'], speak=row['mode'] == 'speak', condition=row['label']) elif parse_transcript == 'emphasis': parseout = parse(blockpath, blockname) try: df = make_df(parseout, 0, subject_id, align_pos=0) except: df = pd.DataFrame(parseout) if not len(df): df = pd.DataFrame(parseout) df = df.loc[df['label'].astype('bool'), :] # handle empty labels nwbfile.add_trial_column('condition', 'word emphasized') nwbfile.add_trial_column( 'speak', 'if True, subject is speaking. If False, subject is listening') for _, row in df.iterrows(): nwbfile.add_trial(start_time=row['start'], stop_time=row['stop'], speak=True, condition=row['label']) elif parse_transcript == 'MOCHA': nwbfile = create_transcription(nwbfile, transcript_path, blockname) # behavior if include_pitch: if behav_module is None: behav_module = nwbfile.create_processing_module( 'behavior', 'processing about behavior') if os.path.isfile( os.path.join(blockpath, 'pitch_' + blockname + '.mat')): fs, data = load_pitch(blockpath) pitch_ts = TimeSeries( data=data, rate=fs, unit='Hz', name='pitch', description= 'Pitch as extracted from Praat. NaNs mark unvoiced regions.') behav_module.add_container( BehavioralTimeSeries(name='pitch', time_series=pitch_ts)) else: print('No pitch file for ' + blockname) if include_intensity: if behav_module is None: behav_module = nwbfile.create_processing_module( 'behavior', 'processing about behavior') if os.path.isfile( os.path.join(blockpath, 'intensity_' + blockname + '.mat')): fs, data = load_pitch(blockpath) intensity_ts = TimeSeries( data=data, rate=fs, unit='dB', name='intensity', description='Intensity of speech in dB extracted from Praat.') behav_module.add_container( BehavioralTimeSeries(name='intensity', time_series=intensity_ts)) else: print('No intensity file for ' + blockname) # Export the NWB file with NWBHDF5IO(outpath, manager=manager, mode='w') as io: io.write(nwbfile) if external_subject: subj_read_io.close() if hilb: file.close() # read check with NWBHDF5IO(outpath, manager=manager, mode='r') as io: io.read()