def make(self, nwb_content: NWBFile): position = Position(name='position') cameras_ids = get_cameras_ids(self.dataset_names, self.metadata) meters_per_pixels = get_meters_per_pixels(cameras_ids, self.metadata) pos_online_paths = [] for dataset in self.datasets: for pos_file in dataset.get_all_data_from_dataset('pos'): if pos_file.endswith('.pos_online.dat'): pos_online_paths.append( os.path.join(dataset.get_data_path_from_dataset('pos'), pos_file)) first_timestamps = [] for series_id, (conversion, pos_online_path) in enumerate( zip(meters_per_pixels, pos_online_paths)): position_df = self.get_position_with_corrected_timestamps( pos_online_path) position.create_spatial_series( name=f'series_{series_id}', description=', '.join(position_df.columns.tolist()), data=np.asarray(position_df), conversion=conversion, reference_frame='Upper left corner of video frame', timestamps=np.asarray(position_df.index), ) first_timestamps.append(position_df.index[0]) # check if timestamps are in order first_timestamps = np.asarray(first_timestamps) assert np.all(first_timestamps[:-1] < first_timestamps[1:]) logger.info('Position: Injecting into Processing Module') nwb_content.processing['behavior'].add(position)
def generate_position_data(filename): ''' Read position data from .bin or .pos file and convert to pynwb.behavior.SpatialSeries objects. Parameters: ------- filename (Path or Str): Full filename of Axona file with any extension. Returns: ------- position (pynwb.behavior.Position) ''' position = Position() position_channel_names = 't,x1,y1,x2,y2,numpix1,numpix2,unused'.split(',') position_data = read_bin_file_position_data(filename) position_timestamps = position_data[:, 0] for ichan in range(0, position_data.shape[1]): spatial_series = SpatialSeries( name=position_channel_names[ichan], timestamps=position_timestamps, data=position_data[:, ichan], reference_frame='start of raw aquisition (.bin file)') position.add_spatial_series(spatial_series) return position
def create(position: Position, series_id: int, fl_position: FlPosition): validate_parameters_not_none(__name__, fl_position.column_labels, fl_position.position_data, fl_position.conversion) position.create_spatial_series( name='series_' + str(series_id), description=fl_position.column_labels, data=fl_position.position_data, conversion=fl_position.conversion, reference_frame='Upper left corner of video frame', timestamps=fl_position.timestamps, )
def add_behavior(nwbfile, expt): bd = expt.behaviorData(imageSync=True) fs = 1 / expt.frame_period() behavior_module = nwbfile.create_processing_module( name='Behavior', description='Data relevant to behavior') # Add Normalized Position pos = Position(name='Normalized Position') pos.create_spatial_series(name='Normalized Position', rate=fs, data=bd['treadmillPosition'][:, np.newaxis], reference_frame='0 is belt start', conversion=0.001 * bd['trackLength']) behavior_module.add_container(pos) # Add Licking licking = BehavioralTimeSeries(name='Licking') licking.create_timeseries( name='Licking', data=bd['licking'], rate=fs, unit='na', description='1 if mouse licked during this imaging frame') behavior_module.add_container(licking) # Add Water Reward Delivery water = BehavioralTimeSeries(name='Water') water.create_timeseries( name='Water', data=bd['water'], rate=fs, unit='na', description='1 if water was delivered during this imaging frame') behavior_module.add_container(water) # Add Lap Times laps = BehavioralEvents(name='Lap Starts') # TODO probably not best to have laps as data and timestamps here laps.create_timeseries(name='Lap Starts', data=bd['lap'], timestamps=bd['lap'], description='Frames at which laps began', unit='na') behavior_module.add_container(laps)
def write_position(nwbfile, f, name='Trajectory 100'): """ Parameters ---------- nwbfile: pynwb.NWBFile f: h5py.File name: str (optional) Returns ------- pynwb.core.ProcessingModule """ obj = f[name] behavior_mod = check_module(nwbfile, 'behavior') spatial_series = SpatialSeries('Position', data=np.array([obj['x'], obj['y']]).T, reference_frame='NA', conversion=1 / 100., resolution=np.nan, rate=float(1 / np.diff(obj['t'][:2]) * 1000)) behavior_mod.add_data_interface(Position(spatial_series)) return behavior_mod
def run_conversion(self, nwbfile: NWBFile, metadata: dict): file_path = self.source_data['file_path'] file = h5py.File(file_path, 'r') cell_info = file['cell_info'] cell_ids = [ ''.join([chr(x[0]) for x in file[x[0]]]) for x in cell_info['cell_id'] ] times = get_data(file, 'time') body_pos = get_data(file, 'body_position') body_speed = get_data(file, 'body_speed') horizontal_eye_pos = get_data(file, 'horizontal_eye_position') vertial_eye_pos = get_data(file, 'vertical_eye_position') horizontal_eye_vel = get_data(file, 'horiztonal_eye_velocity') vertial_eye_vel = get_data(file, 'vertical_eye_velocity') all_spike_times = [ file[x[0]][:].ravel() for x in cell_info['spike_times'] ] behavior = nwbfile.create_processing_module( name='behavior', description='contains processed behavioral data') spatial_series = SpatialSeries( name='position', data=body_pos, timestamps=times, conversion=.01, reference_frame='on track. Position is in VR.') behavior.add(Position(spatial_series=spatial_series)) behavior.add( TimeSeries(name='body_speed', data=body_speed, timestamps=spatial_series, unit='cm/s')) behavior.add( EyeTracking( spatial_series=SpatialSeries(name='eye_position', data=np.c_[horizontal_eye_pos, vertial_eye_pos], timestamps=spatial_series, reference_frame='unknown'))) behavior.add( TimeSeries(name='eye_velocity', data=np.c_[horizontal_eye_vel, vertial_eye_vel], timestamps=spatial_series, unit='unknown')) for spike_times, cell_id in zip(all_spike_times, cell_ids): id_ = int(cell_id.split('_')[-1]) nwbfile.add_unit(spike_times=spike_times, id=id_) return nwbfile
def test_init(self): sS = SpatialSeries('test_sS', np.ones((2, 2)), 'reference_frame', timestamps=[1., 2., 3.]) pc = Position(sS) self.assertEqual(pc.spatial_series.get('test_sS'), sS)
def setUp(self): spatial_series = SpatialSeries( name="position", data=np.linspace(0, 1, 20), rate=50.0, reference_frame="starting gate", ) self.position = Position(spatial_series=spatial_series)
def test_init(self): sS = SpatialSeries('test_sS', 'a hypothetical source', list(), 'reference_frame', timestamps=list()) pc = Position('test_pc', sS) self.assertEqual(pc.source, 'test_pc') self.assertEqual(pc.spatial_series, [sS])
def test_show_position(self): position = Position(spatial_series=self.spatial_series) show_position(position, default_neurodata_vis_spec)
def run_conversion(self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False): session_path = Path(self.source_data["folder_path"]) session_id = session_path.stem # Stimuli [nwbfile.add_stimulus(x) for x in get_events(session_path)] # States sleep_state_fpath = session_path / f"{session_id}.SleepState.states.mat" # label renaming specific to Peyrache state_label_names = dict(WAKEstate="Awake", NREMstate="Non-REM", REMstate="REM") if sleep_state_fpath.is_file(): matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0] table = TimeIntervals(name="states", description="Sleep states of animal.") table.add_column(name="label", description="Sleep state.") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append( dict(start_time=row[0], stop_time=row[1], label=state_label_names[name])) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] check_module(nwbfile, "behavior", "Contains behavioral data.").add(table) # Position pos_names = ["RedLED", "BlueLED"] pos_idx_from = [0, 2] pos_idx_to = [2, 4] # Raw position whlfile_path = session_path / f"{session_id}.whl" whl_data = np.loadtxt(whlfile_path) for name, idx_from, idx_to in zip(pos_names, pos_idx_from, pos_idx_to): nwbfile.add_acquisition( peyrache_spatial_series( name=name, description= "Raw sensor data. Values of -1 indicate that LED detection failed.", data=whl_data[:, idx_from:idx_to], conversion=np.nan, # whl file is in arbitrary grid units )) # Processed position posfile_path = session_path / f"{session_id}.pos" if posfile_path.is_file( ): # at least Mouse32-140820 was missing a .pos file try: pos_data = np.loadtxt(posfile_path) pos_obj = Position(name="SubjectPosition") for name, idx_from, idx_to in zip(pos_names, pos_idx_from, pos_idx_to): pos_obj.add_spatial_series( peyrache_spatial_series( name=name, description= ("(x,y) coordinates tracking subject movement through the maze." "Values of -1 indicate that LED detection failed." ), data=pos_data[:, idx_from:idx_to], conversion=1e-2, # from cm to m )) check_module(nwbfile, "behavior", "Contains behavioral data.").add(pos_obj) except ValueError: # data issue present in at least Mouse17-170201 warn(f"Skipping .pos file for session {session_id}!") # Epochs - only available for sessions with raw data epoch_file = session_path / "raw" / f"{session_id}-raw-info" / f"{session_id}-behaviors.txt" if epoch_file.is_file(): epoch_data = pd.read_csv(epoch_file, header=1)[f"{session_id}:"] epoch_dat_inds = [] epoch_names = [] for epochs in epoch_data: inds, name = epochs.split(": ") epoch_dat_inds.append(inds.split(" ")) epoch_names.append(name) epoch_windows = [0] for epoch in epoch_dat_inds: exp_end_times = [] for dat_ind in epoch: recording_file = session_path / "raw" / f"{session_id}{dat_ind}.dat" info_extractor = NeuroscopeRecordingExtractor( recording_file) dat_end_time = info_extractor.get_num_frames( ) / info_extractor.get_sampling_frequency() # seconds exp_end_times.extend([dat_end_time]) epoch_windows.extend([epoch_windows[-1] + sum(exp_end_times)] * 2) epoch_windows = np.array(epoch_windows[:-1]).reshape(-1, 2) for j, epoch_name in enumerate(epoch_names): nwbfile.add_epoch(start_time=epoch_windows[j][0], stop_time=epoch_windows[j][1], tags=[epoch_name])
# Data interfaces # --------------- # # NWB provides the concept of a *data interface*--an object for a standard # storage location of specific types of data--through the :py:class:`~pynwb.core.NWBDataInterface` class. # For example, :py:class:`~pynwb.behavior.Position` provides a container that holds one or more # :py:class:`~pynwb.behavior.SpatialSeries` objects. :py:class:`~pynwb.behavior.SpatialSeries` is a subtype of # :py:class:`~pynwb.base.TimeSeries` that represents the spatial position of an animal over time. By putting # your position data into a :py:class:`~pynwb.behavior.Position` container, downstream users and # tools know where to look to retrieve position data. For a comprehensive list of available data interfaces, see the # :ref:`overview page <modules_overview>`. Here is how to create a :py:class:`~pynwb.behavior.Position` object # named '`Position'` [#]_. from pynwb.behavior import Position position = Position() #################### # You can add objects to a data interface as a method of the data interface: position.create_spatial_series(name='position1', data=np.linspace(0, 1, 20), rate=50., reference_frame='starting gate') #################### # or you can add pre-existing objects: from pynwb.behavior import SpatialSeries spatial_series = SpatialSeries(name='position2',
def run_conversion(self, nwbfile: NWBFile, metadata: dict): mat_file_path = self.source_data["mat_file_path"] mat_file = loadmat(mat_file_path) trial_info = mat_file["SessionNP"] nwbfile.add_trial_column( name="reward_time", description="Time when subject began consuming reward.") nwbfile.add_trial_column( name="left_or_right", description="Time when subject began consuming reward.") l_r_dict = {1: "Right", 2: "Left"} for trial in trial_info: nwbfile.add_trial(start_time=trial[0], stop_time=trial[1], reward_time=trial[2], left_or_right=l_r_dict[int(trial[3])]) # Position pos_info = mat_file["whlrl"] pos_data = [pos_info[:, 0:1], pos_info[:, 2:3]] starting_time = 0.0 rate = 20000 / 512 # from CRCNS info conversion = np.nan # whl are arbitrary units pos_obj = Position(name="Position") for j in range(2): spatial_series_object = SpatialSeries( name=f"SpatialSeries{j+1}", description= "(x,y) coordinates tracking subject movement through the maze.", data=H5DataIO(pos_data[j], compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) pos_obj.add_spatial_series(spatial_series_object) get_module(nwbfile=nwbfile, name="behavior", description="Contains processed behavioral data." ).add_data_interface(pos_obj) linearized_pos = mat_file["whlrld"][:, 6] lin_pos_obj = Position(name="LinearizedPosition") lin_spatial_series_object = SpatialSeries( name="LinearizedTimeSeries", description= ("Linearized position, with '1' defined as start position (the position at the time of last nose-poking " "in the trial), and d=2 being the end position (position at the tiome just before reward consumption). " "d=0 means subject is not performing working memory trials."), data=H5DataIO(linearized_pos, compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) lin_pos_obj.add_spatial_series(lin_spatial_series_object) get_module(nwbfile=nwbfile, name="behavior").add_data_interface(lin_pos_obj)
for j in np.arange(96,192): nwb.add_unit(electrodes=[j],spike_times=np.ravel(f_info['spikes'][j,1]),electrode_group=electrode_group_S1) nwb.add_unit(electrodes=[j],spike_times=np.ravel(f_info['spikes'][j,2]),electrode_group=electrode_group_S1) # Add behavioral information # SpatialSeries and Position data interfaces to store cursor_pos cursor_pos = SpatialSeries(name='cursor_position', data=f_info['cursor_pos'], reference_frame='0,0', conversion=1e-3, resolution=1e-17, timestamps=np.ravel(f_info['t']), description='The position of the cursor in Cartesian coordinates (x, y) in mm') cursor_position = Position(name='CursorPosition',spatial_series=cursor_pos) # SpatialSeries and Position data interfaces to store finger_pos finger_pos = SpatialSeries(name='finger_position', data=f_info['finger_pos'], reference_frame='0,0', conversion=1e-2, resolution=1e-17, timestamps=np.ravel(f_info['t']), description='The position of the working fingertip in Cartesian coordinates (z, -x, -y), as reported by the hand tracker in cm') finger_position = Position(name='FingerPosition',spatial_series=finger_pos) # SpatialSeries and Position data interfaces to store target_pos target_pos = SpatialSeries(name='target_position', data=f_info['target_pos'], reference_frame='0,0', conversion=1e-3, resolution=1e-17, timestamps=np.ravel(f_info['t']),
def create_all(self, fl_positions: list) -> Position: position = Position(name='position') for series_id, fl_position in enumerate(fl_positions): self.create(position, series_id, fl_position) return position
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903', subject_xls=None, include_spike_waveforms=True, stub=True): subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] identifier = session_id mouse_number = session_id[9:11] if '-' in session_id: subject_id, date_text = session_id.split('-') b = False else: subject_id, date_text = session_id.split('b') b = True if subject_xls is None: subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx') else: if not subject_xls[-4:] == 'xlsx': subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx') session_start_time = dateparse(date_text, yearfirst=True) df = pd.read_excel(subject_xls) subject_data = {} for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']: names = df.iloc[:, 0] if key in names.values: subject_data[key] = df.iloc[np.argmax(names == key), 1] if isinstance(subject_data['DOB'], datetime): age = session_start_time - subject_data['DOB'] else: age = None subject = Subject(subject_id=subject_id, age=str(age), genotype=subject_data['genotype'], species='mouse') nwbfile = NWBFile(session_description='mouse in open exploration and theta maze', identifier=identifier, session_start_time=session_start_time.astimezone(), file_create_date=datetime.now().astimezone(), experimenter='Yuta Senzai', session_id=session_id, institution='NYU', lab='Buzsaki', subject=subject, related_publications='DOI:10.1016/j.neuron.2016.12.011') print('reading and writing raw position data...', end='', flush=True) ns.add_position_data(nwbfile, session_path) shank_channels = ns.get_shank_channels(session_path)[:8] all_shank_channels = np.concatenate(shank_channels) print('setting up electrodes...', end='', flush=True) hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv') lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b) print(lfp_channel) custom_column = [{'name': 'theta_reference', 'description': 'this electrode was used to calculate LFP canonical bands', 'data': all_shank_channels == lfp_channel}] ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks) print('reading LFPs...', end='', flush=True) lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub) lfp_data = all_channels_data[:, all_shank_channels] print('writing LFPs...', flush=True) # lfp_data[:int(len(lfp_data)/4)] lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp', description='lfp signal for all shank electrodes') for name, channel in special_electrode_dict.items(): ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data', data=all_channels_data[channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan) nwbfile.add_acquisition(ts) # compute filtered LFP print('filtering LFP...', end='', flush=True) all_lfp_phases = [] for passband in ('theta', 'gamma'): lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband) lfp_phase, _ = hilbert_lfp(lfp_fft) all_lfp_phases.append(lfp_phase[:, np.newaxis]) data = np.dstack(all_lfp_phases) print('done.', flush=True) if include_spike_waveforms: print('writing waveforms...', end='', flush=True) for shankn in np.arange(1, 9, dtype=int): ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub) print('done.', flush=True) decomp_series = DecompositionSeries(name='LFPDecompositionSeries', description='Theta and Gamma phase for reference LFP', data=data, rate=lfp_fs, source_timeseries=lfp_ts, metric='phase', unit='radians') decomp_series.add_band(band_name='theta', band_limits=(4, 10)) decomp_series.add_band(band_name='gamma', band_limits=(30, 80)) check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series) [nwbfile.add_stimulus(x) for x in ns.get_events(session_path)] # create epochs corresponding to experiments/environments for the mouse sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id)) exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name']))) for task_type in task_types) if exist_pos_data: nwbfile.add_epoch_column('label', 'name of epoch') for task_type in task_types: label = task_type['name'] file = os.path.join(session_path, session_id + '__' + label + '.mat') if os.path.isfile(file): print('loading position for ' + label + '...', end='', flush=True) pos_obj = Position(name=label + '_position') matin = loadmat(file) tt = matin['twhl_norm'][:, 0] exp_times = find_discontinuities(tt) if 'conversion' in task_type: conversion = task_type['conversion'] else: conversion = np.nan for pos_type in ('twhl_norm', 'twhl_linearized'): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + '_{}_spatial_series'.format(pos_type), data=H5DataIO(pos_data_norm, compression='gzip'), reference_frame='unknown', conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression='gzip')) pos_obj.add_spatial_series(spatial_series_object) check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + '_' + str(i)) print('done.') # there are occasional mismatches between the matlab struct and the neuroscope files # regions: 3: 'CA3', 4: 'DG' df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path) celltype_names = [] for celltype_id, region_id in zip(df_unit_features['fineCellType'].values, df_unit_features['region'].values): if celltype_id == 1: if region_id == 3: celltype_names.append('pyramidal cell') elif region_id == 4: celltype_names.append('granule cell') else: raise Exception('unknown type') elif not np.isfinite(celltype_id): celltype_names.append('missing') else: celltype_names.append(celltype_dict[celltype_id]) custom_unit_columns = [ { 'name': 'cell_type', 'description': 'name of cell type', 'data': celltype_names}, { 'name': 'global_id', 'description': 'global id for cell for entire experiment', 'data': df_unit_features['unitID'].values}, { 'name': 'max_electrode', 'description': 'electrode that has the maximum amplitude of the waveform', 'data': get_max_electrodes(nwbfile, session_path), 'table': nwbfile.electrodes }] ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks) trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat') if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat') trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]] features = trialdatainfo[:7] features[:2] = 'start_time', 'stop_time', [nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']] for trial_data in trials_data: if trial_data[3]: cond = 'run_left' else: cond = 'run_right' nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6]) """ mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat') matin = loadmat(mono_syn_fpath) exc = matin['FinalExcMonoSynID'] inh = matin['FinalInhMonoSynID'] #exc_obj = CatCellInfo(name='excitatory_connections', # indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1) #module_cellular.add_container(exc_obj) #inh_obj = CatCellInfo(name='inhibitory_connections', # indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1) #module_cellular.add_container(inh_obj) """ if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)['StatePeriod'] table = TimeIntervals(name='states', description='sleep states of animal') table.add_column(name='label', description='sleep state') data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({'start_time': row[0], 'stop_time': row[1], 'label': name}) [table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])] check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table) if stub: out_fname = session_path + '_stub.nwb' else: out_fname = session_path + '.nwb' print('writing NWB file...', end='', flush=True) with NWBHDF5IO(out_fname, mode='w') as io: io.write(nwbfile) print('done.') print('testing read...', end='', flush=True) # test read with NWBHDF5IO(out_fname, mode='r') as io: io.read() print('done.')
def test_init(self): sS = SpatialSeries('test_sS', list(), 'reference_frame', timestamps=list()) pc = Position(sS) self.assertEqual(pc.spatial_series.get('test_sS'), sS)
### from pynwb.behavior import SpatialSeries, Position position_data = np.array([np.linspace(0, 10, 100), np.linspace(1, 8, 100)]).T tt_position = np.linspace(0, 100) / 200 spatial_series_object = SpatialSeries(name='name', source='source', data=position_data, reference_frame='unknown', conversion=np.nan, resolution=np.nan, timestamps=tt_position) pos_obj = Position(source='source', spatial_series=spatial_series_object, name='name') behavior_module = nwbfile.create_processing_module( name='behavior', source='source', description='data relevant to behavior') behavior_module.add_container(pos_obj) ### from pynwb.file import Subject nwbfile.subject = Subject(age='9 months', description='description', species='rat', genotype='genotype', sex='M',
def add_behavior(nwbfile, behavior_file, metadata_behavior, t0): # The timestamps for behavior data, coming from .mat files, are in milliseconds # and should be transformed to seconds with / 1000 print("adding behavior...") # process raw behavior behavior_data = loadmat(behavior_file) behavior_module = nwbfile.create_processing_module( name='behavior', description='preprocessed behavioral data') # Player Position meta_pos = metadata_behavior['Position'] pos = Position(name=meta_pos['name']) all_pos = np.atleast_1d([[], [], []]).T all_tme = [] for iepoch in range(len(behavior_data["behavior"])): if 'posdat' in behavior_data['behavior'][iepoch]: epoch_data = behavior_data["behavior"][iepoch] all_pos = np.r_[all_pos, epoch_data['posdat']] all_tme = np.r_[all_tme, epoch_data['tme']] # Metadata for SpatialSeries stored in Position pos.create_spatial_series( name=meta_pos['spatial_series'][0]['name'], data=all_pos, reference_frame=meta_pos['spatial_series'][0]['reference_frame'], timestamps=all_tme / 1000. - t0, conversion=np.nan) behavior_module.add(pos) # nlx eye movements nlxeye = EyeTracking(name='EyeTracking') # metadata for SpatialSeries stored in EyeTracking meta_et = metadata_behavior['EyeTracking'] tt = np.array(behavior_data["nlxtme"]) / 1000. - t0 nlxeye.create_spatial_series( name=meta_et['spatial_series'][0]['name'], data=np.array(behavior_data["nlxeye"]).T, reference_frame=meta_et['spatial_series'][0]['reference_frame'], rate=len(tt) / (tt[-1] - tt[0]), starting_time=tt[0], #timestamps=np.array(behavior_data["nlxtme"]) / 1000. - t0, conversion=np.nan) behavior_module.add(nlxeye) # trial columns nwbfile.add_trial_column( name='environment', description='trial environment (calibration if calibration trial)') # nwbfile.add_trial_column(name='trial_vars', description='trial variables - different between calibration and task') # event key dictionary event_dict = { "new_trial": 1000, "right_trial": 1001, "left_trial": 1002, "reward_on": 1, "reward_off": 0, "end_trial": 100, "end_presentation": 101, "successful_trial": 200 } # process behavior here for iepoch in range(len(behavior_data["behavior"])): epoch_data = behavior_data["behavior"][iepoch] if iepoch < 2: process_behavior_calibration(nwbfile, iepoch + 1, epoch_data, t0) else: banana_flag = 1 process_behavior(nwbfile, iepoch + 1, epoch_data, banana_flag, event_dict, t0)
def convert_data( self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False ): session_path = self.input_args["folder_path"] # TODO: check/enforce format? task_types = metadata_dict.get("task_types", []) subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] [nwbfile.add_stimulus(x) for x in get_events(session_path)] exist_pos_data = any( os.path.isfile(os.path.join(session_path, "{}__{}.mat".format(session_id, task_type["name"]))) for task_type in task_types ) if exist_pos_data: nwbfile.add_epoch_column("label", "name of epoch") for task_type in task_types: label = task_type["name"] file = os.path.join(session_path, session_id + "__" + label + ".mat") if os.path.isfile(file): pos_obj = Position(name=label + "_position") matin = loadmat(file) tt = matin["twhl_norm"][:, 0] exp_times = find_discontinuities(tt) if "conversion" in task_type: conversion = task_type["conversion"] else: conversion = np.nan for pos_type in ("twhl_norm", "twhl_linearized"): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + "_{}_spatial_series".format(pos_type), data=H5DataIO(pos_data_norm, compression="gzip"), reference_frame="unknown", conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression="gzip"), ) pos_obj.add_spatial_series(spatial_series_object) check_module(nwbfile, "behavior", "contains processed behavioral data").add_data_interface(pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + "_" + str(i)) trialdata_path = os.path.join(session_path, session_id + "__EightMazeRun.mat") if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)["EightMazeRun"] trialdatainfo_path = os.path.join(fpath_base, "EightMazeRunInfo.mat") trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]] features = trialdatainfo[:7] features[:2] = ( "start_time", "stop_time", ) [nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"]] for trial_data in trials_data: if trial_data[3]: cond = "run_left" else: cond = "run_right" nwbfile.add_trial( start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6], ) sleep_state_fpath = os.path.join(session_path, "{}.SleepState.states.mat".format(session_id)) # label renaming specific to Watson state_label_names = {"WAKEstate": "Awake", "NREMstate": "Non-REM", "REMstate": "REM"} if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0] table = TimeIntervals(name="states", description="Sleep states of animal.") table.add_column(name="label", description="Sleep state.") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({"start_time": row[0], "stop_time": row[1], "label": state_label_names[name]}) [table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"])] check_module(nwbfile, "behavior", "contains behavioral data").add_data_interface(table)
def run_conversion(self, nwbfile: NWBFile, metadata: dict): session_path = Path(self.source_data["folder_path"]) session_id = session_path.stem # Load the file with behavioral data behavior_file_path = Path(session_path) / f"{session_id}.behavior.mat" behavior_mat = read_matlab_file(str(behavior_file_path))["behavior"] # Add trials events = behavior_mat["events"] trial_interval_list = events["trialIntervals"] data = [] for start_time, stop_time in trial_interval_list: data.append( dict( start_time=float(start_time), stop_time=float(stop_time), )) [ nwbfile.add_trial(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] trial_list = events["trials"] direction_list = [trial.get("direction", "") for trial in trial_list] trial_type_list = [trial.get("type", "") for trial in trial_list] if not all([direction == "" for direction in direction_list]): nwbfile.add_trial_column(name="direction", description="direction of the trial", data=direction_list) if not all([trial_type == "" for trial_type in trial_type_list]): nwbfile.add_trial_column(name="trial_type", description="type of trial", data=trial_type_list) # Position module_name = "behavior" module_description = "Contains behavioral data concerning position." processing_module = get_module(nwbfile=nwbfile, name=module_name, description=module_description) timestamps = np.array(behavior_mat["timestamps"])[..., 0] position = behavior_mat["position"] pos_data = [[x, y, z] for (x, y, z) in zip(position["x"], position["y"], position["y"]) ] pos_data = np.array(pos_data)[..., 0] unit = behavior_mat.get("units", "") if unit == ["m", "meter", "meters"]: conversion = 1.0 else: warnings.warn(f"Spatial units {unit} not listed in meters; " "setting conversion to nan.") conversion = np.nan description = behavior_mat.get("description", "generic_position_tracking").replace( "/", "-") rotation_type = behavior_mat.get("rotationType", "non_specified") pos_obj = Position(name=f"{description}_task".replace(" ", "_")) spatial_series_object = SpatialSeries( name="position", description="(x,y,z) coordinates tracking subject movement.", data=H5DataIO(pos_data, compression="gzip"), reference_frame="unknown", unit=unit, conversion=conversion, timestamps=timestamps, resolution=np.nan, ) pos_obj.add_spatial_series(spatial_series_object) # Add error if available errorPerMarker = behavior_mat.get("errorPerMarker", None) if errorPerMarker: error_data = np.array([error for error in errorPerMarker])[..., 0] spatial_series_object = SpatialSeries( name="error_per_marker", description= "Estimated error for marker tracking from optitrack system.", data=H5DataIO(error_data, compression="gzip"), reference_frame="unknown", conversion=conversion, timestamps=timestamps, resolution=np.nan, ) pos_obj.add_spatial_series(spatial_series_object) processing_module.add_data_interface(pos_obj) # Compass try: orientation = behavior_mat["orientation"] orientation_data = [[ x, y, z, w ] for (x, y, z, w) in zip(orientation["x"], orientation["y"], orientation["z"], orientation["w"])] orientation_data = np.array(orientation_data)[..., 0] compass_obj = CompassDirection(name=f"allocentric_frame_tracking") spatial_series_object = SpatialSeries( name="orientation", description= f"(x, y, z, w) orientation coordinates, orientation type: {rotation_type}", data=H5DataIO(orientation_data, compression="gzip"), reference_frame="unknown", conversion=conversion, timestamps=timestamps, resolution=np.nan, ) compass_obj.add_spatial_series(spatial_series_object) processing_module.add_data_interface(compass_obj) except KeyError: warnings.warn(f"Orientation data not found") # States module_name = "ecephys" module_description = "Contains behavioral data concerning classified states." processing_module = get_module(nwbfile=nwbfile, name=module_name, description=module_description) # Sleep states sleep_file_path = session_path / f"{session_id}.SleepState.states.mat" if Path(sleep_file_path).exists(): mat_file = read_matlab_file(sleep_file_path) state_label_names = dict(WAKEstate="Awake", NREMstate="Non-REM", REMstate="REM", MAstate="MA") sleep_state_dic = mat_file["SleepState"]["ints"] table = TimeIntervals(name="sleep_states", description="Sleep state of the animal.") table.add_column(name="label", description="Sleep state.") data = [] for sleep_state in state_label_names: values = sleep_state_dic[sleep_state] if len(values) != 0 and isinstance(values[0], int): values = [values] for start_time, stop_time in values: data.append( dict( start_time=float(start_time), stop_time=float(stop_time), label=state_label_names[sleep_state], )) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] processing_module.add(table) # Add epochs lfp_file_path = session_path / f"{session_path.name}.lfp" raw_file_path = session_path / f"{session_id}.dat" xml_file_path = session_path / f"{session_id}.xml" if raw_file_path.is_file(): recorder = NeuroscopeRecordingExtractor( file_path=raw_file_path, xml_file_path=xml_file_path) else: recorder = NeuroscopeRecordingExtractor( file_path=lfp_file_path, xml_file_path=xml_file_path) num_frames = recorder.get_num_frames() sampling_frequency = recorder.get_sampling_frequency() end_of_the_session = num_frames / sampling_frequency session_start = 0.0 start_trials_time = min( [interval[0] for interval in trial_interval_list]) end_trials_time = max( [interval[1] for interval in trial_interval_list]) end_of_the_session = end_of_the_session nwbfile.add_epoch(start_time=session_start, stop_time=start_trials_time, tags="before trials") nwbfile.add_epoch(start_time=start_trials_time, stop_time=end_trials_time, tags="during trials") nwbfile.add_epoch(start_time=end_trials_time, stop_time=end_of_the_session, tags="after trials")
def run_conversion(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False): session_path = Path(self.source_data["folder_path"]) task_types = [ dict(name="OpenFieldPosition_ExtraLarge"), dict(name="OpenFieldPosition_New_Curtain", conversion=0.46), dict(name="OpenFieldPosition_New", conversion=0.46), dict(name="OpenFieldPosition_Old_Curtain", conversion=0.46), dict(name="OpenFieldPosition_Old", conversion=0.46), dict(name="OpenFieldPosition_Oldlast", conversion=0.46), dict(name="EightMazePosition", conversion=0.65 / 2), ] subject_path = session_path.parent session_id = session_path.stem [nwbfile.add_stimulus(x) for x in get_events(session_path)] sleep_state_fpath = session_path / f"{session_id}--StatePeriod.mat" exist_pos_data = any([ (session_path / "{session_id}__{task_type['name']}.mat").is_file() for task_type in task_types ]) if exist_pos_data: nwbfile.add_epoch_column("label", "Name of epoch.") # Epoch intervals for task_type in task_types: label = task_type["name"] file = session_path / f"{session_id}__{label}.mat" if file.is_file(): pos_obj = Position(name=f"{label}_position") matin = loadmat(file) tt = matin["twhl_norm"][:, 0] exp_times = find_discontinuities(tt) if "conversion" in task_type: conversion = task_type["conversion"] else: conversion = np.nan for pos_type in ("twhl_norm", "twhl_linearized"): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=f"{label}_{pos_type}_spatial_series", data=H5DataIO(pos_data_norm, compression="gzip"), reference_frame="unknown", conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression="gzip"), ) pos_obj.add_spatial_series(spatial_series_object) check_module( nwbfile, "behavior", "Contains processed behavioral data.").add_data_interface( pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch( start_time=window[0], stop_time=window[1], tags=f"{label}_{str(i)}", ) # Trial intervals trialdata_path = session_path / f"{session_id}__EightMazeRun.mat" if trialdata_path.is_file(): trials_data = loadmat(trialdata_path)["EightMazeRun"] trialdatainfo_path = subject_path / "EightMazeRunInfo.mat" trialdatainfo = [ x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0] ] features = trialdatainfo[:7] features[:2] = ( "start_time", "stop_time", ) [ nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"] ] for trial_data in trials_data: if trial_data[3]: cond = "run_left" else: cond = "run_right" nwbfile.add_trial( start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6], ) # SLeep states if sleep_state_fpath.is_file(): matin = loadmat(sleep_state_fpath)["StatePeriod"] table = TimeIntervals(name="states", description="sleep states of animal") table.add_column(name="label", description="sleep state") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append( dict(start_time=row[0], stop_time=row[1], label=name)) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] check_module(nwbfile, "behavior", "Contains behavioral data.").add_data_interface(table)
def convert_data( self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False, ): """Convert the behavioral portion of a particular session of the GrosmarkAD dataset.""" session_path = self.input_args["folder_path"] subject_path, session_id = os.path.split(session_path) # Stimuli [nwbfile.add_stimulus(x) for x in get_events(session_path)] # States sleep_state_fpath = os.path.join(session_path, "{session_id}.SleepState.states.mat") # label renaming specific to Watson state_label_names = dict(WAKEstate="Awake", NREMstate="Non-REM", REMstate="REM") if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0] table = TimeIntervals(name="states", description="Sleep states of animal.") table.add_column(name="label", description="Sleep state.") data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append( dict( start_time=row[0], stop_time=row[1], label=state_label_names[name], )) [ table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"]) ] check_module(nwbfile, "behavior", "contains behavioral data").add_data_interface(table) # Position pos_filepath = Path( session_path) / f"{session_id}.position.behavior.mat" pos_mat = loadmat(str(pos_filepath.absolute())) starting_time = float( pos_mat["position"]["timestamps"][0][0] [0]) # confirmed to be a regularly sampled series rate = float( pos_mat["position"]["timestamps"][0][0][1]) - starting_time if pos_mat["position"]["units"][0][0][0] == "m": conversion = 1.0 else: warnings.warn( f"Spatial units ({pos_mat['position']['units'][0][0][0]}) not listed in meters; " "setting conversion to nan.") conversion = np.nan pos_data = [[x[0], y[0]] for x, y in zip( pos_mat["position"]["position"][0][0]["x"][0][0], pos_mat["position"]["position"][0][0]["y"][0][0], )] linearized_data = [[ lin[0] ] for lin in pos_mat["position"]["position"][0][0]["lin"][0][0]] label = pos_mat["position"]["behaviorinfo"][0][0]["MazeType"][0][0][ 0].replace(" ", "") pos_obj = Position(name=f"{label}Position") spatial_series_object = SpatialSeries( name=f"{label}SpatialSeries", description= "(x,y) coordinates tracking subject movement through the maze.", data=H5DataIO(pos_data, compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) pos_obj.add_spatial_series(spatial_series_object) check_module( nwbfile, "behavior", "contains processed behavioral data").add_data_interface(pos_obj) lin_pos_obj = Position(name=f"{label}LinearizedPosition") lin_spatial_series_object = SpatialSeries( name=f"{label}LinearizedTimeSeries", description= "Linearized position, defined as starting at the edge of reward area, " "and increasing clockwise, terminating at the opposing edge of the reward area.", data=H5DataIO(linearized_data, compression="gzip"), reference_frame="unknown", conversion=conversion, starting_time=starting_time, rate=rate, resolution=np.nan, ) lin_pos_obj.add_spatial_series(lin_spatial_series_object) check_module(nwbfile, "behavior", "contains processed behavioral data").add_data_interface( lin_pos_obj) # Epochs epoch_names = list(pos_mat["position"]["Epochs"][0][0].dtype.names) epoch_windows = [[float(start), float(stop)] for x in pos_mat["position"]["Epochs"][0][0][0][0] for start, stop in x] nwbfile.add_epoch_column("label", "name of epoch") for j, epoch_name in enumerate(epoch_names): nwbfile.add_epoch( start_time=epoch_windows[j][0], stop_time=epoch_windows[j][1], label=epoch_name, )
def test_show_position(self): position = Position(spatial_series=self.spatial_series) show_multi_container_interface(position, default_neurodata_vis_spec)
matin = loadmat(file) tt = matin['twhl_norm'][:, 0] pos_data = matin['twhl_norm'][:, 1:3] exp_times = find_discontinuities(tt) spatial_series_object = SpatialSeries(name=label + ' spatial_series', source='position sensor0', data=gzip(pos_data), reference_frame='unknown', conversion=np.nan, resolution=np.nan, timestamps=gzip(tt)) pos_obj = Position(source=source, spatial_series=spatial_series_object, name=label + '_position') module_behavior.add_container(pos_obj) for i, window in enumerate(exp_times): nwbfile.create_epoch(start_time=window[0], stop_time=window[1], tags=tuple(), description=label + '_' + str(i), timeseries=all_ts + [spatial_series_object]) print('done.') ## load celltypes matin = loadmat(os.path.join(fpath_base, 'DG_all_6__UnitFeatureSummary_add.mat'),
def convert( input_file, session_start_time, subject_date_of_birth, subject_id='I5', subject_description='naive', subject_genotype='wild-type', subject_sex='M', subject_weight='11.6g', subject_species='Mus musculus', subject_brain_region='Medial Entorhinal Cortex', surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible', session_id='npI5_0417_baseline_1', experimenter='Kei Masuda', experiment_description='Virtual Hallway Task', institution='Stanford University School of Medicine', lab_name='Giocomo Lab'): """ Read in the .mat file specified by input_file and convert to .nwb format. Parameters ---------- input_file : np.ndarray (..., n_channels, n_time) the .mat file to be converted subject_id : string the unique subject ID number for the subject of the experiment subject_date_of_birth : datetime ISO 8601 the date and time the subject was born subject_description : string important information specific to this subject that differentiates it from other members of it's species subject_genotype : string the genetic strain of this species. subject_sex : string Male or Female subject_weight : the weight of the subject around the time of the experiment subject_species : string the name of the species of the subject subject_brain_region : basestring the name of the brain region where the electrode probe is recording from surgery : str information about the subject's surgery to implant electrodes session_id: string human-readable ID# for the experiment session that has a one-to-one relationship with a recording session session_start_time : datetime date and time that the experiment started experimenter : string who ran the experiment, first and last name experiment_description : string what task was being run during the session institution : string what institution was the experiment performed in lab_name : string the lab where the experiment was performed Returns ------- nwbfile : NWBFile The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5 """ # input matlab data matfile = hdf5storage.loadmat(input_file) # output path for nwb data def replace_last(source_string, replace_what, replace_with): head, _sep, tail = source_string.rpartition(replace_what) return head + replace_with + tail outpath = replace_last(input_file, '.mat', '.nwb') create_date = datetime.today() timezone_cali = pytz.timezone('US/Pacific') create_date_tz = timezone_cali.localize(create_date) # if loading data from config.yaml, convert string dates into datetime if isinstance(session_start_time, str): session_start_time = datetime.strptime(session_start_time, '%B %d, %Y %I:%M%p') session_start_time = timezone_cali.localize(session_start_time) if isinstance(subject_date_of_birth, str): subject_date_of_birth = datetime.strptime(subject_date_of_birth, '%B %d, %Y %I:%M%p') subject_date_of_birth = timezone_cali.localize(subject_date_of_birth) # create unique identifier for this experimental session uuid_identifier = uuid.uuid1() # Create NWB file nwbfile = NWBFile( session_description=experiment_description, # required identifier=uuid_identifier.hex, # required session_id=session_id, experiment_description=experiment_description, experimenter=experimenter, surgery=surgery, institution=institution, lab=lab_name, session_start_time=session_start_time, # required file_create_date=create_date_tz) # optional # add information about the subject of the experiment experiment_subject = Subject(subject_id=subject_id, species=subject_species, description=subject_description, genotype=subject_genotype, date_of_birth=subject_date_of_birth, weight=subject_weight, sex=subject_sex) nwbfile.subject = experiment_subject # adding constants via LabMetaData container # constants sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0]) n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0]) dat_path = matfile['sp'][0]['dat_path'][0][0][0] offset = int(matfile['sp'][0]['offset'][0][0][0]) data_dtype = matfile['sp'][0]['dtype'][0][0][0] hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0]) vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0] # container lab_metadata = LabMetaData_ext(name='LabMetaData', acquisition_sampling_rate=sample_rate, number_of_electrodes=n_channels_dat, file_path=dat_path, bytes_to_skip=offset, raw_data_dtype=data_dtype, high_pass_filtered=hp_filtered, movie_start_time=vr_session_offset) nwbfile.add_lab_meta_data(lab_metadata) # Adding trial information nwbfile.add_trial_column( 'trial_contrast', 'visual contrast of the maze through which the mouse is running') trial = np.ravel(matfile['trial']) trial_nums = np.unique(trial) position_time = np.ravel(matfile['post']) # matlab trial numbers start at 1. To correctly index trial_contract vector, # subtracting 1 from 'num' so index starts at 0 for num in trial_nums: trial_times = position_time[trial == num] nwbfile.add_trial(start_time=trial_times[0], stop_time=trial_times[-1], trial_contrast=matfile['trial_contrast'][num - 1][0]) # Add mouse position inside: position = Position() position_virtual = np.ravel(matfile['posx']) # position inside the virtual environment sampling_rate = 1 / (position_time[1] - position_time[0]) position.create_spatial_series( name='Position', data=position_virtual, starting_time=position_time[0], rate=sampling_rate, reference_frame='The start of the trial, which begins at the start ' 'of the virtual hallway.', conversion=0.01, description='Subject position in the virtual hallway.', comments='The values should be >0 and <400cm. Values greater than ' '400cm mean that the mouse briefly exited the maze.', ) # physical position on the mouse wheel physical_posx = position_virtual trial_gain = np.ravel(matfile['trial_gain']) for num in trial_nums: physical_posx[trial == num] = physical_posx[trial == num] / trial_gain[num - 1] position.create_spatial_series( name='PhysicalPosition', data=physical_posx, starting_time=position_time[0], rate=sampling_rate, reference_frame='Location on wheel re-referenced to zero ' 'at the start of each trial.', conversion=0.01, description='Physical location on the wheel measured ' 'since the beginning of the trial.', comments='Physical location found by dividing the ' 'virtual position by the "trial_gain"') nwbfile.add_acquisition(position) # Add timing of lick events, as well as mouse's virtual position during lick event lick_events = BehavioralEvents() lick_events.create_timeseries( 'LickEvents', data=np.ravel(matfile['lickx']), timestamps=np.ravel(matfile['lickt']), unit='centimeter', description='Subject position in virtual hallway during the lick.') nwbfile.add_acquisition(lick_events) # Add information on the visual stimulus that was shown to the subject # Assumed rate=60 [Hz]. Update if necessary # Update external_file to link to Unity environment file visualization = ImageSeries( name='ImageSeries', unit='seconds', format='external', external_file=list(['https://unity.com/VR-and-AR-corner']), starting_time=vr_session_offset, starting_frame=[[0]], rate=float(60), description='virtual Unity environment that the mouse navigates through' ) nwbfile.add_stimulus(visualization) # Add the recording device, a neuropixel probe recording_device = nwbfile.create_device(name='neuropixel_probes') electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded' electrode_group_name = 'probe1' electrode_group = nwbfile.create_electrode_group( electrode_group_name, description=electrode_group_description, location=subject_brain_region, device=recording_device) # Add information about each electrode xcoords = np.ravel(matfile['sp'][0]['xcoords'][0]) ycoords = np.ravel(matfile['sp'][0]['ycoords'][0]) data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0] if data_filtered_flag: filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered' else: filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered' num_recording_electrodes = xcoords.shape[0] recording_electrodes = range(0, num_recording_electrodes) # create electrode columns for the x,y location on the neuropixel probe # the standard x,y,z locations are reserved for Allen Brain Atlas location nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe') nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe') for idx in recording_electrodes: nwbfile.add_electrode(id=idx, x=np.nan, y=np.nan, z=np.nan, rel_x=float(xcoords[idx]), rel_y=float(ycoords[idx]), imp=np.nan, location='medial entorhinal cortex', filtering=filter_desc, group=electrode_group) # Add information about each unit, termed 'cluster' in giocomo data # create new columns in unit table nwbfile.add_unit_column( 'quality', 'labels given to clusters during manual sorting in phy (1=MUA, ' '2=Good, 3=Unsorted)') # cluster information cluster_ids = matfile['sp'][0]['cids'][0][0] cluster_quality = matfile['sp'][0]['cgs'][0][0] # spikes in time spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike spike_cluster = np.ravel( matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time for i, cluster_id in enumerate(cluster_ids): unit_spike_times = spike_times[spike_cluster == cluster_id] waveforms = matfile['sp'][0]['temps'][0][cluster_id] nwbfile.add_unit(id=int(cluster_id), spike_times=unit_spike_times, quality=cluster_quality[i], waveform_mean=waveforms, electrode_group=electrode_group) # Trying to add another Units table to hold the results of the automatic spike sorting # create TemplateUnits units table template_units = Units( name='TemplateUnits', description='units assigned during automatic spike sorting') template_units.add_column( 'tempScalingAmps', 'scaling amplitude applied to the template when extracting spike', index=True) # information on extracted spike templates spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0]) spike_template_ids = np.unique(spike_templates) # template scaling amplitudes temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0]) for i, spike_template_id in enumerate(spike_template_ids): template_spike_times = spike_times[spike_templates == spike_template_id] temp_scaling_amps_per_template = temp_scaling_amps[spike_templates == spike_template_id] template_units.add_unit(id=int(spike_template_id), spike_times=template_spike_times, electrode_group=electrode_group, tempScalingAmps=temp_scaling_amps_per_template) # create ecephys processing module spike_template_module = nwbfile.create_processing_module( name='ecephys', description='units assigned during automatic spike sorting') # add template_units table to processing module spike_template_module.add(template_units) print(nwbfile) print('converted to NWB:N') print('saving ...') with NWBHDF5IO(outpath, 'w') as io: io.write(nwbfile) print('saved', outpath)
def main(): import os.path # prerequisites: start import numpy as np rate = 10.0 np.random.seed(1234) data_len = 1000 ephys_data = np.random.rand(data_len) ephys_timestamps = np.arange(data_len) / rate spatial_timestamps = ephys_timestamps[::10] spatial_data = np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T # prerequisites: end # create-nwbfile: start from datetime import datetime from dateutil.tz import tzlocal from pynwb import NWBFile f = NWBFile( 'the PyNWB tutorial', 'my first synthetic recording', 'EXAMPLE_ID', datetime.now(tzlocal()), experimenter='Dr. Bilbo Baggins', lab='Bag End Laboratory', institution='University of Middle Earth at the Shire', experiment_description= 'I went on an adventure with thirteen dwarves to reclaim vast treasures.', session_id='LONELYMTN') # create-nwbfile: end # save-nwbfile: start from pynwb import NWBHDF5IO filename = "example.h5" io = NWBHDF5IO(filename, mode='w') io.write(f) io.close() # save-nwbfile: end os.remove(filename) # create-device: start device = f.create_device(name='trodes_rig123', source="a source") # create-device: end # create-electrode-groups: start electrode_name = 'tetrode1' source = "an hypothetical source" description = "an example tetrode" location = "somewhere in the hippocampus" electrode_group = f.create_electrode_group(electrode_name, source=source, description=description, location=location, device=device) # create-electrode-groups: end # create-electrode-table-region: start for idx in [1, 2, 3, 4]: f.add_electrode(idx, x=1.0, y=2.0, z=3.0, imp=float(-idx), location='CA1', filtering='none', description='channel %s' % idx, group=electrode_group) electrode_table_region = f.create_electrode_table_region( [0, 2], 'the first and third electrodes') # create-electrode-table-region: end # create-timeseries: start from pynwb.ecephys import ElectricalSeries from pynwb.behavior import SpatialSeries ephys_ts = ElectricalSeries( 'test_ephys_data', 'an hypothetical source', ephys_data, electrode_table_region, timestamps=ephys_timestamps, # Alternatively, could specify starting_time and rate as follows # starting_time=ephys_timestamps[0], # rate=rate, resolution=0.001, comments= "This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") f.add_acquisition(ephys_ts) spatial_ts = SpatialSeries( 'test_spatial_timeseries', 'a stumbling rat', spatial_data, 'origin on x,y-plane', timestamps=spatial_timestamps, resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T" ) f.add_acquisition(spatial_ts) # create-timeseries: end # create-data-interface: start from pynwb.ecephys import LFP from pynwb.behavior import Position lfp = f.add_acquisition(LFP('a hypothetical source')) ephys_ts = lfp.create_electrical_series( 'test_ephys_data', 'an hypothetical source', ephys_data, electrode_table_region, timestamps=ephys_timestamps, resolution=0.001, comments= "This data was randomly generated with numpy, using 1234 as the seed", # noqa: E501 description="Random numbers generated with numpy.random.rand") pos = f.add_acquisition(Position('a hypothetical source')) spatial_ts = pos.create_spatial_series( 'test_spatial_timeseries', 'a stumbling rat', spatial_data, 'origin on x,y-plane', timestamps=spatial_timestamps, resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T" ) # noqa: E501 # create-data-interface: end # create-epochs: start epoch_tags = ('example_epoch', ) f.add_epoch(name='epoch1', start_time=0.0, stop_time=1.0, tags=epoch_tags, description="the first test epoch", timeseries=[ephys_ts, spatial_ts]) f.add_epoch(name='epoch2', start_time=0.0, stop_time=1.0, tags=epoch_tags, description="the second test epoch", timeseries=[ephys_ts, spatial_ts]) # create-epochs: end # create-compressed-timeseries: start from pynwb.ecephys import ElectricalSeries from pynwb.behavior import SpatialSeries from pynwb.form.backends.hdf5 import H5DataIO ephys_ts = ElectricalSeries( 'test_compressed_ephys_data', 'an hypothetical source', H5DataIO(ephys_data, compress=True), electrode_table_region, timestamps=H5DataIO(ephys_timestamps, compress=True), resolution=0.001, comments= "This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") f.add_acquisition(ephys_ts) spatial_ts = SpatialSeries( 'test_compressed_spatial_timeseries', 'a stumbling rat', H5DataIO(spatial_data, compress=True), 'origin on x,y-plane', timestamps=H5DataIO(spatial_timestamps, compress=True), resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T" ) f.add_acquisition(spatial_ts)
def conversion_function(source_paths, f_nwb, metadata, add_raw=False, add_processed=True, add_behavior=True, plot_rois=False): """ Copy data stored in a set of .npz files to a single NWB file. Parameters ---------- source_paths : dict Dictionary with paths to source files/directories. e.g.: {'raw_data': {'type': 'file', 'path': ''}, 'raw_info': {'type': 'file', 'path': ''} 'processed_data': {'type': 'file', 'path': ''}, 'sparse_matrix': {'type': 'file', 'path': ''}, 'ref_image',: {'type': 'file', 'path': ''}} f_nwb : str Path to output NWB file, e.g. 'my_file.nwb'. metadata : dict Metadata dictionary add_raw : bool Whether to convert raw data or not. add_processed : bool Whether to convert processed data or not. add_behavior : bool Whether to convert behavior data or not. plot_rois : bool Plot ROIs """ # Source files file_raw = None file_info = None file_processed = None file_sparse_matrix = None file_reference_image = None for k, v in source_paths.items(): if source_paths[k]['path'] != '': fname = source_paths[k]['path'] if k == 'raw_data': file_raw = h5py.File(fname, 'r') if k == 'raw_info': file_info = scipy.io.loadmat(fname, struct_as_record=False, squeeze_me=True) if k == 'processed_data': file_processed = np.load(fname) if k == 'sparse_matrix': file_sparse_matrix = np.load(fname) if k == 'ref_image': file_reference_image = np.load(fname) # Initialize a NWB object nwb = NWBFile(**metadata['NWBFile']) # Create and add device device = Device(name=metadata['Ophys']['Device'][0]['name']) nwb.add_device(device) # Creates one Imaging Plane for each channel fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0]) for meta_ip in metadata['Ophys']['ImagingPlane']: # Optical channel opt_ch = OpticalChannel( name=meta_ip['optical_channel'][0]['name'], description=meta_ip['optical_channel'][0]['description'], emission_lambda=meta_ip['optical_channel'][0]['emission_lambda']) nwb.create_imaging_plane( name=meta_ip['name'], optical_channel=opt_ch, description=meta_ip['description'], device=device, excitation_lambda=meta_ip['excitation_lambda'], imaging_rate=fs, indicator=meta_ip['indicator'], location=meta_ip['location'], ) # Raw optical data if add_raw: print('Adding raw data...') for meta_tps in metadata['Ophys']['TwoPhotonSeries']: if meta_tps['name'][-1] == 'R': raw_data = file_raw['R'] else: raw_data = file_raw['Y'] def data_gen(data): xl, yl, zl, tl = data.shape chunk = 0 while chunk < tl: val = data[:, :, :, chunk] chunk += 1 print('adding data chunk: ', chunk) yield val xl, yl, zl, tl = raw_data.shape tps_data = DataChunkIterator(data=data_gen(data=raw_data), iter_axis=0, maxshape=(tl, xl, yl, zl)) # Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard #raw_data = np.moveaxis(raw_data, -1, 0) tps = TwoPhotonSeries( name=meta_tps['name'], imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']], data=tps_data, rate=file_info['info'].daq.scanRate) nwb.add_acquisition(tps) # Processed data if add_processed: print('Adding processed data...') ophys_module = ProcessingModule( name='Ophys', description='contains optical physiology processed data.', ) nwb.add_processing_module(ophys_module) # Create Image Segmentation compartment img_seg = ImageSegmentation( name=metadata['Ophys']['ImageSegmentation']['name']) ophys_module.add(img_seg) # Create plane segmentation and add ROIs meta_ps = metadata['Ophys']['ImageSegmentation'][ 'plane_segmentations'][0] ps = img_seg.create_plane_segmentation( name=meta_ps['name'], description=meta_ps['description'], imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']], ) # Add ROIs indices = file_sparse_matrix['indices'] indptr = file_sparse_matrix['indptr'] dims = np.squeeze(file_processed['dims']) for start, stop in zip(indptr, indptr[1:]): voxel_mask = make_voxel_mask(indices[start:stop], dims) ps.add_roi(voxel_mask=voxel_mask) # Visualize 3D voxel masks if plot_rois: plot_rois_function(plane_segmentation=ps, indptr=indptr) # DFF measures dff = DfOverF(name=metadata['Ophys']['DfOverF']['name']) ophys_module.add(dff) # create ROI regions n_cells = file_processed['dFF'].shape[0] roi_region = ps.create_roi_table_region(description='RoiTableRegion', region=list(range(n_cells))) # create ROI response series dff_data = file_processed['dFF'] tt = file_processed['time'].ravel() meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0] meta_rrs['data'] = dff_data.T meta_rrs['rois'] = roi_region meta_rrs['timestamps'] = tt dff.create_roi_response_series(**meta_rrs) # Creates GrayscaleVolume containers and add a reference image grayscale_volume = GrayscaleVolume( name=metadata['Ophys']['GrayscaleVolume']['name'], data=file_reference_image['im']) ophys_module.add(grayscale_volume) # Behavior data if add_behavior: print('Adding behavior data...') # Ball motion behavior_mod = nwb.create_processing_module( name='Behavior', description='holds processed behavior data', ) meta_ts = metadata['Behavior']['TimeSeries'][0] meta_ts['data'] = file_processed['ball'].ravel() tt = file_processed['time'].ravel() meta_ts['timestamps'] = tt behavior_ts = TimeSeries(**meta_ts) behavior_mod.add(behavior_ts) # Re-arranges spatial data of body-points positions tracking pos = file_processed['dlc'] n_points = 8 pos_reshaped = pos.reshape( (-1, n_points, 3)) # dims=(nSamples,n_points,3) # Creates a Position object and add one SpatialSeries for each body-point position position = Position() for i in range(n_points): position.create_spatial_series( name='SpatialSeries_' + str(i), data=pos_reshaped[:, i, :], timestamps=tt, reference_frame= 'Description defining what the zero-position is.', conversion=np.nan) behavior_mod.add(position) # Trial times trialFlag = file_processed['trialFlag'].ravel() trial_inds = np.hstack( (0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1)) trial_times = tt[trial_inds] for start, stop in zip(trial_times, trial_times[1:]): nwb.add_trial(start_time=start, stop_time=stop) # Saves to NWB file with NWBHDF5IO(f_nwb, mode='w') as io: io.write(nwb) print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
dimension=[250, 250])) centroid_fname = 'm655_D11_S1_centroids.mat' pos_path = os.path.join(base_dir, centroid_fname) with File(pos_path, 'r') as file: pos_data = np.array(file['c']).T spatial_series = SpatialSeries(name='position', data=pos_data, starting_time=0.0, rate=5.0, units='unknown', reference_frame='unknown') pos = Position(name='position', spatial_series=spatial_series, starting_time=0.0, rate=np.nan) module_pos = nwbfile.create_processing_module(name='position', description='position') module_pos.add_container(pos) data_names = ['cellImages', 'cellTraces', 'centroids', 'eventTimes'] mat_data = {} mat_output_fpath = os.path.join(base_dir, 'm655_D11_S1_output.mat') with File(mat_output_fpath, 'r') as file: for name in data_names: mat_data[name] = file['output'][name][:] optical_channel = OpticalChannel('my channel', 'description', np.nan)
def convert_data(self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False): session_path = self.input_args['folder_path'] # TODO: check/enforce format? task_types = metadata_dict['task_types'] subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] [nwbfile.add_stimulus(x) for x in get_events(session_path)] sleep_state_fpath = os.path.join( session_path, '{}--StatePeriod.mat'.format(session_id)) exist_pos_data = any( os.path.isfile( os.path.join( session_path, '{}__{}.mat'.format(session_id, task_type['name']))) for task_type in task_types) if exist_pos_data: nwbfile.add_epoch_column('label', 'name of epoch') for task_type in task_types: label = task_type['name'] file = os.path.join(session_path, session_id + '__' + label + '.mat') if os.path.isfile(file): pos_obj = Position(name=label + '_position') matin = loadmat(file) tt = matin['twhl_norm'][:, 0] exp_times = find_discontinuities(tt) if 'conversion' in task_type: conversion = task_type['conversion'] else: conversion = np.nan for pos_type in ('twhl_norm', 'twhl_linearized'): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + '_{}_spatial_series'.format(pos_type), data=H5DataIO(pos_data_norm, compression='gzip'), reference_frame='unknown', conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression='gzip')) pos_obj.add_spatial_series(spatial_series_object) check_module( nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface( pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + '_' + str(i)) trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat') if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat') trialdatainfo = [ x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0] ] features = trialdatainfo[:7] features[:2] = 'start_time', 'stop_time', [ nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition'] ] for trial_data in trials_data: if trial_data[3]: cond = 'run_left' else: cond = 'run_right' nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6]) if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)['StatePeriod'] table = TimeIntervals(name='states', description='sleep states of animal') table.add_column(name='label', description='sleep state') data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({ 'start_time': row[0], 'stop_time': row[1], 'label': name }) [ table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time']) ] check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)