def run_conversion(self, nwbfile: NWBFile, metadata: dict): """ Run conversion for this data interface. Reads treadmill experiment behavioral data from csv files and adds it to nwbfile. Parameters ---------- nwbfile : NWBFile metadata : dict """ # Detect relevant files: trials summary, treadmill data and nose data dir_behavior_treadmill = self.source_data['dir_behavior_treadmill'] trials_file = [ f for f in Path(dir_behavior_treadmill).glob('*_tr.csv') if '~lock' not in f.name ][0] treadmill_file = trials_file.name.split('_tr')[0] + '.csv' nose_file = trials_file.name.split('_tr')[0] + '_mk.csv' trials_file = os.path.join(dir_behavior_treadmill, trials_file) treadmill_file = os.path.join(dir_behavior_treadmill, treadmill_file) nose_file = os.path.join(dir_behavior_treadmill, nose_file) # Add trials if nwbfile.trials is not None: print( 'Trials already exist in current nwb file. Treadmill behavior trials not added.' ) else: df_trials_summary = pd.read_csv(trials_file) nwbfile.add_trial_column(name='fail', description='no description') nwbfile.add_trial_column(name='reward_given', description='no description') nwbfile.add_trial_column(name='total_rewards', description='no description') nwbfile.add_trial_column(name='init_dur', description='no description') nwbfile.add_trial_column(name='light_dur', description='no description') nwbfile.add_trial_column(name='motor_dur', description='no description') nwbfile.add_trial_column(name='post_motor', description='no description') nwbfile.add_trial_column(name='speed', description='no description') nwbfile.add_trial_column(name='speed_mode', description='no description') nwbfile.add_trial_column(name='amplitude', description='no description') nwbfile.add_trial_column(name='period', description='no description') nwbfile.add_trial_column(name='deviation', description='no description') t_offset = df_trials_summary.loc[0]['Start Time'] for index, row in df_trials_summary.iterrows(): nwbfile.add_trial( start_time=row['Start Time'] - t_offset, stop_time=row['End Time'] - t_offset, fail=row['Fail'], reward_given=row['Reward Given'], total_rewards=row['Total Rewards'], init_dur=row['Init Dur'], light_dur=row['Light Dur'], motor_dur=row['Motor Dur'], post_motor=row['Post Motor'], speed=row['Speed'], speed_mode=row['Speed Mode'], amplitude=row['Amplitude'], period=row['Period'], deviation=row['+/- Deviation'], ) # Treadmill continuous data df_treadmill = pd.read_csv(treadmill_file, index_col=False) # Nose position continuous data df_nose = pd.read_csv(nose_file, index_col=False) # All behavioral data df_all = pd.concat([df_treadmill, df_nose], axis=1, sort=False) meta_behavioral_ts = metadata['Behavior'] t_offset = df_treadmill.loc[0]['Time'] for meta in meta_behavioral_ts.values(): ts = TimeSeries(name=meta['name'], data=df_all[meta['name']].to_numpy(), timestamps=df_all['Time'].to_numpy() - t_offset, description=meta['description']) nwbfile.add_acquisition(ts)
def run_conversion(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False, external_mode: bool = True, starting_times: Optional[list] = None, chunk_data: bool = True, module_name: Optional[str] = None, module_description: Optional[str] = None): """ Convert the movie data files to ImageSeries and write them in the NWBFile. Parameters ---------- nwbfile : NWBFile metadata : dict stub_test : bool, optional If True, truncates the write operation for fast testing. The default is False. external_mode : bool, optional ImageSeries in NWBFiles may contain either explicit movie data or file paths to external movie files. If True, this utilizes the more efficient method of merely encoding the file path linkage (recommended). For data sharing, the video files must be contained in the same folder as the NWBFile. If the intention of this NWBFile involves an upload to DANDI, the non-NWBFile types are not allowed so this flag would have to be set to False. The default is True. starting_times : list, optional List of start times for each movie. If unspecified, assumes that the movies in the file_paths list are in sequential order and are contiguous. chunk_data : bool, optional If True, uses a DataChunkIterator to read and write the movie, reducing overhead RAM usage at the cost of reduced conversion speed (compared to loading video entirely into RAM as an array). This will also force to True, even if manually set to False, whenever the video file size exceeds available system RAM by a factor of 70 (from compression experiments). Based on experiements for a ~30 FPS system of ~400 x ~600 color frames, the equivalent uncompressed RAM usage is around 2GB per minute of video. The default is True. module_name: str, optional Name of the processing module to add the ImageSeries object to. Default behavior is to add as acquisition. module_description: str, optional If the processing module specified by module_name does not exist, it will be created with this description. The default description is the same as used by the conversion_tools.get_module function. """ file_paths = self.source_data['file_paths'] if stub_test: count_max = 10 else: count_max = np.inf if starting_times is not None: assert isinstance(starting_times, list) and all([isinstance(x, float) for x in starting_times]) \ and len(starting_times) == len(file_paths), \ "Argument 'starting_times' must be a list of floats in one-to-one correspondence with 'file_paths'!" else: starting_times = [0.] for j, file in enumerate(file_paths): timestamps = starting_times[j] + get_movie_timestamps( movie_file=file) if len(starting_times) != len(file_paths): starting_times.append(timestamps[-1]) image_series_kwargs = dict(name=f"Video: {Path(file).stem}", description="Video recorded by camera.", unit="Frames") if check_regular_timestamps(ts=timestamps): fps = get_movie_fps(movie_file=file) image_series_kwargs.update(starting_time=starting_times[j], rate=fps) else: image_series_kwargs.update( timestamps=H5DataIO(timestamps, compression="gzip")) if external_mode: image_series_kwargs.update(format="external", external_file=[file]) else: uncompressed_estimate = Path(file).stat().st_size * 70 available_memory = psutil.virtual_memory().available if not chunk_data and uncompressed_estimate >= available_memory: warn( f"Not enough memory (estimated {round(uncompressed_estimate/1e9, 2)} GB) to load movie file as " f"array ({round(available_memory/1e9, 2)} GB available)! Forcing chunk_data to True." ) chunk_data = True total_frames = len(timestamps) frame_shape = get_frame_shape(movie_file=file) maxshape = [total_frames] maxshape.extend(frame_shape) best_gzip_chunk = (1, frame_shape[0], frame_shape[1], 3) tqdm_pos, tqdm_mininterval = (0, 10) if chunk_data: def data_generator(file, count_max): cap = cv2.VideoCapture(str(file)) for _ in range(min(count_max, total_frames)): success, frame = cap.read() yield frame cap.release() mov = DataChunkIterator( data=tqdm( iterable=data_generator(file=file, count_max=count_max), desc=f"Copying movie data for {Path(file).name}", position=tqdm_pos, total=total_frames, mininterval=tqdm_mininterval), iter_axis=0, # nwb standard is time as zero axis maxshape=tuple(maxshape)) image_series_kwargs.update(data=H5DataIO( mov, compression="gzip", chunks=best_gzip_chunk)) else: cap = cv2.VideoCapture(str(file)) mov = [] with tqdm(desc=f"Reading movie data for {Path(file).name}", position=tqdm_pos, total=total_frames, mininterval=tqdm_mininterval) as pbar: for _ in range(min(count_max, total_frames)): success, frame = cap.read() mov.append(frame) pbar.update(1) cap.release() image_series_kwargs.update(data=H5DataIO( DataChunkIterator( tqdm(iterable=np.array(mov), desc= f"Writing movie data for {Path(file).name}", position=tqdm_pos, mininterval=tqdm_mininterval), iter_axis=0, # nwb standard is time as zero axis maxshape=tuple(maxshape)), compression="gzip", chunks=best_gzip_chunk)) if module_name is None: nwbfile.add_acquisition(ImageSeries(**image_series_kwargs)) else: get_module(nwbfile=nwbfile, name=module_name, description=module_description).add( ImageSeries(**image_series_kwargs))
def writeRecording(recording, save_path, acquisition_name): try: from pynwb import NWBHDF5IO from pynwb import NWBFile from pynwb.ecephys import ElectricalSeries except ModuleNotFoundError: raise ModuleNotFoundError( "To use the Nwb extractors, install pynwb: \n\n" "pip install pynwb\n\n") M = recording.getNumChannels() N = recording.getNumFrames() nwbfile = NWBFile(source='SpikeInterface::NwbRecordingExtractor', session_description='', identifier='', session_start_time=datetime.now(), experimenter='', lab='', institution='', experiment_description='', session_id='') device = nwbfile.create_device(name='device_name', source="device_source") eg_name = 'electrode_group_name' eg_source = "electrode_group_source" eg_description = "electrode_group_description" eg_location = "electrode_group_location" electrode_group = nwbfile.create_electrode_group( name=eg_name, source=eg_source, location=eg_location, device=device, description=eg_description) for m in range(M): id = m location = recording.getChannelProperty(m, 'location') impedence = -1.0 while len(location) < 3: location = np.append(location, [0]) nwbfile.add_electrode(id, x=float(location[0]), y=float(location[1]), z=float(location[2]), imp=impedence, location='electrode_location', filtering='none', group=electrode_group, description='electrode_description') electrode_table_region = nwbfile.create_electrode_table_region( list(range(M)), 'electrode_table_region') rate = recording.getSamplingFrequency() / 1000 ephys_data = recording.getTraces().T ephys_ts = ElectricalSeries( name=acquisition_name, source='acquisition_source', data=ephys_data, electrodes=electrode_table_region, starting_time=recording.frameToTime(0), rate=rate, resolution=1e-6, comments='Generated from SpikeInterface::NwbRecordingExtractor', description='acquisition_description') nwbfile.add_acquisition(ephys_ts) if os.path.exists(save_path): os.remove(save_path) with NWBHDF5IO(save_path, 'w') as io: io.write(nwbfile)
class TestH5DataIO(unittest.TestCase): """ Test that H5DataIO functions correctly on round trip with the HDF5IO backend """ def setUp(self): self.nwbfile = NWBFile('a', 'b', datetime(1970, 1, 1, 12, tzinfo=tzutc())) self.path = "test_pynwb_io_hdf5_h5dataIO.h5" def tearDown(self): if (os.path.exists(self.path)): os.remove(self.path) def test_gzip_timestamps(self): ts = TimeSeries('ts_name', [1, 2, 3], 'A', timestamps=H5DataIO(np.array([1., 2., 3.]), compression='gzip')) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) # confirm that the dataset was indeed compressed infile = File(self.path, 'r') self.assertEquals( infile['/acquisition/ts_name/timestamps'].compression, 'gzip') def test_write_dataset_custom_compress(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), compression='gzip', compression_opts=5, shuffle=True, fletcher32=True) ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) infile = File(self.path, 'r') dset = infile['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.compression, 'gzip') self.assertEqual(dset.compression_opts, 5) self.assertEqual(dset.shuffle, True) self.assertEqual(dset.fletcher32, True) def test_write_dataset_custom_chunks(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), chunks=(1, 1, 3)) ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) infile = File(self.path, 'r') dset = infile['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.chunks, (1, 1, 3)) def test_write_dataset_custom_fillvalue(self): a = H5DataIO(np.arange(20).reshape(5, 4), fillvalue=-1) ts = TimeSeries('ts_name', a, 'A', timestamps=np.arange(5)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) infile = File(self.path, 'r') dset = infile['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.fillvalue, -1) def test_write_dataset_datachunkiterator(self): a = np.arange(30).reshape(5, 2, 3) aiter = iter(a) daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) ts = TimeSeries('ts_name', daiter, 'A', timestamps=np.arange(5)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) infile = File(self.path, 'r') dset = infile['/acquisition/ts_name/data'] self.assertListEqual(dset[:].tolist(), a.tolist()) def test_write_dataset_datachunkiterator_with_compression(self): a = np.arange(30).reshape(5, 2, 3) aiter = iter(a) daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) wrapped_daiter = H5DataIO(data=daiter, compression='gzip', compression_opts=5, shuffle=True, fletcher32=True) ts = TimeSeries('ts_name', wrapped_daiter, 'A', timestamps=np.arange(5)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile) infile = File(self.path, 'r') dset = infile['/acquisition/ts_name/data'] self.assertEqual(dset.shape, a.shape) self.assertListEqual(dset[:].tolist(), a.tolist()) self.assertEqual(dset.compression, 'gzip') self.assertEqual(dset.compression_opts, 5) self.assertEqual(dset.shuffle, True) self.assertEqual(dset.fletcher32, True)
class ABF1Converter: """ Converts Neuron2BrainLab's ABF1 files from a single cell (collected without amplifier settings from the multi-clamp commander) to a collective NeurodataWithoutBorders v2 file. Modeled after ABFConverter created by the Allen Institute. Parameters ---------- inputPath: path to ABF file or a folder of ABF files to be converted outputFilePath: path to the output NWB file acquisitionChannelName: Allows to output only a specific acquisition channel, defaults to all stimulusChannelName: Allows to output only a specific stimulus channel, defaults to all. The name can also be an AD channel name for cases where the stimulus is recorded as well. responseGain: user-input float indicating scalar gain for response channel stimulusGain: user-input float indicating scalar gain for stimulus channel clampMode: 0 or 1 integer indicating clamp mode (0 is VC, 1 is CC). If not None, overwrites clamp mode provided in ABF file """ def __init__(self, inputPath, outputFilePath, acquisitionChannelName=None, stimulusChannelName=None, responseGain = 1, stimulusGain = 1, responseOffset = 0, clampMode = None): self.inputPath = inputPath self.debug=False if os.path.isfile(self.inputPath): print(inputPath) abf = pyabf.ABF(self.inputPath) if abf.abfVersion["major"] != 1: raise ValueError(f"The ABF version for the file {abf} is not supported.") self.fileNames = [os.path.basename(self.inputPath)] self.abfFiles = [abf] elif os.path.isdir(self.inputPath): abfFiles = [] for dirpath, dirnames, filenames in os.walk(self.inputPath): # Find all .abf files in the directory if len(dirnames) == 0 and len(glob.glob(dirpath + "/*.abf")) != 0: abfFiles += glob.glob(dirpath + "/*.abf") if len(abfFiles) == 0: raise ValueError(f"{inputPath} contains no ABF Files.") # Arrange the ABF files in ascending order abfFiles.sort(key=lambda x: os.path.basename(x)) # Collect file names for description self.fileNames = [] for file in abfFiles: self.fileNames += [os.path.basename(file)] self.abfFiles = [] for abfFile in abfFiles: # Load each ABF file using pyabf abf = pyabf.ABF(abfFile) # Check for ABF version if abf.abfVersion["major"] != 1: raise ValueError(f"The ABF version for the file {abf} is not supported.") self.abfFiles += [abf] if clampMode: self.clampMode = clampMode # sometimes the abf-based clamp mode is wrong else: self.clampMode = self.abfFiles[0]._headerV1.nExperimentType self.outputPath = outputFilePath # Take metadata input, and return hard coded values for None self.responseGain = responseGain self.stimulusGain = stimulusGain self.responseOffset = responseOffset self.acquisitionChannelName = acquisitionChannelName self.stimulusChannelName = stimulusChannelName def _outputMetadata(self): """ Create metadata files in HTML format next to the existing ABF files. """ for abfFile in self.abfFiles: root, ext = os.path.splitext(abfFile.abfFilePath) pyabf.abfHeaderDisplay.abfInfoPage(abfFile).generateHTML(saveAs=root + ".html") def _getComments(self, abf): """ Accesses the tag comments created in Clampfit """ return abf.tagComments def _createNWBFile(self): """ Creates the NWB file for the cell, as defined by PyNWB """ self.start_time = self.abfFiles[0].abfDateTime self.inputCellName = os.path.basename(self.inputPath) creatorInfo = self.abfFiles[0]._headerV1.sCreatorInfo creatorVersion = self.abfFiles[0]._headerV1.creatorVersionString experiment_description = (f"{creatorInfo} v{creatorVersion}") self.NWBFile = NWBFile( session_description="", session_start_time=self.start_time, experiment_description = experiment_description, identifier=self.inputCellName, file_create_date= datetime.now(tzlocal()), experimenter=None, notes="" ) return self.NWBFile def _createDevice(self): creatorInfo = self.abfFiles[0]._headerV1.sCreatorInfo creatorVersion = self.abfFiles[0]._headerV1.creatorVersionString self.device = self.NWBFile.create_device(name=f"{creatorInfo} {creatorVersion}") def _createElectrode(self): self.electrode = self.NWBFile.create_ic_electrode(name='elec0', device=self.device, description='PLACEHOLDER') def _unitConversion(self, unit): # Returns a 2-list of base unit and conversion factor if unit == 'V': return 1.0, 'V' elif unit == 'mV': return 1e-3, 'V' elif unit == 'A': return 1.0, 'A' elif unit == 'pA': return 1e-12, 'A' elif unit == 'nA': return 1e-9, 'A' else: # raise ValueError(f"{unit} is not a valid unit.") return 1.0, 'V' # hard coded for units stored as '?' def _getClampMode(self): """ Returns the clamp mode of the experiment. Voltage Clamp Mode = 0 Current Clamp Mode = 1 """ return self.clampMode def _addStimulus(self): """ Adds a stimulus class as defined by PyNWB to the NWB File. Written for experiments conducted from a single channel. For multiple channels, refer to https://github.com/AllenInstitute/ipfx/blob/master/ipfx/x_to_nwb/ABFConverter.py """ for idx, abfFile in enumerate(self.abfFiles): isStimulus = True if self.stimulusChannelName is None: channelList = abfFile.adcNames channelIndices = range(len(channelList)) else: if self.stimulusChannelName in abfFile.dacNames: channelList = abfFile.dacNames channelIndices = [channelList.index(self.stimulusChannelName)] elif self.stimulusChannelName in abfFile.adcNames: isStimulus = False channelList = abfFile.adcNames channelIndices = [channelList.index(self.stimulusChannelName)] else: raise ValueError(f"Channel {self.stimulusChannelName} could not be found.") for i in range(abfFile.sweepCount): for channelIndex in channelIndices: if self.debug: print(f"stimulus: abfFile={abfFile.abfFilePath}, sweep={i}, channelIndex={channelIndex}, channelName={channelList[channelIndex]}") # Collect data from pyABF abfFile.setSweep(i, channel=channelIndex) seriesName = f"Index_{idx}_{i}_{channelIndex}" if isStimulus: data = abfFile.sweepC scaledUnit = abfFile.sweepUnitsC else: data = abfFile.sweepY scaledUnit = abfFile.sweepUnitsY stimulusGain = self.stimulusGain data = data * stimulusGain conversion, unit = self._unitConversion(scaledUnit) electrode = self.electrode resolution = np.nan starting_time = 0.0 rate = float(abfFile.dataRate) # Create a JSON file for the description field description = json.dumps({"file_name": os.path.basename(self.fileNames[idx]), "file_version": abfFile.abfVersionString, "sweep_number": i, "protocol": abfFile.protocol, "protocol_path": abfFile.protocolPath, "comments": self._getComments(abfFile)}, sort_keys=True, indent=4) # Determine the clamp mode if self.clampMode == 0: stimulusClass = VoltageClampStimulusSeries elif self.clampMode == 1: stimulusClass = CurrentClampStimulusSeries else: raise ValueError(f"Unsupported clamp mode {self.clampMode}") data = createCompressedDataset(data) # Create a stimulus class stimulus = stimulusClass(name=seriesName, data=data, sweep_number=i, electrode=electrode, gain=stimulusGain, resolution=resolution, conversion=conversion, starting_time=starting_time, rate=rate, unit=unit, description=description ) self.NWBFile.add_stimulus(stimulus) def _addAcquisition(self): """ Adds an acquisition class as defined by PyNWB to the NWB File. Written for experiments conducted from a single channel. For multiple channels, refer to https://github.com/AllenInstitute/ipfx/blob/master/ipfx/x_to_nwb/ABFConverter.py """ for idx, abfFile in enumerate(self.abfFiles): if self.acquisitionChannelName is None: channelList = abfFile.adcNames channelIndices = range(len(channelList)) else: if self.acquisitionChannelName in abfFile.adcNames: channelList = abfFile.adcNames channelIndices = [channelList.index(self.acquisitionChannelName)] else: raise ValueError(f"Channel {self.acquisitionChannelName} could not be found.") for i in range(abfFile.sweepCount): for channelIndex in channelIndices: if self.debug: print(f"acquisition: abfFile={abfFile.abfFilePath}, sweep={i}, channelIndex={channelIndex}, channelName={channelList[channelIndex]}") # Collect data from pyABF abfFile.setSweep(i, channel=channelIndex) seriesName = f"Index_{idx}_{i}_{channelIndex}" responseGain = self.responseGain responseOffset = self.responseOffset data = abfFile.sweepY * responseGain + responseOffset conversion, unit = self._unitConversion(abfFile.sweepUnitsY) electrode = self.electrode resolution = np.nan starting_time = 0.0 rate = float(abfFile.dataRate) # Create a JSON file for the description field description = json.dumps({"file_name": os.path.basename(self.fileNames[idx]), "file_version": abfFile.abfVersionString, "sweep_number": i, "protocol": abfFile.protocol, "protocol_path": abfFile.protocolPath, "comments": self._getComments(abfFile)}, sort_keys=True, indent=4) # Create an acquisition class # Note: voltage input produces current output; current input produces voltage output data = createCompressedDataset(data) if self.clampMode == 1: acquisition = CurrentClampSeries(name=seriesName, data=data, sweep_number=i, electrode=electrode, gain=responseGain, resolution=resolution, conversion=conversion, starting_time=starting_time, rate=rate, unit=unit, description=description, bias_current=np.nan, bridge_balance=np.nan, capacitance_compensation=np.nan, ) elif self.clampMode == 0: acquisition = VoltageClampSeries(name=seriesName, data=data, sweep_number=i, electrode=electrode, gain=responseGain, resolution=resolution, conversion=conversion, starting_time=starting_time, rate=rate, unit=unit, description=description, capacitance_fast=np.nan, capacitance_slow=np.nan, resistance_comp_bandwidth=np.nan, resistance_comp_correction=np.nan, resistance_comp_prediction=np.nan, whole_cell_capacitance_comp=np.nan, whole_cell_series_resistance_comp=np.nan ) else: raise ValueError(f"Unsupported clamp mode {self.clampMode}") self.NWBFile.add_acquisition(acquisition) def convert(self): """ Iterates through the functions in the specified order. :return: True (for success) """ self._createNWBFile() self._createDevice() self._createElectrode() self._getClampMode() self._addStimulus() self._addAcquisition() with NWBHDF5IO(self.outputPath, "w") as io: io.write(self.NWBFile, cache_spec=True) print(f"Successfully converted to {self.outputPath}.")
filename1 = 'external1_example.nwb' filename2 = 'external2_example.nwb' filename3 = 'external_linkcontainer_example.nwb' filename4 = 'external_linkdataset_example.nwb' # Create the first file nwbfile1 = NWBFile(session_description='demonstrate external files', identifier='NWBE1', session_start_time=start_time, file_create_date=create_date) # Create the second file test_ts1 = TimeSeries(name='test_timeseries1', data=data, unit='SIunit', timestamps=timestamps) nwbfile1.add_acquisition(test_ts1) # Write the first file io = NWBHDF5IO(filename1, 'w') io.write(nwbfile1) io.close() # Create the second file nwbfile2 = NWBFile(session_description='demonstrate external files', identifier='NWBE2', session_start_time=start_time, file_create_date=create_date) # Create the second file test_ts2 = TimeSeries(name='test_timeseries2', data=data, unit='SIunit', timestamps=timestamps)
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=True): this_session = (acquisition.Session & session_key).fetch1() # =============== General ==================== # -- NWB file - a NWB2.0 file for each session nwbfile = NWBFile(session_description=this_session['session_note'], identifier='_'.join([ this_session['subject_id'], this_session['session_time'].strftime('%Y-%m-%d'), this_session['session_id'] ]), session_start_time=this_session['session_time'], file_create_date=datetime.now(tzlocal()), experimenter='; '.join( (acquisition.Session.Experimenter & session_key).fetch('experimenter')), institution=institution, experiment_description=experiment_description, related_publications=related_publications, keywords=keywords) # -- subject subj = (subject.Subject & session_key).fetch1() nwbfile.subject = pynwb.file.Subject( subject_id=this_session['subject_id'], description=subj['subject_description'], genotype=' x '.join( (subject.Subject.Allele & session_key).fetch('allele')), sex=subj['sex'], species=subj['species']) # =============== Intracellular ==================== cell = ((intracellular.Cell & session_key).fetch1() if len(intracellular.Cell & session_key) == 1 else None) if cell: # metadata whole_cell_device = nwbfile.create_device(name=cell['device_name']) ic_electrode = nwbfile.create_ic_electrode( name=cell['cell_id'], device=whole_cell_device, description='N/A', filtering='N/A', location='; '.join([ f'{k}: {str(v)}' for k, v in dict((reference.BrainLocation & cell).fetch1(), depth=cell['recording_depth']).items() ])) # acquisition - membrane potential mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1( 'membrane_potential', 'membrane_potential_timestamps') nwbfile.add_acquisition( pynwb.icephys.PatchClampSeries(name='PatchClampSeries', electrode=ic_electrode, unit='mV', conversion=1e-3, gain=1.0, data=mp, timestamps=mp_timestamps)) # acquisition - spike train spk, spk_timestamps = (intracellular.SpikeTrain & cell).fetch1( 'spike_train', 'spike_timestamps') nwbfile.add_acquisition( pynwb.icephys.PatchClampSeries(name='SpikeTrain', electrode=ic_electrode, unit='a.u.', conversion=1e1, gain=1.0, data=spk, timestamps=spk_timestamps)) # =============== Behavior ==================== behavior_data = ((behavior.Behavior & session_key).fetch1() if len(behavior.Behavior & session_key) == 1 else None) if behavior_data: behav_acq = pynwb.behavior.BehavioralTimeSeries(name='behavior') nwbfile.add_acquisition(behav_acq) [behavior_data.pop(k) for k in behavior.Behavior.primary_key] timestamps = behavior_data.pop('behavior_timestamps') # get behavior data description from the comments of table definition behavior_descriptions = { attr: re.search(f'(?<={attr})(.*)#(.*)', str(behavior.Behavior.heading)).groups()[-1].strip() for attr in behavior_data } for b_k, b_v in behavior_data.items(): behav_acq.create_timeseries(name=b_k, description=behavior_descriptions[b_k], unit='a.u.', conversion=1.0, data=b_v, timestamps=timestamps) # =============== Photostimulation ==================== photostim = ((stimulation.PhotoStimulation & session_key).fetch1() if len(stimulation.PhotoStimulation & session_key) == 1 else None) if photostim: photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1() stim_device = nwbfile.create_device( name=photostim_device['device_name']) stim_site = pynwb.ogen.OptogeneticStimulusSite( name='-'.join([photostim['hemisphere'], photostim['brain_region']]), device=stim_device, excitation_lambda=float( (stimulation.PhotoStimulationProtocol & photostim).fetch1('photo_stim_excitation_lambda')), location='; '.join([ f'{k}: {str(v)}' for k, v in (reference.ActionLocation & photostim).fetch1().items() ]), description=(stimulation.PhotoStimulationProtocol & photostim).fetch1('photo_stim_notes')) nwbfile.add_ogen_site(stim_site) if photostim['photostim_timeseries'] is not None: nwbfile.add_stimulus( pynwb.ogen.OptogeneticSeries( name='_'.join([ 'photostim_on', photostim['photostim_datetime'].strftime( '%Y-%m-%d_%H-%M-%S') ]), site=stim_site, resolution=0.0, conversion=1e-3, data=photostim['photostim_timeseries'], starting_time=photostim['photostim_start_time'], rate=photostim['photostim_sampling_rate'])) # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: # 'id', 'start_time' and 'stop_time'. # Other trial-related information needs to be added in to the trial-table as additional columns (with column name # and column description) if len((acquisition.TrialSet & session_key).fetch()) == 1: # Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any) trial_columns = [{ 'name': tag.replace('trial_', ''), 'description': re.search( f'(?<={tag})(.*)#(.*)', str((acquisition.TrialSet.Trial * stimulation.TrialPhotoStimInfo ).heading)).groups()[-1].strip() } for tag in acquisition.TrialSet.Trial.heading.names if tag not in acquisition.TrialSet.Trial.primary_key + ['start_time', 'stop_time']] # Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time # also add `_time` suffix to all events trial_events = set(((acquisition.TrialSet.EventTime & session_key) - [{ 'trial_event': 'trial_start' }, { 'trial_event': 'trial_stop' }]).fetch('trial_event')) event_names = [{ 'name': e + '_time', 'description': d } for e, d in zip(*(reference.ExperimentalEvent & [{ 'event': k } for k in trial_events]).fetch('event', 'description'))] # Add new table columns to nwb trial-table for trial-label for c in trial_columns + event_names: nwbfile.add_trial_column(**c) # Add entry to the trial-table for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True): events = dict( zip(*(acquisition.TrialSet.EventTime & trial & [{ 'trial_event': e } for e in trial_events] ).fetch('trial_event', 'event_time'))) # shift event times to be relative to session_start (currently relative to trial_start) events = {k: v + trial['start_time'] for k, v in events.items()} trial_tag_value = {**trial, **events} # rename 'trial_id' to 'id' trial_tag_value['id'] = trial_tag_value['trial_id'] [ trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key ] # Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix events = {k + '_time': trial_tag_value.pop(k) for k in events} trial_attrs = { k.replace('trial_', ''): trial_tag_value.pop(k) for k in [n for n in trial_tag_value if n.startswith('trial_')] } nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs) # =============== Write NWB 2.0 file =============== if save: save_file_name = ''.join([nwbfile.identifier, '.nwb']) if not os.path.exists(nwb_output_dir): os.makedirs(nwb_output_dir) if not overwrite and os.path.exists( os.path.join(nwb_output_dir, save_file_name)): return nwbfile with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io: io.write(nwbfile) print(f'Write NWB 2.0 file: {save_file_name}') return nwbfile
device = nwbfile.create_device(device_label) electrode_group = nwbfile.create_electrode_group(name=device_label + ' electrode group', description=' ', device=device, location='unknown') nwbfile.add_electrode(i, x, y, z, imp=np.nan, location='unknown', filtering='unknown', description=label, group=electrode_group) electrode_table_region = nwbfile.create_electrode_table_region( list(range(len(electrode_positions))), 'all ECoG electrodes') nwbfile.add_acquisition( LFP(electrical_series=ElectricalSeries('lfp', 'lfp signal for all electrodes', lfp, electrode_table_region, starting_time=0.0, rate=lfp_rate))) with NWBHDF5IO('resting_state.nwb') as io: io.write(nwbfile)
ts = TetrodeSeries( 'test_ephys_data', 'an hypothetical source', data, electrode_table_region, timestamps=timestamps, trode_id=1, # Alternatively, could specify starting_time and rate as follows # starting_time=ephys_timestamps[0], # rate=rate, resolution=0.001, comments= "This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") nwbfile.add_acquisition(ts) #################### # .. note:: # # For more information on writing :py:class:`~pynwb.ecephys.ElectricalSeries`, # see :ref:`ecephys_tutorial`. # # Now that we have some data, lets write our file while caching our spec: from pynwb import NWBHDF5IO io = NWBHDF5IO('cache_spec_example.nwb', mode='w') io.write(nwbfile, cache_spec=True) io.close()
def main(): import os.path # prerequisites: start import numpy as np rate = 10.0 np.random.seed(1234) data_len = 1000 ephys_data = np.random.rand(data_len) ephys_timestamps = np.arange(data_len) / rate spatial_timestamps = ephys_timestamps[::10] spatial_data = np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T # prerequisites: end # create-nwbfile: start from datetime import datetime from pynwb import NWBFile f = NWBFile('the PyNWB tutorial', 'my first synthetic recording', 'EXAMPLE_ID', datetime.now(), experimenter='Dr. Bilbo Baggins', lab='Bag End Laboratory', institution='University of Middle Earth at the Shire', experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.', session_id='LONELYMTN') # create-nwbfile: end # save-nwbfile: start from pynwb import get_manager from pynwb.form.backends.hdf5 import HDF5IO filename = "example.h5" io = HDF5IO(filename, manager=get_manager(), mode='w') io.write(f) io.close() # save-nwbfile: end os.remove(filename) # create-device: start device = f.create_device(name='trodes_rig123', source="a source") # create-device: end # create-electrode-groups: start electrode_name = 'tetrode1' source = "an hypothetical source" description = "an example tetrode" location = "somewhere in the hippocampus" electrode_group = f.create_electrode_group(electrode_name, source=source, description=description, location=location, device=device) # create-electrode-groups: end # create-electrode-table-region: start for idx in [1, 2, 3, 4]: f.add_electrode(idx, x=1.0, y=2.0, z=3.0, imp=float(-idx), location='CA1', filtering='none', description='channel %s' % idx, group=electrode_group) electrode_table_region = f.create_electrode_table_region([0, 2], 'the first and third electrodes') # create-electrode-table-region: end # create-timeseries: start from pynwb.ecephys import ElectricalSeries from pynwb.behavior import SpatialSeries ephys_ts = ElectricalSeries('test_ephys_data', 'an hypothetical source', ephys_data, electrode_table_region, timestamps=ephys_timestamps, # Alternatively, could specify starting_time and rate as follows # starting_time=ephys_timestamps[0], # rate=rate, resolution=0.001, comments="This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") f.add_acquisition(ephys_ts) spatial_ts = SpatialSeries('test_spatial_timeseries', 'a stumbling rat', spatial_data, 'origin on x,y-plane', timestamps=spatial_timestamps, resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T") f.add_acquisition(spatial_ts) # create-timeseries: end # create-data-interface: start from pynwb.ecephys import LFP from pynwb.behavior import Position lfp = f.add_acquisition(LFP('a hypothetical source')) ephys_ts = lfp.create_electrical_series('test_ephys_data', 'an hypothetical source', ephys_data, electrode_table_region, timestamps=ephys_timestamps, resolution=0.001, comments="This data was randomly generated with numpy, using 1234 as the seed", # noqa: E501 description="Random numbers generated with numpy.random.rand") pos = f.add_acquisition(Position('a hypothetical source')) spatial_ts = pos.create_spatial_series('test_spatial_timeseries', 'a stumbling rat', spatial_data, 'origin on x,y-plane', timestamps=spatial_timestamps, resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T") # noqa: E501 # create-data-interface: end # create-epochs: start epoch_tags = ('example_epoch',) f.create_epoch(name='epoch1', start_time=0.0, stop_time=1.0, tags=epoch_tags, description="the first test epoch", timeseries=[ephys_ts, spatial_ts]) f.create_epoch(name='epoch2', start_time=0.0, stop_time=1.0, tags=epoch_tags, description="the second test epoch", timeseries=[ephys_ts, spatial_ts]) # create-epochs: end # create-compressed-timeseries: start from pynwb.ecephys import ElectricalSeries from pynwb.behavior import SpatialSeries from pynwb.form.backends.hdf5 import H5DataIO ephys_ts = ElectricalSeries('test_compressed_ephys_data', 'an hypothetical source', H5DataIO(ephys_data, compress=True), electrode_table_region, timestamps=H5DataIO(ephys_timestamps, compress=True), resolution=0.001, comments="This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") f.add_acquisition(ephys_ts) spatial_ts = SpatialSeries('test_compressed_spatial_timeseries', 'a stumbling rat', H5DataIO(spatial_data, compress=True), 'origin on x,y-plane', timestamps=H5DataIO(spatial_timestamps, compress=True), resolution=0.1, comments="This data was generated with numpy, using 1234 as the seed", description="This 2D Brownian process generated with " "np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T") f.add_acquisition(spatial_ts)
nwbfile = NWBFile(session_start_time=session_start_time, identifier=this_dir, session_description='unknown') device = nwbfile.create_device(name='Neuronexus Probe Buzsaki32/H32Package') group = nwbfile.create_electrode_group(name='all_channels_group', description='all channels', device=device, location='unknown') for i in range(nchannels): nwbfile.add_electrode(np.nan, np.nan, np.nan, # position imp=np.nan, location='unknown', filtering='unknown', group=group) electrode_table_region = nwbfile.create_electrode_table_region( list(range(nchannels)), 'all electrodes') electrical_series = ElectricalSeries(data=amp_data, rate=amp_fs, electrodes=electrode_table_region, name='amp_data') nwbfile.add_acquisition(LFP(name='amp_data', electrical_series=electrical_series)) nwbfile.add_acquisition(TimeSeries('auxiliary', data=aux_data, rate=amp_fs, unit='na')) nwbfile.add_acquisition(TimeSeries('supply', data=supply_data, rate=amp_fs, unit='na')) out_fname = this_dir + '.nwb' with NWBHDF5IO(out_fname, 'w') as io: io.write(nwbfile)
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903', subject_xls=None, include_spike_waveforms=True, stub=True, cache_spec=True): subject_path, session_id = os.path.split(session_path) fpath_base = os.path.split(subject_path)[0] identifier = session_id mouse_number = session_id[9:11] if '-' in session_id: subject_id, date_text = session_id.split('-') b = False else: subject_id, date_text = session_id.split('b') b = True if subject_xls is None: subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx') else: if not subject_xls[-4:] == 'xlsx': subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx') session_start_time = dateparse(date_text, yearfirst=True) df = pd.read_excel(subject_xls) subject_data = {} for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']: names = df.iloc[:, 0] if key in names.values: subject_data[key] = df.iloc[np.argmax(names == key), 1] if isinstance(subject_data['DOB'], datetime): age = session_start_time - subject_data['DOB'] else: age = None subject = Subject(subject_id=subject_id, age=str(age), genotype=subject_data['genotype'], species='mouse') nwbfile = NWBFile(session_description='mouse in open exploration and theta maze', identifier=identifier, session_start_time=session_start_time.astimezone(), file_create_date=datetime.now().astimezone(), experimenter='Yuta Senzai', session_id=session_id, institution='NYU', lab='Buzsaki', subject=subject, related_publications='DOI:10.1016/j.neuron.2016.12.011') print('reading and writing raw position data...', end='', flush=True) ns.add_position_data(nwbfile, session_path) shank_channels = ns.get_shank_channels(session_path)[:8] all_shank_channels = np.concatenate(shank_channels) print('setting up electrodes...', end='', flush=True) hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv') lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b) print(lfp_channel) custom_column = [{'name': 'theta_reference', 'description': 'this electrode was used to calculate LFP canonical bands', 'data': all_shank_channels == lfp_channel}] ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks) print('reading LFPs...', end='', flush=True) lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub) lfp_data = all_channels_data[:, all_shank_channels] print('writing LFPs...', flush=True) # lfp_data[:int(len(lfp_data)/4)] lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp', description='lfp signal for all shank electrodes') for name, channel in special_electrode_dict.items(): ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data', data=all_channels_data[:, channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan) nwbfile.add_acquisition(ts) # compute filtered LFP print('filtering LFP...', end='', flush=True) all_lfp_phases = [] for passband in ('theta', 'gamma'): lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband) lfp_phase, _ = hilbert_lfp(lfp_fft) all_lfp_phases.append(lfp_phase[:, np.newaxis]) data = np.dstack(all_lfp_phases) print('done.', flush=True) if include_spike_waveforms: print('writing waveforms...', end='', flush=True) nshanks = min((max_shanks, len(ns.get_shank_channels(session_path)))) for shankn in np.arange(nshanks, dtype=int) + 1: ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub) print('done.', flush=True) decomp_series = DecompositionSeries(name='LFPDecompositionSeries', description='Theta and Gamma phase for reference LFP', data=data, rate=lfp_fs, source_timeseries=lfp_ts, metric='phase', unit='radians') decomp_series.add_band(band_name='theta', band_limits=(4, 10)) decomp_series.add_band(band_name='gamma', band_limits=(30, 80)) check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series) [nwbfile.add_stimulus(x) for x in ns.get_events(session_path)] # create epochs corresponding to experiments/environments for the mouse sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id)) exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name']))) for task_type in task_types) if exist_pos_data: nwbfile.add_epoch_column('label', 'name of epoch') for task_type in task_types: label = task_type['name'] file = os.path.join(session_path, session_id + '__' + label + '.mat') if os.path.isfile(file): print('loading position for ' + label + '...', end='', flush=True) pos_obj = Position(name=label + '_position') matin = loadmat(file) tt = matin['twhl_norm'][:, 0] exp_times = find_discontinuities(tt) if 'conversion' in task_type: conversion = task_type['conversion'] else: conversion = np.nan for pos_type in ('twhl_norm', 'twhl_linearized'): if pos_type in matin: pos_data_norm = matin[pos_type][:, 1:] spatial_series_object = SpatialSeries( name=label + '_{}_spatial_series'.format(pos_type), data=H5DataIO(pos_data_norm, compression='gzip'), reference_frame='unknown', conversion=conversion, resolution=np.nan, timestamps=H5DataIO(tt, compression='gzip')) pos_obj.add_spatial_series(spatial_series_object) check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj) for i, window in enumerate(exp_times): nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + '_' + str(i)) print('done.') # there are occasional mismatches between the matlab struct and the neuroscope files # regions: 3: 'CA3', 4: 'DG' df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path) celltype_names = [] for celltype_id, region_id in zip(df_unit_features['fineCellType'].values, df_unit_features['region'].values): if celltype_id == 1: if region_id == 3: celltype_names.append('pyramidal cell') elif region_id == 4: celltype_names.append('granule cell') else: raise Exception('unknown type') elif not np.isfinite(celltype_id): celltype_names.append('missing') else: celltype_names.append(celltype_dict[celltype_id]) custom_unit_columns = [ { 'name': 'cell_type', 'description': 'name of cell type', 'data': celltype_names}, { 'name': 'global_id', 'description': 'global id for cell for entire experiment', 'data': df_unit_features['unitID'].values}, { 'name': 'max_electrode', 'description': 'electrode that has the maximum amplitude of the waveform', 'data': get_max_electrodes(nwbfile, session_path), 'table': nwbfile.electrodes }] ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks) trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat') if os.path.isfile(trialdata_path): trials_data = loadmat(trialdata_path)['EightMazeRun'] trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat') trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]] features = trialdatainfo[:7] features[:2] = 'start_time', 'stop_time', [nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']] for trial_data in trials_data: if trial_data[3]: cond = 'run_left' else: cond = 'run_right' nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond, error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6]) """ mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat') matin = loadmat(mono_syn_fpath) exc = matin['FinalExcMonoSynID'] inh = matin['FinalInhMonoSynID'] #exc_obj = CatCellInfo(name='excitatory_connections', # indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1) #module_cellular.add_container(exc_obj) #inh_obj = CatCellInfo(name='inhibitory_connections', # indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1) #module_cellular.add_container(inh_obj) """ if os.path.isfile(sleep_state_fpath): matin = loadmat(sleep_state_fpath)['StatePeriod'] table = TimeIntervals(name='states', description='sleep states of animal') table.add_column(name='label', description='sleep state') data = [] for name in matin.dtype.names: for row in matin[name][0][0]: data.append({'start_time': row[0], 'stop_time': row[1], 'label': name}) [table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])] check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table) if stub: out_fname = session_path + '_stub.nwb' else: out_fname = session_path + '.nwb' print('writing NWB file...', end='', flush=True) with NWBHDF5IO(out_fname, mode='w') as io: io.write(nwbfile, cache_spec=cache_spec) print('done.') print('testing read...', end='', flush=True) # test read with NWBHDF5IO(out_fname, mode='r') as io: io.read() print('done.')
class TestH5DataIO(TestCase): """ Test that H5DataIO functions correctly on round trip with the HDF5IO backend """ def setUp(self): self.nwbfile = NWBFile(session_description='a', identifier='b', session_start_time=datetime(1970, 1, 1, 12, tzinfo=tzutc())) self.path = "test_pynwb_io_hdf5_h5dataIO.h5" def tearDown(self): remove_test_file(self.path) def test_gzip_timestamps(self): ts = TimeSeries(name='ts_name', data=[1, 2, 3], unit='A', timestamps=H5DataIO(np.array([1., 2., 3.]), compression='gzip')) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) # confirm that the dataset was indeed compressed with File(self.path, 'r') as f: self.assertEqual(f['/acquisition/ts_name/timestamps'].compression, 'gzip') def test_write_dataset_custom_compress(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), compression='gzip', compression_opts=5, shuffle=True, fletcher32=True) ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.compression, 'gzip') self.assertEqual(dset.compression_opts, 5) self.assertEqual(dset.shuffle, True) self.assertEqual(dset.fletcher32, True) def test_write_dataset_custom_chunks(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), chunks=(1, 1, 3)) ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.chunks, (1, 1, 3)) def test_write_dataset_custom_fillvalue(self): a = H5DataIO(np.arange(20).reshape(5, 4), fillvalue=-1) ts = TimeSeries(name='ts_name', data=a, unit='A', timestamps=np.arange(5.)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertTrue(np.all(dset[:] == a.data)) self.assertEqual(dset.fillvalue, -1) def test_write_dataset_datachunkiterator_data_and_time(self): a = np.arange(30).reshape(5, 2, 3) aiter = iter(a) daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) tstamps = np.arange(5.) tsiter = DataChunkIterator.from_iterable(tstamps) ts = TimeSeries(name='ts_name', data=daiter, unit='A', timestamps=tsiter) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertListEqual(dset[:].tolist(), a.tolist()) def test_write_dataset_datachunkiterator_data_only(self): a = np.arange(30).reshape(5, 2, 3) aiter = iter(a) daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) tstamps = np.arange(5.) ts = TimeSeries(name='ts_name', data=daiter, unit='A', timestamps=tstamps) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertListEqual(dset[:].tolist(), a.tolist()) def test_write_dataset_datachunkiterator_with_compression(self): a = np.arange(30).reshape(5, 2, 3) aiter = iter(a) daiter = DataChunkIterator.from_iterable(aiter, buffer_size=2) wrapped_daiter = H5DataIO(data=daiter, compression='gzip', compression_opts=5, shuffle=True, fletcher32=True) ts = TimeSeries(name='ts_name', data=wrapped_daiter, unit='A', timestamps=np.arange(5.)) self.nwbfile.add_acquisition(ts) with NWBHDF5IO(self.path, 'w') as io: io.write(self.nwbfile, cache_spec=False) with File(self.path, 'r') as f: dset = f['/acquisition/ts_name/data'] self.assertEqual(dset.shape, a.shape) self.assertListEqual(dset[:].tolist(), a.tolist()) self.assertEqual(dset.compression, 'gzip') self.assertEqual(dset.compression_opts, 5) self.assertEqual(dset.shuffle, True) self.assertEqual(dset.fletcher32, True)
class TestHDF5Writer(TestCase): _required_tests = ('test_nwbio', 'test_write_clobber', 'test_write_cache_spec', 'test_write_no_cache_spec') @property def required_tests(self): return self._required_tests def setUp(self): self.manager = get_manager() self.path = "test_pynwb_io_hdf5.nwb" self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) self.container = NWBFile(session_description='a test NWB File', identifier='TEST123', session_start_time=self.start_time, file_create_date=self.create_date) ts = TimeSeries(name='test_timeseries', data=list(range(100, 200, 10)), unit='SIunit', timestamps=np.arange(10.), resolution=0.1) self.container.add_acquisition(ts) ts_builder = GroupBuilder('test_timeseries', attributes={'neurodata_type': 'TimeSeries'}, datasets={'data': DatasetBuilder('data', list(range(100, 200, 10)), attributes={'unit': 'SIunit', 'conversion': 1.0, 'resolution': 0.1}), 'timestamps': DatasetBuilder('timestamps', np.arange(10.), attributes={'unit': 'seconds', 'interval': 1})}) self.builder = GroupBuilder( 'root', groups={'acquisition': GroupBuilder('acquisition', groups={'test_timeseries': ts_builder}), 'analysis': GroupBuilder('analysis'), 'general': GroupBuilder('general'), 'processing': GroupBuilder('processing'), 'stimulus': GroupBuilder( 'stimulus', groups={'presentation': GroupBuilder('presentation'), 'templates': GroupBuilder('templates')})}, datasets={'file_create_date': DatasetBuilder('file_create_date', [self.create_date.isoformat()]), 'identifier': DatasetBuilder('identifier', 'TEST123'), 'session_description': DatasetBuilder('session_description', 'a test NWB File'), 'nwb_version': DatasetBuilder('nwb_version', '1.0.6'), 'session_start_time': DatasetBuilder('session_start_time', self.start_time.isoformat())}, attributes={'neurodata_type': 'NWBFile'}) def tearDown(self): remove_test_file(self.path) def test_nwbio(self): with HDF5IO(self.path, manager=self.manager, mode='a') as io: io.write(self.container) with File(self.path, 'r') as f: self.assertIn('acquisition', f) self.assertIn('analysis', f) self.assertIn('general', f) self.assertIn('processing', f) self.assertIn('file_create_date', f) self.assertIn('identifier', f) self.assertIn('session_description', f) self.assertIn('session_start_time', f) acq = f.get('acquisition') self.assertIn('test_timeseries', acq) def test_write_clobber(self): with HDF5IO(self.path, manager=self.manager, mode='a') as io: io.write(self.container) with self.assertRaisesWith(UnsupportedOperation, "Unable to open file %s in 'w-' mode. File already exists." % self.path): with HDF5IO(self.path, manager=self.manager, mode='w-') as io: pass def test_write_cache_spec(self): ''' Round-trip test for writing spec and reading it back in ''' with HDF5IO(self.path, manager=self.manager, mode="a") as io: io.write(self.container) with File(self.path, 'r') as f: self.assertIn('specifications', f) ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) HDF5IO.load_namespaces(ns_catalog, self.path) original_ns = self.manager.namespace_catalog.get_namespace('core') cached_ns = ns_catalog.get_namespace('core') self.maxDiff = None for key in ('author', 'contact', 'doc', 'full_name', 'name'): with self.subTest(namespace_field=key): self.assertEqual(original_ns[key], cached_ns[key]) for dt in original_ns.get_registered_types(): with self.subTest(neurodata_type=dt): original_spec = original_ns.get_spec(dt) cached_spec = cached_ns.get_spec(dt) with self.subTest(test='data_type spec read back in'): self.assertIsNotNone(cached_spec) with self.subTest(test='cached spec preserved original spec'): self.assertDictEqual(original_spec, cached_spec) def test_write_no_cache_spec(self): ''' Round-trip test for not writing spec ''' with HDF5IO(self.path, manager=self.manager, mode="a") as io: io.write(self.container, cache_spec=False) with File(self.path, 'r') as f: self.assertNotIn('specifications', f)
class NWBFileTest(TestCase): def setUp(self): self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal()) self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) self.create = [ datetime(2017, 5, 1, 12, tzinfo=tzlocal()), datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()), datetime(2017, 5, 2, 14, tzinfo=tzutc()) ] self.path = 'nwbfile_test.h5' self.nwbfile = NWBFile( 'a test session description for a test NWBFile', 'FILE123', self.start, file_create_date=self.create, timestamps_reference_time=self.ref_time, experimenter='A test experimenter', lab='a test lab', institution='a test institution', experiment_description='a test experiment description', session_id='test1', notes='my notes', pharmacology='drugs', protocol='protocol', related_publications='my pubs', slices='my slices', surgery='surgery', virus='a virus', source_script='noscript', source_script_file_name='nofilename', stimulus_notes='test stimulus notes', data_collection='test data collection notes', keywords=('these', 'are', 'keywords')) def test_constructor(self): self.assertEqual(self.nwbfile.session_description, 'a test session description for a test NWBFile') self.assertEqual(self.nwbfile.identifier, 'FILE123') self.assertEqual(self.nwbfile.session_start_time, self.start) self.assertEqual(self.nwbfile.file_create_date, self.create) self.assertEqual(self.nwbfile.lab, 'a test lab') self.assertEqual(self.nwbfile.experimenter, ('A test experimenter', )) self.assertEqual(self.nwbfile.institution, 'a test institution') self.assertEqual(self.nwbfile.experiment_description, 'a test experiment description') self.assertEqual(self.nwbfile.session_id, 'test1') self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes') self.assertEqual(self.nwbfile.data_collection, 'test data collection notes') self.assertEqual(self.nwbfile.related_publications, ('my pubs', )) self.assertEqual(self.nwbfile.source_script, 'noscript') self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename') self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords')) self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time) def test_create_electrode_group(self): name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' d = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d) self.assertEqual(elecgrp.description, desc) self.assertEqual(elecgrp.location, loc) self.assertIs(elecgrp.device, d) def test_create_custom_intervals(self): df_words = pd.DataFrame({ 'start_time': [.1, 2.], 'stop_time': [.8, 2.3], 'label': ['hello', 'there'] }) words = TimeIntervals.from_dataframe(df_words, name='words') self.nwbfile.add_time_intervals(words) self.assertEqual(self.nwbfile.intervals['words'], words) def test_create_electrode_group_invalid_index(self): """ Test the case where the user creates an electrode table region with indexes that are out of range of the amount of electrodes added. """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i) with self.assertRaises(IndexError): nwbfile.create_electrode_table_region(list(range(6)), 'test') def test_access_group_after_io(self): """ Motivated by #739 """ nwbfile = NWBFile('a', 'b', datetime.now(tzlocal())) device = nwbfile.create_device('a') elecgrp = nwbfile.create_electrode_group('a', 'b', device=device, location='a') nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) for i in range(4): nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=i + 1) with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io: io.write(nwbfile) with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io: nwbfile_i = io.read() for aa, bb in zip(nwbfile_i.electrodes['group'][:], nwbfile.electrodes['group'][:]): self.assertEqual(aa.name, bb.name) remove_test_file("electrodes_mwe.nwb") def test_access_processing(self): self.nwbfile.create_processing_module('test_mod', 'test_description') # test deprecate .modules with self.assertWarnsWith( DeprecationWarning, 'NWBFile.modules has been replaced by NWBFile.processing.'): modules = self.nwbfile.modules['test_mod'] self.assertIs(self.nwbfile.processing['test_mod'], modules) def test_epoch_tags(self): tags1 = ['t1', 't2'] tags2 = ['t3', 't4'] tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float64) ts = TimeSeries("test_ts", list(range(len(tstamps))), 'unit', timestamps=tstamps) expected_tags = tags1 + tags2 self.nwbfile.add_epoch(0.0, 1.0, tags1, ts) self.nwbfile.add_epoch(0.0, 1.0, tags2, ts) tags = self.nwbfile.epoch_tags self.assertEqual(set(expected_tags), set(tags)) def test_add_acquisition(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.acquisition), 1) def test_add_stimulus(self): self.nwbfile.add_stimulus( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus), 1) def test_add_stimulus_template(self): self.nwbfile.add_stimulus_template( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.stimulus_template), 1) def test_add_analysis(self): self.nwbfile.add_analysis( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.assertEqual(len(self.nwbfile.analysis), 1) def test_add_acquisition_check_dups(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) with self.assertRaises(ValueError): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) def test_get_acquisition_empty(self): with self.assertRaisesWith(ValueError, "acquisition of NWBFile 'root' is empty."): self.nwbfile.get_acquisition() def test_get_acquisition_multiple_elements(self): self.nwbfile.add_acquisition( TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) self.nwbfile.add_acquisition( TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "More than one element in acquisition of NWBFile 'root' -- must specify a name." with self.assertRaisesWith(ValueError, msg): self.nwbfile.get_acquisition() def test_add_acquisition_invalid_name(self): self.nwbfile.add_acquisition( TimeSeries('test_ts', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])) msg = "\"'TEST_TS' not found in acquisition of NWBFile 'root'.\"" with self.assertRaisesWith(KeyError, msg): self.nwbfile.get_acquisition("TEST_TS") def test_set_electrode_table(self): table = ElectrodeTable() dev1 = self.nwbfile.create_device('dev1') group = self.nwbfile.create_electrode_group('tetrode1', 'tetrode description', 'tetrode location', dev1) table.add_row(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-2.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-3.0, location='CA1', filtering='none', group=group, group_name='tetrode1') table.add_row(x=1.0, y=2.0, z=3.0, imp=-4.0, location='CA1', filtering='none', group=group, group_name='tetrode1') self.nwbfile.set_electrode_table(table) self.assertIs(self.nwbfile.electrodes, table) self.assertIs(table.parent, self.nwbfile) def test_add_unit_column(self): self.nwbfile.add_unit_column('unit_type', 'the type of unit') self.assertEqual(self.nwbfile.units.colnames, ('unit_type', )) def test_add_unit(self): self.nwbfile.add_unit(id=1) self.assertEqual(len(self.nwbfile.units), 1) self.nwbfile.add_unit(id=2) self.nwbfile.add_unit(id=3) self.assertEqual(len(self.nwbfile.units), 3) def test_add_trial_column(self): self.nwbfile.add_trial_column('trial_type', 'the type of trial') self.assertEqual(self.nwbfile.trials.colnames, ('start_time', 'stop_time', 'trial_type')) def test_add_trial(self): self.nwbfile.add_trial(start_time=10.0, stop_time=20.0) self.assertEqual(len(self.nwbfile.trials), 1) self.nwbfile.add_trial(start_time=30.0, stop_time=40.0) self.nwbfile.add_trial(start_time=50.0, stop_time=70.0) self.assertEqual(len(self.nwbfile.trials), 3) def test_add_invalid_times_column(self): self.nwbfile.add_invalid_times_column( 'comments', 'description of reason for omitting time') self.assertEqual(self.nwbfile.invalid_times.colnames, ('start_time', 'stop_time', 'comments')) def test_add_invalid_time_interval(self): self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0) self.assertEqual(len(self.nwbfile.invalid_times), 1) self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0) self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5) self.assertEqual(len(self.nwbfile.invalid_times), 3) def test_add_invalid_time_w_ts(self): ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na') self.nwbfile.add_invalid_time_interval(start_time=18.0, stop_time=20.6, timeseries=ts, tags=('hi', 'there')) def test_add_electrode(self): dev1 = self.nwbfile.create_device(name='dev1') group = self.nwbfile.create_electrode_group( name='tetrode1', description='tetrode description', location='tetrode location', device=dev1) self.nwbfile.add_electrode(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, id=1) elec = self.nwbfile.electrodes[0] self.assertEqual(elec.index[0], 1) self.assertEqual(elec.iloc[0]['x'], 1.0) self.assertEqual(elec.iloc[0]['y'], 2.0) self.assertEqual(elec.iloc[0]['z'], 3.0) self.assertEqual(elec.iloc[0]['location'], 'CA1') self.assertEqual(elec.iloc[0]['filtering'], 'none') self.assertEqual(elec.iloc[0]['group'], group) def test_add_electrode_some_opt(self): dev1 = self.nwbfile.create_device(name='dev1') group = self.nwbfile.create_electrode_group( name='tetrode1', description='tetrode description', location='tetrode location', device=dev1) self.nwbfile.add_electrode(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, id=1, rel_x=4.0, rel_y=5.0, rel_z=6.0, reference='ref1') self.nwbfile.add_electrode(x=1.0, y=2.0, z=3.0, imp=-1.0, location='CA1', filtering='none', group=group, id=2, rel_x=7.0, rel_y=8.0, rel_z=9.0, reference='ref2') elec = self.nwbfile.electrodes[0] self.assertEqual(elec.iloc[0]['rel_x'], 4.0) self.assertEqual(elec.iloc[0]['rel_y'], 5.0) self.assertEqual(elec.iloc[0]['rel_z'], 6.0) self.assertEqual(elec.iloc[0]['reference'], 'ref1') elec = self.nwbfile.electrodes[1] self.assertEqual(elec.iloc[0]['rel_x'], 7.0) self.assertEqual(elec.iloc[0]['rel_y'], 8.0) self.assertEqual(elec.iloc[0]['rel_z'], 9.0) self.assertEqual(elec.iloc[0]['reference'], 'ref2') def test_all_children(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) name = 'example_electrode_group' desc = 'An example electrode' loc = 'an example location' device = self.nwbfile.create_device('a fake device') elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device) children = self.nwbfile.all_children() self.assertIn(ts1, children) self.assertIn(ts2, children) self.assertIn(device, children) self.assertIn(elecgrp, children) def test_fail_if_source_script_file_name_without_source_script(self): with self.assertRaises(ValueError): # <-- source_script_file_name without source_script is not allowed NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, source_script=None, source_script_file_name='nofilename') def test_get_neurodata_type(self): ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) p1 = ts1.get_ancestor(neurodata_type='NWBFile') self.assertIs(p1, self.nwbfile) p2 = ts2.get_ancestor(neurodata_type='NWBFile') self.assertIs(p2, self.nwbfile) def test_print_units(self): self.nwbfile.add_unit(spike_times=[1., 2., 3.]) expected = """units pynwb.misc.Units at 0x%d Fields: colnames: ['spike_times'] columns: ( spike_times_index <class 'hdmf.common.table.VectorIndex'>, spike_times <class 'hdmf.common.table.VectorData'> ) description: Autogenerated by NWBFile id: id <class 'hdmf.common.table.ElementIdentifiers'> waveform_unit: volts """ expected = expected % id(self.nwbfile.units) self.assertEqual(str(self.nwbfile.units), expected) def test_copy(self): self.nwbfile.add_unit(spike_times=[1., 2., 3.]) device = self.nwbfile.create_device('a') elecgrp = self.nwbfile.create_electrode_group('a', 'b', device=device, location='a') self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'a', 'a', elecgrp, id=0) self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'b', 'b', elecgrp) elec_region = self.nwbfile.create_electrode_table_region([1], 'name') ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5], 'grams', timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) ts2 = ElectricalSeries('test_ts2', [0, 1, 2, 3, 4, 5], electrodes=elec_region, timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]) self.nwbfile.add_acquisition(ts1) self.nwbfile.add_acquisition(ts2) self.nwbfile.add_trial(start_time=50.0, stop_time=70.0) self.nwbfile.add_invalid_times_column( 'comments', 'description of reason for omitting time') self.nwbfile.create_processing_module('test_mod', 'test_description') self.nwbfile.create_time_intervals('custom_interval', 'a custom time interval') self.nwbfile.intervals['custom_interval'].add_interval(start_time=10., stop_time=20.) newfile = self.nwbfile.copy() # test dictionaries self.assertIs(self.nwbfile.devices['a'], newfile.devices['a']) self.assertIs(self.nwbfile.acquisition['test_ts1'], newfile.acquisition['test_ts1']) self.assertIs(self.nwbfile.acquisition['test_ts2'], newfile.acquisition['test_ts2']) self.assertIs(self.nwbfile.processing['test_mod'], newfile.processing['test_mod']) # test dynamic tables self.assertIsNot(self.nwbfile.electrodes, newfile.electrodes) self.assertIs(self.nwbfile.electrodes['x'], newfile.electrodes['x']) self.assertIsNot(self.nwbfile.units, newfile.units) self.assertIs(self.nwbfile.units['spike_times'], newfile.units['spike_times']) self.assertIsNot(self.nwbfile.trials, newfile.trials) self.assertIsNot(self.nwbfile.trials.parent, newfile.trials.parent) self.assertIs(self.nwbfile.trials.id, newfile.trials.id) self.assertIs(self.nwbfile.trials['start_time'], newfile.trials['start_time']) self.assertIs(self.nwbfile.trials['stop_time'], newfile.trials['stop_time']) self.assertIsNot(self.nwbfile.invalid_times, newfile.invalid_times) self.assertTupleEqual(self.nwbfile.invalid_times.colnames, newfile.invalid_times.colnames) self.assertIsNot(self.nwbfile.intervals['custom_interval'], newfile.intervals['custom_interval']) self.assertTupleEqual( self.nwbfile.intervals['custom_interval'].colnames, newfile.intervals['custom_interval'].colnames) self.assertIs(self.nwbfile.intervals['custom_interval']['start_time'], newfile.intervals['custom_interval']['start_time']) self.assertIs(self.nwbfile.intervals['custom_interval']['stop_time'], newfile.intervals['custom_interval']['stop_time']) def test_multi_experimenters(self): self.nwbfile = NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, experimenter=('experimenter1', 'experimenter2')) self.assertTupleEqual(self.nwbfile.experimenter, ('experimenter1', 'experimenter2')) def test_multi_publications(self): self.nwbfile = NWBFile('a test session description for a test NWBFile', 'FILE123', self.start, related_publications=('pub1', 'pub2')) self.assertTupleEqual(self.nwbfile.related_publications, ('pub1', 'pub2'))
def convert_from_old_neo(old_file, bird_name, electrode_df): root_dir, fname = os.path.split(old_file) rec_date = bird_info[bird_name]['recording_date'] rec_start = bird_info[bird_name]['recording_start'] rec_end = bird_info[bird_name]['recording_end'] rec_datetime = date_parser.parse('{} {}'.format(rec_date, rec_start)) # create an electrode array, electrode groups, and electrode table for the electrodes i = electrode_df.bird == bird_name edf = electrode_df[i] for block, gdf in edf.groupby(['block']): lfp_series = dict() spike_series = dict() print('*************** Processing block {}'.format(block)) if bird_name == 'WhiWhi4522M' and block == 'Site1': continue # get the LFP and spike data for the block hf = h5py.File(old_file, 'r') block_data = read_old_neo(hf, block) hf.close() recording_name = '{}_{}_{}'.format(bird_name, block, block_data['protocol']) nwb_file = os.path.join(root_dir, '{}.nwb'.format(recording_name)) session_desc = """ A single recording session, roughly one hour long, at a single depth, in a series of recordings from the auditory "cortex" of zebra finch {}. Block {}, stimulus protocol {}. """.format(bird_name, block, block_data['protocol']) exp_desc = """ A randomly interleaved mixture of Zebra finch vocalizations, songs, and modulation limited noise played to a Zebra finch under urethane anaesthesia in an acute experiment. Two 16 microwire electrode arrays were implanted, one in each hemisphere. Experiments designed and performed by Julie Elie, vocal repertoire recorded by Julie Elie. Data converted to NWB by Mike Schachter. For a full description of methods, please consult and also cite the following publications: Elie, J. E., & Theunissen, F. E. (2015). Meaning in the avian auditory cortex: neural representation of communication calls. European Journal of Neuroscience, 41(5), 546-567. Elie, J. E., & Theunissen, F. E. (2016). The vocal repertoire of the domesticated zebra finch: a data-driven approach to decipher the information-bearing acoustic features of communication signals. Animal cognition, 19(2), 285-315. """ nf = NWBFile(recording_name, session_desc, bird_name, rec_datetime, experimenter='Julie Elie', lab='Theunissen Lab', institution='UC Berkeley', experiment_description=exp_desc, session_id=bird_name) # create the electrodes and electrode tables for hemi, ggdf in gdf.groupby(['hemisphere']): electrode_array_name = '16 electrode microwire array on {} hemisphere'.format( hemi) electrode_array = nf.create_device(name=electrode_array_name, source='') # create an electrode group egrp_desc = """ The (x,y) locations of the electrodes refer to the distance from region midline and distance from L2A, respectively, in mm. """ electrode_group = nf.create_electrode_group( hemi, source=electrode_array_name, description=egrp_desc, location='{} Hemisphere, Field L, CM, NCM'.format(hemi), device=electrode_array) # add electrodes to electrode group for row_idx, row in ggdf.iterrows(): electrode_number = row['electrode'] - 1 dist_l2a = row['dist_l2a'] dist_midline = row['dist_midline'] region = row['region'] if bird_name == 'GreBlu9508M': dist_l2a *= 4 # correct for the error in the original data nf.add_electrode(electrode_number, x=dist_midline, y=dist_l2a, z=0.0, imp=0.0, location=region, filtering='none', description='Row {}, Column {}'.format( row['row'], row['col']), group=electrode_group) # create an electrode table region electrode_numbers = list( np.array(sorted(ggdf.electrode.unique())) - 1) print('electrode_numbers=', electrode_numbers) etable = nf.create_electrode_table_region( electrode_numbers, 'All electrodes in array for hemisphere {} with LFP'.format( hemi)) lfp_data = np.array([ block_data['electrodes'][e + 1]['lfp'] for e in electrode_numbers ]) sr = block_data['sample_rate'] t = np.arange(lfp_data.shape[1]) / sr # add the raw LFP lfp_series_name = 'Multi-electrode LFP on {} hemisphere from block {}'.format( hemi, block) lfp = ElectricalSeries( lfp_series_name, electrode_array_name, lfp_data, etable, timestamps=t, resolution=1e-12, comments='', description='Low-passed LFP recorded from microwire array') lfp_series[hemi] = lfp # add the spikes and their waveforms for row_idx, row in ggdf.iterrows(): # electrode_number ranges from 1-32, same as the keys in block_data['electrodes'] electrode_number = row['electrode'] for k, unit_data in enumerate( block_data['electrodes'][electrode_number]['units']): spike_times = unit_data['spike_times'] waveforms = unit_data['waveforms'] unit_name = unit_data['name'] e_num = unit_data['electrode'] print('{} electrode_number={}({}), e_num={}, k={}'.format( unit_name, electrode_number, electrode_number - 1, e_num, k)) assert e_num == electrode_number if unit_name.startswith('RF Sort'): xarr = unit_name.split(' ') unit_num = xarr[-1] sort_type = 'Spike-sorted unit {}'.format(unit_num) elif unit_name.startswith('Chan'): if 'Code0' not in unit_name: print( 'Skipping multi-unit channel with non-zero sort code' ) continue sort_type = 'Multi-unit' else: raise Exception( 'Unknown unit name: {}'.format(unit_name)) full_unit_name = '{} on block {} and electrode {}'.format( sort_type, block, e_num - 1) spikes = SpikeEventSeries( full_unit_name, lfp_series_name, waveforms, spike_times, etable, resolution=1e-12, conversion=1e6, comments='', description='', ) print( '\tAdding spikes acquisition: {} ({}), waveforms.shape={}' .format(full_unit_name, sort_type, str(waveforms.shape))) spike_series[(hemi, electrode_number, unit_name)] = spikes all_series = list() all_series.extend(lfp_series.values()) all_series.extend(spike_series.values()) print('len(all_series)=', len(all_series)) nf.add_acquisition(all_series) # create trials for each stim presentation nf.add_trial_column('stim_id', 'The ID of the sound played during the trial.') nf.add_trial_column( 'trial', 'The number of times this stimulus has been presented.') for stim_epoch_data in block_data['epochs']: stim_id = stim_epoch_data['stim_id'] trial_num = stim_epoch_data['trial'] stime = stim_epoch_data['start'] etime = stim_epoch_data['end'] nf.add_trial({ 'start': stime, 'end': etime, 'stim_id': stim_id, 'trial': trial_num }) print('Writing to {}'.format(nwb_file)) with NWBHDF5IO(nwb_file, mode='w') as io: io.write(nf) del nf
def save(self, file_name, to32=True, order='F', imagej=False, bigtiff=True, excitation_lambda=488.0, compress=0, var_name_hdf5='mov', sess_desc='some_description', identifier='some identifier', imaging_plane_description='some imaging plane description', emission_lambda=520.0, indicator='OGB-1', location='brain', starting_time=0., experimenter='Dr Who', lab_name=None, institution=None, experiment_description='Experiment Description', session_id='Session ID'): """ Save the timeseries in single precision. Supported formats include TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB Args: file_name: str name of file. Possible formats are tif, avi, npz, mmap and hdf5 to32: Bool whether to transform to 32 bits order: 'F' or 'C' C or Fortran order var_name_hdf5: str Name of hdf5 file subdirectory Raises: Exception 'Extension Unknown' """ name, extension = os.path.splitext(file_name)[:2] extension = extension.lower() logging.debug("Parsing extension " + str(extension)) if extension == '.tif': with tifffile.TiffWriter(file_name, bigtiff=bigtiff, imagej=imagej) as tif: for i in range(self.shape[0]): if i % 200 == 0: logging.debug(str(i) + ' frames saved') curfr = self[i].copy() if to32 and not ('float32' in str(self.dtype)): curfr = curfr.astype(np.float32) tif.save(curfr, compress=compress) elif extension == '.npz': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) np.savez(file_name, input_arr=input_arr, start_time=self.start_time, fr=self.fr, meta_data=self.meta_data, file_name=self.file_name) elif extension == '.avi': codec = None try: codec = cv2.FOURCC('I', 'Y', 'U', 'V') except AttributeError: codec = cv2.VideoWriter_fourcc(*'IYUV') np.clip(self, np.percentile(self, 1), np.percentile(self, 99), self) minn, maxx = np.min(self), np.max(self) data = 255 * (self - minn) / (maxx - minn) data = data.astype(np.uint8) y, x = data[0].shape vw = cv2.VideoWriter(file_name, codec, self.fr, (x, y), isColor=True) for d in data: vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR)) vw.release() elif extension == '.mat': if self.file_name[0] is not None: f_name = self.file_name else: f_name = '' if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) if self.meta_data[0] is None: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': [], 'file_name': f_name }) else: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': self.meta_data, 'file_name': f_name }) elif extension in ('.hdf5', '.h5'): with h5py.File(file_name, "w") as f: if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) dset = f.create_dataset(var_name_hdf5, data=input_arr) dset.attrs["fr"] = self.fr dset.attrs["start_time"] = self.start_time try: dset.attrs["file_name"] = [ a.encode('utf8') for a in self.file_name ] except: logging.warning('No file saved') if self.meta_data[0] is not None: logging.debug("Metadata for saved file: " + str(self.meta_data)) dset.attrs["meta_data"] = cpk.dumps(self.meta_data) elif extension == '.mmap': base_name = name T = self.shape[0] dims = self.shape[1:] if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) input_arr = np.transpose(input_arr, list(range(1, len(dims) + 1)) + [0]) input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F') fname_tot = memmap_frames_filename(base_name, dims, T, order) fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=(np.uint64(np.prod(dims)), np.uint64(T)), order=order) big_mov[:] = np.asarray(input_arr, dtype=np.float32) big_mov.flush() del big_mov, input_arr return fname_tot elif extension == '.nwb': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) # Create NWB file nwbfile = NWBFile(sess_desc, identifier, datetime.now(tzlocal()), experimenter=experimenter, lab=lab_name, institution=institution, experiment_description=experiment_description, session_id=session_id) # Get the device device = Device('imaging_device') nwbfile.add_device(device) # OpticalChannel optical_channel = OpticalChannel('OpticalChannel', 'main optical channel', emission_lambda=emission_lambda) imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, description=imaging_plane_description, device=device, excitation_lambda=excitation_lambda, imaging_rate=self.fr, indicator=indicator, location=location) # Images image_series = TwoPhotonSeries(name=var_name_hdf5, dimension=self.shape[1:], data=input_arr, imaging_plane=imaging_plane, starting_frame=[0], starting_time=starting_time, rate=self.fr) nwbfile.add_acquisition(image_series) with NWBHDF5IO(file_name, 'w') as io: io.write(nwbfile) return file_name else: logging.error("Extension " + str(extension) + " unknown") raise Exception('Extension Unknown')
class TestNWBFileHDF5IO(TestCase): """ Test reading/writing an NWBFile using HDF5IO """ def setUp(self): """ Set up an NWBFile object with an acquisition TimeSeries, analysis TimeSeries, and a processing module """ self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) self.manager = get_manager() self.filename = 'test_nwbfileio.h5' self.nwbfile = NWBFile( session_description='a test NWB File', identifier='TEST123', session_start_time=self.start_time, timestamps_reference_time=self.ref_time, file_create_date=self.create_date, experimenter='test experimenter', stimulus_notes='test stimulus notes', data_collection='test data collection notes', experiment_description='test experiment description', institution='nomad', lab='nolab', notes='nonotes', pharmacology='nopharmacology', protocol='noprotocol', related_publications='nopubs', session_id='007', slices='noslices', source_script='nosources', surgery='nosurgery', virus='novirus', source_script_file_name='nofilename') self.ts = TimeSeries(name='test_timeseries', data=list(range(100, 200, 10)), unit='SIunit', timestamps=np.arange(10.), resolution=0.1) self.nwbfile.add_acquisition(self.ts) self.ts2 = TimeSeries(name='test_timeseries2', data=list(range(200, 300, 10)), unit='SIunit', timestamps=np.arange(10.), resolution=0.1) self.nwbfile.add_analysis(self.ts2) self.mod = self.nwbfile.create_processing_module( 'test_module', 'a test module') self.ts3 = TimeSeries(name='test_timeseries2', data=list(range(100, 200, 10)), unit='SIunit', timestamps=np.arange(10.), resolution=0.1) self.mod.add(self.ts3) def tearDown(self): """ Delete the created test file """ remove_test_file(self.filename) def test_children(self): """ Test that the TimeSeries and processing module are children of their respective parents """ self.assertIn(self.ts, self.nwbfile.children) self.assertIn(self.ts2, self.nwbfile.children) self.assertIn(self.mod, self.nwbfile.children) self.assertIn(self.ts3, self.mod.children) def test_write(self): """ Test writing the NWBFile using HDF5IO """ hdf5io = HDF5IO(self.filename, manager=self.manager, mode='a') hdf5io.write(self.nwbfile) hdf5io.close() # TODO add some asserts def test_read(self): """ Test reading the NWBFile using HDF5IO """ hdf5io = HDF5IO(self.filename, manager=self.manager, mode='w') hdf5io.write(self.nwbfile) hdf5io.close() hdf5io = HDF5IO(self.filename, manager=self.manager, mode='r') container = hdf5io.read() self.assertIsInstance(container, NWBFile) self.assertEqual(len(container.acquisition), 1) self.assertEqual(len(container.analysis), 1) for v in container.acquisition.values(): self.assertIsInstance(v, TimeSeries) self.assertContainerEqual(container, self.nwbfile) hdf5io.close()
unit='m', starting_time=0.0, rate=1.0) #################### # Using this scheme says that this :py:class:`~pynwb.base.TimeSeries` started recording 0 seconds after # *start_time* stored in the :py:class:`~pynwb.file.NWBFile` and sampled every second. # # :py:class:`~pynwb.base.TimeSeries` objects can be added directly to your :py:class:`~pynwb.file.NWBFile` using # the methods :py:meth:`~pynwb.file.NWBFile.add_acquisition`, :py:meth:`~pynwb.file.NWBFile.add_stimulus` # and :py:meth:`~pynwb.file.NWBFile.add_stimulus_template`. Which method you use depends on the source of the # data: use :py:meth:`~pynwb.file.NWBFile.add_acquisition` to indicated *acquisition* data, # :py:meth:`~pynwb.file.NWBFile.add_stimulus` to indicate *stimulus* data, and # :py:meth:`~pynwb.file.NWBFile.add_stimulus_template` to store stimulus templates. nwbfile.add_acquisition(test_ts) #################### # Access the :py:class:`~pynwb.base.TimeSeries` object `'test_timeseries'` from *acquisition* using nwbfile.acquisition['test_timeseries'] #################### # or nwbfile.get_acquisition('test_timeseries') #################### # .. _basic_writing: # # Writing an NWB file # ------------------- #
from pynwb.image import ImageSeries from pynwb import NWBHDF5IO, NWBFile from h5py import File import numpy as np from datetime import datetime base_dir = '/Users/bendichter/Desktop/Schnitzer/data/Example Data' nwbfile = NWBFile(session_start_time=datetime(1900, 1, 1), session_description=' ', identifier='m655_D11_S1') nwbfile.add_acquisition( ImageSeries(name='video', format='external', external_file=['m655_D11_S1.avi'], starting_time=0.0, rate=np.nan, starting_frame=[0], dimension=[250, 250])) centroid_fname = 'm655_D11_S1_centroids.mat' pos_path = os.path.join(base_dir, centroid_fname) with File(pos_path, 'r') as file: pos_data = np.array(file['c']).T spatial_series = SpatialSeries(name='position', data=pos_data, starting_time=0.0, rate=5.0, units='unknown', reference_frame='unknown')
class TestHDF5Writer(unittest.TestCase): _required_tests = ('test_nwbio', 'test_write_clobber', 'test_write_cache_spec') @property def required_tests(self): return self._required_tests def setUp(self): self.manager = get_manager() self.path = "test_pynwb_io_hdf5.h5" self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) self.container = NWBFile('a test NWB File', 'TEST123', self.start_time, file_create_date=self.create_date) ts = TimeSeries('test_timeseries', list(range(100, 200, 10)), 'SIunit', timestamps=list(range(10)), resolution=0.1) self.container.add_acquisition(ts) ts_builder = GroupBuilder('test_timeseries', attributes={ 'neurodata_type': 'TimeSeries', 'help': 'General purpose TimeSeries' }, datasets={ 'data': DatasetBuilder('data', list(range(100, 200, 10)), attributes={ 'unit': 'SIunit', 'conversion': 1.0, 'resolution': 0.1 }), 'timestamps': DatasetBuilder('timestamps', list(range(10)), attributes={ 'unit': 'Seconds', 'interval': 1 }) }) self.builder = GroupBuilder( 'root', groups={ 'acquisition': GroupBuilder('acquisition', groups={'test_timeseries': ts_builder}), 'analysis': GroupBuilder('analysis'), 'general': GroupBuilder('general'), 'processing': GroupBuilder('processing'), 'stimulus': GroupBuilder('stimulus', groups={ 'presentation': GroupBuilder('presentation'), 'templates': GroupBuilder('templates') }) }, datasets={ 'file_create_date': DatasetBuilder('file_create_date', [self.create_date.isoformat()]), 'identifier': DatasetBuilder('identifier', 'TEST123'), 'session_description': DatasetBuilder('session_description', 'a test NWB File'), 'nwb_version': DatasetBuilder('nwb_version', '1.0.6'), 'session_start_time': DatasetBuilder('session_start_time', self.start_time.isoformat()) }, attributes={'neurodata_type': 'NWBFile'}) def tearDown(self): os.remove(self.path) def test_nwbio(self): io = HDF5IO(self.path, manager=self.manager, mode='a') io.write(self.container) io.close() f = File(self.path) self.assertIn('acquisition', f) self.assertIn('analysis', f) self.assertIn('general', f) self.assertIn('processing', f) self.assertIn('file_create_date', f) self.assertIn('identifier', f) self.assertIn('session_description', f) self.assertIn('session_start_time', f) acq = f.get('acquisition') self.assertIn('test_timeseries', acq) def test_write_clobber(self): io = HDF5IO(self.path, manager=self.manager, mode='a') io.write(self.container) io.close() f = File(self.path) # noqa: F841 if six.PY2: assert_file_exists = IOError elif six.PY3: assert_file_exists = OSError with self.assertRaises(assert_file_exists): io = HDF5IO(self.path, manager=self.manager, mode='w-') io.write(self.container) io.close() def test_write_cache_spec(self): ''' Round-trip test for writing spec and reading it back in ''' io = HDF5IO(self.path, manager=self.manager, mode="a") io.write(self.container, cache_spec=True) io.close() f = File(self.path) self.assertIn('specifications', f) ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) HDF5IO.load_namespaces(ns_catalog, self.path, namespaces=['core']) original_ns = self.manager.namespace_catalog.get_namespace('core') cached_ns = ns_catalog.get_namespace('core') self.maxDiff = None for key in ('author', 'contact', 'doc', 'full_name', 'name'): with self.subTest(namespace_field=key): self.assertEqual(original_ns[key], cached_ns[key]) for dt in original_ns.get_registered_types(): with self.subTest(neurodata_type=dt): original_spec = original_ns.get_spec(dt) cached_spec = cached_ns.get_spec(dt) with self.subTest(test='data_type spec read back in'): self.assertIsNotNone(cached_spec) with self.subTest(test='cached spec preserved original spec'): self.assertDictEqual(original_spec, cached_spec)
deconv_roi_response_series = DeconvolvedRoiResponseSeries( name="deconvolved_fluorescence_trace", description="my roi response series", data=np.random.randn(100, 1), unit='F', rate=30.0, rois=fibers_ref, raw=roi_response_series, ) ophys_module = nwbfile.create_processing_module( name="ophys", description="fiber photometry" ) ophys_module.add(multi_commanded_voltage) nwbfile.add_acquisition(roi_response_series) ophys_module.add(deconv_roi_response_series) # write nwb file filename = 'test.nwb' with NWBHDF5IO(filename, 'w') as io: io.write(nwbfile) # read nwb file and check its contents with NWBHDF5IO(filename, 'r', load_namespaces=True) as io: nwbfile = io.read() # Access and print information about the acquisition print(nwbfile.acquisition["raw_fluorescence_trace"]) # Access and print information about the processed data print(nwbfile.processing['ophys']["deconvolved_fluorescence_trace"]) # Access and print all of the metadata
def run_conversion(self, nwbfile: NWBFile, metadata: dict, stub_test: bool = False): super().run_conversion(nwbfile=nwbfile, metadata=metadata, stub_test=stub_test) session_path = Path(self.source_data["file_path"]).parent session_id = session_path.name subject_path = session_path.parent xml_filepath = session_path / f"{session_id}.xml" root = et.parse(str(xml_filepath)).getroot() n_total_channels = int(root.find("acquisitionSystem").find("nChannels").text) lfp_sampling_rate = float(root.find("fieldPotentials").find("lfpSamplingRate").text) shank_channels = [ [int(channel.text) for channel in group.find("channels")] for group in root.find("spikeDetection").find("channelGroups").findall("group") ] all_shank_channels = np.concatenate(shank_channels) # Flattened # Special electrodes special_electrode_mapping = dict( ch_wait=79, ch_arm=78, ch_solL=76, ch_solR=77, ch_dig1=65, ch_dig2=68, ch_entL=72, ch_entR=71, ch_SsolL=73, ch_SsolR=70, ) special_electrodes = [] for special_electrode_name, channel in special_electrode_mapping.items(): if channel <= n_total_channels - 1: special_electrodes.append( dict( name=special_electrode_name, channel=channel, description="Environmental electrode recorded inline with neural data.", ) ) _, all_channels_lfp_data = read_lfp(session_path, stub=stub_test) for special_electrode in special_electrodes: ts = TimeSeries( name=special_electrode["name"], description=special_electrode["description"], data=all_channels_lfp_data[:, special_electrode["channel"]], rate=lfp_sampling_rate, unit="V", resolution=np.nan, ) nwbfile.add_acquisition(ts) # DecompositionSeries mouse_number = session_id[-9:-7] subject_xls = str(subject_path / f"DGProject/YM{mouse_number} exp_sheet.xlsx") hilus_csv_path = str(subject_path / "DGProject/early_session_hilus_chans.csv") session_start = metadata["NWBFile"]["session_start_time"] if "-" in session_id: b = False else: b = True lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start, session_id, b=b) if lfp_channel is not None: lfp_data = all_channels_lfp_data[:, all_shank_channels] all_lfp_phases = [] for passband in ("theta", "gamma"): lfp_fft = filter_lfp( lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_sampling_rate, passband=passband, ) lfp_phase, _ = hilbert_lfp(lfp_fft) all_lfp_phases.append(lfp_phase[:, np.newaxis]) decomp_series_data = np.dstack(all_lfp_phases) ecephys_mod = check_module( nwbfile, "ecephys", "Intermediate data from extracellular electrophysiology recordings, e.g., LFP.", ) lfp_ts = ecephys_mod.data_interfaces["LFP"]["LFP"] decomp_series = DecompositionSeries( name="LFPDecompositionSeries", description="Theta and Gamma phase for reference LFP", data=decomp_series_data, rate=lfp_sampling_rate, source_timeseries=lfp_ts, metric="phase", unit="radians", ) # TODO: the band limits should be extracted from parse_passband in band_analysis? decomp_series.add_band(band_name="theta", band_limits=(4, 10)) decomp_series.add_band(band_name="gamma", band_limits=(30, 80)) check_module( nwbfile, "ecephys", "Contains processed extracellular electrophysiology data.", ).add(decomp_series)
session_description='demonstrate NWBFile scratch', # required identifier='NWB456', # required session_start_time=start_time, # required file_create_date=create_date) # optional # make some fake data timestamps = np.linspace(0, 100, 1024) data = np.sin(0.333 * timestamps) + np.cos(0.1 * timestamps) + np.random.randn( len(timestamps)) test_ts = TimeSeries(name='raw_timeseries', data=data, unit='m', timestamps=timestamps) # add it to the NWBFile nwb.add_acquisition(test_ts) with NWBHDF5IO('raw_data.nwb', mode='w') as io: io.write(nwb) #################### # .. _basic_copying: # # Copying an NWB file # ------------------- # # To copy a file, we must first read the file. # raw_io = NWBHDF5IO('raw_data.nwb', 'r') nwb_in = raw_io.read()
group=electrode_group) ephys_table_region = nwbfile.create_electrode_table_region(list(range(5)), 'all ephys electrodes') ophys_device = nwbfile.create_device('ophys_device', source=source) ogen_site = nwbfile.create_ogen_site('oPhys', ophys_device, description='unknown', excitation_lambda='unknown', location='unknown') module = nwbfile.create_processing_module(name='0', description=source) for i, trial_data in enumerate(data): nwbfile.add_acquisition( ElectricalSeries('ePhys trial' + str(i), gzip(trial_data.ephys[:, [0, 1, 2, 6, 7]]), ephys_table_region, timestamps=trial_data.time) ) nwbfile.add_acquisition( OptogeneticSeries('oPhys trial' + str(i), gzip(trial_data.tempo_data[:, [0, 6, 7]]), ogen_site, description='laser, reference, voltage', timestamps=trial_data.time)) with NWBHDF5IO('/Users/bendichter/Desktop/Schnitzer/data/simon_out.nwb', 'w') as io: io.write(nwbfile) #trial_data.ephys
def chang2nwb(blockpath, out_file_path=None, save_to_file=False, htk_config=None): """ Parameters ---------- blockpath: str out_file_path: None | str if None, output = [blockpath]/[blockname].nwb save_to_file : bool If True, saves to file. If False, just returns nwbfile object htk_config : dict Dictionary cotaining HTK conversion paths and options. Example: { ecephys_path: 'path_to/ecephys_htk_files', ecephys_type: 'raw', 'preprocessed' or 'high_gamma', analog_path: 'path_to/analog_htk_files', anin1: {present: True, name: 'microphone', type: 'acquisition'}, anin2: {present: True, name: 'speaker1', type: 'stimulus'}, anin3: {present: False, name: 'speaker2', type: 'stimulus'}, anin4: {present: False, name: 'custom', type: 'acquisition'}, metadata: metadata, electrodes_file: electrodes_file, bipolar_file: bipolar_file } Returns ------- """ metadata = {} if htk_config is None: blockpath = Path(blockpath) else: blockpath = Path(htk_config['ecephys_path']) metadata = htk_config['metadata'] blockname = blockpath.parent.name subject_id = blockpath.parent.parent.name[2:] if out_file_path is None: out_file_path = blockpath.resolve().parent / ''.join(['EC', subject_id, '_', blockname, '.nwb']) # file paths ecog_path = blockpath anin_path = htk_config['analog_path'] bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat') # Create the NWB file object nwbfile_dict = { 'session_description': blockname, 'identifier': blockname, 'session_start_time': datetime.now().astimezone(), 'institution': 'University of California, San Francisco', 'lab': 'Chang Lab' } if 'NWBFile' in metadata: nwbfile_dict.update(metadata['NWBFile']) nwbfile = NWBFile(**nwbfile_dict) # Read electrophysiology data from HTK files print('reading htk acquisition...', flush=True) ecog_rate, data = readhtks(ecog_path) data = data.squeeze() print('done', flush=True) # Get electrodes info from mat file if htk_config['electrodes_file'] is not None: nwbfile = elecs_to_electrode_table( nwbfile=nwbfile, elecspath=htk_config['electrodes_file'], ) n_electrodes = nwbfile.electrodes[:].shape[0] all_elecs = list(range(n_electrodes)) elecs_region = nwbfile.create_electrode_table_region( region=all_elecs, description='ECoG electrodes on brain' ) else: ecephys_dict = { 'Device': [{'name': 'auto_device'}], 'ElectricalSeries': [{'name': 'ECoG', 'description': 'description'}], 'ElectrodeGroup': [{'name': 'auto_group', 'description': 'auto_group', 'location': 'location', 'device': 'auto_device'}] } if 'Ecephys' in metadata: ecephys_dict.update(metadata['Ecephys']) # Create devices for dev in ecephys_dict['Device']: device = nwbfile.create_device(dev['name']) # Electrode groups for el_grp in ecephys_dict['ElectrodeGroup']: device = nwbfile.devices[el_grp['device']] electrode_group = nwbfile.create_electrode_group( name=el_grp['name'], description=el_grp['description'], location=el_grp['location'], device=device ) # Electrodes table n_electrodes = data.shape[1] nwbfile.add_electrode_column('label', 'label of electrode') nwbfile.add_electrode_column('bad', 'electrode identified as too noisy') nwbfile.add_electrode_column('x_warped', 'x warped onto cvs_avg35_inMNI152') nwbfile.add_electrode_column('y_warped', 'y warped onto cvs_avg35_inMNI152') nwbfile.add_electrode_column('z_warped', 'z warped onto cvs_avg35_inMNI152') nwbfile.add_electrode_column('null', 'if not connected to real electrode') bad_elecs_inds = get_bad_elecs(blockpath) for elec_counter in range(n_electrodes): bad = elec_counter in bad_elecs_inds nwbfile.add_electrode( id=elec_counter, x=np.nan, y=np.nan, z=np.nan, imp=np.nan, x_warped=np.nan, y_warped=np.nan, z_warped=np.nan, location='', filtering='none', group=electrode_group, label='', bad=bad, null=False, ) all_elecs = list(range(n_electrodes)) elecs_region = nwbfile.create_electrode_table_region( region=all_elecs, description='ECoG electrodes on brain' ) # Get Bipolar table from file if htk_config['bipolar_file'] is not None: df = pd.read_csv(htk_config['bipolar_file'], index_col='id', sep='\t') # Create bipolar scheme table bipolar_scheme_table = BipolarSchemeTable( name='bipolar_scheme_table', description='desc' ) # Columns for bipolar scheme - all anodes and cathodes within the same # bipolar row are considered to have the same group and location bipolar_scheme_table.add_column( name='group_name', description='electrode group name' ) bipolar_scheme_table.add_column( name='location', description='electrode location' ) # Iterate over anode / cathode rows for i, r in df.iterrows(): if isinstance(r['anodes'], str): anodes = [int(a) for a in r['anodes'].split(',')] else: anodes = [int(r['anodes'])] if isinstance(r['cathodes'], str): cathodes = [int(a) for a in r['cathodes'].split(',')] else: cathodes = [int(r['cathodes'])] bipolar_scheme_table.add_row( anodes=anodes, cathodes=cathodes, group_name=nwbfile.electrodes['group_name'][anodes[0]], location=nwbfile.electrodes['location'][anodes[0]] ) bipolar_scheme_table.anodes.table = nwbfile.electrodes bipolar_scheme_table.cathodes.table = nwbfile.electrodes # Creates bipolar table region elecs_region = DynamicTableRegion( name='electrodes', data=np.arange(0, df.shape[0]), description='desc', table=bipolar_scheme_table ) ecephys_ext = EcephysExt(name='ecephys_ext') ecephys_ext.bipolar_scheme_table = bipolar_scheme_table nwbfile.add_lab_meta_data(ecephys_ext) # Stores HTK electrophysiology data as raw, preprocessed or high gamma if htk_config['ecephys_type'] == 'raw': ecog_es = ElectricalSeries(name='ECoG', data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'), electrodes=elecs_region, rate=ecog_rate, description='all Wav data') nwbfile.add_acquisition(ecog_es) elif htk_config['ecephys_type'] == 'preprocessed': lfp = LFP() ecog_es = ElectricalSeries(name='preprocessed', data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'), electrodes=elecs_region, rate=ecog_rate, description='all Wav data') lfp.add_electrical_series(ecog_es) # Creates the ecephys processing module ecephys_module = nwbfile.create_processing_module( name='ecephys', description='preprocessed electrophysiology data' ) ecephys_module.add_data_interface(lfp) elif htk_config['ecephys_type'] == 'high_gamma': ecog_es = ElectricalSeries(name='high_gamma', data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'), electrodes=elecs_region, rate=ecog_rate, description='all Wav data') # Creates the ecephys processing module ecephys_module = nwbfile.create_processing_module( name='ecephys', description='preprocessed electrophysiology data' ) ecephys_module.add_data_interface(ecog_es) # Add ANIN 1 if htk_config['anin1']['present']: fs, data = get_analog(anin_path, 1) ts = TimeSeries( name=htk_config['anin1']['name'], data=data, unit='NA', rate=fs, ) if htk_config['anin1']['type'] == 'acquisition': nwbfile.add_acquisition(ts) else: nwbfile.add_stimulus(ts) print('ANIN1 saved with name "', htk_config['anin1']['name'], '" in ', htk_config['anin1']['type']) # Add ANIN 2 if htk_config['anin2']['present']: fs, data = get_analog(anin_path, 2) ts = TimeSeries( name=htk_config['anin2']['name'], data=data, unit='NA', rate=fs, ) if htk_config['anin2']['type'] == 'acquisition': nwbfile.add_acquisition(ts) else: nwbfile.add_stimulus(ts) print('ANIN2 saved with name "', htk_config['anin2']['name'], '" in ', htk_config['anin2']['type']) # Add ANIN 3 if htk_config['anin3']['present']: fs, data = get_analog(anin_path, 3) ts = TimeSeries( name=htk_config['anin3']['name'], data=data, unit='NA', rate=fs, ) if htk_config['anin3']['type'] == 'acquisition': nwbfile.add_acquisition(ts) else: nwbfile.add_stimulus(ts) print('ANIN3 saved with name "', htk_config['anin3']['name'], '" in ', htk_config['anin3']['type']) # Add ANIN 4 if htk_config['anin4']['present']: fs, data = get_analog(anin_path, 4) ts = TimeSeries( name=htk_config['anin4']['name'], data=data, unit='NA', rate=fs, ) if htk_config['anin4']['type'] == 'acquisition': nwbfile.add_acquisition(ts) else: nwbfile.add_stimulus(ts) print('ANIN4 saved with name "', htk_config['anin4']['name'], '" in ', htk_config['anin4']['type']) # Add bad time segments if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size: bad_time = sio.loadmat(bad_time_file)['badTimeSegments'] for row in bad_time: nwbfile.add_invalid_time_interval(start_time=row[0], stop_time=row[1], tags=('ECoG artifact',), timeseries=ecog_es) # Subject subject_dict = {'subject_id': subject_id} if 'Subject' in metadata: subject_dict.update(metadata['Subject']) subject = ECoGSubject(**subject_dict) nwbfile.subject = subject if save_to_file: print('Saving HTK content to NWB file...') # Export the NWB file with NWBHDF5IO(str(out_file_path), manager=manager, mode='w') as io: io.write(nwbfile) # read check with NWBHDF5IO(str(out_file_path), manager=manager, mode='r') as io: io.read() print('NWB file saved: ', str(out_file_path)) return nwbfile, out_file_path, subject_id, blockname
def convert( input_file, session_start_time, subject_date_of_birth, subject_id='I5', subject_description='naive', subject_genotype='wild-type', subject_sex='M', subject_weight='11.6g', subject_species='Mus musculus', subject_brain_region='Medial Entorhinal Cortex', surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible', session_id='npI5_0417_baseline_1', experimenter='Kei Masuda', experiment_description='Virtual Hallway Task', institution='Stanford University School of Medicine', lab_name='Giocomo Lab'): """ Read in the .mat file specified by input_file and convert to .nwb format. Parameters ---------- input_file : np.ndarray (..., n_channels, n_time) the .mat file to be converted subject_id : string the unique subject ID number for the subject of the experiment subject_date_of_birth : datetime ISO 8601 the date and time the subject was born subject_description : string important information specific to this subject that differentiates it from other members of it's species subject_genotype : string the genetic strain of this species. subject_sex : string Male or Female subject_weight : the weight of the subject around the time of the experiment subject_species : string the name of the species of the subject subject_brain_region : basestring the name of the brain region where the electrode probe is recording from surgery : str information about the subject's surgery to implant electrodes session_id: string human-readable ID# for the experiment session that has a one-to-one relationship with a recording session session_start_time : datetime date and time that the experiment started experimenter : string who ran the experiment, first and last name experiment_description : string what task was being run during the session institution : string what institution was the experiment performed in lab_name : string the lab where the experiment was performed Returns ------- nwbfile : NWBFile The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5 """ # input matlab data matfile = hdf5storage.loadmat(input_file) # output path for nwb data def replace_last(source_string, replace_what, replace_with): head, _sep, tail = source_string.rpartition(replace_what) return head + replace_with + tail outpath = replace_last(input_file, '.mat', '.nwb') create_date = datetime.today() timezone_cali = pytz.timezone('US/Pacific') create_date_tz = timezone_cali.localize(create_date) # if loading data from config.yaml, convert string dates into datetime if isinstance(session_start_time, str): session_start_time = datetime.strptime(session_start_time, '%B %d, %Y %I:%M%p') session_start_time = timezone_cali.localize(session_start_time) if isinstance(subject_date_of_birth, str): subject_date_of_birth = datetime.strptime(subject_date_of_birth, '%B %d, %Y %I:%M%p') subject_date_of_birth = timezone_cali.localize(subject_date_of_birth) # create unique identifier for this experimental session uuid_identifier = uuid.uuid1() # Create NWB file nwbfile = NWBFile( session_description=experiment_description, # required identifier=uuid_identifier.hex, # required session_id=session_id, experiment_description=experiment_description, experimenter=experimenter, surgery=surgery, institution=institution, lab=lab_name, session_start_time=session_start_time, # required file_create_date=create_date_tz) # optional # add information about the subject of the experiment experiment_subject = Subject(subject_id=subject_id, species=subject_species, description=subject_description, genotype=subject_genotype, date_of_birth=subject_date_of_birth, weight=subject_weight, sex=subject_sex) nwbfile.subject = experiment_subject # adding constants via LabMetaData container # constants sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0]) n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0]) dat_path = matfile['sp'][0]['dat_path'][0][0][0] offset = int(matfile['sp'][0]['offset'][0][0][0]) data_dtype = matfile['sp'][0]['dtype'][0][0][0] hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0]) vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0] # container lab_metadata = LabMetaData_ext(name='LabMetaData', acquisition_sampling_rate=sample_rate, number_of_electrodes=n_channels_dat, file_path=dat_path, bytes_to_skip=offset, raw_data_dtype=data_dtype, high_pass_filtered=hp_filtered, movie_start_time=vr_session_offset) nwbfile.add_lab_meta_data(lab_metadata) # Adding trial information nwbfile.add_trial_column( 'trial_contrast', 'visual contrast of the maze through which the mouse is running') trial = np.ravel(matfile['trial']) trial_nums = np.unique(trial) position_time = np.ravel(matfile['post']) # matlab trial numbers start at 1. To correctly index trial_contract vector, # subtracting 1 from 'num' so index starts at 0 for num in trial_nums: trial_times = position_time[trial == num] nwbfile.add_trial(start_time=trial_times[0], stop_time=trial_times[-1], trial_contrast=matfile['trial_contrast'][num - 1][0]) # Add mouse position inside: position = Position() position_virtual = np.ravel(matfile['posx']) # position inside the virtual environment sampling_rate = 1 / (position_time[1] - position_time[0]) position.create_spatial_series( name='Position', data=position_virtual, starting_time=position_time[0], rate=sampling_rate, reference_frame='The start of the trial, which begins at the start ' 'of the virtual hallway.', conversion=0.01, description='Subject position in the virtual hallway.', comments='The values should be >0 and <400cm. Values greater than ' '400cm mean that the mouse briefly exited the maze.', ) # physical position on the mouse wheel physical_posx = position_virtual trial_gain = np.ravel(matfile['trial_gain']) for num in trial_nums: physical_posx[trial == num] = physical_posx[trial == num] / trial_gain[num - 1] position.create_spatial_series( name='PhysicalPosition', data=physical_posx, starting_time=position_time[0], rate=sampling_rate, reference_frame='Location on wheel re-referenced to zero ' 'at the start of each trial.', conversion=0.01, description='Physical location on the wheel measured ' 'since the beginning of the trial.', comments='Physical location found by dividing the ' 'virtual position by the "trial_gain"') nwbfile.add_acquisition(position) # Add timing of lick events, as well as mouse's virtual position during lick event lick_events = BehavioralEvents() lick_events.create_timeseries( 'LickEvents', data=np.ravel(matfile['lickx']), timestamps=np.ravel(matfile['lickt']), unit='centimeter', description='Subject position in virtual hallway during the lick.') nwbfile.add_acquisition(lick_events) # Add information on the visual stimulus that was shown to the subject # Assumed rate=60 [Hz]. Update if necessary # Update external_file to link to Unity environment file visualization = ImageSeries( name='ImageSeries', unit='seconds', format='external', external_file=list(['https://unity.com/VR-and-AR-corner']), starting_time=vr_session_offset, starting_frame=[[0]], rate=float(60), description='virtual Unity environment that the mouse navigates through' ) nwbfile.add_stimulus(visualization) # Add the recording device, a neuropixel probe recording_device = nwbfile.create_device(name='neuropixel_probes') electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded' electrode_group_name = 'probe1' electrode_group = nwbfile.create_electrode_group( electrode_group_name, description=electrode_group_description, location=subject_brain_region, device=recording_device) # Add information about each electrode xcoords = np.ravel(matfile['sp'][0]['xcoords'][0]) ycoords = np.ravel(matfile['sp'][0]['ycoords'][0]) data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0] if data_filtered_flag: filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered' else: filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered' num_recording_electrodes = xcoords.shape[0] recording_electrodes = range(0, num_recording_electrodes) # create electrode columns for the x,y location on the neuropixel probe # the standard x,y,z locations are reserved for Allen Brain Atlas location nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe') nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe') for idx in recording_electrodes: nwbfile.add_electrode(id=idx, x=np.nan, y=np.nan, z=np.nan, rel_x=float(xcoords[idx]), rel_y=float(ycoords[idx]), imp=np.nan, location='medial entorhinal cortex', filtering=filter_desc, group=electrode_group) # Add information about each unit, termed 'cluster' in giocomo data # create new columns in unit table nwbfile.add_unit_column( 'quality', 'labels given to clusters during manual sorting in phy (1=MUA, ' '2=Good, 3=Unsorted)') # cluster information cluster_ids = matfile['sp'][0]['cids'][0][0] cluster_quality = matfile['sp'][0]['cgs'][0][0] # spikes in time spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike spike_cluster = np.ravel( matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time for i, cluster_id in enumerate(cluster_ids): unit_spike_times = spike_times[spike_cluster == cluster_id] waveforms = matfile['sp'][0]['temps'][0][cluster_id] nwbfile.add_unit(id=int(cluster_id), spike_times=unit_spike_times, quality=cluster_quality[i], waveform_mean=waveforms, electrode_group=electrode_group) # Trying to add another Units table to hold the results of the automatic spike sorting # create TemplateUnits units table template_units = Units( name='TemplateUnits', description='units assigned during automatic spike sorting') template_units.add_column( 'tempScalingAmps', 'scaling amplitude applied to the template when extracting spike', index=True) # information on extracted spike templates spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0]) spike_template_ids = np.unique(spike_templates) # template scaling amplitudes temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0]) for i, spike_template_id in enumerate(spike_template_ids): template_spike_times = spike_times[spike_templates == spike_template_id] temp_scaling_amps_per_template = temp_scaling_amps[spike_templates == spike_template_id] template_units.add_unit(id=int(spike_template_id), spike_times=template_spike_times, electrode_group=electrode_group, tempScalingAmps=temp_scaling_amps_per_template) # create ecephys processing module spike_template_module = nwbfile.create_processing_module( name='ecephys', description='units assigned during automatic spike sorting') # add template_units table to processing module spike_template_module.add(template_units) print(nwbfile) print('converted to NWB:N') print('saving ...') with NWBHDF5IO(outpath, 'w') as io: io.write(nwbfile) print('saved', outpath)
# using external data. only the file paths will be stored inside the NWB file image_series2 = TwoPhotonSeries(name='TwoPhotonSeries2', dimension=[100, 100], external_file=['images.tiff'], imaging_plane=imaging_plane, starting_frame=[0], format='external', starting_time=0.0, rate=1.0) #################### # Since these two-photon data are raw, acquired data, we will add the # :py:class:`~pynwb.ophys.TwoPhotonSeries` objects to the :py:class:`~pynwb.NWBFile` # as acquired data. nwbfile.add_acquisition(image_series1) nwbfile.add_acquisition(image_series2) #################### # Motion Correction (optional) # --------------------------------- # # You can also store the result of motion correction. # These should be stored in a :py:class:`~pynwb.ophys.MotionCorrection` object, # which is a :py:class:`~pynwb.core.MultiContainerInterface` (similar to pynwb.behavior.Position) # which holds 1 or more :py:class:`~pynwb.ophys.CorrectedImageStack` objects. corrected = ImageSeries( name='corrected', # this must be named "corrected" data=np.ones((1000, 100, 100)), unit='na',
ephys_timestamps = np.arange(data_len) / rate ephys_ts = ElectricalSeries( 'test_ephys_data', 'an hypothetical source', ephys_data, electrode_table_region, timestamps=ephys_timestamps, # Alternatively, could specify starting_time and rate as follows # starting_time=ephys_timestamps[0], # rate=rate, resolution=0.001, comments= "This data was randomly generated with numpy, using 1234 as the seed", description="Random numbers generated with numpy.random.rand") nwbfile.add_acquisition(ephys_ts) ####################### # Designating electrophysiology data # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # As mentioned above, :py:class:`~pynwb.ecephys.ElectricalSeries` and :py:class:`~pynwb.ecephys.SpikeEventSeries` # are meant for storing specific types of extracellular recordings. In addition to these two # :py:class:`~pynwb.base.TimeSeries` classes, NWB provides some :ref:`data interfaces <basic_data_interfaces>` # for designating the type of data you are storing. We will briefly discuss them here, and refer the reader to # :py:mod:`API documentation <pynwb.ecephys>` and :ref:`PyNWB Basics tutorial <basics>` for more details on # using these objects. # # For storing spike data, there are two options. Which one you choose depends on what data you have available. # If you need to store the raw voltage traces, you should store your the traces with # :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
def run_conversion(self, nwbfile: NWBFile, metadata: dict): """ Run conversion for this data interface. Reads labview experiment behavioral data and adds it to nwbfile. Parameters ---------- nwbfile : NWBFile metadata : dict """ print("Converting Labview data...") # Get list of trial summary files dir_behavior_labview = self.source_data['dir_behavior_labview'] all_files = os.listdir(dir_behavior_labview) trials_files = [f for f in all_files if '_sum.txt' in f] trials_files.sort() # Get session_start_time from first file timestamps fpath = os.path.join(dir_behavior_labview, trials_files[0]) colnames = [ 'Trial', 'StartT', 'EndT', 'Result', 'InitT', 'SpecificResults', 'ProbLeft', 'OptoDur', 'LRew', 'RRew', 'InterT', 'LTrial', 'ReactionTime', 'OptoCond', 'OptoTrial' ] df_0 = pd.read_csv(fpath, sep='\t', index_col=False, names=colnames) t0 = df_0['StartT'][0] # initial time in Labview seconds # Add trials print("Converting Labview trials data...") if nwbfile.trials is not None: print( 'Trials already exist in current nwb file. Labview behavior trials not added.' ) else: # Make dataframe frames = [] for f in trials_files: fpath = os.path.join(dir_behavior_labview, f) frames.append( pd.read_csv(fpath, sep='\t', index_col=False, names=colnames)) df_trials_summary = pd.concat(frames) nwbfile.add_trial_column( name='results', description= "0 means sucess (rewarded trial), 1 means licks during intitial " "period, which leads to a failed trial. 2 means early lick failure. 3 means " "wrong lick or no response.") nwbfile.add_trial_column( name='init_t', description="duration of initial delay period.") nwbfile.add_trial_column( name='specific_results', description= "Possible outcomes classified based on raw data & meta file (_tr.m)." ) nwbfile.add_trial_column( name='prob_left', description= "probability for left trials in order to keep the number of " "left and right trials balanced within the session. ") nwbfile.add_trial_column( name='opto_dur', description="the duration of optical stimulation.") nwbfile.add_trial_column( name='l_rew_n', description="counting the number of left rewards.") nwbfile.add_trial_column( name='r_rew_n', description="counting the number of rightrewards.") nwbfile.add_trial_column(name='inter_t', description="inter-trial delay period.") nwbfile.add_trial_column( name='l_trial', description= "trial type (which side the air-puff is applied). 1 means " "left-trial, 0 means right-trial") nwbfile.add_trial_column( name='reaction_time', description= "if it is a successful trial or wrong lick during response " "period trial: ReactionTime = time between the first decision " "lick and the beginning of the response period. If it is a failed " "trial due to early licks: reaction time = the duration of " "the air-puff period (in other words, when the animal licks " "during the sample period).") nwbfile.add_trial_column( name='opto_cond', description="0: no opto. 1: opto is on during sample period. " "2: opto is on half way through the sample period (0.5s) " "and 0.5 during the response period. 3. opto is on during " "the response period.") nwbfile.add_trial_column( name='opto_trial', description="1: opto trials. 0: Non-opto trials.") for index, row in df_trials_summary.iterrows(): nwbfile.add_trial( start_time=row['StartT'] - t0, stop_time=row['EndT'] - t0, results=int(row['Result']), init_t=row['InitT'], specific_results=int(row['SpecificResults']), prob_left=row['ProbLeft'], opto_dur=row['OptoDur'], l_rew_n=int(row['LRew']), r_rew_n=int(row['RRew']), inter_t=row['InterT'], l_trial=int(row['LTrial']), reaction_time=int(row['ReactionTime']), opto_cond=int(row['OptoCond']), opto_trial=int(row['OptoTrial']), ) # Get list of files: continuous data continuous_files = [f.replace('_sum', '') for f in trials_files] # Adds continuous behavioral data frames = [] for f in continuous_files: fpath_lick = os.path.join(dir_behavior_labview, f) frames.append(pd.read_csv(fpath_lick, sep='\t', index_col=False)) df_continuous = pd.concat(frames) # Behavioral data print("Converting Labview behavior data...") l1_ts = TimeSeries(name="left_lick", data=df_continuous['Lick 1'].to_numpy(), timestamps=df_continuous['Time'].to_numpy() - t0, description="no description") l2_ts = TimeSeries(name="right_lick", data=df_continuous['Lick 2'].to_numpy(), timestamps=df_continuous['Time'].to_numpy() - t0, description="no description") nwbfile.add_acquisition(l1_ts) nwbfile.add_acquisition(l2_ts) # Optogenetics stimulation data print("Converting Labview optogenetics data...") ogen_device = nwbfile.create_device( name=metadata['Ogen']['Device']['name'], description=metadata['Ogen']['Device']['description']) meta_ogen_site = metadata['Ogen']['OptogeneticStimulusSite'] ogen_stim_site = OptogeneticStimulusSite( name=meta_ogen_site['name'], device=ogen_device, description=meta_ogen_site['description'], excitation_lambda=float(meta_ogen_site['excitation_lambda']), location=meta_ogen_site['location']) nwbfile.add_ogen_site(ogen_stim_site) meta_ogen_series = metadata['Ogen']['OptogeneticSeries'] ogen_series = OptogeneticSeries( name=meta_ogen_series['name'], data=df_continuous['Opto'].to_numpy(), site=ogen_stim_site, timestamps=df_continuous['Time'].to_numpy() - t0, description=meta_ogen_series['description'], ) nwbfile.add_stimulus(ogen_series)