def test_raises_on_empty_h5_file(): filename = os.path.join(TEST_DATA_PATH, "empty.nwb") with h5py.File(filename, 'w'): pass with pytest.raises(ValueError, match=r'unknown NWB major'): create_nwb_reader(filename)
def test_valid_v1_but_unknown_sweep_naming(): filename = os.path.join(TEST_DATA_PATH, 'invalid_sweep_naming_convention.nwb') with h5py.File(filename, 'w') as fh: dt = h5py.special_dtype(vlen=bytes) dset = fh.create_dataset("nwb_version", (1, ), dtype=dt) dset[:] = "NWB-1" with pytest.raises(ValueError, match=r'sweep naming convention'): create_nwb_reader(filename)
def __init__(self, input_file, output_file): """ Convert NWB v1 to v2 """ self.nwb_data = nwb_reader.create_nwb_reader(input_file) self.notebook = lab_notebook_reader.create_lab_notebook_reader( input_file) nwb_file = self.create_nwb_file() device = nwb_file.create_device(name='electrode_0') electrode = nwb_file.create_ic_electrode( name="elec0", description=' some kind of electrode', device=device) for i in self.create_stimulus_series(electrode): nwb_file.add_stimulus(i) for i in self.create_acquisition_series(electrode): nwb_file.add_acquisition(i) with NWBHDF5IO(output_file, "w") as io: io.write(nwb_file, cache_spec=True)
def __init__( self, sweep_info=None, nwb_file=None, h5_file=None, ontology=None, api_sweeps=True, validate_stim=True, deprecation_warning=True ): super(AibsDataSet, self).__init__( ontology, validate_stim, deprecation_warning=deprecation_warning ) self._nwb_data = nwb_reader.create_nwb_reader(nwb_file) if sweep_info: sweep_info = sp.modify_sweep_info_keys(sweep_info) if api_sweeps else sweep_info # Remove sweeps not found in nwb_data sweep map sweep_numbers_in_map = self._nwb_data.sweep_map_table["sweep_number"].tolist() sweep_info = [si for si in sweep_info if si["sweep_number"] in sweep_numbers_in_map] else: self.notebook = lab_notebook_reader.create_lab_notebook_reader(nwb_file, h5_file) sweep_info = self.extract_sweep_stim_info() self.build_sweep_table(sweep_info, deprecation_warning=False)
def test_valid_v1_skeleton_X_NWB(): filename = os.path.join(TEST_DATA_PATH, 'valid_v2.nwb') with h5py.File(filename, 'w') as fh: fh.attrs["nwb_version"] = "2" reader = create_nwb_reader(filename) assert isinstance(reader, NwbXReader)
def test_valid_v1_full_MIES_3(NWB_file): reader = create_nwb_reader(NWB_file) assert isinstance(reader, NwbMiesReader) sweep_names_ref = [u'data_00000_AD0'] sweep_names = reader.get_sweep_names() assert sorted(sweep_names_ref) == sorted(sweep_names) assert reader.get_pipeline_version() == (0, 0) assert reader.get_sweep_number("data_00000_AD0") == 0 assert reader.get_stim_code(0) == "EXTPSMOKET180424" # ignore very long comment sweep_attrs_ref = { u'ancestry': np.array([u'TimeSeries', u'PatchClampSeries', u'VoltageClampSeries'], dtype=object), u'comment': None, u'description': u'PLACEHOLDER', u'missing_fields': np.array([ u'resistance_comp_bandwidth', u'resistance_comp_correction', u'resistance_comp_prediction', u'whole_cell_capacitance_comp', u'whole_cell_series_resistance_comp' ], dtype=object), u'neurodata_type': u'TimeSeries', u'source': u'Device=ITC18USB_Dev_0;Sweep=0;AD=0;ElectrodeNumber=0;ElectrodeName=0' } sweep_attrs = reader.get_sweep_attrs(0) sweep_attrs['comment'] = None compare_dicts(sweep_attrs_ref, sweep_attrs) # assume the data itself is correct and replace it with None sweep_data_ref = { 'response': None, 'sampling_rate': 200000.0, 'stimulus': None, 'stimulus_unit': 'Volts' } sweep_data = reader.get_sweep_data(0) sweep_data['response'] = None sweep_data['stimulus'] = None assert sweep_data_ref == sweep_data
def test_sweep_map_sweep_numbers(NWB_file): sweep_numbers_ref = np.arange(0, 71) reader = create_nwb_reader(NWB_file) sweep_map_table = reader.sweep_map_table sweep_numbers = sweep_map_table["sweep_number"].values print(sweep_numbers_ref) assert (sweep_numbers == sweep_numbers_ref).all()
def test_valid_v1_with_no_sweeps(): filename = os.path.join(TEST_DATA_PATH, 'no_sweeps.nwb') with h5py.File(filename, 'w') as fh: dt = h5py.special_dtype(vlen=bytes) dset = fh.create_dataset("nwb_version", (1, ), dtype=dt) dset[:] = "NWB-1" fh.create_group("acquisition/timeseries") reader = create_nwb_reader(filename) assert isinstance(reader, NwbMiesReader)
def test_valid_v2_full_DAT(NWB_file): reader = create_nwb_reader(NWB_file) assert isinstance(reader, NwbXReader) sweep_names_ref = ['index_{:02d}'.format(x) for x in range(0, 78)] sweep_names = reader.get_sweep_names() assert sorted(sweep_names_ref) == sorted(sweep_names) assert reader.get_pipeline_version() == (0, 0) assert reader.get_sweep_number("index_00") == 10101 assert reader.get_stim_code(10101) == "extpinbath" # ignore very long description sweep_attrs_ref = { u'capacitance_fast': 0.0, u'capacitance_slow': np.nan, u'comments': u'no comments', u'description': None, u'gain': 5000000.0, u'help': u'Current recorded from cell during voltage-clamp recording', u'namespace': u'core', u'neurodata_type': u'VoltageClampSeries', u'resistance_comp_bandwidth': np.nan, u'resistance_comp_correction': np.nan, u'resistance_comp_prediction': np.nan, u'starting_time': 3768.2174599999998, u'stimulus_description': u'extpinbath', u'sweep_number': 10101, u'whole_cell_capacitance_comp': np.nan, u'whole_cell_series_resistance_comp': np.nan } sweep_attrs = reader.get_sweep_attrs(10101) sweep_attrs['description'] = None compare_dicts(sweep_attrs_ref, sweep_attrs) # assume the data itself is correct and replace it with None sweep_data_ref = { 'response': None, 'sampling_rate': 200000.00000000003, 'stimulus': None, 'stimulus_unit': 'Volts' } sweep_data = reader.get_sweep_data(10101) sweep_data['response'] = None sweep_data['stimulus'] = None assert sweep_data_ref == sweep_data
def test_sweep_map_sweep_0(NWB_file): reader = create_nwb_reader(NWB_file) sweep_map_ref = { 'acquisition_group': u'data_00046_AD0', 'stimulus_group': u'data_00046_DA0', 'sweep_number': 0, 'starting_time': 2740.1590003967285 } sweep_map = reader.get_sweep_map(0) assert sweep_map == sweep_map_ref
def __init__(self, sweep_info=None, nwb_file=None, ontology=None, api_sweeps=True, validate_stim=True): super(HBGDataSet, self).__init__(ontology, validate_stim) self._nwb_data = nwb_reader.create_nwb_reader(nwb_file) if sweep_info is None: sweep_info = self.extract_sweep_stim_info() self.build_sweep_table(sweep_info)
def test_valid_v1_skeleton_Pipeline(): filename = os.path.join(TEST_DATA_PATH, 'valid_v1_Pipeline.nwb') with h5py.File(filename, 'w') as fh: dt = h5py.special_dtype(vlen=bytes) dset = fh.create_dataset("nwb_version", (1, ), dtype=dt) dset[:] = "NWB-1" dset = fh.create_dataset("acquisition/timeseries/Sweep_0", (1, ), dtype="f") reader = create_nwb_reader(filename) assert isinstance(reader, NwbPipelineReader)
def test_valid_v2_full_ABF(NWB_file): reader = create_nwb_reader(NWB_file) assert isinstance(reader, NwbXReader) sweep_names_ref = [u'index_0'] sweep_names = reader.get_sweep_names() assert sorted(sweep_names_ref) == sorted(sweep_names) assert reader.get_pipeline_version() == (0, 0) assert reader.get_sweep_number("index_0") == 0 assert reader.get_stim_code(0) == "RAMP1" # ignore very long description sweep_attrs_ref = { u'bias_current': np.nan, u'bridge_balance': np.nan, u'capacitance_compensation': np.nan, u'comments': u'no comments', u'description': None, u'gain': 1.0, u'help': u'Voltage recorded from cell during current-clamp recording', u'namespace': u'core', u'neurodata_type': u'CurrentClampSeries', u'starting_time': 0.0, u'stimulus_description': u'RAMP1', u'sweep_number': 0 } sweep_attrs = reader.get_sweep_attrs(0) sweep_attrs['description'] = None compare_dicts(sweep_attrs_ref, sweep_attrs) # assume the data itself is correct and replace it with None sweep_data_ref = { 'response': None, 'sampling_rate': 50000.0, 'stimulus': None, 'stimulus_unit': 'Amps' } sweep_data = reader.get_sweep_data(0) sweep_data['response'] = None sweep_data['stimulus'] = None assert sweep_data_ref == sweep_data
def test_embed_spike_times_into_nwb(make_skeleton_nwb_file, tmpdir_factory): sweep_spike_times = { 3: [56.0, 44.6, 661.1], 4: [156.0, 144.6, 61.1, 334.944] } tmp_dir = tmpdir_factory.mktemp("embed_spikes_into_nwb") input_nwb_file_name = str(tmp_dir.join("input.nwb")) output_nwb_file_name = str(tmp_dir.join("output.nwb")) make_skeleton_nwb_file(input_nwb_file_name) embed_spike_times(input_nwb_file_name, output_nwb_file_name, sweep_spike_times) nwb_data = nwb_reader.create_nwb_reader(output_nwb_file_name) for sweep_num, spike_times in sweep_spike_times.items(): assert np.allclose(nwb_data.get_spike_times(sweep_num), spike_times)
def main(paths, sweeps, dendrite_type, bridge_avg, passive_fit_start_time, electrode_capacitance, junction_potential, random_seeds, output_json, **kwargs): """Main sequence of pre-processing and passive fitting""" # Extract Sweep objects (from IPFX package) from NWB file nwb_path = paths["nwb"] # nwb - neurodata without borders (ephys data) nwb_data = create_nwb_reader(nwb_path) core_1_lsq, c1_start, c1_end = sweeps_from_nwb( nwb_data, sweeps["core_1_long_squares"]) core_2_lsq, c2_start, c2_end = sweeps_from_nwb( nwb_data, sweeps["core_2_long_squares"]) # Choose sweeps to train the model sweep_set_to_fit, start, end = preprocess.select_core_1_or_core_2_sweeps( core_1_lsq, c1_start, c1_end, core_2_lsq, c2_start, c2_end) if len(sweep_set_to_fit.sweeps) == 0: ju.write(output_json, { 'error': "No usable sweeps found" }) return # Calculate the target features from the training sweeps step_analysis = StepAnalysis(start, end) step_analysis.analyze(sweep_set_to_fit) target_info = preprocess.target_features( step_analysis.sweep_features(), step_analysis.spikes_data()) stim_amp = step_analysis.sweep_features()["stim_amp"].values[0] stim_dur = end - start v_baseline = target_info.at["v_baseline", "mean"] # Determine maximum current used for depolarization block checks # during optimization # Load noise sweeps to check highest current used noise_1, _, _ = sweeps_from_nwb( nwb_data, sweeps["seed_1_noise"]) noise_2, _, _ = sweeps_from_nwb( nwb_data, sweeps["seed_2_noise"]) max_i = preprocess.max_i_for_depol_block_check( core_1_lsq, core_2_lsq, noise_1, noise_2) # Prepare inputs for passive fitting is_spiny = dendrite_type == "spiny" cap_checks, _, _ = sweeps_from_nwb( nwb_data, sweeps["cap_checks"]) if len(cap_checks.sweeps) == 0: logging.info("No cap check traces found") should_run_passive_fit = False passive_info = { "should_run": False, } else: grand_up, grand_down, t = preprocess.cap_check_grand_averages(cap_checks) up_file, down_file = preprocess.save_grand_averages( grand_up, grand_down, t, paths["storage_directory"]) escape_time = preprocess.passive_fit_window(grand_up, grand_down, t, start_time=passive_fit_start_time) passive_info = { "should_run": True, "bridge": bridge_avg, "fit_window_start": passive_fit_start_time, "fit_window_end": escape_time, "electrode_cap": electrode_capacitance, "is_spiny": is_spiny, } paths["up"] = up_file paths["down"] = down_file passive_info_path = os.path.join( paths["storage_directory"], "passive_info.json") ju.write(passive_info_path, passive_info) # Determine whether morphology has an apical dendrite has_apical = preprocess.swc_has_apical_compartments(paths["swc"]) # Decide which fits to run based on morphology and AP width fit_types = preprocess.FitStyle.get_fit_types( has_apical=has_apical, is_spiny=is_spiny, width=target_info.at["width", "mean"]) stage_1_tasks = [{"fit_type": fit_type, "seed": seed} for seed in random_seeds for fit_type in fit_types] stage_2_tasks = [{"fit_type": preprocess.FitStyle.map_stage_2(fit_type), "seed": seed} for seed in random_seeds for fit_type in fit_types] preprocess_results_path = os.path.join( paths["storage_directory"], "preprocess_results.json") ju.write(preprocess_results_path, { "is_spiny": is_spiny, "has_apical": has_apical, "junction_potential": junction_potential, "max_stim_test_na": max_i, "v_baseline": v_baseline, "stimulus": { "amplitude": 1e-3 * stim_amp, # to nA "delay": 1e3, "duration": 1e3 * stim_dur, # to ms }, "target_features": target_info.to_dict(orient="index"), "sweeps": sweeps, "sweeps_to_fit": [s.sweep_number for s in sweep_set_to_fit.sweeps], }) paths.update({ "preprocess_results": preprocess_results_path, "passive_info": passive_info_path, }) output = { "paths": paths, "stage_1_task_list": stage_1_tasks, "stage_2_task_list": stage_2_tasks, } ju.write(module.args["output_json"], output)
def test_assumed_sweep_number_fallback(NWB_file): reader = create_nwb_reader(NWB_file) assert isinstance(reader, NwbPipelineReader) assert reader.get_sweep_number("Sweep_10") == 10
def test_get_recording_date(NWB_file): reader = create_nwb_reader(NWB_file) assert "2018-03-20 20:59:48" == reader.get_recording_date()
def test_valid_v1_full_Pipeline(fetch_pipeline_file): reader = create_nwb_reader(fetch_pipeline_file) assert isinstance(reader, NwbPipelineReader) sweep_names_ref = [ u'Sweep_10', u'Sweep_12', u'Sweep_13', u'Sweep_14', u'Sweep_15', u'Sweep_16', u'Sweep_17', u'Sweep_19', u'Sweep_20', u'Sweep_25', u'Sweep_28', u'Sweep_29', u'Sweep_30', u'Sweep_32', u'Sweep_33', u'Sweep_34', u'Sweep_35', u'Sweep_36', u'Sweep_37', u'Sweep_38', u'Sweep_39', u'Sweep_40', u'Sweep_41', u'Sweep_42', u'Sweep_43', u'Sweep_44', u'Sweep_45', u'Sweep_46', u'Sweep_47', u'Sweep_5', u'Sweep_51', u'Sweep_52', u'Sweep_53', u'Sweep_54', u'Sweep_55', u'Sweep_57', u'Sweep_58', u'Sweep_59', u'Sweep_6', u'Sweep_61', u'Sweep_62', u'Sweep_63', u'Sweep_64', u'Sweep_65', u'Sweep_66', u'Sweep_67', u'Sweep_68', u'Sweep_69', u'Sweep_7', u'Sweep_70', u'Sweep_74', u'Sweep_8', u'Sweep_9' ] sweep_names = reader.get_sweep_names() assert sorted(sweep_names_ref) == sorted(sweep_names) assert reader.get_pipeline_version() == (1, 0) assert reader.get_sweep_number("Sweep_10") == 10 assert reader.get_stim_code(10) == "Short Square" sweep_attrs_ref = { u'ancestry': np.array(['TimeSeries', 'PatchClampSeries', 'CurrentClampSeries'], dtype='|S18'), u'comments': u'', u'description': u'', u'help': u'Voltage recorded from cell during current-clamp recording', u'missing_fields': np.array(['gain'], dtype='|S4'), u'neurodata_type': u'TimeSeries', u'source': u'' } sweep_attrs = reader.get_sweep_attrs(10) compare_dicts(sweep_attrs_ref, sweep_attrs) # assume the data itself is correct and replace it with None sweep_data_ref = { 'response': None, 'sampling_rate': 50000.0, 'stimulus': None, 'stimulus_unit': 'Amps' } sweep_data = reader.get_sweep_data(10) sweep_data['response'] = None sweep_data['stimulus'] = None assert sweep_data_ref == sweep_data
def test_raises_on_missing_file(): with pytest.raises(IOError): create_nwb_reader('I_DONT_EXIST.nwb')