def test_identify_isi_shape_one_spike(): min_spike = 5 test_input_amplitudes = [10, 20, 30, 40] test_avg_rate = [0, 1, 1, 1] test_features = { "sweeps": pd.DataFrame({ "stim_amp": test_input_amplitudes, "avg_rate": test_avg_rate, }), "spikes_set": [None] * len(test_avg_rate), } # Random test sweeps np.random.seed(42) n_points = 100 t = np.arange(n_points) i = np.zeros_like(t) epochs = {"sweep": (0, n_points - 1), "test": None, "recording": None, "experiment": None, "stim": None} sampling_rate = 1 clamp_mode = "CurrentClamp" sweep_list = [] for a in test_input_amplitudes: v = np.random.randn(n_points) test_sweep = Sweep(t, v, i, clamp_mode, sampling_rate, epochs=epochs) sweep_list.append(test_sweep) test_sweep_set = SweepSet(sweep_list) selected_sweep, _ = fv.identify_sweep_for_isi_shape( test_sweep_set, test_features, duration=1, min_spike=min_spike) assert np.array_equal(selected_sweep.v, sweep_list[1].v)
def test_isi_shape(feature_vector_input): sweeps, features, start, end = feature_vector_input isi_sweep, isi_sweep_spike_info = fv.identify_sweep_for_isi_shape( sweeps, features, end - start) temp_data = fv.isi_shape(isi_sweep, isi_sweep_spike_info, end) test_data = np.load(os.path.join(TEST_OUTPUT_DIR, "isi_shape.npy")) assert np.array_equal(test_data, temp_data)
def data_for_specimen_id(specimen_id, sweep_qc_option, data_source, ap_window_length=0.006, target_sampling_rate=10000, nfiles=None): logging.debug("specimen_id: {}".format(specimen_id)) lsq_fail = False ssq_fail = False ramp_fail = False # Find or retrieve NWB file and ancillary info and construct an AibsDataSet object ontology = StimulusOntology( ju.read(StimulusOntology.DEFAULT_STIMULUS_ONTOLOGY_FILE)) if data_source == "local": nwb_path = nfiles[specimen_id] if type(nwb_path) is dict and "error" in nwb_path: logging.warning( "Problem getting NWB file for specimen {:d}".format( specimen_id)) return nwb_path data_set = HBGDataSet(nwb_file=nwb_path, ontology=ontology) else: logging.error("invalid data source specified ({})".format(data_source)) # Identify and preprocess long square sweeps try: lsq_sweep_numbers = categorize_iclamp_sweeps( data_set, ontology.long_square_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) (lsq_sweeps, lsq_features, lsq_start, lsq_end, lsq_spx) = preprocess_long_square_sweeps(data_set, lsq_sweep_numbers) except Exception as detail: lsq_fail = True logging.warning( "Exception when preprocessing long square sweeps from specimen {:d}" .format(specimen_id)) logging.warning(detail) return { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Identify and preprocess short square sweeps try: ssq_sweep_numbers = categorize_iclamp_sweeps( data_set, ontology.short_square_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) ssq_sweeps, ssq_features = preprocess_short_square_sweeps( data_set, ssq_sweep_numbers) except Exception as detail: ssq_fail = True logging.warning( "Exception when preprocessing short square sweeps from specimen {:d}" .format(specimen_id)) logging.warning(detail) { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Identify and preprocess ramp sweeps try: ramp_sweep_numbers = categorize_iclamp_sweeps( data_set, ontology.ramp_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) ramp_sweeps, ramp_features = preprocess_ramp_sweeps( data_set, ramp_sweep_numbers) except Exception as detail: ramp_fail = True logging.warning( "Exception when preprocessing ramp sweeps from specimen {:d}". format(specimen_id)) logging.warning(detail) { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Calculate desired feature vectors result = {} try: (subthresh_hyperpol_dict, hyperpol_deflect_dict ) = fv.identify_subthreshold_hyperpol_with_amplitudes( lsq_features, lsq_sweeps) target_amps_for_step_subthresh = [-90, -70, -50, -30, -10] result["step_subthresh"] = fv.step_subthreshold( subthresh_hyperpol_dict, target_amps_for_step_subthresh, lsq_start, lsq_end, amp_tolerance=5) result["subthresh_norm"] = fv.subthresh_norm(subthresh_hyperpol_dict, hyperpol_deflect_dict, lsq_start, lsq_end) (subthresh_depol_dict, depol_deflect_dict) = fv.identify_subthreshold_depol_with_amplitudes( lsq_features, lsq_sweeps) result["subthresh_depol_norm"] = fv.subthresh_depol_norm( subthresh_depol_dict, depol_deflect_dict, lsq_start, lsq_end) isi_sweep, isi_sweep_spike_info = fv.identify_sweep_for_isi_shape( lsq_sweeps, lsq_features, lsq_end - lsq_start) result["isi_shape"] = fv.isi_shape(isi_sweep, isi_sweep_spike_info, lsq_end) if ssq_fail == False: # Calculate waveforms from each type of sweep spiking_ssq_sweep_list = [ ssq_sweeps.sweeps[swp_ind] for swp_ind in ssq_features["common_amp_sweeps"].index ] spiking_ssq_info_list = [ ssq_features["spikes_set"][swp_ind] for swp_ind in ssq_features["common_amp_sweeps"].index ] ssq_ap_v, ssq_ap_dv = fv.first_ap_vectors( spiking_ssq_sweep_list, spiking_ssq_info_list, target_sampling_rate=target_sampling_rate, window_length=ap_window_length, skip_clipped=True) else: ssq_ap_v, ssq_ap_dv = np.nan, np.nan rheo_ind = lsq_features["rheobase_sweep"].name sweep = lsq_sweeps.sweeps[rheo_ind] lsq_ap_v, lsq_ap_dv = fv.first_ap_vectors( [sweep], [lsq_features["spikes_set"][rheo_ind]], target_sampling_rate=target_sampling_rate, window_length=ap_window_length) if ramp_fail == False: spiking_ramp_sweep_list = [ ramp_sweeps.sweeps[swp_ind] for swp_ind in ramp_features["spiking_sweeps"].index ] spiking_ramp_info_list = [ ramp_features["spikes_set"][swp_ind] for swp_ind in ramp_features["spiking_sweeps"].index ] ramp_ap_v, ramp_ap_dv = fv.first_ap_vectors( spiking_ramp_sweep_list, spiking_ramp_info_list, target_sampling_rate=target_sampling_rate, window_length=ap_window_length, skip_clipped=True) else: ramp_ap_v, ramp_ap_dv = np.nan, np.nan if ramp_fail == True: ramp_ap_dv = np.copy(lsq_ap_dv) ramp_ap_v = np.copy(lsq_ap_v) if ssq_fail == True: ssq_ap_dv = np.copy(lsq_ap_dv) ssq_ap_v = np.copy(lsq_ap_v) # Combine so that differences can be assessed by analyses like sPCA result["first_ap_v"] = np.hstack([ssq_ap_v, lsq_ap_v, ramp_ap_v]) result["first_ap_dv"] = np.hstack([ssq_ap_dv, lsq_ap_dv, ramp_ap_dv]) target_amplitudes = np.arange(0, 120, 20) supra_info_list = fv.identify_suprathreshold_spike_info( lsq_features, target_amplitudes, shift=10) result["psth"] = fv.psth_vector(supra_info_list, lsq_start, lsq_end) result["inst_freq"] = fv.inst_freq_vector(supra_info_list, lsq_start, lsq_end) spike_feature_list = [ "upstroke_downstroke_ratio", "peak_v", "fast_trough_v", "threshold_v", "width", ] for feature in spike_feature_list: result["spiking_" + feature] = fv.spike_feature_vector( feature, supra_info_list, lsq_start, lsq_end) if feature == 'width': result["spiking_width"] = result["spiking_width"] / 2 except Exception as detail: logging.warning( "Exception when processing specimen {:d}".format(specimen_id)) logging.warning(detail) return { "error": { "type": "processing", "details": traceback.format_exc(limit=None) } } return result
def main(nwb_file, output_dir, project, **kwargs): nwb = MiesNwb(nwb_file) # SPECIFICS FOR EXAMPLE NWB ========= # Only analyze one channel at a time channel = 0 # We can work out code to automatically extract these based on stimulus names later. if_sweep_inds = [39, 45] targetv_sweep_inds = [15, 21] # END SPECIFICS ===================== # Assemble all Recordings and convert to Sweeps supra_sweep_ids = list(range(*if_sweep_inds)) sub_sweep_ids = list(range(*targetv_sweep_inds)) supra_recs = [nwb.contents[i][channel] for i in supra_sweep_ids] sub_recs = [nwb.contents[i][channel] for i in sub_sweep_ids] # Build sweep sets lsq_supra_sweep_list, lsq_supra_dur = recs_to_sweeps(supra_recs) lsq_sub_sweep_list, lsq_sub_dur = recs_to_sweeps(sub_recs) lsq_supra_sweeps = SweepSet(lsq_supra_sweep_list) lsq_sub_sweeps = SweepSet(lsq_sub_sweep_list) lsq_supra_start = 0 lsq_supra_end = lsq_supra_dur lsq_sub_start = 0 lsq_sub_end = lsq_sub_dur # Pre-process sweeps lsq_supra_spx, lsq_supra_spfx = dsf.extractors_for_sweeps( lsq_supra_sweeps, start=lsq_supra_start, end=lsq_supra_end) lsq_supra_an = spa.LongSquareAnalysis(lsq_supra_spx, lsq_supra_spfx, subthresh_min_amp=-100., require_subthreshold=False) lsq_supra_features = lsq_supra_an.analyze(lsq_supra_sweeps) lsq_sub_spx, lsq_sub_spfx = dsf.extractors_for_sweeps(lsq_sub_sweeps, start=lsq_sub_start, end=lsq_sub_end) lsq_sub_an = spa.LongSquareAnalysis(lsq_sub_spx, lsq_sub_spfx, subthresh_min_amp=-100., require_suprathreshold=False) lsq_sub_features = lsq_sub_an.analyze(lsq_sub_sweeps) # Calculate feature vectors result = {} (subthresh_hyperpol_dict, hyperpol_deflect_dict ) = fv.identify_subthreshold_hyperpol_with_amplitudes( lsq_sub_features, lsq_sub_sweeps) target_amps_for_step_subthresh = [-90, -70, -50, -30, -10] result["step_subthresh"] = fv.step_subthreshold( subthresh_hyperpol_dict, target_amps_for_step_subthresh, lsq_sub_start, lsq_sub_end, amp_tolerance=5) result["subthresh_norm"] = fv.subthresh_norm(subthresh_hyperpol_dict, hyperpol_deflect_dict, lsq_sub_start, lsq_sub_end) (subthresh_depol_dict, depol_deflect_dict) = fv.identify_subthreshold_depol_with_amplitudes( lsq_supra_features, lsq_supra_sweeps) result["subthresh_depol_norm"] = fv.subthresh_depol_norm( subthresh_depol_dict, depol_deflect_dict, lsq_supra_start, lsq_supra_end) isi_sweep, isi_sweep_spike_info = fv.identify_sweep_for_isi_shape( lsq_supra_sweeps, lsq_supra_features, lsq_supra_end - lsq_supra_start) result["isi_shape"] = fv.isi_shape(isi_sweep, isi_sweep_spike_info, lsq_supra_end) # Calculate AP waveform from long squares rheo_ind = lsq_supra_features["rheobase_sweep"].name sweep = lsq_supra_sweeps.sweeps[rheo_ind] lsq_ap_v, lsq_ap_dv = fv.first_ap_vectors( [sweep], [lsq_supra_features["spikes_set"][rheo_ind]], window_length=ap_window_length) result["first_ap_v"] = lsq_ap_v result["first_ap_dv"] = lsq_ap_dv target_amplitudes = np.arange(0, 120, 20) supra_info_list = fv.identify_suprathreshold_sweep_sequence( lsq_supra_features, target_amplitudes, shift=10) result["psth"] = fv.psth_vector(supra_info_list, lsq_supra_start, lsq_supra_end) result["inst_freq"] = fv.inst_freq_vector(supra_info_list, lsq_supra_start, lsq_supra_end) spike_feature_list = [ "upstroke_downstroke_ratio", "peak_v", "fast_trough_v", "threshold_v", "width", ] for feature in spike_feature_list: result["spiking_" + feature] = fv.spike_feature_vector( feature, supra_info_list, lsq_supra_start, lsq_supra_end) # Save the results specimen_ids = [0] results = [result] filtered_set = [(i, r) for i, r in zip(specimen_ids, results) if not "error" in r.keys()] error_set = [{ "id": i, "error": d } for i, d in zip(specimen_ids, results) if "error" in d.keys()] if len(filtered_set) == 0: logging.info("No specimens had results") return with open(os.path.join(output_dir, "fv_errors_{:s}.json".format(project)), "w") as f: json.dump(error_set, f, indent=4) used_ids, results = zip(*filtered_set) logging.info("Finished with {:d} processed specimens".format( len(used_ids))) k_sizes = {} for k in results[0].keys(): if k not in k_sizes and results[0][k] is not None: k_sizes[k] = len(results[0][k]) data = np.array([ r[k] if k in r else np.nan * np.zeros(k_sizes[k]) for r in results ]) if len(data.shape) == 1: # it'll be 1D if there's just one specimen data = np.reshape(data, (1, -1)) if data.shape[0] < len(used_ids): logging.warn("Missing data!") missing = np.array([k not in r for r in results]) print(k, np.array(used_ids)[missing]) np.save( os.path.join(output_dir, "fv_{:s}_{:s}.npy".format(k, project)), data) np.save(os.path.join(output_dir, "fv_ids_{:s}.npy".format(project)), used_ids)
def test_identify_sweep_for_isi_shape(): sweeps = { "index": [0, 1, 2, 3, 4, 5, 6, 7, 8], "columns": [ "avg_rate", "peak_deflect", "stim_amp", "v_baseline", "sag", "adapt", "latency", "isi_cv", "mean_isi", "median_isi", "first_isi", ], "data": [ [ 0.0, (-70.65, 35892), -29.999998092651367, -68.25825500488281, 0.054702017456293106, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, ], [ 0.0, (-72.2, 33470), -50.0, -68.39005279541016, 0.01751580648124218, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, ], [ 0.0, (-73.66875, 79087), -70.0, -68.42821502685547, 0.004760173615068197, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, ], [ 0.0, (-30.550001, 31322), 469.9999694824219, -68.29730987548828, -8.956585884094238, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, ], [ 0.0, (-28.54375, 31625), 479.9999694824219, -68.26968383789062, -9.093791961669922, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, ], [ 5.000000000000001, (14.98125, 30920), 489.9999694824219, -67.90441131591797, -8.981210708618164, 0.00839006531426236, 0.018079999999999985, 0.026166652896782595, 0.016109999999999985, 0.01608000000000004, 0.015880000000000005, ], [ 30.000000000000004, (15.19375, 32169), 520.0, -67.4559097290039, -8.814754486083984, 0.00801722722893242, 0.01963999999999999, 0.1548801847128805, 0.015706896551724133, 0.0156400000000001, 0.011479999999999935, ], [ 72.00000000000001, (15.31875, 31168), 540.0, -67.43457794189453, -6.910084247589111, 0.001789278456852954, 0.012699999999999934, 0.09433347064888947, 0.013758591549295774, 0.013900000000000023, 0.010319999999999996, ], [ 81.00000000000001, (15.7125, 52887), 560.0, -67.80497741699219, -6.805542945861816, 0.002374929272995164, 0.011459999999999915, 0.0921604402099859, 0.0123335, 0.012390000000000012, 0.009160000000000057, ], ], } end = 1.5 start = 0.5 class SomeSweeps: @property def sweeps(self): ndata = len(sweeps["data"][0]) nsweeps = len(sweeps["index"]) return np.arange(nsweeps * ndata).reshape(nsweeps, ndata) isi_sweep, isi_sweep_spike_info = fv.identify_sweep_for_isi_shape( SomeSweeps(), { "sweeps": pd.DataFrame(**sweeps), "spikes_set": { 5: "foo" } }, end - start, ) assert isi_sweep_spike_info == "foo" assert np.allclose(isi_sweep, np.arange(55, 66))
def data_for_specimen_id( specimen_id, sweep_qc_option, data_source, ontology, ap_window_length=0.005, target_sampling_rate=50000, file_list=None, ): """ Extract feature vector from given cell identified by the specimen_id Parameters ---------- specimen_id : int cell identified sweep_qc_option : str see CollectFeatureVectorParameters input schema for details data_source: str see CollectFeatureVectorParameters input schema for details ontology : stimulus.StimulusOntology mapping of stimuli names to stimulus codes ap_window_length : float see CollectFeatureVectorParameters input schema for details target_sampling_rate : float sampling rate file_list : list of str nwbfile names Returns ------- dict : features for a given cell specimen_id """ logging.debug("specimen_id: {}".format(specimen_id)) # Find or retrieve NWB file and ancillary info and construct an AibsDataSet object data_set = su.dataset_for_specimen_id(specimen_id, data_source, ontology, file_list) if type(data_set) is dict and "error" in data_set: logging.warning( "Problem getting AibsDataSet for specimen {:d} from LIMS".format( specimen_id)) return data_set # Identify and preprocess long square sweeps try: lsq_sweep_numbers = su.categorize_iclamp_sweeps( data_set, ontology.long_square_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) (lsq_sweeps, lsq_features, _, lsq_start, lsq_end) = su.preprocess_long_square_sweeps(data_set, lsq_sweep_numbers) except Exception as detail: logging.warning( "Exception when preprocessing long square sweeps from specimen {:d}" .format(specimen_id)) logging.warning(detail) return { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Identify and preprocess short square sweeps try: ssq_sweep_numbers = su.categorize_iclamp_sweeps( data_set, ontology.short_square_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) ssq_sweeps, ssq_features, _ = su.preprocess_short_square_sweeps( data_set, ssq_sweep_numbers) except Exception as detail: logging.warning( "Exception when preprocessing short square sweeps from specimen {:d}" .format(specimen_id)) logging.warning(detail) return { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Identify and preprocess ramp sweeps try: ramp_sweep_numbers = su.categorize_iclamp_sweeps( data_set, ontology.ramp_names, sweep_qc_option=sweep_qc_option, specimen_id=specimen_id) ramp_sweeps, ramp_features, _ = su.preprocess_ramp_sweeps( data_set, ramp_sweep_numbers) except Exception as detail: logging.warning( "Exception when preprocessing ramp sweeps from specimen {:d}". format(specimen_id)) logging.warning(detail) return { "error": { "type": "sweep_table", "details": traceback.format_exc(limit=None) } } # Calculate desired feature vectors result = {} if data_source == "filesystem": result["id"] = [specimen_id] try: (subthresh_hyperpol_dict, hyperpol_deflect_dict ) = fv.identify_subthreshold_hyperpol_with_amplitudes( lsq_features, lsq_sweeps) target_amps_for_step_subthresh = [-90, -70, -50, -30, -10] result["step_subthresh"] = fv.step_subthreshold( subthresh_hyperpol_dict, target_amps_for_step_subthresh, lsq_start, lsq_end, amp_tolerance=5) result["subthresh_norm"] = fv.subthresh_norm(subthresh_hyperpol_dict, hyperpol_deflect_dict, lsq_start, lsq_end) (subthresh_depol_dict, depol_deflect_dict) = fv.identify_subthreshold_depol_with_amplitudes( lsq_features, lsq_sweeps) result["subthresh_depol_norm"] = fv.subthresh_depol_norm( subthresh_depol_dict, depol_deflect_dict, np.round(lsq_start, decimals=3), np.round(lsq_end, decimals=3)) isi_sweep, isi_sweep_spike_info = fv.identify_sweep_for_isi_shape( lsq_sweeps, lsq_features, lsq_end - lsq_start) result["isi_shape"] = fv.isi_shape(isi_sweep, isi_sweep_spike_info, lsq_end) # Calculate waveforms from each type of sweep spiking_ssq_sweep_list = [ ssq_sweeps.sweeps[swp_ind] for swp_ind in ssq_features["common_amp_sweeps"].index ] spiking_ssq_info_list = [ ssq_features["spikes_set"][swp_ind] for swp_ind in ssq_features["common_amp_sweeps"].index ] ssq_ap_v, ssq_ap_dv = fv.first_ap_vectors( spiking_ssq_sweep_list, spiking_ssq_info_list, target_sampling_rate=target_sampling_rate, window_length=ap_window_length, skip_clipped=True) rheo_ind = lsq_features["rheobase_sweep"].name sweep = lsq_sweeps.sweeps[rheo_ind] lsq_ap_v, lsq_ap_dv = fv.first_ap_vectors( [sweep], [lsq_features["spikes_set"][rheo_ind]], target_sampling_rate=target_sampling_rate, window_length=ap_window_length) spiking_ramp_sweep_list = [ ramp_sweeps.sweeps[swp_ind] for swp_ind in ramp_features["spiking_sweeps"].index ] spiking_ramp_info_list = [ ramp_features["spikes_set"][swp_ind] for swp_ind in ramp_features["spiking_sweeps"].index ] ramp_ap_v, ramp_ap_dv = fv.first_ap_vectors( spiking_ramp_sweep_list, spiking_ramp_info_list, target_sampling_rate=target_sampling_rate, window_length=ap_window_length, skip_clipped=True) # Combine so that differences can be assessed by analyses like sPCA result["first_ap_v"] = np.hstack([ssq_ap_v, lsq_ap_v, ramp_ap_v]) result["first_ap_dv"] = np.hstack([ssq_ap_dv, lsq_ap_dv, ramp_ap_dv]) target_amplitudes = np.arange(0, 120, 20) supra_info_list = fv.identify_suprathreshold_spike_info( lsq_features, target_amplitudes, shift=10) result["psth"] = fv.psth_vector(supra_info_list, lsq_start, lsq_end) result["inst_freq"] = fv.inst_freq_vector(supra_info_list, lsq_start, lsq_end) spike_feature_list = [ "upstroke_downstroke_ratio", "peak_v", "fast_trough_v", "threshold_v", "width", ] for feature in spike_feature_list: result["spiking_" + feature] = fv.spike_feature_vector( feature, supra_info_list, lsq_start, lsq_end) except Exception as detail: logging.warning( "Exception when processing specimen {:d}".format(specimen_id)) logging.warning(detail) return { "error": { "type": "processing", "details": traceback.format_exc(limit=None) } } return result