def save_nwb(output_path, v, sweep, sweep_by_type= None): '''Save a single voltage output result into an existing sweep in a NWB file. This is intended to overwrite a recorded trace with a simulated voltage. Parameters ---------- output_path : string file name of a pre-existing NWB file. v : numpy array voltage sweep : integer which entry to overwrite in the file. ''' output = NwbDataSet(output_path) output.set_sweep(sweep, None, v) if sweep_by_type is not None: sweep_by_type = {t: [sweep] for t, ss in sweeps_by_type.items() if sweep in ss} sweep_features = extract_cell_features.extract_sweep_features(output, sweep_by_type) try: spikes = sweep_features[sweep]['spikes'] spike_times = [s['threshold_t'] for s in spikes] output.set_spike_times(sweep, spike_times) except Exception as e: logging.info("sweep %d has no sweep features. %s" % (sweep, e.args))
def save_nwb(output_path, v, sweep, sweeps_by_type): '''Save a single voltage output result into an existing sweep in a NWB file. This is intended to overwrite a recorded trace with a simulated voltage. Parameters ---------- output_path : string file name of a pre-existing NWB file. v : numpy array voltage sweep : integer which entry to overwrite in the file. ''' output = NwbDataSet(output_path) output.set_sweep(sweep, None, v) sweep_by_type = {t: [ sweep ] for t, ss in sweeps_by_type.items() if sweep in ss } sweep_features = extract_cell_features.extract_sweep_features(output, sweep_by_type) try: spikes = sweep_features[sweep]['spikes'] spike_times = [ s['threshold_t'] for s in spikes ] output.set_spike_times(sweep, spike_times) except Exception, e: logging.info("sweep %d has no sweep features. %s" % (sweep, e.message) )
def save_nwb(output_path, v, sweep): '''Save a single voltage output result into an existing sweep in a NWB file. This is intended to overwrite a recorded trace with a simulated voltage. Parameters ---------- output_path : string file name of a pre-existing NWB file. v : numpy array voltage sweep : integer which entry to overwrite in the file. ''' output = NwbDataSet(output_path) output.set_sweep(sweep, None, v) sweep_features = extract_cell_features.extract_sweep_features(output_path, [sweep]) spikes = sweep_features[sweep]['mean']['spikes'] spike_times = [ s['t'] for s in spikes ] output.set_spike_times(sweep, spike_times)
def main(jin): infile = jin[0]["nwb_file"] outfile = jin[0]["publish_nwb"] tmpfile = outfile + ".working" metafile = local_dir + jin[0]["metadata_file"] # load metadata stored in YML file metadata_desc_file = os.path.join(os.path.dirname(__file__), metafile) rsrc = resource_file.ResourceFile() rsrc.load(metadata_desc_file) # metadata = organize_metadata(jin[0]) # TODO dig deeper here # only fetching metadata for passing sweeps passing_sweeps = metadata['sweeps'].keys() copy_file(infile, outfile, passing_sweeps, rsrc, metadata) # try: # shutil.copyfile(infile, tmpfile) # except: # print("Unable to copy '%s' to %s" % (infile, tmpfile)) # print("----------------------------") # raise # # open NWB file so the modification date is updated # # add metadata then close file and do remaining manipulations using # # HDF5 library (except DF's legacy code that interfaces w/ nwb file # # using nwb library) # args = {} # args["filename"] = tmpfile # args["modify"] = True # try: # nwb_file = nwb.NWB(**args) # except: # print("Error opening NWB file '%s'" % args["filename"]) # raise # write_metadata(nwb_file, rsrc, metadata) # nwb_file.close() # open publish file directlya using HDF5 library # 1) remove hdf5 groups corresponding to failed sweeps # 2) add sweep-specific metadata data to file to match original publish # format. this includes (acquisition and stimulus): # aibs_stimulus_amplitude_pa # aibs_stimulus_interval # aibs_stimulus_name # initial_access_resistance # seal hdf = h5py.File(outfile, "r+") # ################################ # # delete epochs, stim, recordings for non-passed sweeps # epochs = hdf["epochs/"] # for grp in epochs: # try: # num = int(str(grp).split('_')[-1]) # except: # continue # if num not in passing_sweeps: # del epochs[str(grp)] # stim = hdf["stimulus/presentation"] # for grp in stim: # try: # num = int(str(grp).split('_')[-1]) # except: # continue # if num not in passing_sweeps: # del stim[str(grp)] # acq = hdf["acquisition/timeseries"] # for grp in acq: # try: # num = int(str(grp).split('_')[-1]) # except: # continue # if num not in passing_sweeps: # del acq[str(grp)] ################################ # add data acq = hdf["acquisition/timeseries"] stim = hdf["stimulus/presentation"] sweeps = jin[0]["specimens"][0]["ephys_sweeps"] for grp in acq: try: num = int(str(grp).split('_')[-1]) except: continue try: for sweep in sweeps: if sweep["sweep_number"] == num: break if sweep["sweep_number"] != num: print(sweep) print(num) raise Exception("WTF") # stim amplitude amp = sweep["stimulus_amplitude"] if amp is None: amp = float('nan') else: amp = float(amp) ds = acq["Sweep_%d" % num].create_dataset( "aibs_stimulus_amplitude_pa", data=amp) ds.attrs["neurodata_type"] = "Custom" ds = stim["Sweep_%d" % num].create_dataset( "aibs_stimulus_amplitude_pa", data=amp) ds.attrs["neurodata_type"] = "Custom" # stim interval interval = sweep["stimulus_interval"] if interval is None: interval = float('nan') else: interval = float(interval) ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_interval", data=interval) ds.attrs["neurodata_type"] = "Custom" ds = stim["Sweep_%d" % num].create_dataset( "aibs_stimulus_interval", data=interval) ds.attrs["neurodata_type"] = "Custom" # stim name name = sweep["ephys_stimulus"]["ephys_stimulus_type"]["name"] ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_name", data=name) ds.attrs["neurodata_type"] = "Custom" ds = stim["Sweep_%d" % num].create_dataset("aibs_stimulus_name", data=name) ds.attrs["neurodata_type"] = "Custom" # seal seal = jin[0]["seal_gohm"] if seal is None: seal = float('nan') else: seal = float(seal) ds = acq["Sweep_%d" % num].create_dataset("seal", data=seal) ds.attrs["neurodata_type"] = "Custom" ds = stim["Sweep_%d" % num].create_dataset("seal", data=seal) ds.attrs["neurodata_type"] = "Custom" # initial access resistance res = jin[0]["initial_access_resistance_mohm"] if res is None: res = float('nan') else: res = float(res) ds = acq["Sweep_%d" % num].create_dataset( "initial_access_resistance", data=res) ds.attrs["neurodata_type"] = "Custom" ds = stim["Sweep_%d" % num].create_dataset( "initial_access_resistance", data=res) ds.attrs["neurodata_type"] = "Custom" # # # recycle code from old publish module for custom sweep metadata # if num in metadata['sweeps']: # sweep_md = metadata['sweeps'][num] # stimulus_interval = sweep_md['stimulus_interval'] # if stimulus_interval is None: # stimulus_interval = float('nan') # ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_interval", data=stimulus_interval) # ds.attrs["neurodata_type"] = "Custom" # # # ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_name", data=sweep_md['stimulus_type_name']) # ds.attrs["neurodata_type"] = "Custom" # # # ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_amplitude_%s" % stim_units, sweep_md['stimulus_amplitude']) # ds.attrs["neurodata_type"] = "Custom" # # # ds = acq["Sweep_%d" % num].create_dataset("seal", sweep_md['seal_gohm']) # ds.attrs["neurodata_type"] = "Custom" except: print("json parse error for sweep %d" % num) raise # all done hdf.close() # TODO describe what's happening here sweeps_by_type = defaultdict(list) for sweep_number, sweep_data in iteritems(metadata['sweeps']): if sweep_data["stimulus_units"] in [ "pA", "Amps" ]: # only compute spikes for current clamp sweeps sweeps_by_type[sweep_data['stimulus_type_name']].append( sweep_number) sweep_features = extract_cell_features.extract_sweep_features( NwbDataSet(outfile), sweeps_by_type) # TODO describe what's happening here for sweep_num in passing_sweeps: try: spikes = sweep_features[sweep_num]['spikes'] spike_times = [s['threshold_t'] for s in spikes] NwbDataSet(outfile).set_spike_times(sweep_num, spike_times) except Exception as e: logging.info("sweep %d has no sweep features. %s" % (sweep_num, e.message)) # try: # # remove spike times for non-passing sweeps # spk = hdf["analysis/spike_times"] # for grp in spk: # try: # num = int(str(grp).split('_')[-1]) # except: # continue # if num not in passing_sweeps: # del spk[str(grp)] # except: # # # rescaling the contents of the data arrays causes the file to grow # # execute hdf5-repack to get it back to its original size # try: # print("Repacking hdf5 file with compression") # process = subprocess.Popen(["h5repack", "-f", "GZIP=4", tmpfile, outfile], stdout=subprocess.PIPE) # process.wait() # except: # print("Unable to run h5repack on temporary nwb file") # print("--------------------------------------------") # raise # try: # print("Removing temporary file") # os.remove(tmpfile) # except: # print("Unable to delete temporary file ('%s')" % tmpfile) # raise empty = {} return empty
def extract_data(data, nwb_file): ########################################################## #### alings with ephys_sweep_qc_tool extract_features #### cell_specimen = data['specimens'][0] sweep_list = cell_specimen['ephys_sweeps'] sweep_index = {s['sweep_number']: s for s in sweep_list} data_set = NwbDataSet(nwb_file) # extract sweep-level features logging.debug("Computing sweep features") iclamp_sweep_list = filter_sweeps(sweep_list, iclamp_only=True, passed_only=False) iclamp_sweeps = defaultdict(list) for s in iclamp_sweep_list: try: stimulus_type_name = s['ephys_stimulus']['ephys_stimulus_type'][ 'name'] except KeyError as e: raise Exception( "Sweep %d has no ephys stimulus record in features JSON file: %s" % (s['sweep_number'], json.dumps(s, indent=3, default=json_handler))) if stimulus_type_name == "Unknown": raise Exception(( "Sweep %d (%s) has 'Unknown' stimulus type." + "Please update the EpysStimuli and EphysRawStimulusNames associations in LIMS." ) % (s['sweep_number'], s['ephys_stimulus']['description'])) iclamp_sweeps[stimulus_type_name].append(s['sweep_number']) passed_iclamp_sweep_list = filter_sweeps(sweep_list, iclamp_only=True, passed_only=True) num_passed_sweeps = len(passed_iclamp_sweep_list) logging.info("%d of %d sweeps passed QC", num_passed_sweeps, len(iclamp_sweep_list)) if num_passed_sweeps == 0: raise FeatureError( "There are no QC-passed sweeps available to analyze") # compute sweep features logging.info("Computing sweep features") sweep_features = extract_sweep_features(data_set, iclamp_sweeps) cell_specimen['sweep_ephys_features'] = sweep_features # extract cell-level features logging.info("Computing cell features") long_square_sweep_numbers = filtered_sweep_numbers( sweep_list, [LONG_SQUARE_COARSE, LONG_SQUARE_FINE]) short_square_sweep_numbers = filtered_sweep_numbers( sweep_list, [SHORT_SQUARE]) ramp_sweep_numbers = filtered_sweep_numbers(sweep_list, [RAMP]) logging.debug("long square sweeps: %s", str(long_square_sweep_numbers)) logging.debug("short square sweeps: %s", str(short_square_sweep_numbers)) logging.debug("ramp sweeps: %s", str(ramp_sweep_numbers)) # PBS-262 -- have variable subthreshold minimum for human cells subthresh_min_amp = None # None means default (mouse) behavior long_square_amp_delta = find_coarse_long_square_amp_delta(sweep_list) if long_square_amp_delta != 20.0: subthresh_min_amp = -200 logging.info( "Long squares using %fpA step size. Using subthreshold minimum amplitude of %s.", long_square_amp_delta, str(subthresh_min_amp) if subthresh_min_amp is not None else "[default]") stim_start = find_sweep_stim_start(data_set, long_square_sweep_numbers[0]) if stim_start > 0: logging.info("resetting long square start time to: %f", stim_start) reset_long_squares_start(stim_start) cell_features = extract_cell_features(data_set, ramp_sweep_numbers, short_square_sweep_numbers, long_square_sweep_numbers, subthresh_min_amp) # shuffle peak deflection for the subthreshold long squares for s in cell_features["long_squares"]["subthreshold_sweeps"]: sweep_features[s['id']]['peak_deflect'] = s['peak_deflect'] cell_specimen['cell_ephys_features'] = cell_features update_output_sweep_features(cell_features, sweep_features, sweep_index) ephys_features = generate_output_cell_features(cell_features, sweep_features, sweep_index) try: out_ephys_features = cell_specimen.get('ephys_features', [])[0] out_ephys_features.update(ephys_features) except IndexError: cell_specimen['ephys_features'] = [ephys_features] #### breaks with ephys_sweep_qc_tool extract_features #### ########################################################## return sweep_list, sweep_features