def save_nwb(output_path, v, sweep, sweep_by_type= None):
'''Save a single voltage output result into an existing sweep in a NWB file.
This is intended to overwrite a recorded trace with a simulated voltage.
Parameters
----------
output_path : string
file name of a pre-existing NWB file.
v : numpy array
voltage
sweep : integer
which entry to overwrite in the file.
'''
output = NwbDataSet(output_path)
output.set_sweep(sweep, None, v)
if sweep_by_type is not None:
sweep_by_type = {t: [sweep]
for t, ss in sweeps_by_type.items() if sweep in ss}
sweep_features = extract_cell_features.extract_sweep_features(output,
sweep_by_type)
try:
spikes = sweep_features[sweep]['spikes']
spike_times = [s['threshold_t'] for s in spikes]
output.set_spike_times(sweep, spike_times)
except Exception as e:
logging.info("sweep %d has no sweep features. %s" % (sweep, e.args))
def save_nwb(output_path, v, sweep, sweeps_by_type):
'''Save a single voltage output result into an existing sweep in a NWB file.
This is intended to overwrite a recorded trace with a simulated voltage.
Parameters
----------
output_path : string
file name of a pre-existing NWB file.
v : numpy array
voltage
sweep : integer
which entry to overwrite in the file.
'''
output = NwbDataSet(output_path)
output.set_sweep(sweep, None, v)
sweep_by_type = {t: [ sweep ] for t, ss in sweeps_by_type.items() if sweep in ss }
sweep_features = extract_cell_features.extract_sweep_features(output,
sweep_by_type)
try:
spikes = sweep_features[sweep]['spikes']
spike_times = [ s['threshold_t'] for s in spikes ]
output.set_spike_times(sweep, spike_times)
except Exception, e:
logging.info("sweep %d has no sweep features. %s" % (sweep, e.message) )
def save_nwb(output_path, v, sweep):
'''Save a single voltage output result into an existing sweep in a NWB file.
This is intended to overwrite a recorded trace with a simulated voltage.
Parameters
----------
output_path : string
file name of a pre-existing NWB file.
v : numpy array
voltage
sweep : integer
which entry to overwrite in the file.
'''
output = NwbDataSet(output_path)
output.set_sweep(sweep, None, v)
sweep_features = extract_cell_features.extract_sweep_features(output_path,
[sweep])
spikes = sweep_features[sweep]['mean']['spikes']
spike_times = [ s['t'] for s in spikes ]
output.set_spike_times(sweep, spike_times)
def main(jin):
infile = jin[0]["nwb_file"]
outfile = jin[0]["publish_nwb"]
tmpfile = outfile + ".working"
metafile = local_dir + jin[0]["metadata_file"]
# load metadata stored in YML file
metadata_desc_file = os.path.join(os.path.dirname(__file__), metafile)
rsrc = resource_file.ResourceFile()
rsrc.load(metadata_desc_file)
#
metadata = organize_metadata(jin[0])
# TODO dig deeper here
# only fetching metadata for passing sweeps
passing_sweeps = metadata['sweeps'].keys()
copy_file(infile, outfile, passing_sweeps, rsrc, metadata)
# try:
# shutil.copyfile(infile, tmpfile)
# except:
# print("Unable to copy '%s' to %s" % (infile, tmpfile))
# print("----------------------------")
# raise
# # open NWB file so the modification date is updated
# # add metadata then close file and do remaining manipulations using
# # HDF5 library (except DF's legacy code that interfaces w/ nwb file
# # using nwb library)
# args = {}
# args["filename"] = tmpfile
# args["modify"] = True
# try:
# nwb_file = nwb.NWB(**args)
# except:
# print("Error opening NWB file '%s'" % args["filename"])
# raise
# write_metadata(nwb_file, rsrc, metadata)
# nwb_file.close()
# open publish file directlya using HDF5 library
# 1) remove hdf5 groups corresponding to failed sweeps
# 2) add sweep-specific metadata data to file to match original publish
# format. this includes (acquisition and stimulus):
# aibs_stimulus_amplitude_pa
# aibs_stimulus_interval
# aibs_stimulus_name
# initial_access_resistance
# seal
hdf = h5py.File(outfile, "r+")
# ################################
# # delete epochs, stim, recordings for non-passed sweeps
# epochs = hdf["epochs/"]
# for grp in epochs:
# try:
# num = int(str(grp).split('_')[-1])
# except:
# continue
# if num not in passing_sweeps:
# del epochs[str(grp)]
# stim = hdf["stimulus/presentation"]
# for grp in stim:
# try:
# num = int(str(grp).split('_')[-1])
# except:
# continue
# if num not in passing_sweeps:
# del stim[str(grp)]
# acq = hdf["acquisition/timeseries"]
# for grp in acq:
# try:
# num = int(str(grp).split('_')[-1])
# except:
# continue
# if num not in passing_sweeps:
# del acq[str(grp)]
################################
# add data
acq = hdf["acquisition/timeseries"]
stim = hdf["stimulus/presentation"]
sweeps = jin[0]["specimens"][0]["ephys_sweeps"]
for grp in acq:
try:
num = int(str(grp).split('_')[-1])
except:
continue
try:
for sweep in sweeps:
if sweep["sweep_number"] == num:
break
if sweep["sweep_number"] != num:
print(sweep)
print(num)
raise Exception("WTF")
# stim amplitude
amp = sweep["stimulus_amplitude"]
if amp is None:
amp = float('nan')
else:
amp = float(amp)
ds = acq["Sweep_%d" % num].create_dataset(
"aibs_stimulus_amplitude_pa", data=amp)
ds.attrs["neurodata_type"] = "Custom"
ds = stim["Sweep_%d" % num].create_dataset(
"aibs_stimulus_amplitude_pa", data=amp)
ds.attrs["neurodata_type"] = "Custom"
# stim interval
interval = sweep["stimulus_interval"]
if interval is None:
interval = float('nan')
else:
interval = float(interval)
ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_interval",
data=interval)
ds.attrs["neurodata_type"] = "Custom"
ds = stim["Sweep_%d" % num].create_dataset(
"aibs_stimulus_interval", data=interval)
ds.attrs["neurodata_type"] = "Custom"
# stim name
name = sweep["ephys_stimulus"]["ephys_stimulus_type"]["name"]
ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_name",
data=name)
ds.attrs["neurodata_type"] = "Custom"
ds = stim["Sweep_%d" % num].create_dataset("aibs_stimulus_name",
data=name)
ds.attrs["neurodata_type"] = "Custom"
# seal
seal = jin[0]["seal_gohm"]
if seal is None:
seal = float('nan')
else:
seal = float(seal)
ds = acq["Sweep_%d" % num].create_dataset("seal", data=seal)
ds.attrs["neurodata_type"] = "Custom"
ds = stim["Sweep_%d" % num].create_dataset("seal", data=seal)
ds.attrs["neurodata_type"] = "Custom"
# initial access resistance
res = jin[0]["initial_access_resistance_mohm"]
if res is None:
res = float('nan')
else:
res = float(res)
ds = acq["Sweep_%d" % num].create_dataset(
"initial_access_resistance", data=res)
ds.attrs["neurodata_type"] = "Custom"
ds = stim["Sweep_%d" % num].create_dataset(
"initial_access_resistance", data=res)
ds.attrs["neurodata_type"] = "Custom"
#
# # recycle code from old publish module for custom sweep metadata
# if num in metadata['sweeps']:
# sweep_md = metadata['sweeps'][num]
# stimulus_interval = sweep_md['stimulus_interval']
# if stimulus_interval is None:
# stimulus_interval = float('nan')
# ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_interval", data=stimulus_interval)
# ds.attrs["neurodata_type"] = "Custom"
# #
# ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_name", data=sweep_md['stimulus_type_name'])
# ds.attrs["neurodata_type"] = "Custom"
# #
# ds = acq["Sweep_%d" % num].create_dataset("aibs_stimulus_amplitude_%s" % stim_units, sweep_md['stimulus_amplitude'])
# ds.attrs["neurodata_type"] = "Custom"
# #
# ds = acq["Sweep_%d" % num].create_dataset("seal", sweep_md['seal_gohm'])
# ds.attrs["neurodata_type"] = "Custom"
except:
print("json parse error for sweep %d" % num)
raise
# all done
hdf.close()
# TODO describe what's happening here
sweeps_by_type = defaultdict(list)
for sweep_number, sweep_data in iteritems(metadata['sweeps']):
if sweep_data["stimulus_units"] in [
"pA", "Amps"
]: # only compute spikes for current clamp sweeps
sweeps_by_type[sweep_data['stimulus_type_name']].append(
sweep_number)
sweep_features = extract_cell_features.extract_sweep_features(
NwbDataSet(outfile), sweeps_by_type)
# TODO describe what's happening here
for sweep_num in passing_sweeps:
try:
spikes = sweep_features[sweep_num]['spikes']
spike_times = [s['threshold_t'] for s in spikes]
NwbDataSet(outfile).set_spike_times(sweep_num, spike_times)
except Exception as e:
logging.info("sweep %d has no sweep features. %s" %
(sweep_num, e.message))
# try:
# # remove spike times for non-passing sweeps
# spk = hdf["analysis/spike_times"]
# for grp in spk:
# try:
# num = int(str(grp).split('_')[-1])
# except:
# continue
# if num not in passing_sweeps:
# del spk[str(grp)]
# except:
#
# # rescaling the contents of the data arrays causes the file to grow
# # execute hdf5-repack to get it back to its original size
# try:
# print("Repacking hdf5 file with compression")
# process = subprocess.Popen(["h5repack", "-f", "GZIP=4", tmpfile, outfile], stdout=subprocess.PIPE)
# process.wait()
# except:
# print("Unable to run h5repack on temporary nwb file")
# print("--------------------------------------------")
# raise
# try:
# print("Removing temporary file")
# os.remove(tmpfile)
# except:
# print("Unable to delete temporary file ('%s')" % tmpfile)
# raise
empty = {}
return empty
def extract_data(data, nwb_file):
##########################################################
#### alings with ephys_sweep_qc_tool extract_features ####
cell_specimen = data['specimens'][0]
sweep_list = cell_specimen['ephys_sweeps']
sweep_index = {s['sweep_number']: s for s in sweep_list}
data_set = NwbDataSet(nwb_file)
# extract sweep-level features
logging.debug("Computing sweep features")
iclamp_sweep_list = filter_sweeps(sweep_list,
iclamp_only=True,
passed_only=False)
iclamp_sweeps = defaultdict(list)
for s in iclamp_sweep_list:
try:
stimulus_type_name = s['ephys_stimulus']['ephys_stimulus_type'][
'name']
except KeyError as e:
raise Exception(
"Sweep %d has no ephys stimulus record in features JSON file: %s"
% (s['sweep_number'],
json.dumps(s, indent=3, default=json_handler)))
if stimulus_type_name == "Unknown":
raise Exception((
"Sweep %d (%s) has 'Unknown' stimulus type." +
"Please update the EpysStimuli and EphysRawStimulusNames associations in LIMS."
) % (s['sweep_number'], s['ephys_stimulus']['description']))
iclamp_sweeps[stimulus_type_name].append(s['sweep_number'])
passed_iclamp_sweep_list = filter_sweeps(sweep_list,
iclamp_only=True,
passed_only=True)
num_passed_sweeps = len(passed_iclamp_sweep_list)
logging.info("%d of %d sweeps passed QC", num_passed_sweeps,
len(iclamp_sweep_list))
if num_passed_sweeps == 0:
raise FeatureError(
"There are no QC-passed sweeps available to analyze")
# compute sweep features
logging.info("Computing sweep features")
sweep_features = extract_sweep_features(data_set, iclamp_sweeps)
cell_specimen['sweep_ephys_features'] = sweep_features
# extract cell-level features
logging.info("Computing cell features")
long_square_sweep_numbers = filtered_sweep_numbers(
sweep_list, [LONG_SQUARE_COARSE, LONG_SQUARE_FINE])
short_square_sweep_numbers = filtered_sweep_numbers(
sweep_list, [SHORT_SQUARE])
ramp_sweep_numbers = filtered_sweep_numbers(sweep_list, [RAMP])
logging.debug("long square sweeps: %s", str(long_square_sweep_numbers))
logging.debug("short square sweeps: %s", str(short_square_sweep_numbers))
logging.debug("ramp sweeps: %s", str(ramp_sweep_numbers))
# PBS-262 -- have variable subthreshold minimum for human cells
subthresh_min_amp = None # None means default (mouse) behavior
long_square_amp_delta = find_coarse_long_square_amp_delta(sweep_list)
if long_square_amp_delta != 20.0:
subthresh_min_amp = -200
logging.info(
"Long squares using %fpA step size. Using subthreshold minimum amplitude of %s.",
long_square_amp_delta,
str(subthresh_min_amp)
if subthresh_min_amp is not None else "[default]")
stim_start = find_sweep_stim_start(data_set, long_square_sweep_numbers[0])
if stim_start > 0:
logging.info("resetting long square start time to: %f", stim_start)
reset_long_squares_start(stim_start)
cell_features = extract_cell_features(data_set, ramp_sweep_numbers,
short_square_sweep_numbers,
long_square_sweep_numbers,
subthresh_min_amp)
# shuffle peak deflection for the subthreshold long squares
for s in cell_features["long_squares"]["subthreshold_sweeps"]:
sweep_features[s['id']]['peak_deflect'] = s['peak_deflect']
cell_specimen['cell_ephys_features'] = cell_features
update_output_sweep_features(cell_features, sweep_features, sweep_index)
ephys_features = generate_output_cell_features(cell_features,
sweep_features, sweep_index)
try:
out_ephys_features = cell_specimen.get('ephys_features', [])[0]
out_ephys_features.update(ephys_features)
except IndexError:
cell_specimen['ephys_features'] = [ephys_features]
#### breaks with ephys_sweep_qc_tool extract_features ####
##########################################################
return sweep_list, sweep_features
