def run_csd(args: dict) -> dict:
stimulus_table = pd.read_csv(args['stimulus']['stimulus_table_path'])
probewise_outputs = []
for probe_idx, probe in enumerate(args['probes']):
logging.info('Processing probe: {} (index: {})'.format(
probe['name'], probe_idx))
time_step = 1.0 / probe['sampling_rate']
logging.info('Calculated time step: {}'.format(time_step))
logging.info('Extracting trial windows')
trial_windows, relative_window = extract_trial_windows(
stimulus_table=stimulus_table,
stimulus_name=args['stimulus']['key'],
time_step=time_step,
pre_stimulus_time=args['pre_stimulus_time'],
post_stimulus_time=args['post_stimulus_time'],
num_trials=args['num_trials'],
stimulus_index=args['stimulus']['index'])
logging.info('Loading LFP data')
lfp_data_file = ContinuousFile(probe['lfp_data_path'],
probe['lfp_timestamps_path'],
probe['total_channels'])
lfp_raw, timestamps = lfp_data_file.load(
memmap=args['memmap'], memmap_thresh=args['memmap_thresh'])
if probe['phase'].lower() == '3a':
lfp_channels = lfp_data_file.get_lfp_channel_order()
else:
lfp_channels = np.arange(0, probe['total_channels'])
logging.info('Accumulating LFP data')
accumulated_lfp_data = accumulate_lfp_data(
timestamps=timestamps,
lfp_raw=lfp_raw,
lfp_channels=lfp_channels,
trial_windows=trial_windows,
volts_per_bit=args['volts_per_bit'])
logging.info('Removing noisy and reference channels')
clean_lfp, clean_channels = select_good_channels(
lfp=accumulated_lfp_data,
reference_channels=probe['reference_channels'],
noisy_channel_threshold=args['noisy_channel_threshold'])
logging.info('Bandpass filtering LFP channel data')
filt_lfp = filter_lfp_channels(lfp=clean_lfp,
sampling_rate=probe['sampling_rate'],
filter_cuts=args['filter_cuts'],
filter_order=args['filter_order'])
logging.info('Interpolating LFP channel locations')
actual_locs = make_actual_channel_locations(
0, accumulated_lfp_data.shape[1])
clean_actual_locs = actual_locs[clean_channels, :]
interp_locs = make_interp_channel_locations(
0, accumulated_lfp_data.shape[1])
interp_lfp, spacing = interp_channel_locs(
lfp=filt_lfp,
actual_locs=clean_actual_locs,
interp_locs=interp_locs)
logging.info('Averaging LFPs over trials')
trial_mean_lfp = np.nanmean(interp_lfp, axis=0)
logging.info('Computing CSD')
current_source_density, csd_channels = compute_csd(
trial_mean_lfp=trial_mean_lfp, spacing=spacing)
logging.info('Saving data')
write_csd_to_h5(path=probe["csd_output_path"],
csd=current_source_density,
relative_window=relative_window,
channels=csd_channels,
csd_locations=interp_locs,
stimulus_name=args['stimulus']['key'],
stimulus_index=args["stimulus"]["index"],
num_trials=args["num_trials"])
probewise_outputs.append({
'name': probe['name'],
'csd_path': probe['csd_output_path'],
})
return {
'probe_outputs': probewise_outputs,
}
def write_probe_lfp_file(session_id, session_metadata, session_start_time,
log_level, probe):
""" Writes LFP data (and associated channel information) for one
probe to a standalone nwb file
"""
logging.getLogger('').setLevel(log_level)
logging.info(f"writing lfp file for probe {probe['id']}")
nwbfile = pynwb.NWBFile(
session_description=
'LFP data and associated channel info for a single Ecephys probe',
identifier=f"{probe['id']}",
session_id=f"{session_id}",
session_start_time=session_start_time,
institution="Allen Institute for Brain Science")
if session_metadata is not None:
nwbfile = add_metadata_to_nwbfile(nwbfile, session_metadata)
if probe.get("temporal_subsampling_factor", None) is not None:
probe["lfp_sampling_rate"] = probe["lfp_sampling_rate"] / probe[
"temporal_subsampling_factor"]
nwbfile, probe_nwb_device, probe_nwb_electrode_group = add_probe_to_nwbfile(
nwbfile,
probe_id=probe["id"],
name=probe["name"],
sampling_rate=probe["sampling_rate"],
lfp_sampling_rate=probe["lfp_sampling_rate"],
has_lfp_data=probe["lfp"] is not None)
lfp_channels = np.load(probe['lfp']['input_channels_path'],
allow_pickle=False)
add_ecephys_electrodes(nwbfile,
probe["channels"],
probe_nwb_electrode_group,
local_index_whitelist=lfp_channels)
electrode_table_region = nwbfile.create_electrode_table_region(
region=np.arange(len(
nwbfile.electrodes)).tolist(), # must use raw indices here
name='electrodes',
description=f"lfp channels on probe {probe['id']}")
lfp_data, lfp_timestamps = ContinuousFile(
data_path=probe['lfp']['input_data_path'],
timestamps_path=probe['lfp']['input_timestamps_path'],
total_num_channels=len(nwbfile.electrodes)).load(memmap=False)
lfp_data = lfp_data.astype(np.float32)
lfp_data = lfp_data * probe["amplitude_scale_factor"]
lfp = pynwb.ecephys.LFP(name=f"probe_{probe['id']}_lfp")
nwbfile.add_acquisition(
lfp.create_electrical_series(name=f"probe_{probe['id']}_lfp_data",
data=H5DataIO(data=lfp_data,
compression='gzip',
compression_opts=9),
timestamps=H5DataIO(data=lfp_timestamps,
compression='gzip',
compression_opts=9),
electrodes=electrode_table_region))
nwbfile.add_acquisition(lfp)
csd, csd_times, csd_locs = read_csd_data_from_h5(probe["csd_path"])
nwbfile = add_csd_to_nwbfile(nwbfile, csd, csd_times, csd_locs)
with pynwb.NWBHDF5IO(probe['lfp']['output_path'], 'w') as lfp_writer:
logging.info(
f"writing probe lfp file to {probe['lfp']['output_path']}")
lfp_writer.write(nwbfile, cache_spec=True)
return {"id": probe["id"], "nwb_path": probe["lfp"]["output_path"]}
def subsample(args):
"""
:param args:
:return:
"""
params = args['lfp_subsampling']
probe_outputs = []
for probe in args['probes']:
logging.info("Sub-sampling LFP for " + probe['name'])
lfp_data_file = ContinuousFile(probe['lfp_input_file_path'],
probe['lfp_timestamps_input_path'],
probe['total_channels'])
logging.info("loading lfp data...")
lfp_raw, timestamps = lfp_data_file.load()
if params['reorder_channels']:
lfp_channel_order = lfp_data_file.get_lfp_channel_order()
else:
lfp_channel_order = np.arange(0, probe['total_channels'])
logging.info("selecting channels...")
channels_to_save, actual_channels = select_channels(
probe['total_channels'], probe['surface_channel'],
params['surface_padding'], params['start_channel_offset'],
params['channel_stride'], lfp_channel_order,
probe.get('noisy_channels', []), params['remove_noisy_channels'],
probe['reference_channels'], params['remove_reference_channels'])
ts_subsampled = subsample_timestamps(
timestamps, params['temporal_subsampling_factor'])
logging.info("subsampling data...")
lfp_subsampled = subsample_lfp(lfp_raw, channels_to_save,
params['temporal_subsampling_factor'])
del lfp_raw
logging.info("removing offset...")
lfp_filtered = remove_lfp_offset(
lfp_subsampled,
probe['lfp_sampling_rate'] / params['temporal_subsampling_factor'],
params['cutoff_frequency'], params['filter_order'])
del lfp_subsampled
logging.info("Surface channel: " + str(probe['surface_channel']))
logging.info("removing noise...")
lfp = remove_lfp_noise(lfp_filtered, probe['surface_channel'],
actual_channels)
del lfp_filtered
if params['remove_channels_out_of_brain']:
channels_to_keep = actual_channels < (probe['surface_channel'] +
10)
actual_channels = actual_channels[channels_to_keep]
lfp = lfp[:, channels_to_keep]
logging.info('Writing to disk...')
lfp.tofile(probe['lfp_data_path'])
np.save(probe['lfp_timestamps_path'], ts_subsampled)
np.save(probe['lfp_channel_info_path'], actual_channels)
probe_outputs.append({
'name':
probe['name'],
'lfp_data_path':
probe['lfp_data_path'],
'lfp_timestamps_path':
probe['lfp_timestamps_path'],
'lfp_channel_info_path':
probe['lfp_channel_info_path']
})
return {'probe_outputs': probe_outputs}
def write_probe_lfp_file(session_start_time, log_level, probe):
""" Writes LFP data (and associated channel information) for one probe to a standalone nwb file
"""
logging.getLogger('').setLevel(log_level)
logging.info(f"writing lfp file for probe {probe['id']}")
nwbfile = pynwb.NWBFile(session_description='EcephysProbe',
identifier=f"{probe['id']}",
session_start_time=session_start_time)
if probe.get("temporal_subsampling_factor", None) is not None:
probe["lfp_sampling_rate"] = probe["lfp_sampling_rate"] / probe[
"temporal_subsampling_factor"]
nwbfile, probe_nwb_device, probe_nwb_electrode_group = add_probe_to_nwbfile(
nwbfile,
probe_id=probe["id"],
description=probe["name"],
sampling_rate=probe["sampling_rate"],
lfp_sampling_rate=probe["lfp_sampling_rate"],
has_lfp_data=probe["lfp"] is not None)
channels = prepare_probewise_channel_table(probe['channels'],
probe_nwb_electrode_group)
channel_li_id_map = {
row["local_index"]: cid
for cid, row in channels.iterrows()
}
lfp_channels = np.load(probe['lfp']['input_channels_path'],
allow_pickle=False)
channels.reset_index(inplace=True)
channels.set_index("local_index", inplace=True)
channels = channels.loc[lfp_channels, :]
channels.reset_index(inplace=True)
channels.set_index("id", inplace=True)
channels = fill_df(channels)
nwbfile.electrodes = pynwb.file.ElectrodeTable().from_dataframe(
channels, name='electrodes')
electrode_table_region = nwbfile.create_electrode_table_region(
region=np.arange(
channels.shape[0]).tolist(), # must use raw indices here
name='electrodes',
description=f"lfp channels on probe {probe['id']}")
lfp_data, lfp_timestamps = ContinuousFile(
data_path=probe['lfp']['input_data_path'],
timestamps_path=probe['lfp']['input_timestamps_path'],
total_num_channels=channels.shape[0]).load(memmap=False)
lfp_data = lfp_data.astype(np.float32)
lfp_data = lfp_data * probe["amplitude_scale_factor"]
lfp = pynwb.ecephys.LFP(name=f"probe_{probe['id']}_lfp")
nwbfile.add_acquisition(
lfp.create_electrical_series(name=f"probe_{probe['id']}_lfp_data",
data=H5DataIO(data=lfp_data,
compression='gzip',
compression_opts=9),
timestamps=H5DataIO(data=lfp_timestamps,
compression='gzip',
compression_opts=9),
electrodes=electrode_table_region))
nwbfile.add_acquisition(lfp)
csd, csd_times, csd_locs = read_csd_data_from_h5(probe["csd_path"])
nwbfile = add_csd_to_nwbfile(nwbfile, csd, csd_times, csd_locs)
with pynwb.NWBHDF5IO(probe['lfp']['output_path'], 'w') as lfp_writer:
logging.info(
f"writing probe lfp file to {probe['lfp']['output_path']}")
lfp_writer.write(nwbfile)
return {"id": probe["id"], "nwb_path": probe["lfp"]["output_path"]}