def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
# ===============================================================================
# ============================== META INFORMATION ===============================
# ===============================================================================
# -- NWB file - a NWB2.0 file for each session
experimenter = (acquisition.Session.Experimenter & session_key).fetch1('experimenter')
nwbfile = NWBFile(identifier='_'.join(
[this_session['subject'],
this_session['session_time'].strftime('%Y-%m-%d %H:%M:%S')]),
related_publications='https://www.nature.com/articles/s41586-018-0633-x',
experiment_description='Extracelluar recording in ALM',
session_description='',
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter=experimenter,
institution=institution,
keywords=['motor planning', 'anterior lateral cortex',
'ALM', 'Extracellular recording', 'optogenetics'])
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=str(this_session['subject']),
genotype=' x '.join((subject.Zygosity &
subj).fetch('allele')) \
if len(subject.Zygosity & subj) else 'unknown',
sex=subj['sex'],
species=subj['species'],
date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time) if subj['date_of_birth'] else None)
# -- virus
nwbfile.virus = json.dumps([{k: str(v) for k, v in virus_injection.items() if k not in subj}
for virus_injection in action.VirusInjection * reference.Virus & session_key])
# ===============================================================================
# ======================== EXTRACELLULAR & CLUSTERING ===========================
# ===============================================================================
"""
In the event of multiple probe recording (i.e. multiple probe insertions), the clustering results
(and the associated units) are associated with the corresponding probe.
Each probe insertion is associated with one ElectrodeConfiguration (which may define multiple electrode groups)
"""
dj_insert_location = ephys.ProbeInsertion
for probe_insertion in ephys.ProbeInsertion & session_key:
probe = (reference.Probe & probe_insertion).fetch1()
electrode_group = nwbfile.create_electrode_group(
name=probe['probe_type'] + '_g1',
description='N/A',
device=nwbfile.create_device(name=probe['probe_type']),
location=json.dumps({k: str(v) for k, v in (dj_insert_location & session_key).fetch1().items()
if k not in dj_insert_location.primary_key}))
for chn in (reference.Probe.Channel & probe).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['channel_id']-1,
group=electrode_group,
filtering=hardware_filter,
imp=-1.,
x=np.nan,
y=np.nan,
z=np.nan,
location=(dj_insert_location & session_key).fetch1('brain_location'))
# --- unit spike times ---
nwbfile.add_unit_column(name='cell_type', description='cell type (e.g. fast spiking or pyramidal)')
nwbfile.add_unit_column(name='sampling_rate', description='sampling rate of the waveform, Hz')
spk_times_all = np.hstack((ephys.UnitSpikeTimes & probe_insertion).fetch('spike_times'))
obs_min = np.min(spk_times_all)
obs_max = np.max(spk_times_all)
for unit in (ephys.UnitSpikeTimes & probe_insertion).fetch(as_dict=True):
nwbfile.add_unit(id=unit['unit_id'],
electrodes=[unit['channel']-1],
electrode_group=electrode_group,
cell_type=unit['unit_cell_type'],
spike_times=unit['spike_times'],
obs_intervals=np.array([[obs_min - 0.001, obs_max + 0.001]]),
waveform_mean=np.mean(unit['spike_waveform'], axis=0),
waveform_sd=np.std(unit['spike_waveform'], axis=0),
sampling_rate=20000)
# ===============================================================================
# ============================= PHOTO-STIMULATION ===============================
# ===============================================================================
stim_sites = {}
for photostim in acquisition.PhotoStim * reference.BrainLocation & session_key:
stim_device = (nwbfile.get_device(photostim['photo_stim_method'])
if photostim['photo_stim_method'] in nwbfile.devices
else nwbfile.create_device(name=photostim['photo_stim_method']))
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name=photostim['brain_location'],
device=stim_device,
excitation_lambda=float(photostim['photo_stim_wavelength']),
location=json.dumps({k: str(v) for k, v in photostim.items()
if k in acquisition.PhotoStim.heading.names and k not in acquisition.PhotoStim.primary_key + ['photo_stim_method', 'photo_stim_wavelength']}),
description='')
nwbfile.add_ogen_site(stim_site)
# ===============================================================================
# =============================== BEHAVIOR TRIALS ===============================
# ===============================================================================
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time'
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
dj_trial = acquisition.Session * behavior.TrialSet.Trial
skip_adding_columns = acquisition.Session.primary_key + \
['trial_id', 'trial_start_idx', 'trial_end_idx', 'trial_start_time', 'session_note']
if behavior.TrialSet.Trial & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{'name': tag.replace('trial_', ''),
'description': re.sub('\s+:|\s+', ' ', re.search(
f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip()}
for tag in dj_trial.heading.names
if tag not in skip_adding_columns]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (dj_trial & session_key).fetch(as_dict=True):
trial['start_time'] = float(trial['trial_start_time'])
trial['stop_time'] = float(trial['trial_start_time']) + 5.0
trial['trial_pole_in_time'] = trial['start_time'] + trial['trial_pole_in_time']
trial['trial_pole_out_time'] = trial['start_time'] + trial['trial_pole_out_time']
trial['trial_cue_time'] = trial['start_time'] + trial['trial_cue_time']
[trial.pop(k) for k in skip_adding_columns]
for k in trial.keys():
if 'trial_' in k:
trial[k.replace('trial_', '')] = trial.pop(k)
nwbfile.add_trial(**trial)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversion for this data interface.
Reads labview experiment behavioral data and adds it to nwbfile.
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
print("Converting Labview data...")
# Get list of trial summary files
dir_behavior_labview = self.source_data['dir_behavior_labview']
all_files = os.listdir(dir_behavior_labview)
trials_files = [f for f in all_files if '_sum.txt' in f]
trials_files.sort()
# Get session_start_time from first file timestamps
fpath = os.path.join(dir_behavior_labview, trials_files[0])
colnames = [
'Trial', 'StartT', 'EndT', 'Result', 'InitT', 'SpecificResults',
'ProbLeft', 'OptoDur', 'LRew', 'RRew', 'InterT', 'LTrial',
'ReactionTime', 'OptoCond', 'OptoTrial'
]
df_0 = pd.read_csv(fpath, sep='\t', index_col=False, names=colnames)
t0 = df_0['StartT'][0] # initial time in Labview seconds
# Add trials
print("Converting Labview trials data...")
if nwbfile.trials is not None:
print(
'Trials already exist in current nwb file. Labview behavior trials not added.'
)
else:
# Make dataframe
frames = []
for f in trials_files:
fpath = os.path.join(dir_behavior_labview, f)
frames.append(
pd.read_csv(fpath,
sep='\t',
index_col=False,
names=colnames))
df_trials_summary = pd.concat(frames)
nwbfile.add_trial_column(
name='results',
description=
"0 means sucess (rewarded trial), 1 means licks during intitial "
"period, which leads to a failed trial. 2 means early lick failure. 3 means "
"wrong lick or no response.")
nwbfile.add_trial_column(
name='init_t', description="duration of initial delay period.")
nwbfile.add_trial_column(
name='specific_results',
description=
"Possible outcomes classified based on raw data & meta file (_tr.m)."
)
nwbfile.add_trial_column(
name='prob_left',
description=
"probability for left trials in order to keep the number of "
"left and right trials balanced within the session. ")
nwbfile.add_trial_column(
name='opto_dur',
description="the duration of optical stimulation.")
nwbfile.add_trial_column(
name='l_rew_n',
description="counting the number of left rewards.")
nwbfile.add_trial_column(
name='r_rew_n',
description="counting the number of rightrewards.")
nwbfile.add_trial_column(name='inter_t',
description="inter-trial delay period.")
nwbfile.add_trial_column(
name='l_trial',
description=
"trial type (which side the air-puff is applied). 1 means "
"left-trial, 0 means right-trial")
nwbfile.add_trial_column(
name='reaction_time',
description=
"if it is a successful trial or wrong lick during response "
"period trial: ReactionTime = time between the first decision "
"lick and the beginning of the response period. If it is a failed "
"trial due to early licks: reaction time = the duration of "
"the air-puff period (in other words, when the animal licks "
"during the sample period).")
nwbfile.add_trial_column(
name='opto_cond',
description="0: no opto. 1: opto is on during sample period. "
"2: opto is on half way through the sample period (0.5s) "
"and 0.5 during the response period. 3. opto is on during "
"the response period.")
nwbfile.add_trial_column(
name='opto_trial',
description="1: opto trials. 0: Non-opto trials.")
for index, row in df_trials_summary.iterrows():
nwbfile.add_trial(
start_time=row['StartT'] - t0,
stop_time=row['EndT'] - t0,
results=int(row['Result']),
init_t=row['InitT'],
specific_results=int(row['SpecificResults']),
prob_left=row['ProbLeft'],
opto_dur=row['OptoDur'],
l_rew_n=int(row['LRew']),
r_rew_n=int(row['RRew']),
inter_t=row['InterT'],
l_trial=int(row['LTrial']),
reaction_time=int(row['ReactionTime']),
opto_cond=int(row['OptoCond']),
opto_trial=int(row['OptoTrial']),
)
# Get list of files: continuous data
continuous_files = [f.replace('_sum', '') for f in trials_files]
# Adds continuous behavioral data
frames = []
for f in continuous_files:
fpath_lick = os.path.join(dir_behavior_labview, f)
frames.append(pd.read_csv(fpath_lick, sep='\t', index_col=False))
df_continuous = pd.concat(frames)
# Behavioral data
print("Converting Labview behavior data...")
l1_ts = TimeSeries(name="left_lick",
data=df_continuous['Lick 1'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
l2_ts = TimeSeries(name="right_lick",
data=df_continuous['Lick 2'].to_numpy(),
timestamps=df_continuous['Time'].to_numpy() - t0,
description="no description")
nwbfile.add_acquisition(l1_ts)
nwbfile.add_acquisition(l2_ts)
# Optogenetics stimulation data
print("Converting Labview optogenetics data...")
ogen_device = nwbfile.create_device(
name=metadata['Ogen']['Device']['name'],
description=metadata['Ogen']['Device']['description'])
meta_ogen_site = metadata['Ogen']['OptogeneticStimulusSite']
ogen_stim_site = OptogeneticStimulusSite(
name=meta_ogen_site['name'],
device=ogen_device,
description=meta_ogen_site['description'],
excitation_lambda=float(meta_ogen_site['excitation_lambda']),
location=meta_ogen_site['location'])
nwbfile.add_ogen_site(ogen_stim_site)
meta_ogen_series = metadata['Ogen']['OptogeneticSeries']
ogen_series = OptogeneticSeries(
name=meta_ogen_series['name'],
data=df_continuous['Opto'].to_numpy(),
site=ogen_stim_site,
timestamps=df_continuous['Time'].to_numpy() - t0,
description=meta_ogen_series['description'],
)
nwbfile.add_stimulus(ogen_series)
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier='_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']
]),
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join(
(subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell
& session_key).fetch1() if len(intracellular.Cell
& session_key) == 1 else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([
f'{k}: {str(v)}'
for k, v in dict((reference.BrainLocation & cell).fetch1(),
depth=cell['recording_depth']).items()
]))
# acquisition - membrane potential
mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
timestamps=mp_timestamps))
# acquisition - spike train
spk, spk_timestamps = (intracellular.SpikeTrain & cell).fetch1(
'spike_train', 'spike_timestamps')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='SpikeTrain',
electrode=ic_electrode,
unit='a.u.',
conversion=1e1,
gain=1.0,
data=spk,
timestamps=spk_timestamps))
# =============== Behavior ====================
behavior_data = ((behavior.Behavior & session_key).fetch1()
if len(behavior.Behavior & session_key) == 1 else None)
if behavior_data:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='behavior')
nwbfile.add_acquisition(behav_acq)
[behavior_data.pop(k) for k in behavior.Behavior.primary_key]
timestamps = behavior_data.pop('behavior_timestamps')
# get behavior data description from the comments of table definition
behavior_descriptions = {
attr:
re.search(f'(?<={attr})(.*)#(.*)',
str(behavior.Behavior.heading)).groups()[-1].strip()
for attr in behavior_data
}
for b_k, b_v in behavior_data.items():
behav_acq.create_timeseries(name=b_k,
description=behavior_descriptions[b_k],
unit='a.u.',
conversion=1.0,
data=b_v,
timestamps=timestamps)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if len(stimulation.PhotoStimulation
& session_key) == 1 else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(
(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_excitation_lambda')),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimulationProtocol
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if len((acquisition.TrialSet & session_key).fetch()) == 1:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any)
trial_columns = [{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimInfo
).heading)).groups()[-1].strip()
} for tag in acquisition.TrialSet.Trial.heading.names
if tag not in acquisition.TrialSet.Trial.primary_key +
['start_time', 'stop_time']]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
# shift event times to be relative to session_start (currently relative to trial_start)
events = {k: v + trial['start_time'] for k, v in events.items()}
trial_tag_value = {**trial, **events}
# rename 'trial_id' to 'id'
trial_tag_value['id'] = trial_tag_value['trial_id']
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=False):
this_session = (experiment.Session & session_key).fetch1()
print(f'Exporting to NWB 2.0 for session: {this_session}...')
# ===============================================================================
# ============================== META INFORMATION ===============================
# ===============================================================================
sess_desc = session_description_mapper[(
experiment.ProjectSession & session_key).fetch1('project_name')]
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(
identifier='_'.join([
'ANM' + str(this_session['subject_id']),
this_session['session_date'].strftime('%Y-%m-%d'),
str(this_session['session'])
]),
session_description='',
session_start_time=datetime.combine(this_session['session_date'],
zero_zero_time),
file_create_date=datetime.now(tzlocal()),
experimenter=this_session['username'],
institution=institution,
experiment_description=sess_desc['experiment_description'],
related_publications=sess_desc['related_publications'],
keywords=sess_desc['keywords'])
# -- subject
subj = (lab.Subject & session_key).aggr(
lab.Subject.Strain, ...,
strains='GROUP_CONCAT(animal_strain)').fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=str(this_session['subject_id']),
description=
f'source: {subj["animal_source"]}; strains: {subj["strains"]}',
genotype=' x '.join((lab.Subject.GeneModification
& subj).fetch('gene_modification')),
sex=subj['sex'],
species=subj['species'],
date_of_birth=datetime.combine(subj['date_of_birth'], zero_zero_time)
if subj['date_of_birth'] else None)
# -- virus
nwbfile.virus = json.dumps([{
k: str(v)
for k, v in virus_injection.items() if k not in subj
} for virus_injection in virus.VirusInjection * virus.Virus & session_key])
# ===============================================================================
# ======================== EXTRACELLULAR & CLUSTERING ===========================
# ===============================================================================
"""
In the event of multiple probe recording (i.e. multiple probe insertions), the clustering results
(and the associated units) are associated with the corresponding probe.
Each probe insertion is associated with one ElectrodeConfiguration (which may define multiple electrode groups)
"""
dj_insert_location = ephys.ProbeInsertion.InsertionLocation.aggr(
ephys.ProbeInsertion.RecordableBrainRegion.proj(
brain_region='CONCAT(hemisphere, " ", brain_area)'), ...,
brain_regions='GROUP_CONCAT(brain_region)')
for probe_insertion in ephys.ProbeInsertion & session_key:
electrode_config = (lab.ElectrodeConfig & probe_insertion).fetch1()
electrode_groups = {}
for electrode_group in lab.ElectrodeConfig.ElectrodeGroup & electrode_config:
electrode_groups[electrode_group[
'electrode_group']] = nwbfile.create_electrode_group(
name=electrode_config['electrode_config_name'] + '_g' +
str(electrode_group['electrode_group']),
description='N/A',
device=nwbfile.create_device(
name=electrode_config['probe']),
location=json.dumps({
k: str(v)
for k, v in (dj_insert_location
& session_key).fetch1().items()
if k not in dj_insert_location.primary_key
}))
for chn in (lab.ElectrodeConfig.Electrode * lab.Probe.Electrode
& electrode_config).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['electrode'],
group=electrode_groups[chn['electrode_group']],
filtering=hardware_filter,
imp=-1.,
x=chn['x_coord'] if chn['x_coord'] else np.nan,
y=chn['y_coord'] if chn['y_coord'] else np.nan,
z=chn['z_coord'] if chn['z_coord'] else np.nan,
location=electrode_groups[chn['electrode_group']].location)
# --- unit spike times ---
nwbfile.add_unit_column(
name='sampling_rate',
description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='quality',
description='unit quality from clustering')
nwbfile.add_unit_column(
name='posx',
description=
'estimated x position of the unit relative to probe (0,0) (um)')
nwbfile.add_unit_column(
name='posy',
description=
'estimated y position of the unit relative to probe (0,0) (um)')
nwbfile.add_unit_column(
name='cell_type',
description='cell type (e.g. fast spiking or pyramidal)')
for unit_key in (ephys.Unit * ephys.UnitCellType
& probe_insertion).fetch('KEY'):
unit = (ephys.Unit * ephys.UnitCellType & probe_insertion
& unit_key).proj(..., '-spike_times').fetch1()
if ephys.TrialSpikes & unit_key:
obs_intervals = np.array(
list(
zip(*(ephys.TrialSpikes * experiment.SessionTrial
& unit_key).fetch('start_time',
'stop_time')))).astype(float)
tr_spike_times, tr_start, tr_go = (
ephys.TrialSpikes * experiment.SessionTrial *
(experiment.TrialEvent & 'trial_event_type = "go"')
& unit_key).fetch('spike_times', 'start_time',
'trial_event_time')
spike_times = np.hstack([
spks + float(t_go) + float(t_start) for spks, t_go, t_start
in zip(tr_spike_times, tr_start, tr_go)
])
else: # the case of unavailable `TrialSpikes`
spike_times = (ephys.Unit & unit_key).fetch1('spike_times')
obs_intervals = np.array(
list(
zip(*(experiment.SessionTrial & unit_key).fetch(
'start_time', 'stop_time')))).astype(float)
obs_intervals = [
interval for interval in obs_intervals
if np.logical_and(spike_times >= interval[0],
spike_times <= interval[-1]).any()
]
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(
id=unit['unit'],
electrodes=np.where(
np.array(nwbfile.electrodes.id.data) ==
unit['electrode'])[0],
electrode_group=electrode_groups[unit['electrode_group']],
obs_intervals=obs_intervals,
sampling_rate=ecephys_fs,
quality=unit['unit_quality'],
posx=unit['unit_posx'],
posy=unit['unit_posy'],
cell_type=unit['cell_type'],
spike_times=spike_times,
waveform_mean=np.mean(unit['waveform'], axis=0),
waveform_sd=np.std(unit['waveform'], axis=0))
# ===============================================================================
# ============================= BEHAVIOR TRACKING ===============================
# ===============================================================================
if tracking.LickTrace * experiment.SessionTrial & session_key:
# re-concatenating trialized tracking traces
lick_traces, time_vecs, trial_starts = (
tracking.LickTrace * experiment.SessionTrial & session_key).fetch(
'lick_trace', 'lick_trace_timestamps', 'start_time')
behav_acq = pynwb.behavior.BehavioralTimeSeries(
name='BehavioralTimeSeries')
nwbfile.add_acquisition(behav_acq)
behav_acq.create_timeseries(
name='lick_trace',
unit='a.u.',
conversion=1.0,
data=np.hstack(lick_traces),
description=
"Time-series of the animal's tongue movement when licking",
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
# ===============================================================================
# ============================= PHOTO-STIMULATION ===============================
# ===============================================================================
stim_sites = {}
for photostim in experiment.Photostim * experiment.PhotostimBrainRegion * lab.PhotostimDevice & session_key:
stim_device = (nwbfile.get_device(photostim['photostim_device'])
if photostim['photostim_device'] in nwbfile.devices else
nwbfile.create_device(
name=photostim['photostim_device']))
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name=photostim['stim_laterality'] + ' ' +
photostim['stim_brain_area'],
device=stim_device,
excitation_lambda=float(photostim['excitation_wavelength']),
location=json.dumps([{
k: v
for k, v in stim_locs.items()
if k not in experiment.Photostim.primary_key
} for stim_locs in (experiment.Photostim.PhotostimLocation.proj(
..., '-brain_area')
& photostim).fetch(as_dict=True)],
default=str),
description='')
nwbfile.add_ogen_site(stim_site)
stim_sites[photostim['photo_stim']] = stim_site
# re-concatenating trialized photostim traces
dj_photostim = (experiment.PhotostimTrace * experiment.SessionTrial *
experiment.PhotostimEvent * experiment.Photostim
& session_key)
for photo_stim, stim_site in stim_sites.items():
if dj_photostim & {'photo_stim': photo_stim}:
aom_input_trace, laser_power, time_vecs, trial_starts = (
dj_photostim & {
'photo_stim': photo_stim
}).fetch('aom_input_trace', 'laser_power',
'photostim_timestamps', 'start_time')
aom_series = pynwb.ogen.OptogeneticSeries(
name=stim_site.name + '_aom_input_trace',
site=stim_site,
conversion=1e-3,
data=np.hstack(aom_input_trace),
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
laser_series = pynwb.ogen.OptogeneticSeries(
name=stim_site.name + '_laser_power',
site=stim_site,
conversion=1e-3,
data=np.hstack(laser_power),
timestamps=np.hstack(time_vecs + trial_starts.astype(float)))
nwbfile.add_stimulus(aom_series)
nwbfile.add_stimulus(laser_series)
# ===============================================================================
# =============================== BEHAVIOR TRIALS ===============================
# ===============================================================================
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time'
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
dj_trial = experiment.SessionTrial * experiment.BehaviorTrial
skip_adding_columns = experiment.Session.primary_key + [
'trial_uid', 'trial'
]
if experiment.SessionTrial & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{
'name':
tag,
'description':
re.sub('\s+:|\s+', ' ',
re.search(f'(?<={tag})(.*)',
str(dj_trial.heading)).group()).strip()
} for tag in dj_trial.heading.names if tag not in skip_adding_columns +
['start_time', 'stop_time']]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns:
nwbfile.add_trial_column(**c)
# Add entry to the trial-table
for trial in (dj_trial & session_key).fetch(as_dict=True):
trial['start_time'] = float(trial['start_time'])
trial['stop_time'] = float(
trial['stop_time']) if trial['stop_time'] else np.nan
trial['id'] = trial['trial'] # rename 'trial_id' to 'id'
[trial.pop(k) for k in skip_adding_columns]
nwbfile.add_trial(**trial)
# ===============================================================================
# =============================== BEHAVIOR TRIAL EVENTS ==========================
# ===============================================================================
behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents')
nwbfile.add_acquisition(behav_event)
for trial_event_type in (experiment.TrialEventType & experiment.TrialEvent
& session_key).fetch('trial_event_type'):
event_times, trial_starts = (
experiment.TrialEvent * experiment.SessionTrial
& session_key & {
'trial_event_type': trial_event_type
}).fetch('trial_event_time', 'start_time')
if len(event_times) > 0:
event_times = np.hstack(
event_times.astype(float) + trial_starts.astype(float))
behav_event.create_timeseries(name=trial_event_type,
unit='a.u.',
conversion=1.0,
data=np.full_like(event_times, 1),
timestamps=event_times)
photostim_event_time, trial_starts, photo_stim, power, duration = (
experiment.PhotostimEvent * experiment.SessionTrial
& session_key).fetch('photostim_event_time', 'start_time',
'photo_stim', 'power', 'duration')
if len(photostim_event_time) > 0:
behav_event.create_timeseries(
name='photostim_start_time',
unit='a.u.',
conversion=1.0,
data=power,
timestamps=photostim_event_time.astype(float) +
trial_starts.astype(float),
control=photo_stim.astype('uint8'),
control_description=stim_sites)
behav_event.create_timeseries(
name='photostim_stop_time',
unit='a.u.',
conversion=1.0,
data=np.full_like(photostim_event_time, 0),
timestamps=photostim_event_time.astype(float) +
duration.astype(float) + trial_starts.astype(float),
control=photo_stim.astype('uint8'),
control_description=stim_sites)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
identifier = '_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']
])
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(session_description=this_session['session_note'],
identifier=identifier,
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join(
(acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join(
(subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell
& session_key).fetch1() if len(intracellular.Cell
& session_key) == 1 else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([
f'{k}: {str(v)}'
for k, v in dict((reference.BrainLocation & cell).fetch1(),
depth=cell['cell_depth']).items()
]))
# acquisition - membrane potential
mp, mp_wo_spike, mp_start_time, mp_fs = (
intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_wo_spike',
'membrane_potential_start_time',
'membrane_potential_sampling_rate')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
starting_time=mp_start_time,
rate=mp_fs))
# acquisition - current injection
if (intracellular.CurrentInjection & cell):
current_injection, ci_start_time, ci_fs = (
intracellular.CurrentInjection & cell).fetch1(
'current_injection', 'current_injection_start_time',
'current_injection_sampling_rate')
nwbfile.add_stimulus(
pynwb.icephys.CurrentClampStimulusSeries(
name='CurrentClampStimulus',
electrode=ic_electrode,
conversion=1e-9,
gain=1.0,
data=current_injection,
starting_time=ci_start_time,
rate=ci_fs))
# analysis - membrane potential without spike
mp_rmv_spike = nwbfile.create_processing_module(
name='icephys', description='Spike removal')
mp_rmv_spike.add_data_interface(
pynwb.icephys.PatchClampSeries(name='icephys',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp_wo_spike,
starting_time=mp_start_time,
rate=mp_fs))
# =============== Extracellular ====================
probe_insertion = ((extracellular.ProbeInsertion
& session_key).fetch1() if extracellular.ProbeInsertion
& session_key else None)
if probe_insertion:
probe = nwbfile.create_device(name=probe_insertion['probe_name'])
electrode_group = nwbfile.create_electrode_group(name='; '.join([
f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}'
]),
description='N/A',
device=probe,
location='; '.join([
f'{k}: {str(v)}'
for k, v in
(reference.
BrainLocation
& probe_insertion
).fetch1().items(
)
]))
for chn in (reference.Probe.Channel
& probe_insertion).fetch(as_dict=True):
nwbfile.add_electrode(
id=chn['channel_id'],
group=electrode_group,
filtering=hardware_filter,
imp=-1.,
x=0.0, # not available from data
y=0.0, # not available from data
z=0.0, # not available from data
location=electrode_group.location)
# --- unit spike times ---
nwbfile.add_unit_column(
name='sampling_rate',
description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='depth',
description='depth this unit (mm)')
nwbfile.add_unit_column(name='spike_width',
description='spike width of this unit (ms)')
nwbfile.add_unit_column(
name='cell_type',
description='cell type (e.g. wide width, narrow width spiking)')
for unit in (extracellular.UnitSpikeTimes
& probe_insertion).fetch(as_dict=True):
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(
id=unit['unit_id'],
electrodes=(unit['channel_id'] if isinstance(
unit['channel_id'], np.ndarray) else [unit['channel_id']]),
depth=unit['unit_depth'],
sampling_rate=ecephys_fs,
spike_width=unit['unit_spike_width'],
cell_type=unit['unit_cell_type'],
spike_times=unit['spike_times'],
waveform_mean=unit['spike_waveform'])
# =============== Behavior ====================
# Note: for this study, raw behavioral data were not available, only trialized data were provided
# here, we reconstruct raw behavioral data by concatenation
trial_seg_setting = (analysis.TrialSegmentationSetting
& 'trial_seg_setting=0').fetch1()
seg_behav_query = (
behavior.TrialSegmentedLickTrace * acquisition.TrialSet.Trial *
(analysis.RealignedEvent.RealignedEventTime
& 'trial_event="trial_start"')
& session_key & trial_seg_setting)
if seg_behav_query:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='lick_times')
nwbfile.add_acquisition(behav_acq)
seg_behav = pd.DataFrame(
seg_behav_query.fetch('start_time', 'realigned_event_time',
'segmented_lick_left_on',
'segmented_lick_left_off',
'segmented_lick_right_on',
'segmented_lick_right_off')).T
seg_behav.columns = [
'start_time', 'realigned_event_time', 'segmented_lick_left_on',
'segmented_lick_left_off', 'segmented_lick_right_on',
'segmented_lick_right_off'
]
for behav_name in [
'lick_left_on', 'lick_left_off', 'lick_right_on',
'lick_right_off'
]:
lick_times = np.hstack(r['segmented_' + behav_name] -
r.realigned_event_time + r.start_time
for _, r in seg_behav.iterrows())
behav_acq.create_timeseries(name=behav_name,
unit='a.u.',
conversion=1.0,
data=np.full_like(lick_times, 1),
timestamps=lick_times)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if stimulation.PhotoStimulation
& session_key else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(
(stimulation.PhotoStimProtocol
& photostim).fetch1('photo_stim_excitation_lambda')),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimProtocol
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if acquisition.TrialSet & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo - remove '_trial' prefix (if any)
trial_columns = [
{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimParam
).heading)).groups()[-1].strip()
} for tag in (acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimParam).heading.names
if tag not in (acquisition.TrialSet.Trial
& stimulation.TrialPhotoStimParam).primary_key +
['start_time', 'stop_time']
]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d + ' - (s) relative to trial start time'
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
photostim_tag_default = {
tag: ''
for tag in stimulation.TrialPhotoStimParam.heading.names
if tag not in stimulation.TrialPhotoStimParam.primary_key
}
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
trial_tag_value = ({
**trial,
**events,
**(stimulation.TrialPhotoStimParam & trial).fetch1()
} if (stimulation.TrialPhotoStimParam & trial) else {
**trial,
**events,
**photostim_tag_default
})
trial_tag_value['id'] = trial_tag_value[
'trial_id'] # rename 'trial_id' to 'id'
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# convert None to np.nan since nwb fields does not take None
for k, v in trial_tag_value.items():
trial_tag_value[k] = v if v is not None else np.nan
trial_tag_value['delay_duration'] = float(
trial_tag_value['delay_duration']) # convert Decimal to float
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(photostim['photo_stim_excitation_lambda']),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimulationInfo
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
unit='mW',
resolution=0.0,
conversion=1e-6,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
def export_to_nwb(session_key,
nwb_output_dir=default_nwb_output_dir,
save=False,
overwrite=True):
this_session = (acquisition.Session & session_key).fetch1()
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(
session_description=this_session['session_note'],
identifier='_'.join([
this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d_%H-%M-%S')
]),
session_start_time=this_session['session_time'],
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join((acquisition.Session.Experimenter
& session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join((subject.Subject.Allele
& session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell & session_key).fetch1() if intracellular.Cell
& session_key else None)
if cell:
# metadata
cell = (intracellular.Cell & session_key).fetch1()
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['cell_id'],
device=whole_cell_device,
description='N/A',
filtering='low-pass: 10kHz',
location='; '.join([
f'{k}: {str(v)}'
for k, v in (reference.ActionLocation & cell).fetch1().items()
]))
# acquisition - membrane potential
mp, mp_wo_spike, mp_start_time, mp_fs = (
intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_wo_spike',
'membrane_potential_start_time',
'membrane_potential_sampling_rate')
nwbfile.add_acquisition(
pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp,
starting_time=mp_start_time,
rate=mp_fs))
# acquisition - current injection
current_injection, ci_start_time, ci_fs = (
intracellular.CurrentInjection & cell).fetch1(
'current_injection', 'current_injection_start_time',
'current_injection_sampling_rate')
nwbfile.add_stimulus(
pynwb.icephys.CurrentClampStimulusSeries(
name='CurrentClampStimulus',
electrode=ic_electrode,
conversion=1e-9,
gain=1.0,
data=current_injection,
starting_time=ci_start_time,
rate=ci_fs))
# analysis - membrane potential without spike
mp_rmv_spike = nwbfile.create_processing_module(
name='icephys', description='Spike removal')
mp_rmv_spike.add_data_interface(
pynwb.icephys.PatchClampSeries(name='icephys',
electrode=ic_electrode,
unit='mV',
conversion=1e-3,
gain=1.0,
data=mp_wo_spike,
starting_time=mp_start_time,
rate=mp_fs))
# =============== Extracellular ====================
probe_insertion = ((extracellular.ProbeInsertion
& session_key).fetch1() if extracellular.ProbeInsertion
& session_key else None)
if probe_insertion:
probe = nwbfile.create_device(name=probe_insertion['probe_name'])
electrode_group = nwbfile.create_electrode_group(name='; '.join([
f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}'
]),
description='N/A',
device=probe,
location='; '.join([
f'{k}: {str(v)}'
for k, v in
(reference.
ActionLocation
& probe_insertion
).fetch1().items(
)
]))
for chn in (reference.Probe.Channel
& probe_insertion).fetch(as_dict=True):
nwbfile.add_electrode(id=chn['channel_id'],
group=electrode_group,
filtering=hardware_filter,
imp=-1.,
x=chn['channel_x_pos'],
y=chn['channel_y_pos'],
z=chn['channel_z_pos'],
location=electrode_group.location)
# --- unit spike times ---
nwbfile.add_unit_column(
name='sampling_rate',
description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='unit_x',
description='x-coordinate of this unit (mm)')
nwbfile.add_unit_column(name='unit_y',
description='y-coordinate of this unit (mm)')
nwbfile.add_unit_column(name='unit_z',
description='z-coordinate of this unit (mm)')
nwbfile.add_unit_column(
name='cell_type',
description='cell type (e.g. wide width, narrow width spiking)')
for unit in (extracellular.UnitSpikeTimes
& probe_insertion).fetch(as_dict=True):
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(
id=unit['unit_id'],
electrodes=(unit['channel_id'] if isinstance(
unit['channel_id'], np.ndarray) else [unit['channel_id']]),
sampling_rate=ecephys_fs,
unit_x=unit['unit_x'],
unit_y=unit['unit_y'],
unit_z=unit['unit_z'],
cell_type=unit['unit_cell_type'],
spike_times=unit['spike_times'],
waveform_mean=np.mean(unit['spike_waveform'], axis=0),
waveform_sd=np.std(unit['spike_waveform'], axis=0))
# =============== Behavior ====================
behavior_data = ((behavior.LickTrace
& session_key).fetch1() if behavior.LickTrace
& session_key else None)
if behavior_data:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='lick_trace')
nwbfile.add_acquisition(behav_acq)
[behavior_data.pop(k) for k in behavior.LickTrace.primary_key]
lt_start_time = behavior_data.pop('lick_trace_start_time')
lt_fs = behavior_data.pop('lick_trace_sampling_rate')
for b_k, b_v in behavior_data.items():
behav_acq.create_timeseries(name=b_k,
unit='a.u.',
conversion=1.0,
data=b_v,
starting_time=lt_start_time,
rate=lt_fs)
# =============== Photostimulation ====================
photostim = ((stimulation.PhotoStimulation
& session_key).fetch1() if stimulation.PhotoStimulation
& session_key else None)
if photostim:
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = nwbfile.create_device(
name=photostim_device['device_name'])
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name='-'.join([photostim['hemisphere'],
photostim['brain_region']]),
device=stim_device,
excitation_lambda=float(photostim['photo_stim_excitation_lambda']),
location='; '.join([
f'{k}: {str(v)}' for k, v in (reference.ActionLocation
& photostim).fetch1().items()
]),
description=(stimulation.PhotoStimulationInfo
& photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(
pynwb.ogen.OptogeneticSeries(
name='_'.join([
'photostim_on',
photostim['photostim_datetime'].strftime(
'%Y-%m-%d_%H-%M-%S')
]),
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
starting_time=photostim['photostim_start_time'],
rate=photostim['photostim_sampling_rate']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if acquisition.TrialSet & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [
{
'name':
tag.replace('trial_', ''),
'description':
re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimInfo
).heading)).groups()[-1].strip()
} for tag in (acquisition.TrialSet.Trial *
stimulation.TrialPhotoStimInfo).heading.names
if tag not in (acquisition.TrialSet.Trial
& stimulation.TrialPhotoStimInfo).primary_key +
['start_time', 'stop_time']
]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key) -
[{
'trial_event': 'trial_start'
}, {
'trial_event': 'trial_stop'
}]).fetch('trial_event'))
event_names = [{
'name': e + '_time',
'description': d
} for e, d in zip(*(reference.ExperimentalEvent & [{
'event': k
} for k in trial_events]).fetch('event', 'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
photostim_tag_default = {
tag: ''
for tag in stimulation.TrialPhotoStimInfo.heading.names
if tag not in stimulation.TrialPhotoStimInfo.primary_key
}
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial
& session_key).fetch(as_dict=True):
events = dict(
zip(*(acquisition.TrialSet.EventTime & trial
& [{
'trial_event': e
} for e in trial_events]
).fetch('trial_event', 'event_time')))
trial_tag_value = ({
**trial,
**events,
**(stimulation.TrialPhotoStimInfo & trial).fetch1()
} if (stimulation.TrialPhotoStimInfo & trial) else {
**trial,
**events,
**photostim_tag_default
})
# rename 'trial_id' to 'id'
trial_tag_value['id'] = trial_tag_value['trial_id']
[
trial_tag_value.pop(k)
for k in acquisition.TrialSet.Trial.primary_key
]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {
k.replace('trial_', ''): trial_tag_value.pop(k)
for k in
[n for n in trial_tag_value if n.startswith('trial_')]
}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(
os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name),
mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
def export_to_nwb(session_key, nwb_output_dir=default_nwb_output_dir, save=False, overwrite=False):
this_session = (acquisition.Session & session_key).fetch1()
identifier = '_'.join([this_session['subject_id'],
this_session['session_time'].strftime('%Y-%m-%d'),
this_session['session_id']])
# =============== General ====================
# -- NWB file - a NWB2.0 file for each session
nwbfile = NWBFile(
session_description=this_session['session_note'],
identifier=identifier,
session_start_time=datetime.combine(this_session['session_time'], zero_zero_time),
file_create_date=datetime.now(tzlocal()),
experimenter='; '.join((acquisition.Session.Experimenter & session_key).fetch('experimenter')),
institution=institution,
experiment_description=experiment_description,
related_publications=related_publications,
keywords=keywords)
# -- subject
subj = (subject.Subject & session_key).fetch1()
nwbfile.subject = pynwb.file.Subject(
subject_id=this_session['subject_id'],
description=subj['subject_description'],
genotype=' x '.join((subject.Subject.Allele & session_key).fetch('allele')),
sex=subj['sex'],
species=subj['species'])
# =============== Intracellular ====================
cell = ((intracellular.Cell & session_key).fetch1()
if intracellular.Cell & session_key
else None)
if cell:
# metadata
whole_cell_device = nwbfile.create_device(name=cell['device_name'])
ic_electrode = nwbfile.create_ic_electrode(
name=cell['session_id'],
device=whole_cell_device,
description='N/A',
filtering='N/A',
location='; '.join([f'{k}: {str(v)}'
for k, v in (reference.ActionLocation & cell).fetch1().items()]))
# acquisition - membrane potential
mp, mp_timestamps = (intracellular.MembranePotential & cell).fetch1(
'membrane_potential', 'membrane_potential_timestamps')
nwbfile.add_acquisition(pynwb.icephys.PatchClampSeries(name='PatchClampSeries',
electrode=ic_electrode,
unit='mV',
conversion=1.0,
gain=1.0,
data=mp,
timestamps=mp_timestamps))
# acquisition - current injection
if (intracellular.CurrentInjection & cell):
current_injection, ci_timestamps = (intracellular.CurrentInjection & cell).fetch1(
'current_injection', 'current_injection_timestamps')
nwbfile.add_stimulus(pynwb.icephys.CurrentClampStimulusSeries(name='CurrentClampStimulus',
electrode=ic_electrode,
conversion=1e-6,
gain=1.0,
data=current_injection,
timestamps=ci_timestamps))
# =============== Extracellular ====================
probe_insertion = ((extracellular.ProbeInsertion & session_key).fetch1()
if extracellular.ProbeInsertion & session_key
else None)
if probe_insertion:
probe = nwbfile.create_device(name=probe_insertion['probe_name'])
electrode_group = nwbfile.create_electrode_group(
name='; '.join([f'{probe_insertion["probe_name"]}: {str(probe_insertion["channel_counts"])}']),
description = 'N/A',
device = probe,
location = '; '.join([f'{k}: {str(v)}' for k, v in
(reference.BrainLocation & (extracellular.ProbeInsertion.InsertLocation
& probe_insertion)).fetch1().items()]))
for chn in (reference.Probe.Channel & probe_insertion).fetch(as_dict=True):
nwbfile.add_electrode(id=chn['channel_id'],
group=electrode_group,
filtering=hardware_filter,
imp=-1.,
x=chn['channel_x_pos'],
y=chn['channel_y_pos'],
z=chn['channel_z_pos'],
location=electrode_group.location)
# --- unit spike times ---
nwbfile.add_unit_column(name='sampling_rate', description='Sampling rate of the raw voltage traces (Hz)')
nwbfile.add_unit_column(name='cell_desc', description='description of this unit (e.g. cell type)')
for unit in (extracellular.UnitSpikeTimes & probe_insertion).fetch(as_dict=True):
# waveforms - mean and std over spikes per electrode
wfs = (extracellular.UnitSpikeTimes.SpikeWaveform & unit).fetch('spike_waveform')
wfs_mean = np.vstack([wf.mean(axis=0) for wf in wfs]).T
wfs_sd = np.vstack([wf.std(axis=0) for wf in wfs]).T
# make an electrode table region (which electrode(s) is this unit coming from)
nwbfile.add_unit(id=unit['unit_id'],
electrodes=(extracellular.UnitSpikeTimes.UnitChannel & unit).fetch('channel_id') - 1,
sampling_rate=ecephys_fs,
cell_desc=unit['cell_desc'],
spike_times=unit['spike_times'],
waveform_mean=wfs_mean,
waveform_sd=wfs_sd)
# =============== Behavior ====================
lick_trace_data = ((behavior.LickTrace & session_key).fetch1()
if behavior.LickTrace & session_key
else None)
if lick_trace_data:
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='lick_trace')
nwbfile.add_acquisition(behav_acq)
[lick_trace_data.pop(k) for k in behavior.LickTrace.primary_key]
timestamps = lick_trace_data.pop('lick_trace_timestamps')
for b_k, b_v in lick_trace_data.items():
behav_acq.create_timeseries(name=b_k,
unit='a.u.',
conversion=1.0,
data=b_v,
timestamps=timestamps)
if behavior.Whisker & session_key:
for whisker_data in (behavior.Whisker & session_key).fetch(as_dict=True):
behav_acq = pynwb.behavior.BehavioralTimeSeries(name='principal_'*whisker_data.pop('principal_whisker') + 'whisker_' + whisker_data['whisker_config'])
nwbfile.add_acquisition(behav_acq)
[whisker_data.pop(k) for k in behavior.Whisker.primary_key]
timestamps = whisker_data.pop('behavior_timestamps')
# get behavior data description from the comments of table definition
behavior_descriptions = {attr: re.search(f'(?<={attr})(.*)#(.*)',
str(behavior.Whisker.heading)).groups()[-1].strip()
for attr in whisker_data}
for b_k, b_v in whisker_data.items():
behav_acq.create_timeseries(name=b_k,
description=behavior_descriptions[b_k],
unit='a.u.',
conversion=1.0,
data=b_v,
timestamps=timestamps)
# =============== Photostimulation ====================
if stimulation.PhotoStimulation & session_key:
for photostim in (stimulation.PhotoStimulation & session_key).fetch(as_dict=True):
photostim_device = (stimulation.PhotoStimDevice & photostim).fetch1()
stim_device = (nwbfile.devices.get(photostim_device['device_name'])
if photostim_device['device_name'] in nwbfile.devices.keys()
else nwbfile.create_device(name=photostim_device['device_name']))
stim_site = pynwb.ogen.OptogeneticStimulusSite(
name=photostim['photostim_id'],
device=stim_device,
excitation_lambda=float((stimulation.PhotoStimProtocol & photostim).fetch1('photo_stim_excitation_lambda')),
location = '; '.join([f'{k}: {str(v)}' for k, v in
(reference.ActionLocation & photostim).fetch1().items()]),
description=(stimulation.PhotoStimProtocol & photostim).fetch1('photo_stim_notes'))
nwbfile.add_ogen_site(stim_site)
if photostim['photostim_timeseries'] is not None:
nwbfile.add_stimulus(pynwb.ogen.OptogeneticSeries(
name='photostimulation_' + photostim['photostim_id'],
site=stim_site,
resolution=0.0,
conversion=1e-3,
data=photostim['photostim_timeseries'],
timestamps=photostim['photostim_timestamps']))
# =============== TrialSet ====================
# NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes:
# 'id', 'start_time' and 'stop_time'.
# Other trial-related information needs to be added in to the trial-table as additional columns (with column name
# and column description)
if acquisition.TrialSet & session_key:
# Get trial descriptors from TrialSet.Trial and TrialStimInfo
trial_columns = [{'name': tag.replace('trial_', ''),
'description': re.search(
f'(?<={tag})(.*)#(.*)',
str((acquisition.TrialSet.Trial
* stimulation.TrialPhotoStimParam).heading)).groups()[-1].strip()}
for tag in (acquisition.TrialSet.Trial * stimulation.TrialPhotoStimParam).heading.names
if tag not in (acquisition.TrialSet.Trial
& stimulation.TrialPhotoStimParam).primary_key + ['start_time', 'stop_time']]
# Trial Events - discard 'trial_start' and 'trial_stop' as we already have start_time and stop_time
# also add `_time` suffix to all events
trial_events = set(((acquisition.TrialSet.EventTime & session_key)
- [{'trial_event': 'trial_start'}, {'trial_event': 'trial_stop'}]).fetch('trial_event'))
event_names = [{'name': e + '_time', 'description': d}
for e, d in zip(*(reference.ExperimentalEvent & [{'event': k}
for k in trial_events]).fetch('event',
'description'))]
# Add new table columns to nwb trial-table for trial-label
for c in trial_columns + event_names:
nwbfile.add_trial_column(**c)
photostim_tag_default = {tag: '' for tag in stimulation.TrialPhotoStimParam.heading.names
if tag not in stimulation.TrialPhotoStimParam.primary_key}
# Add entry to the trial-table
for trial in (acquisition.TrialSet.Trial & session_key).fetch(as_dict=True):
events = dict(zip(*(acquisition.TrialSet.EventTime & trial
& [{'trial_event': e} for e in trial_events]).fetch('trial_event', 'event_time')))
# shift event times to be relative to session_start (currently relative to trial_start)
events = {k: v + trial['start_time'] for k, v in events.items()}
trial_tag_value = ({**trial, **events, **(stimulation.TrialPhotoStimParam & trial).fetch1()}
if (stimulation.TrialPhotoStimParam & trial)
else {**trial, **events, **photostim_tag_default})
trial_tag_value['id'] = trial_tag_value['trial_id'] # rename 'trial_id' to 'id'
# convert None to np.nan since nwb fields does not take None
for k, v in trial_tag_value.items():
trial_tag_value[k] = v if v is not None else np.nan
[trial_tag_value.pop(k) for k in acquisition.TrialSet.Trial.primary_key]
# Final tweaks: i) add '_time' suffix and ii) remove 'trial_' prefix
events = {k + '_time': trial_tag_value.pop(k) for k in events}
trial_attrs = {k.replace('trial_', ''): trial_tag_value.pop(k)
for k in [n for n in trial_tag_value if n.startswith('trial_')]}
nwbfile.add_trial(**trial_tag_value, **events, **trial_attrs)
# =============== Write NWB 2.0 file ===============
if save:
save_file_name = ''.join([nwbfile.identifier, '.nwb'])
if not os.path.exists(nwb_output_dir):
os.makedirs(nwb_output_dir)
if not overwrite and os.path.exists(os.path.join(nwb_output_dir, save_file_name)):
return nwbfile
with NWBHDF5IO(os.path.join(nwb_output_dir, save_file_name), mode='w') as io:
io.write(nwbfile)
print(f'Write NWB 2.0 file: {save_file_name}')
return nwbfile
