def test_append(self):
proc_mod = self.nwbfile.create_processing_module(name='test_proc_mod',
description='')
proc_inter = LFP(name='test_proc_dset')
proc_mod.add(proc_inter)
device = self.nwbfile.create_device(name='test_device')
e_group = self.nwbfile.create_electrode_group(
name='test_electrode_group',
description='',
location='',
device=device)
self.nwbfile.add_electrode(x=0.0,
y=0.0,
z=0.0,
imp=np.nan,
location='',
filtering='',
group=e_group)
electrodes = self.nwbfile.create_electrode_table_region(region=[0],
description='')
e_series = ElectricalSeries(
name='test_es',
electrodes=electrodes,
data=np.ones(shape=(100, )),
rate=10000.0,
)
proc_inter.add_electrical_series(e_series)
with NWBHDF5IO(self.path, mode='w') as io:
io.write(self.nwbfile, cache_spec=False)
with NWBHDF5IO(self.path, mode='a') as io:
nwb = io.read()
link_electrodes = nwb.processing['test_proc_mod'][
'LFP'].electrical_series['test_es'].electrodes
ts2 = ElectricalSeries(name='timeseries2',
data=[4., 5., 6.],
rate=1.0,
electrodes=link_electrodes)
nwb.add_acquisition(ts2)
io.write(nwb) # also attempt to write same spec again
self.assertIs(
nwb.processing['test_proc_mod']
['LFP'].electrical_series['test_es'].electrodes,
nwb.acquisition['timeseries2'].electrodes)
with NWBHDF5IO(self.path, mode='r') as io:
nwb = io.read()
np.testing.assert_equal(nwb.acquisition['timeseries2'].data[:],
ts2.data)
self.assertIs(
nwb.processing['test_proc_mod']
['LFP'].electrical_series['test_es'].electrodes,
nwb.acquisition['timeseries2'].electrodes)
errors = validate(io)
for e in errors:
print('ERROR', e)
def test_add_electrical_series(self):
lfp = LFP()
table = make_electrode_table()
region = DynamicTableRegion('electrodes', [0, 2],
'the first and third electrodes', table)
eS = ElectricalSeries('test_eS', [0, 1, 2, 3],
region,
timestamps=[0.1, 0.2, 0.3, 0.4])
lfp.add_electrical_series(eS)
self.assertEqual(lfp.electrical_series.get('test_eS'), eS)
def test_append(self):
FILENAME = 'test_append.nwb'
nwb = NWBFile(session_description='hi',
identifier='hi',
session_start_time=datetime(1970,
1,
1,
12,
tzinfo=tzutc()))
proc_mod = nwb.create_processing_module(name='test_proc_mod',
description='')
proc_inter = LFP(name='test_proc_dset')
proc_mod.add_data_interface(proc_inter)
device = nwb.create_device(name='test_device')
e_group = nwb.create_electrode_group(name='test_electrode_group',
description='',
location='',
device=device)
nwb.add_electrode(x=0.0,
y=0.0,
z=0.0,
imp=np.nan,
location='',
filtering='',
group=e_group)
electrodes = nwb.create_electrode_table_region(region=[0],
description='')
e_series = ElectricalSeries(
name='test_device',
electrodes=electrodes,
data=np.ones(shape=(100, )),
rate=10000.0,
)
proc_inter.add_electrical_series(e_series)
with NWBHDF5IO(FILENAME, mode='w') as io:
io.write(nwb)
with NWBHDF5IO(FILENAME, mode='a') as io:
nwb = io.read()
elec = nwb.modules['test_proc_mod']['LFP'].electrical_series[
'test_device'].electrodes
ts2 = ElectricalSeries(name='timeseries2',
data=[4, 5, 6],
rate=1.0,
electrodes=elec)
nwb.add_acquisition(ts2)
io.write(nwb)
with NWBHDF5IO(FILENAME, mode='r') as io:
nwb = io.read()
np.testing.assert_equal(nwb.acquisition['timeseries2'].data[:],
ts2.data)
def test_add_electrical_series(self):
lfp = LFP() # noqa: F405
dev1 = Device('dev1') # noqa: F405
group = ElectrodeGroup( # noqa: F405, F841
'tetrode1', 'tetrode description', 'tetrode location', dev1)
table = make_electrode_table()
region = DynamicTableRegion('electrodes', [0, 2], 'the first and third electrodes', table)
eS = ElectricalSeries( # noqa: F405
'test_eS', [0, 1, 2, 3], region, timestamps=[0.1, 0.2, 0.3, 0.4])
lfp.add_electrical_series(eS)
self.assertEqual(lfp.electrical_series.get('test_eS'), eS)
self.assertEqual(lfp['test_eS'], lfp.electrical_series.get('test_eS'))
def chang2nwb(blockpath, out_file_path=None, save_to_file=False, htk_config=None):
"""
Parameters
----------
blockpath: str
out_file_path: None | str
if None, output = [blockpath]/[blockname].nwb
save_to_file : bool
If True, saves to file. If False, just returns nwbfile object
htk_config : dict
Dictionary cotaining HTK conversion paths and options. Example:
{
ecephys_path: 'path_to/ecephys_htk_files',
ecephys_type: 'raw', 'preprocessed' or 'high_gamma',
analog_path: 'path_to/analog_htk_files',
anin1: {present: True, name: 'microphone', type: 'acquisition'},
anin2: {present: True, name: 'speaker1', type: 'stimulus'},
anin3: {present: False, name: 'speaker2', type: 'stimulus'},
anin4: {present: False, name: 'custom', type: 'acquisition'},
metadata: metadata,
electrodes_file: electrodes_file,
bipolar_file: bipolar_file
}
Returns
-------
"""
metadata = {}
if htk_config is None:
blockpath = Path(blockpath)
else:
blockpath = Path(htk_config['ecephys_path'])
metadata = htk_config['metadata']
blockname = blockpath.parent.name
subject_id = blockpath.parent.parent.name[2:]
if out_file_path is None:
out_file_path = blockpath.resolve().parent / ''.join(['EC', subject_id, '_', blockname, '.nwb'])
# file paths
ecog_path = blockpath
anin_path = htk_config['analog_path']
bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat')
# Create the NWB file object
nwbfile_dict = {
'session_description': blockname,
'identifier': blockname,
'session_start_time': datetime.now().astimezone(),
'institution': 'University of California, San Francisco',
'lab': 'Chang Lab'
}
if 'NWBFile' in metadata:
nwbfile_dict.update(metadata['NWBFile'])
nwbfile = NWBFile(**nwbfile_dict)
# Read electrophysiology data from HTK files
print('reading htk acquisition...', flush=True)
ecog_rate, data = readhtks(ecog_path)
data = data.squeeze()
print('done', flush=True)
# Get electrodes info from mat file
if htk_config['electrodes_file'] is not None:
nwbfile = elecs_to_electrode_table(
nwbfile=nwbfile,
elecspath=htk_config['electrodes_file'],
)
n_electrodes = nwbfile.electrodes[:].shape[0]
all_elecs = list(range(n_electrodes))
elecs_region = nwbfile.create_electrode_table_region(
region=all_elecs,
description='ECoG electrodes on brain'
)
else:
ecephys_dict = {
'Device': [{'name': 'auto_device'}],
'ElectricalSeries': [{'name': 'ECoG', 'description': 'description'}],
'ElectrodeGroup': [{'name': 'auto_group', 'description': 'auto_group',
'location': 'location', 'device': 'auto_device'}]
}
if 'Ecephys' in metadata:
ecephys_dict.update(metadata['Ecephys'])
# Create devices
for dev in ecephys_dict['Device']:
device = nwbfile.create_device(dev['name'])
# Electrode groups
for el_grp in ecephys_dict['ElectrodeGroup']:
device = nwbfile.devices[el_grp['device']]
electrode_group = nwbfile.create_electrode_group(
name=el_grp['name'],
description=el_grp['description'],
location=el_grp['location'],
device=device
)
# Electrodes table
n_electrodes = data.shape[1]
nwbfile.add_electrode_column('label', 'label of electrode')
nwbfile.add_electrode_column('bad', 'electrode identified as too noisy')
nwbfile.add_electrode_column('x_warped', 'x warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('y_warped', 'y warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('z_warped', 'z warped onto cvs_avg35_inMNI152')
nwbfile.add_electrode_column('null', 'if not connected to real electrode')
bad_elecs_inds = get_bad_elecs(blockpath)
for elec_counter in range(n_electrodes):
bad = elec_counter in bad_elecs_inds
nwbfile.add_electrode(
id=elec_counter,
x=np.nan,
y=np.nan,
z=np.nan,
imp=np.nan,
x_warped=np.nan,
y_warped=np.nan,
z_warped=np.nan,
location='',
filtering='none',
group=electrode_group,
label='',
bad=bad,
null=False,
)
all_elecs = list(range(n_electrodes))
elecs_region = nwbfile.create_electrode_table_region(
region=all_elecs,
description='ECoG electrodes on brain'
)
# Get Bipolar table from file
if htk_config['bipolar_file'] is not None:
df = pd.read_csv(htk_config['bipolar_file'], index_col='id', sep='\t')
# Create bipolar scheme table
bipolar_scheme_table = BipolarSchemeTable(
name='bipolar_scheme_table',
description='desc'
)
# Columns for bipolar scheme - all anodes and cathodes within the same
# bipolar row are considered to have the same group and location
bipolar_scheme_table.add_column(
name='group_name',
description='electrode group name'
)
bipolar_scheme_table.add_column(
name='location',
description='electrode location'
)
# Iterate over anode / cathode rows
for i, r in df.iterrows():
if isinstance(r['anodes'], str):
anodes = [int(a) for a in r['anodes'].split(',')]
else:
anodes = [int(r['anodes'])]
if isinstance(r['cathodes'], str):
cathodes = [int(a) for a in r['cathodes'].split(',')]
else:
cathodes = [int(r['cathodes'])]
bipolar_scheme_table.add_row(
anodes=anodes,
cathodes=cathodes,
group_name=nwbfile.electrodes['group_name'][anodes[0]],
location=nwbfile.electrodes['location'][anodes[0]]
)
bipolar_scheme_table.anodes.table = nwbfile.electrodes
bipolar_scheme_table.cathodes.table = nwbfile.electrodes
# Creates bipolar table region
elecs_region = DynamicTableRegion(
name='electrodes',
data=np.arange(0, df.shape[0]),
description='desc',
table=bipolar_scheme_table
)
ecephys_ext = EcephysExt(name='ecephys_ext')
ecephys_ext.bipolar_scheme_table = bipolar_scheme_table
nwbfile.add_lab_meta_data(ecephys_ext)
# Stores HTK electrophysiology data as raw, preprocessed or high gamma
if htk_config['ecephys_type'] == 'raw':
ecog_es = ElectricalSeries(name='ECoG',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
nwbfile.add_acquisition(ecog_es)
elif htk_config['ecephys_type'] == 'preprocessed':
lfp = LFP()
ecog_es = ElectricalSeries(name='preprocessed',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
lfp.add_electrical_series(ecog_es)
# Creates the ecephys processing module
ecephys_module = nwbfile.create_processing_module(
name='ecephys',
description='preprocessed electrophysiology data'
)
ecephys_module.add_data_interface(lfp)
elif htk_config['ecephys_type'] == 'high_gamma':
ecog_es = ElectricalSeries(name='high_gamma',
data=H5DataIO(data[:, 0:n_electrodes], compression='gzip'),
electrodes=elecs_region,
rate=ecog_rate,
description='all Wav data')
# Creates the ecephys processing module
ecephys_module = nwbfile.create_processing_module(
name='ecephys',
description='preprocessed electrophysiology data'
)
ecephys_module.add_data_interface(ecog_es)
# Add ANIN 1
if htk_config['anin1']['present']:
fs, data = get_analog(anin_path, 1)
ts = TimeSeries(
name=htk_config['anin1']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin1']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN1 saved with name "', htk_config['anin1']['name'], '" in ',
htk_config['anin1']['type'])
# Add ANIN 2
if htk_config['anin2']['present']:
fs, data = get_analog(anin_path, 2)
ts = TimeSeries(
name=htk_config['anin2']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin2']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN2 saved with name "', htk_config['anin2']['name'], '" in ',
htk_config['anin2']['type'])
# Add ANIN 3
if htk_config['anin3']['present']:
fs, data = get_analog(anin_path, 3)
ts = TimeSeries(
name=htk_config['anin3']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin3']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN3 saved with name "', htk_config['anin3']['name'], '" in ',
htk_config['anin3']['type'])
# Add ANIN 4
if htk_config['anin4']['present']:
fs, data = get_analog(anin_path, 4)
ts = TimeSeries(
name=htk_config['anin4']['name'],
data=data,
unit='NA',
rate=fs,
)
if htk_config['anin4']['type'] == 'acquisition':
nwbfile.add_acquisition(ts)
else:
nwbfile.add_stimulus(ts)
print('ANIN4 saved with name "', htk_config['anin4']['name'], '" in ',
htk_config['anin4']['type'])
# Add bad time segments
if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size:
bad_time = sio.loadmat(bad_time_file)['badTimeSegments']
for row in bad_time:
nwbfile.add_invalid_time_interval(start_time=row[0],
stop_time=row[1],
tags=('ECoG artifact',),
timeseries=ecog_es)
# Subject
subject_dict = {'subject_id': subject_id}
if 'Subject' in metadata:
subject_dict.update(metadata['Subject'])
subject = ECoGSubject(**subject_dict)
nwbfile.subject = subject
if save_to_file:
print('Saving HTK content to NWB file...')
# Export the NWB file
with NWBHDF5IO(str(out_file_path), manager=manager, mode='w') as io:
io.write(nwbfile)
# read check
with NWBHDF5IO(str(out_file_path), manager=manager, mode='r') as io:
io.read()
print('NWB file saved: ', str(out_file_path))
return nwbfile, out_file_path, subject_id, blockname
def nlx_to_nwb(nev_fp,
ncs_paths,
desc='',
trim_buffer=60 * 10,
practice_incl=False,
electrode_locations=None):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ncs = nlx.load_ncs(ncs_paths.pop(0), load_time=False)
nev = nlx.load_nev(nev_fp)
uuid = nev['header']['SessionUUID']
start_time = pd.to_datetime(ncs['timestamp'][0], unit='us',
utc=True).to_pydatetime()
start_time_sec = start_time.timestamp()
nwbfile = NWBFile(session_description=desc,
identifier=uuid,
session_start_time=start_time)
dev = nwbfile.create_device(name='Neuralynx')
# ts = ncs_to_timeseries(ncs)
if electrode_locations is not None:
add_electrodes(nwbfile, electrode_locations, dev)
ev = pd.DataFrame.from_records(nev_as_records(nev), index='TimeStamp')
ev['EventString'] = [str(v, 'utf-8') for v in ev.EventString.values]
ev['time'] = pd.to_datetime(ev.index.values, unit='us', utc=True)
ev = ev[ev.ttl == 1]
label_blockstart(ev)
n_blocks = int(len(ev[ev.label == 'block_start']) / 2)
n_ttls = int(n_blocks * 17)
if practice_incl:
n_ttls -= 5
# ev['TimeStamp'] = ev.index.values/10**6 - start_time.timestamp()
ev_ts = np.array([t.timestamp() for t in ev.time]) - start_time.timestamp()
# start_stop = ev[ev.EventString.isin(['Starting Recording','Stopping Recording'])]
events = AnnotationSeries(name='ttl',
data=ev.label.values[:n_ttls],
timestamps=ev_ts[:n_ttls])
data_time_len = events.timestamps[-1] + trim_buffer
nwbfile.add_acquisition(events)
lfp = LFP(name='LFP')
nwbfile.add_acquisition(ncs_to_timeseries(ncs, data_time_len))
for ch, ts_kwargs in iter_ncs_to_timeseries(ncs_paths, data_time_len):
if electrode_locations is not None:
# ts_kwargs['electrodes']=electrode_table_region
try:
electrode_table_region = nwbfile.create_electrode_table_region(
[ch], 'Channel')
row = electrode_locations.where(
electrode_locations.chan_num == ch + 1).dropna().iloc[0]
ts_kwargs['name'] = 'wire_{}_electrode_{}'.format(
int(row.wire_num), int(row.electrode))
ts_kwargs['electrodes'] = electrode_table_region
ts = ElectricalSeries(**ts_kwargs)
if ts.name not in lfp.electrical_series.keys():
lfp.add_electrical_series(ts)
except IndexError:
pass
else:
ts = TimeSeries(**ts_kwargs)
if ts.name not in nwbfile.acquisition.keys():
nwbfile.add_acquisition(ts)
else:
print('Failed to load: ', ts.name)
ecephys_module = ProcessingModule(
name='ecephys',
description='preprocessed extracellular electrophysiology')
nwbfile.create_processing_module(
name='ecephys',
description='preprocessed extracellular electrophysiology')
nwbfile.processing['ecephys'].add(lfp)
return nwbfile
def ncs_to_nwb(
ncs_paths,
desc='',
nev_path=None,
electrode_locations=None,
lab='Thompson Lab',
institution='University of Colorado Anschutz',
trim_buffer=60 * 10,
):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
first_ncs = ncs_paths.pop(0)
ncs = nlx.load_ncs(first_ncs, load_time=False)
uuid = ncs['header']['SessionUUID']
start_time = pd.to_datetime(ncs['timestamp'][0], unit='us',
utc=True).to_pydatetime()
# start_time = ncs['header']['TimeCreated_dt']
start_time_sec = start_time.timestamp()
nwbfile = NWBFile(session_description=desc,
identifier=uuid,
session_id=uuid,
lab=lab,
institution=institution,
session_start_time=start_time)
dev = nwbfile.create_device(name='Neuralynx')
# ts = ncs_to_timeseries(ncs)
if electrode_locations is not None:
add_electrodes(nwbfile, electrode_locations, dev)
if nev_path is not None:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
nev = load_nev(nev_path)
if len(nev['events']) > 0:
ev = pd.DataFrame.from_records(nev_as_records(nev),
index='TimeStamp')
ev['EventString'] = [
str(v, 'utf-8') for v in ev.EventString.values
]
ev['time'] = pd.to_datetime(ev.index.values,
unit='us',
utc=True)
ev = ev[ev.ttl == 1]
ev_ts = np.array([t.timestamp()
for t in ev.time]) - start_time.timestamp()
data_time_len = ev_ts[-1] + trim_buffer
else:
data_time_len = len(ncs['data']) / int(ncs['sampling_rate'])
ncs_paths.append(first_ncs)
lfp = LFP(name='LFP')
for ch, ts_kwargs in iter_ncs_to_timeseries(ncs_paths, data_time_len):
if electrode_locations is not None:
# ts_kwargs['electrodes']=electrode_table_region
try:
electrode_table_region = nwbfile.create_electrode_table_region(
[ch], 'Channel')
row = electrode_locations.where(
electrode_locations.chan_num == ch + 1).dropna().iloc[0]
ts_kwargs['name'] = 'wire_{}_electrode_{}'.format(
int(row.wire_num), int(row.electrode))
ts_kwargs['electrodes'] = electrode_table_region
ts = ElectricalSeries(**ts_kwargs)
if ts.name not in lfp.electrical_series.keys():
lfp.add_electrical_series(ts)
else:
print('Failed to load: ', ts.name)
except IndexError:
pass
else:
ts = TimeSeries(**ts_kwargs)
if ts.name not in nwbfile.acquisition.keys():
nwbfile.add_acquisition(ts)
else:
print('Failed to load: ', ts.name)
nwbfile.create_processing_module(
name='ecephys',
description='preprocessed extracellular electrophysiology')
nwbfile.processing['ecephys'].add(lfp)
return nwbfile