def test_nd_array_to_df(self):
data = np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]])
col = VectorData(name='name', description='desc', data=data)
df = DynamicTable('test', 'desc', np.arange(3, dtype='int'), (col, )).to_dataframe()
df2 = pd.DataFrame({'name': [x for x in data]},
index=pd.Index(name='id', data=[0, 1, 2]))
assert_frame_equal(df, df2)
def with_table_columns(self):
cols = [TableColumn(**d) for d in self.spec]
table = DynamicTable("with_table_columns",
'PyNWB unit test',
'a test table',
columns=cols)
return table
def infer_columns_to_plot(dynamic_table: DynamicTable):
"""Infer which columns can be plotted in summary widgets
Parameters
----------
dynamic_table : DynamicTable
Returns
-------
column_names_to_plot: list
Columns that can be plotted with the dynamic table summary
"""
categorical_cols = infer_categorical_columns(dynamic_table)
column_names_to_plot = []
df = dynamic_table.to_dataframe()
categorical_columns = []
for name in dynamic_table.colnames:
# if name not in categorical_cols.keys(): # categorical columns can always be plotted
value = df[name].values[0]
if isinstance(value, (int, float, np.integer)):
column_names_to_plot.append(name)
elif isinstance(value, (str, bool, bytes)):
column_names_to_plot.append(name)
categorical_columns.append(name)
return column_names_to_plot, categorical_columns
def test_constructor_ids_default(self):
columns = [
VectorData(name=s['name'], description=s['description'], data=d)
for s, d in zip(self.spec, self.data)
]
table = DynamicTable("with_spec", 'a test table', columns=columns)
self.check_table(table)
def addContainer(self, nwbfile):
nwbfile.units = DynamicTable.from_dataframe(pd.DataFrame({
'a': [1, 2, 3],
'b': ['4', '5', '6']
}), 'units')
# reset the thing
self.container = nwbfile.units
def setUpContainer(self):
self.timeseries = TimeSeries(name='dummy timeseries',
description='desc',
data=np.ones((3, 3)),
unit='flibs',
timestamps=np.ones((3, )))
bands = DynamicTable(name='bands',
description='band info for LFPSpectralAnalysis',
columns=[
VectorData(name='band_name',
description='name of bands',
data=['alpha', 'beta', 'gamma']),
VectorData(
name='band_limits',
description='low and high cutoffs in Hz',
data=np.ones((3, 2)))
])
spec_anal = DecompositionSeries(name='LFPSpectralAnalysis',
description='my description',
data=np.ones((3, 3, 3)),
timestamps=np.ones((3, )),
source_timeseries=self.timeseries,
metric='amplitude',
bands=bands)
return spec_anal
def setUpContainer(self):
# this will get ignored
return DynamicTable.from_dataframe(
pd.DataFrame({
'a': [[1, 2, 3], [1, 2, 3], [1, 2, 3]],
'b': ['4', '5', '6']
}), 'test_table')
def with_columns_and_data(self):
columns = [
VectorData(name=s['name'], description=s['description'], data=d)
for s, d in zip(self.spec, self.data)
]
return DynamicTable("with_columns_and_data",
'a test table',
columns=columns)
def test_from_dataframe(self):
df = pd.DataFrame({
'foo': [1, 2, 3, 4, 5],
'bar': [10.0, 20.0, 30.0, 40.0, 50.0],
'baz': ['cat', 'dog', 'bird', 'fish', 'lizard']
}).loc[:, ('foo', 'bar', 'baz')]
obtained_table = DynamicTable.from_dataframe(df, 'test')
self.check_table(obtained_table)
def test_pandas_roundtrip(self):
df = pd.DataFrame({
'a': [1, 2, 3, 4],
'b': ['a', 'b', 'c', '4']
}, index=pd.Index(name='an_index', data=[2, 4, 6, 8]))
table = DynamicTable.from_dataframe(df, 'foo')
obtained = table.to_dataframe()
assert df.equals(obtained)
def test_constructor_ids(self):
columns = [
TableColumn(name=s['name'], description=s['description'], data=d)
for s, d in zip(self.spec, self.data)
]
table = DynamicTable("with_columns",
'a test table',
id=[0, 1, 2, 3, 4],
columns=columns)
self.check_table(table)
def test_constructor_ids_bad_ids(self):
columns = [
VectorData(name=s['name'], description=s['description'], data=d)
for s, d in zip(self.spec, self.data)
]
msg = "must provide same number of ids as length of columns"
with self.assertRaisesRegex(ValueError, msg):
DynamicTable("with_columns",
'a test table',
id=[0, 1],
columns=columns)
def test_constructor_ElementIdentifier_ids(self):
columns = [
VectorData(name=s['name'], description=s['description'], data=d)
for s, d in zip(self.spec, self.data)
]
ids = ElementIdentifiers('ids', [0, 1, 2, 3, 4])
table = DynamicTable("with_columns",
'a test table',
id=ids,
columns=columns)
self.check_table(table)
def test_roundtrip(self):
"""Test writing and reading the ontology_terms and ontology_objects tables."""
container = TimeSeries(
name='test_ts',
data=[1, 2, 3],
unit='si_unit',
timestamps=[0.1, 0.2, 0.3],
)
self.nwbfile.add_acquisition(container)
table = DynamicTable(name='test_table',
description='test table description')
table.add_column(name='test_col',
description='test column description')
table.add_row(test_col='Mouse')
self.nwbfile.add_acquisition(table)
self.nwbfile.ontology_terms.add_row(
id=np.uint64(1),
key='meter',
ontology='si_ontology',
uri='si_ontology:m',
)
self.nwbfile.ontology_objects.add_row(
id=np.uint64(5),
object_id=container.object_id,
field='unit',
item=np.uint64(1),
)
self.nwbfile.ontology_terms.add_row(
id=np.uint64(2),
key='Mouse',
ontology='species_ontology',
uri='species_ontology:Mus musculus',
)
self.nwbfile.ontology_objects.add_row(
id=np.uint64(6),
object_id=table.object_id,
field='test_col',
item=np.uint64(2),
)
with NWBHDF5IO(self.path, mode='w') as io:
io.write(self.nwbfile)
with NWBHDF5IO(self.path, mode='r', load_namespaces=True) as io:
read_nwbfile = io.read()
self.assertContainerEqual(self.nwbfile.ontology_objects,
read_nwbfile.ontology_objects)
self.assertContainerEqual(self.nwbfile.ontology_terms,
read_nwbfile.ontology_terms)
def test_infer_categorical_columns():
data1 = np.array([1, 2, 2, 3, 1, 1, 3, 2, 3])
data2 = np.array([3, 4, 2, 4, 3, 2, 2, 4, 4])
vd1 = VectorData('Data1',
'vector data for creating a DynamicTable',
data=data1)
vd2 = VectorData('Data2',
'vector data for creating a DynamicTable',
data=data2)
vd = [vd1, vd2]
dynamic_table = DynamicTable(name='test table',
description='This is a test table',
columns=vd,
colnames=['Data1', 'Data2'])
assert dicts_exact_equal(infer_categorical_columns(dynamic_table), {
'Data1': np.array([1, 2, 3]),
'Data2': np.array([2, 3, 4])
})
def test_infer_categorical_columns():
data1 = np.array([1, 2, 2, 3, 1, 1, 3, 2, 3])
data2 = np.array([3, 4, 2, 4, 3, 2, 2, 4, 4])
device = Device(name="device")
eg_1 = ElectrodeGroup(name="electrodegroup1",
description="desc",
location="brain",
device=device)
eg_2 = ElectrodeGroup(name="electrodegroup2",
description="desc",
location="brain",
device=device)
data3 = [eg_1, eg_2, eg_1, eg_1, eg_1, eg_1, eg_1, eg_1, eg_1]
vd1 = VectorData("Data1",
"vector data for creating a DynamicTable",
data=data1)
vd2 = VectorData("Data2",
"vector data for creating a DynamicTable",
data=data2)
vd3 = VectorData("ElectrodeGroup",
"vector data for creating a DynamicTable",
data=data3)
vd = [vd1, vd2, vd3]
dynamic_table = DynamicTable(
name="test table",
description="This is a test table",
columns=vd,
colnames=["Data1", "Data2", "ElectrodeGroup"],
)
assert dicts_exact_equal(
infer_categorical_columns(dynamic_table),
{
"Data1": data1,
"Data2": data2,
"ElectrodeGroup": [i.name for i in data3]
},
)
def transform(block_path, filter='default', bands_vals=None):
"""
Takes raw LFP data and does the standard Hilbert algorithm:
1) CAR
2) notch filters
3) Hilbert transform on different bands
Takes about 20 minutes to run on 1 10-min block.
Parameters
----------
block_path : str
subject file path
filter: str, optional
Frequency bands to filter the signal.
'default' for Chang lab default values (Gaussian filters)
'custom' for user defined (Gaussian filters)
bands_vals: 2D array, necessary only if filter='custom'
[2,nBands] numpy array with Gaussian filter parameters, where:
bands_vals[0,:] = filter centers [Hz]
bands_vals[1,:] = filter sigmas [Hz]
Returns
-------
Saves preprocessed signals (LFP) and spectral power (DecompositionSeries) in
the current NWB file. Only if containers for these data do not exist in the
file.
"""
write_file = 1
rate = 400.
# Define filter parameters
if filter == 'default':
band_param_0 = bands.chang_lab['cfs']
band_param_1 = bands.chang_lab['sds']
elif filter == 'high_gamma':
band_param_0 = bands.chang_lab['cfs'][(bands.chang_lab['cfs'] > 70)
& (bands.chang_lab['cfs'] < 150)]
band_param_1 = bands.chang_lab['sds'][(bands.chang_lab['cfs'] > 70)
& (bands.chang_lab['cfs'] < 150)]
#band_param_0 = [ bands.neuro['min_freqs'][-1] ] #for hamming window filter
#band_param_1 = [ bands.neuro['max_freqs'][-1] ]
#band_param_0 = bands.chang_lab['cfs'][29:37] #for average of gaussian filters
#band_param_1 = bands.chang_lab['sds'][29:37]
elif filter == 'custom':
band_param_0 = bands_vals[0, :]
band_param_1 = bands_vals[1, :]
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'a') as io:
nwb = io.read()
# Storage of processed signals on NWB file -----------------------------
if 'ecephys' not in nwb.modules:
# Add module to NWB file
nwb.create_processing_module(
name='ecephys',
description='Extracellular electrophysiology data.')
ecephys_module = nwb.modules['ecephys']
# LFP: Downsampled and power line signal removed
if 'LFP' in nwb.modules['ecephys'].data_interfaces:
lfp_ts = nwb.modules['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
X = lfp_ts.data[:].T
rate = lfp_ts.rate
else:
# 1e6 scaling helps with numerical accuracy
X = nwb.acquisition['ECoG'].data[:].T * 1e6
fs = nwb.acquisition['ECoG'].rate
bad_elects = load_bad_electrodes(nwb)
print('Load time for h5 {}: {} seconds'.format(
block_name,
time.time() - start))
print('rates {}: {} {}'.format(block_name, rate, fs))
if not np.allclose(rate, fs):
assert rate < fs
start = time.time()
X = resample(X, rate, fs)
print('resample time for {}: {} seconds'.format(
block_name,
time.time() - start))
if bad_elects.sum() > 0:
X[bad_elects] = np.nan
# Subtract CAR
start = time.time()
X = subtract_CAR(X)
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Notch filters
start = time.time()
X = linenoise_notch(X, rate)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
lfp = LFP()
# Add preprocessed downsampled signals as an electrical_series
lfp_ts = lfp.create_electrical_series(
name='preprocessed',
data=X.T,
electrodes=nwb.acquisition['ECoG'].electrodes,
rate=rate,
description='')
ecephys_module.add_data_interface(lfp)
# Spectral band power
if 'Bandpower_' + filter not in nwb.modules['ecephys'].data_interfaces:
# Apply Hilbert transform
X = X.astype('float32') # signal (nChannels,nSamples)
nChannels = X.shape[0]
nSamples = X.shape[1]
nBands = len(band_param_0)
Xp = np.zeros((nBands, nChannels,
nSamples)) # power (nBands,nChannels,nSamples)
X_fft_h = None
for ii, (bp0, bp1) in enumerate(zip(band_param_0, band_param_1)):
# if filter=='high_gamma':
# kernel = hamming(X, rate, bp0, bp1)
# else:
kernel = gaussian(X, rate, bp0, bp1)
X_analytic, X_fft_h = hilbert_transform(X,
rate,
kernel,
phase=None,
X_fft_h=X_fft_h)
Xp[ii] = abs(X_analytic).astype('float32')
# Scales signals back to Volt
X /= 1e6
band_param_0V = VectorData(
name='filter_param_0',
description='frequencies for bandpass filters',
data=band_param_0)
band_param_1V = VectorData(
name='filter_param_1',
description='frequencies for bandpass filters',
data=band_param_1)
bandsTable = DynamicTable(
name='bands',
description='Series of filters used for Hilbert transform.',
columns=[band_param_0V, band_param_1V],
colnames=['filter_param_0', 'filter_param_1'])
# data: (ndarray) dims: num_times * num_channels * num_bands
Xp = np.swapaxes(Xp, 0, 2)
decs = DecompositionSeries(
name='Bandpower_' + filter,
data=Xp,
description='Band power estimated with Hilbert transform.',
metric='power',
unit='V**2/Hz',
bands=bandsTable,
rate=rate,
source_timeseries=lfp_ts)
ecephys_module.add_data_interface(decs)
io.write(nwb)
print('done', flush=True)
def spectral_decomposition(block_path, bands_vals):
"""
Takes preprocessed LFP data and does the standard Hilbert transform on
different bands. Takes about 20 minutes to run on 1 10-min block.
Parameters
----------
block_path : str
subject file path
bands_vals : [2,nBands] numpy array with Gaussian filter parameters, where:
bands_vals[0,:] = filter centers [Hz]
bands_vals[1,:] = filter sigmas [Hz]
Returns
-------
Saves spectral power (DecompositionSeries) in the current NWB file.
Only if container for this data do not exist in the file.
"""
# Get filter parameters
band_param_0 = bands_vals[0, :]
band_param_1 = bands_vals[1, :]
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
lfp = nwb.processing['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
rate = lfp.rate
nBands = len(band_param_0)
nSamples = lfp.data.shape[0]
nChannels = lfp.data.shape[1]
Xp = np.zeros(
(nBands, nChannels, nSamples)) #power (nBands,nChannels,nSamples)
# Apply Hilbert transform ----------------------------------------------
print('Running Spectral Decomposition...')
start = time.time()
for ch in np.arange(nChannels):
Xch = lfp.data[:,
ch] * 1e6 # 1e6 scaling helps with numerical accuracy
Xch = Xch.reshape(1, -1)
Xch = Xch.astype('float32') # signal (nChannels,nSamples)
X_fft_h = None
for ii, (bp0, bp1) in enumerate(zip(band_param_0, band_param_1)):
kernel = gaussian(Xch, rate, bp0, bp1)
X_analytic, X_fft_h = hilbert_transform(Xch,
rate,
kernel,
phase=None,
X_fft_h=X_fft_h)
Xp[ii, ch, :] = abs(X_analytic).astype('float32')
print('Spectral Decomposition finished in {} seconds'.format(
time.time() - start))
# data: (ndarray) dims: num_times * num_channels * num_bands
Xp = np.swapaxes(Xp, 0, 2)
# Spectral band power
# bands: (DynamicTable) frequency bands that signal was decomposed into
band_param_0V = VectorData(
name='filter_param_0',
description='frequencies for bandpass filters',
data=band_param_0)
band_param_1V = VectorData(
name='filter_param_1',
description='frequencies for bandpass filters',
data=band_param_1)
bandsTable = DynamicTable(
name='bands',
description='Series of filters used for Hilbert transform.',
columns=[band_param_0V, band_param_1V],
colnames=['filter_param_0', 'filter_param_1'])
decs = DecompositionSeries(
name='DecompositionSeries',
data=Xp,
description='Analytic amplitude estimated with Hilbert transform.',
metric='amplitude',
unit='V',
bands=bandsTable,
rate=rate,
source_timeseries=lfp)
# Storage of spectral decomposition on NWB file ------------------------
ecephys_module = nwb.processing['ecephys']
ecephys_module.add_data_interface(decs)
io.write(nwb)
print('Spectral decomposition saved in ' + block_path)
def with_spec(self):
table = DynamicTable("with_spec", 'a test table', columns=self.spec)
return table
def setUpContainer(self):
return DynamicTable('electrodes',
'metadata about extracellular electrodes',
'autogenerated by PyNWB API')
def with_table_columns(self):
cols = [VectorData(**d) for d in self.spec]
table = DynamicTable("with_table_columns",
'a test table',
columns=cols)
return table
def get_bipolar_referenced_electrodes(X,
electrodes,
rate,
grid_size=None,
grid_step=1):
'''
Bipolar referencing of electrodes according to the scheme of Dr. John Burke
Each electrode (with obvious exceptions at the edges) yields two bipolar-
referenced channels, one for the "right" and one for the "below" neighbors.
These are returned as an ElectricalSeries.
Input arguments:
--------
X:
numpy array containing (raw) electrode traces (Nelectrodes x T)
electrodes:
DynamicTableRegion containing the metadata for the electrodes whose
traces are in X
rate:
sampling rate of X; for storage in ElectricalSeries
grid_size:
numpy array with the two dimensions of the grid (2, )
Returns:
--------
bipolarElectrodes:
ElectricalSeries containing the bipolar-referenced (pseudo) electrodes
and associated metadata. (NB that a, e.g., 16x16 grid yields 960 of
these pseudo-electrodes.)
'''
# set mutable default argument(s)
if grid_size is None:
grid_size = np.array([16, 16])
# malloc
elec_layout = np.arange(np.prod(grid_size) - 1, -1,
-1).reshape(grid_size).T
elec_layout = elec_layout[::grid_step, ::grid_step]
grid_size = elec_layout.T.shape # in case grid_step > 1
Nchannels = 2 * np.prod(grid_size) - np.sum(grid_size)
XX = np.zeros((Nchannels, X.shape[1]))
# create a new dynamic table to hold the metadata
column_names = ['x', 'y', 'z', 'imp', 'location', 'label', 'bad']
columns = [
VectorData(name=name, description=electrodes.table[name].description)
for name in column_names
]
bipolarTable = DynamicTable(
name='bipolar-referenced metadata',
description=('pseudo-channels derived via John Burke style'
' bipolar referencing'),
colnames=column_names,
columns=columns,
)
# compute bipolar-ref'd channel and add a new row of metadata to table
def add_new_channel(iChannel, iElectrode, jElectrode):
jElectrode = int(jElectrode)
# "bipolar referencing": the difference of neighboring electrodes
XX[iChannel, :] = X[iElectrode, :] - X[jElectrode, :]
# add a row to the table for this new pseudo-electrode
bipolarTable.add_row({
'location':
'_'.join({
electrodes.table['location'][iElectrode],
electrodes.table['location'][jElectrode]
}),
'label':
'-'.join([
electrodes.table['label'][iElectrode],
electrodes.table['label'][jElectrode]
]),
'bad': (electrodes.table['bad'][iElectrode]
or electrodes.table['bad'][jElectrode]),
**{
name: electrodes.table[name][iElectrode]
for name in ['x', 'y', 'z', 'imp']
},
})
return iChannel + 1
iChannel = 0
# loop across columns and rows (remembering that grid is transposed)
for i in range(grid_size[1]):
for j in range(grid_size[0]):
if j < grid_size[0] - 1:
iChannel = add_new_channel(iChannel, elec_layout[i, j],
elec_layout[i, j + 1])
if i < grid_size[1] - 1:
iChannel = add_new_channel(iChannel, elec_layout[i, j],
elec_layout[i + 1, j])
# create one big region for the entire table
bipolarTableRegion = bipolarTable.create_region(
'electrodes', [i for i in range(Nchannels)], 'all bipolar electrodes')
return XX, bipolarTable, bipolarTableRegion
# a "category". This is similar to a table with "sub-headings". In the case of the
# :py:class:`~pynwb.icephys.IntracellularRecordingsTable`, we have three predefined categories,
# i.e., electrodes, stimuli, and responses. We can also dynamically add new categories to
# the table. As each category corresponds to a ``DynamicTable``, this means we have to create a
# new ``DynamicTable`` and add it to our table.
# Create a new DynamicTable for our category that contains a location column of type VectorData
location_column = VectorData(name='location',
data=['Mordor', 'Gondor', 'Rohan'],
description='Recording location in Middle Earth')
lab_category = DynamicTable(
name='recording_lab_data',
description='category table for lab-specific recording metadata',
colnames=[
'location',
],
columns=[
location_column,
])
# Add the table as a new category to our intracellular_recordings
nwbfile.intracellular_recordings.add_category(category=lab_category)
# Note, the name of the category is name of the table, i.e., 'recording_lab_data'
#####################################################################
# .. note:: In an ``AlignedDynamicTable`` all category tables MUST align with the main table,
# i.e., all tables must have the same number of rows and rows are expected to
# correspond to each other by index
#####################################################################
# We can also add custom columns to any of the subcategory tables, i.e.,
def setUpContainer(self):
# this will get ignored
return DynamicTable('units', 'unit table integration test',
'a placeholder table')
def setUpContainer(self):
return DynamicTable('electrodes', 'metadata about extracellular electrodes')
def test_auto_ragged_array(self):
df = pd.DataFrame({'a': [[1], [1, 2]]})
df2 = DynamicTable.from_dataframe(df, name='test').to_dataframe()
df.equals(df2)
def setUpContainer(self):
# this will get ignored
return DynamicTable('trials', 'a placeholder table')
def test_show_dynamic_table():
d = {'col1': [1, 2], 'col2': [3, 4]}
dt = DynamicTable.from_dataframe(df=pd.DataFrame(data=d),
name='Test Dtable',
table_description='no description')
show_dynamic_table(dt)
def test_show_dynamic_table():
d = {"col1": [1, 2], "col2": [3, 4]}
dt = DynamicTable.from_dataframe(df=pd.DataFrame(data=d),
name="Test Dtable",
table_description="no description")
show_dynamic_table(dt)
def setUpContainer(self):
return DynamicTable('trials', 'DynamicTable integration test',
'a test table')
