def test_no_tags(self):
nwbfile = NWBFile("a file with header data", "NB123A",
datetime(1970, 1, 1, tzinfo=tz.tzutc()))
df = self.get_dataframe()
for i, row in df.iterrows():
nwbfile.add_epoch(start_time=row['start_time'],
stop_time=row['stop_time'])
def test_print_file(self):
nwbfile = NWBFile(session_description='session_description',
identifier='identifier', session_start_time=datetime.now(tzlocal()))
ts = TimeSeries('name', [1., 2., 3.] * 1000, timestamps=[1, 2, 3])
ts2 = TimeSeries('name2', [1, 2, 3] * 1000, timestamps=[1, 2, 3])
self.assertEqual(str(ts), """
name <class 'pynwb.base.TimeSeries'>
Fields:
comments: no comments
conversion: 1.0
data: [1. 2. 3. ... 1. 2. 3.]
description: no description
interval: 1
resolution: 0.0
timestamps: [1 2 3]
timestamps_unit: Seconds
"""
)
nwbfile.add_acquisition(ts)
nwbfile.add_acquisition(ts2)
nwbfile.add_epoch(start_time=1.0, stop_time=10.0, tags=['tag1', 'tag2'])
self.assertEqual(str(nwbfile),
"""
root <class 'pynwb.file.NWBFile'>
Fields:
acquisition: {
name <class 'pynwb.base.TimeSeries'>,
name2 <class 'pynwb.base.TimeSeries'>
}
epoch_tags: {
tag1,
tag2
}
epochs: epochs <class 'pynwb.epoch.TimeIntervals'>
""")
def test_print_file(self):
nwbfile = NWBFile(session_description='session_description',
identifier='identifier',
session_start_time=datetime.now(tzlocal()))
ts1 = TimeSeries(name='name1',
data=[1., 2., 3.] * 1000,
unit='unit',
timestamps=[1, 2, 3])
ts2 = TimeSeries(name='name2',
data=[1, 2, 3] * 1000,
unit='unit',
timestamps=[1, 2, 3])
expected = """name1 pynwb.base.TimeSeries at 0x%d
Fields:
comments: no comments
conversion: 1.0
data: [1. 2. 3. ... 1. 2. 3.]
description: no description
interval: 1
resolution: -1.0
timestamps: [1 2 3]
timestamps_unit: seconds
unit: unit
"""
expected %= id(ts1)
self.assertEqual(str(ts1), expected)
nwbfile.add_acquisition(ts1)
nwbfile.add_acquisition(ts2)
nwbfile.add_epoch(start_time=1.0,
stop_time=10.0,
tags=['tag1', 'tag2'])
expected_re = r"""root pynwb\.file\.NWBFile at 0x\d+
Fields:
acquisition: {
name1 <class 'pynwb\.base\.TimeSeries'>,
name2 <class 'pynwb\.base\.TimeSeries'>
}
epoch_tags: {
tag1,
tag2
}
epochs: epochs <class 'pynwb.epoch.TimeIntervals'>
file_create_date: \[datetime.datetime\(.*\)\]
identifier: identifier
session_description: session_description
session_start_time: .*
timestamps_reference_time: .*
"""
self.assertRegex(str(nwbfile), expected_re)
def test_print_file(self):
nwbfile = NWBFile(session_description='session_description',
identifier='identifier',
session_start_time=datetime.now(tzlocal()))
ts = TimeSeries('name', [1., 2., 3.] * 1000, timestamps=[1, 2, 3])
ts2 = TimeSeries('name2', [1, 2, 3] * 1000, timestamps=[1, 2, 3])
self.assertEqual(
str(ts), """
name <class 'pynwb.base.TimeSeries'>
Fields:
comments: no comments
conversion: 1.0
data: [1. 2. 3. ... 1. 2. 3.]
description: no description
interval: 1
num_samples: 3000
resolution: 0.0
timestamps: [1 2 3]
timestamps_unit: Seconds
""")
nwbfile.add_acquisition(ts)
nwbfile.add_acquisition(ts2)
nwbfile.add_epoch(start_time=1.0,
stop_time=10.0,
tags=['tag1', 'tag2'])
self.assertEqual(
str(nwbfile), """
root <class 'pynwb.file.NWBFile'>
Fields:
acquisition: { name <class 'pynwb.base.TimeSeries'>, name2 <class 'pynwb.base.TimeSeries'> }
analysis: { }
devices: { }
electrode_groups: { }
epoch_tags: {'tag1', 'tag2'}
epochs: epochs <class 'pynwb.epoch.TimeIntervals'>
ic_electrodes: { }
imaging_planes: { }
lab_meta_data: { }
modules: { }
ogen_sites: { }
stimulus: { }
stimulus_template: { }
time_intervals: { }
""")
def nwb_copy_file(old_file, new_file, cp_objs={}):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path
String such as '/path/to/old_file.nwb'.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
"""
manager = get_manager()
# Open original signal file
with NWBHDF5IO(old_file, 'r', manager=manager,
load_namespaces=True) as io1:
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(session_description=str(nwb_old.session_description),
identifier='',
session_start_time=datetime.now(tzlocal()))
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[
aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name)
)
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux])
)
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering',
'group', 'group_name']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [str(elem) for elem in nwb_old.electrodes[
var].data[:]]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(name=str(var),
description=
str(nwb_old.electrodes[
var].description),
data=var_data)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags',
'timeseries']
[new_vars.remove(var) for var in default_vars if
var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(name=var,
description=nwb_old.epochs[
var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(
start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(name=var,
description=nwb_old.trials[
var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
if cp_objs['ecephys'] is True:
interfaces = nwb_old.processing[
'ecephys'].data_interfaces.keys()
else: # list of items
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.'
)
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
if 'acquisition' in cp_objs:
if cp_objs['acquisition'] is True:
all_acq_names = list(nwb_old.acquisition.keys())
else: # list of items
all_acq_names = cp_objs['acquisition']
for acq_name in all_acq_names:
obj_old = nwb_old.acquisition[acq_name]
obj = copy_obj(obj_old, nwb_old, nwb_new)
if obj is not None:
nwb_new.add_acquisition(obj)
# Subject ---------------------------------------------------------
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
subject_id=nwb_old.subject.subject_id,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
# Write new file with copied fields
io2.write(nwb_new, link_data=False)
# 1 3.0 5.0 ocean
# 2 6.0 8.0 desert
#
# .. _basic_epochs:
#
# Epochs
# ~~~~~~~~~~~~~~~~~~~~~~
#
# Epochs can be added to an NWB file using the method :py:meth:`~pynwb.file.NWBFile.add_epoch`.
# The first and second arguments are the start time and stop times, respectively.
# The third argument is one or more tags for labeling the epoch, and the fourth argument is a
# list of all the :py:class:`~pynwb.base.TimeSeries` that the epoch applies
# to.
nwbfile.add_epoch(2.0, 4.0, ['first', 'example'], [
test_ts,
])
nwbfile.add_epoch(6.0, 8.0, ['second', 'example'], [
test_ts,
])
####################
# Other time intervals
# ~~~~~~~~~~~~~~~~~~~~~~
# Both ``epochs`` and ``trials`` are of of data type :py:class:`~pynwb.epoch.TimeIntervals`, which is a type of
# ``DynamicTable`` for storing information about time intervals. ``"epochs"`` and ``"trials"``
# are the two default names for :py:class:`~pynwb.base.TimeIntervals` objects, but you can also add your own
from pynwb.epoch import TimeIntervals
sleep_stages = TimeIntervals(
'test_spatial_data',
'a stumbling rat',
spatial_data,
'origin on x,y-plane',
timestamps=spatial_timestamps,
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with \
numpy.cumsum(numpy.random.normal(size=(2,len(spatial_timestamps))), axis=-1).T"
)
# Create experimental epochs
epoch_tags = ('test_example', )
ep1 = f.add_epoch('epoch1',
ephys_timestamps[100],
ephys_timestamps[200],
tags=epoch_tags,
description="the first test epoch")
ep2 = f.add_epoch('epoch2',
ephys_timestamps[600],
ephys_timestamps[700],
tags=epoch_tags,
description="the second test epoch")
# Add the time series data and include the epochs it is apart of
f.add_raw_timeseries(ephys_ts, [ep1, ep2])
f.add_raw_timeseries(spatial_ts, [ep1, ep2])
# Write the NWB file
manager = get_build_manager()
def run_conversion(
fpath_in='/Volumes/easystore5T/data/Brunton/subj_01_day_4.h5',
fpath_out='/Volumes/easystore5T/data/Brunton/subj_01_day_4.nwb',
events_path='C:/Users/micha/Desktop/Brunton Lab Data/event_times.csv',
r2_path='C:/Users/micha/Desktop/Brunton Lab Data/full_model_r2.npy',
coarse_events_path='C:/Users/micha/Desktop/Brunton Lab Data/coarse_labels/coarse_labels',
reach_features_path='C:/Users/micha/Desktop/Brunton Lab Data/behavioral_features.csv',
elec_loc_labels_path='elec_loc_labels.csv',
special_chans=SPECIAL_CHANNELS,
session_description='no description'
):
print(f"Converting {fpath_in}...")
fname = os.path.split(os.path.splitext(fpath_in)[0])[1]
_, subject_id, _, session = fname.split('_')
file = File(fpath_in, 'r')
nwbfile = NWBFile(
session_description=session_description,
identifier=str(uuid.uuid4()),
session_start_time=datetime.fromtimestamp(file['start_timestamp'][()]),
subject=Subject(subject_id=subject_id, species="H**o sapiens"),
session_id=session
)
# extract electrode groups
file_elec_col_names = file['chan_info']['axis1'][:]
elec_data = file['chan_info']['block0_values']
re_exp = re.compile("([ a-zA-Z]+)([0-9]+)")
channel_labels_dset = file['chan_info']['axis0']
group_names, group_nums = [], []
for i, bytes_ in enumerate(channel_labels_dset):
if bytes_ not in special_chans:
str_ = bytes_.decode()
res = re_exp.match(str_).groups()
group_names.append(res[0])
group_nums.append(int(res[1]))
is_elec = ~np.isin(channel_labels_dset, special_chans)
dset = DatasetView(file['dataset']).lazy_transpose()
# add special channels
for kwargs in (
dict(
name='EOGL',
description='Electrooculography for tracking saccades - left',
),
dict(
name='EOGR',
description='Electrooculography for tracking saccades - right',
),
dict(
name='ECGL',
description='Electrooculography for tracking saccades - left',
),
dict(
name='ECGR',
description='Electrooculography for tracking saccades - right',
)
):
if kwargs['name'].encode() in channel_labels_dset:
nwbfile.add_acquisition(
TimeSeries(
rate=file['f_sample'][()],
conversion=np.nan,
unit='V',
data=dset[:, list(channel_labels_dset).index(kwargs['name'].encode())],
**kwargs
)
)
# add electrode groups
df = pd.read_csv(elec_loc_labels_path)
df_subject = df[df['subject_ID'] == 'subj' + subject_id]
electrode_group_descriptions = {row['label']: row['long_name'] for _, row in df_subject.iterrows()}
groups_map = dict()
for group_name, group_description in electrode_group_descriptions.items():
device = nwbfile.create_device(name=group_name)
groups_map[group_name] = nwbfile.create_electrode_group(
name=group_name,
description=group_description,
device=device,
location='unknown'
)
# add required cols to electrodes table
for row, group_name in zip(elec_data[:].T, group_names):
nwbfile.add_electrode(
x=row[file_elec_col_names == b'X'][0],
y=row[file_elec_col_names == b'Y'][0],
z=row[file_elec_col_names == b'Z'][0],
imp=np.nan,
location='unknown',
filtering='250 Hz lowpass',
group=groups_map[group_name],
)
# load r2 values to input into custom cols in electrodes table
r2 = np.load(r2_path)
low_freq_r2 = np.ravel(r2[int(subject_id)-1, :len(group_names), 0])
high_freq_r2 = np.ravel(r2[int(subject_id)-1, :len(group_names), 1])
# add custom cols to electrodes table
elecs_dset = file['chan_info']['block0_values']
def get_data(label):
return elecs_dset[file_elec_col_names == label, :].ravel()[is_elec]
[nwbfile.add_electrode_column(**kwargs) for kwargs in (
dict(
name='standard_deviation',
description="standard deviation of each electrode's data for the entire recording period",
data=get_data(b'SD_channels')
),
dict(
name='kurtosis',
description="kurtosis of each electrode's data for the entire recording period",
data=get_data(b'Kurt_channels')
),
dict(
name='median_deviation',
description="median absolute deviation estimator for standard deviation for each electrode",
data=get_data(b'standardizeDenoms')
),
dict(
name='good',
description='good electrodes',
data=get_data(b'goodChanInds').astype(bool)
),
dict(
name='low_freq_R2',
description='R^2 for low frequency band on each electrode',
data=low_freq_r2
),
dict(
name='high_freq_R2',
description='R^2 for high frequency band on each electrode',
data=high_freq_r2
)
)]
# confirm that electrodes table looks right
# nwbfile.electrodes.to_dataframe()
# add ElectricalSeries
elecs_data = dset.lazy_slice[:, is_elec]
n_bytes = np.dtype(elecs_data).itemsize
nwbfile.add_acquisition(
ElectricalSeries(
name='ElectricalSeries',
data=H5DataIO(
data=DataChunkIterator(
data=elecs_data,
maxshape=elecs_data.shape,
buffer_size=int(5000 * 1e6) // elecs_data.shape[1] * n_bytes
),
compression='gzip'
),
rate=file['f_sample'][()],
conversion=1e-6, # data is in uV
electrodes=nwbfile.create_electrode_table_region(
region=list(range(len(nwbfile.electrodes))),
description='all electrodes'
)
)
)
# add pose data
pose_dset = file['pose_data']['block0_values']
nwbfile.create_processing_module(
name='behavior',
description='pose data').add(
Position(
spatial_series=[
SpatialSeries(
name=file['pose_data']['axis0'][x_ind][:-2].decode(),
data=H5DataIO(
data=pose_dset[:, [x_ind, y_ind]],
compression='gzip'
),
reference_frame='unknown',
conversion=np.nan,
rate=30.
) for x_ind, y_ind in zip(
range(0, pose_dset.shape[1], 2),
range(1, pose_dset.shape[1], 2))
]
)
)
# add events
events = pd.read_csv(events_path)
mask = (events['Subject'] == int(subject_id)) & (events['Recording day'] == int(session))
events = events[mask]
timestamps = events['Event time'].values
events = events.reset_index()
events = Events(
name='ReachEvents',
description=events['Event type'][0], # Specifies which arm was used
timestamps=timestamps,
resolution=2e-3, # resolution of the timestamps, i.e., smallest possible difference between timestamps
)
# add the Events type to the processing group of the NWB file
nwbfile.processing['behavior'].add(events)
# add coarse behavioral labels
event_fp = f'sub{subject_id}_fullday_{session}'
full_fp = coarse_events_path + '//' + event_fp + '.npy'
coarse_events = np.load(full_fp, allow_pickle=True)
label, data = np.unique(coarse_events, return_inverse=True)
transition_idx = np.where(np.diff(data) != 0)
start_t = nwbfile.processing["behavior"].data_interfaces["Position"]['L_Wrist'].starting_time
rate = nwbfile.processing["behavior"].data_interfaces["Position"]['L_Wrist'].rate
times = np.divide(transition_idx, rate) + start_t # 30Hz sampling rate
max_time = (np.shape(coarse_events)[0] / rate) + start_t
times = np.hstack([start_t, np.ravel(times), max_time])
transition_labels = np.hstack([label[data[transition_idx]], label[data[-1]]])
nwbfile.add_epoch_column(name='labels', description='Coarse behavioral labels')
for start_time, stop_time, label in zip(times[:-1], times[1:], transition_labels):
nwbfile.add_epoch(start_time=start_time, stop_time=stop_time, labels=label)
# add additional reaching features
reach_features = pd.read_csv(reach_features_path)
mask = (reach_features['Subject'] == int(subject_id)) & (reach_features['Recording day'] == int(session))
reach_features = reach_features[mask]
reaches = TimeIntervals(name='reaches', description='Features of each reach')
reaches.add_column(name='Reach_magnitude_px', description='Magnitude of reach in pixels')
reaches.add_column(name='Reach_angle_degrees', description='Reach angle in degrees')
reaches.add_column(name='Onset_speed_px_per_sec', description='Onset speed in pixels / second)')
reaches.add_column(name='Speech_ratio', description='rough estimation of whether someone is likely to be speaking '
'based on a power ratio of audio data; ranges from 0 (no '
'speech) to 1 (high likelihood of speech)h')
reaches.add_column(name='Bimanual_ratio', description='ratio of ipsilateral wrist reach magnitude to the sum of '
'ipsilateral and contralateral wrist magnitudes; ranges from '
'0 (unimanual/contralateral move only) to 1 (only ipsilateral'
' arm moving); 0.5 indicates bimanual movement')
reaches.add_column(name='Bimanual_overlap', description='The amount of ipsilateral and contralateral wrist temporal'
'overlap as a fraction of the entire contralateral movement'
' duration')
reaches.add_column(name='Bimanual_class', description='binary feature that classifies each movement event as '
'unimanual (0) or bimanual (1) based on how close in time a '
'ipsilateral wrist movement started relative to each '
'contralateral wrist movement events')
for row in reach_features.iterrows():
row_data = row[1]
start_time = row_data['Time of day (sec)']
stop_time = start_time + row_data['Reach duration (sec)']
reaches.add_row(start_time=start_time,
stop_time=stop_time,
Reach_magnitude_px=row_data['Reach magnitude (px)'],
Reach_angle_degrees=row_data['Reach angle (degrees)'],
Onset_speed_px_per_sec=row_data['Onset speed (px/sec)'],
Speech_ratio=row_data['Speech ratio'],
Bimanual_ratio=row_data['Bimanual ratio'],
Bimanual_overlap=row_data['Bimanual overlap (sec)'],
Bimanual_class=row_data['Bimanual class']
)
nwbfile.add_time_intervals(reaches)
with NWBHDF5IO(fpath_out, 'w') as io:
io.write(nwbfile)
class NWBFileTest(TestCase):
def setUp(self):
self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create = [
datetime(2017, 5, 1, 12, tzinfo=tzlocal()),
datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()),
datetime(2017, 5, 2, 14, tzinfo=tzutc())
]
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile(
'a test session description for a test NWBFile',
'FILE123',
self.start,
file_create_date=self.create,
timestamps_reference_time=self.ref_time,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
notes='my notes',
pharmacology='drugs',
protocol='protocol',
related_publications='my pubs',
slices='my slices',
surgery='surgery',
virus='a virus',
source_script='noscript',
source_script_file_name='nofilename',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
keywords=('these', 'are', 'keywords'))
def test_constructor(self):
self.assertEqual(self.nwbfile.session_description,
'a test session description for a test NWBFile')
self.assertEqual(self.nwbfile.identifier, 'FILE123')
self.assertEqual(self.nwbfile.session_start_time, self.start)
self.assertEqual(self.nwbfile.file_create_date, self.create)
self.assertEqual(self.nwbfile.lab, 'a test lab')
self.assertEqual(self.nwbfile.experimenter, ('A test experimenter', ))
self.assertEqual(self.nwbfile.institution, 'a test institution')
self.assertEqual(self.nwbfile.experiment_description,
'a test experiment description')
self.assertEqual(self.nwbfile.session_id, 'test1')
self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes')
self.assertEqual(self.nwbfile.data_collection,
'test data collection notes')
self.assertEqual(self.nwbfile.related_publications, ('my pubs', ))
self.assertEqual(self.nwbfile.source_script, 'noscript')
self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename')
self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords'))
self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time)
def test_create_electrode_group(self):
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
d = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d)
self.assertEqual(elecgrp.description, desc)
self.assertEqual(elecgrp.location, loc)
self.assertIs(elecgrp.device, d)
def test_create_custom_intervals(self):
df_words = pd.DataFrame({
'start_time': [.1, 2.],
'stop_time': [.8, 2.3],
'label': ['hello', 'there']
})
words = TimeIntervals.from_dataframe(df_words, name='words')
self.nwbfile.add_time_intervals(words)
self.assertEqual(self.nwbfile.intervals['words'], words)
def test_create_electrode_group_invalid_index(self):
"""
Test the case where the user creates an electrode table region with
indexes that are out of range of the amount of electrodes added.
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i)
with self.assertRaises(IndexError):
nwbfile.create_electrode_table_region(list(range(6)), 'test')
def test_access_group_after_io(self):
"""
Motivated by #739
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i + 1)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
remove_test_file("electrodes_mwe.nwb")
def test_access_processing(self):
self.nwbfile.create_processing_module('test_mod', 'test_description')
# test deprecate .modules
with self.assertWarnsWith(DeprecationWarning,
'replaced by NWBFile.processing'):
modules = self.nwbfile.modules['test_mod']
self.assertIs(self.nwbfile.processing['test_mod'], modules)
def test_epoch_tags(self):
tags1 = ['t1', 't2']
tags2 = ['t3', 't4']
tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float)
ts = TimeSeries("test_ts",
list(range(len(tstamps))),
'unit',
timestamps=tstamps)
expected_tags = tags1 + tags2
self.nwbfile.add_epoch(0.0, 1.0, tags1, ts)
self.nwbfile.add_epoch(0.0, 1.0, tags2, ts)
tags = self.nwbfile.epoch_tags
self.assertEqual(set(expected_tags), set(tags))
def test_add_acquisition(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.acquisition), 1)
def test_add_stimulus(self):
self.nwbfile.add_stimulus(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus), 1)
def test_add_stimulus_template(self):
self.nwbfile.add_stimulus_template(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus_template), 1)
def test_add_analysis(self):
self.nwbfile.add_analysis(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.analysis), 1)
def test_add_acquisition_check_dups(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
with self.assertRaises(ValueError):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
def test_get_acquisition_empty(self):
with self.assertRaisesWith(ValueError,
"acquisition of NWBFile 'root' is empty"):
self.nwbfile.get_acquisition()
def test_get_acquisition_multiple_elements(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.nwbfile.add_acquisition(
TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name"
with self.assertRaisesWith(ValueError, msg):
self.nwbfile.get_acquisition()
def test_add_acquisition_invalid_name(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "\"'TEST_TS' not found in acquisition of NWBFile 'root'\""
with self.assertRaisesWith(KeyError, msg):
self.nwbfile.get_acquisition("TEST_TS")
def test_set_electrode_table(self):
table = ElectrodeTable()
dev1 = self.nwbfile.create_device('dev1')
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-2.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-3.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-4.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
self.nwbfile.set_electrode_table(table)
self.assertIs(self.nwbfile.electrodes, table)
self.assertIs(table.parent, self.nwbfile)
def test_add_unit_column(self):
self.nwbfile.add_unit_column('unit_type', 'the type of unit')
self.assertEqual(self.nwbfile.units.colnames, ('unit_type', ))
def test_add_unit(self):
self.nwbfile.add_unit(id=1)
self.assertEqual(len(self.nwbfile.units), 1)
self.nwbfile.add_unit(id=2)
self.nwbfile.add_unit(id=3)
self.assertEqual(len(self.nwbfile.units), 3)
def test_add_trial_column(self):
self.nwbfile.add_trial_column('trial_type', 'the type of trial')
self.assertEqual(self.nwbfile.trials.colnames,
('start_time', 'stop_time', 'trial_type'))
def test_add_trial(self):
self.nwbfile.add_trial(start_time=10.0, stop_time=20.0)
self.assertEqual(len(self.nwbfile.trials), 1)
self.nwbfile.add_trial(start_time=30.0, stop_time=40.0)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.assertEqual(len(self.nwbfile.trials), 3)
def test_add_invalid_times_column(self):
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.assertEqual(self.nwbfile.invalid_times.colnames,
('start_time', 'stop_time', 'comments'))
def test_add_invalid_time_interval(self):
self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0)
self.assertEqual(len(self.nwbfile.invalid_times), 1)
self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0)
self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5)
self.assertEqual(len(self.nwbfile.invalid_times), 3)
def test_add_invalid_time_w_ts(self):
ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na')
self.nwbfile.add_invalid_time_interval(start_time=18.0,
stop_time=20.6,
timeseries=ts,
tags=('hi', 'there'))
def test_add_electrode(self):
dev1 = self.nwbfile.create_device('dev1')
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
self.nwbfile.add_electrode(1.0,
2.0,
3.0,
-1.0,
'CA1',
'none',
group=group,
id=1)
elec = self.nwbfile.electrodes[0]
self.assertEqual(elec.index[0], 1)
self.assertEqual(elec.iloc[0]['x'], 1.0)
self.assertEqual(elec.iloc[0]['y'], 2.0)
self.assertEqual(elec.iloc[0]['z'], 3.0)
self.assertEqual(elec.iloc[0]['location'], 'CA1')
self.assertEqual(elec.iloc[0]['filtering'], 'none')
self.assertEqual(elec.iloc[0]['group'], group)
def test_all_children(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
device = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device)
children = self.nwbfile.all_children()
self.assertIn(ts1, children)
self.assertIn(ts2, children)
self.assertIn(device, children)
self.assertIn(elecgrp, children)
def test_fail_if_source_script_file_name_without_source_script(self):
with self.assertRaises(ValueError):
# <-- source_script_file_name without source_script is not allowed
NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
source_script=None,
source_script_file_name='nofilename')
def test_get_neurodata_type(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
p1 = ts1.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p1, self.nwbfile)
p2 = ts2.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p2, self.nwbfile)
def test_print_units(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
expected = """units pynwb.misc.Units at 0x%d
Fields:
colnames: ['spike_times']
columns: (
spike_times_index <class 'hdmf.common.table.VectorIndex'>,
spike_times <class 'hdmf.common.table.VectorData'>
)
description: Autogenerated by NWBFile
id: id <class 'hdmf.common.table.ElementIdentifiers'>
"""
expected = expected % id(self.nwbfile.units)
self.assertEqual(str(self.nwbfile.units), expected)
def test_copy(self):
self.nwbfile.add_unit(spike_times=[1., 2., 3.])
device = self.nwbfile.create_device('a')
elecgrp = self.nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
self.nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
self.nwbfile.add_electrode(np.nan, np.nan, np.nan, np.nan, 'b', 'b',
elecgrp)
elec_region = self.nwbfile.create_electrode_table_region([1], 'name')
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = ElectricalSeries('test_ts2', [0, 1, 2, 3, 4, 5],
electrodes=elec_region,
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.nwbfile.create_processing_module('test_mod', 'test_description')
self.nwbfile.create_time_intervals('custom_interval',
'a custom time interval')
self.nwbfile.intervals['custom_interval'].add_interval(start_time=10.,
stop_time=20.)
newfile = self.nwbfile.copy()
# test dictionaries
self.assertIs(self.nwbfile.devices['a'], newfile.devices['a'])
self.assertIs(self.nwbfile.acquisition['test_ts1'],
newfile.acquisition['test_ts1'])
self.assertIs(self.nwbfile.acquisition['test_ts2'],
newfile.acquisition['test_ts2'])
self.assertIs(self.nwbfile.processing['test_mod'],
newfile.processing['test_mod'])
# test dynamic tables
self.assertIsNot(self.nwbfile.electrodes, newfile.electrodes)
self.assertIs(self.nwbfile.electrodes['x'], newfile.electrodes['x'])
self.assertIsNot(self.nwbfile.units, newfile.units)
self.assertIs(self.nwbfile.units['spike_times'],
newfile.units['spike_times'])
self.assertIsNot(self.nwbfile.trials, newfile.trials)
self.assertIsNot(self.nwbfile.trials.parent, newfile.trials.parent)
self.assertIs(self.nwbfile.trials.id, newfile.trials.id)
self.assertIs(self.nwbfile.trials['start_time'],
newfile.trials['start_time'])
self.assertIs(self.nwbfile.trials['stop_time'],
newfile.trials['stop_time'])
self.assertIsNot(self.nwbfile.invalid_times, newfile.invalid_times)
self.assertTupleEqual(self.nwbfile.invalid_times.colnames,
newfile.invalid_times.colnames)
self.assertIsNot(self.nwbfile.intervals['custom_interval'],
newfile.intervals['custom_interval'])
self.assertTupleEqual(
self.nwbfile.intervals['custom_interval'].colnames,
newfile.intervals['custom_interval'].colnames)
self.assertIs(self.nwbfile.intervals['custom_interval']['start_time'],
newfile.intervals['custom_interval']['start_time'])
self.assertIs(self.nwbfile.intervals['custom_interval']['stop_time'],
newfile.intervals['custom_interval']['stop_time'])
def test_multi_experimenters(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
experimenter=('experimenter1', 'experimenter2'))
self.assertTupleEqual(self.nwbfile.experimenter,
('experimenter1', 'experimenter2'))
def test_multi_publications(self):
self.nwbfile = NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
related_publications=('pub1', 'pub2'))
self.assertTupleEqual(self.nwbfile.related_publications,
('pub1', 'pub2'))
def create_nwb_file(Sess, start_time):
sr = 30000 #30kHz
if sys.platform == 'win32':
SaveDir = os.path.join(r'C:\Users\slashchevam\Desktop\NPx\Results',
Sess)
RawDataDir = r'C:\Users\slashchevam\Desktop\NPx'
ExcelInfoPath = RawDataDir
PathToUpload = os.path.join(RawDataDir, Sess)
if sys.platform == 'linux':
SaveDir = os.path.join('/mnt/gs/departmentN4/Marina/NPx/Results', Sess)
RawDataDir = '/mnt/gs/projects/OWVinckNatIm/NPx_recordings/'
PAthToAnalyzed = '/experiment1/recording1/continuous/Neuropix-PXI-100.0/'
MatlabOutput = '/mnt/gs/projects/OWVinckNatIm/NPx_processed/Lev0_condInfo/'
ExcelInfoPath = '/mnt/gs/departmentN4/Marina/'
PathToUpload = RawDataDir + Sess + PAthToAnalyzed
if not os.path.exists(SaveDir):
os.makedirs(SaveDir)
os.chdir(SaveDir)
# Upload all the data
spike_stamps = np.load(os.path.join(PathToUpload, "spike_times.npy"))
spike_times = spike_stamps / sr
spike_clusters = np.load(os.path.join(PathToUpload, "spike_clusters.npy"))
cluster_group = pd.read_csv(os.path.join(PathToUpload,
"cluster_group.tsv"),
sep="\t")
cluster_info = pd.read_csv(os.path.join(PathToUpload, "cluster_info.tsv"),
sep="\t")
if len(cluster_group) != len(cluster_info):
print('Cluster group (manual labeling) and claster info do not match!')
#excel_info = pd.read_excel((ExcelInfoPath + '\\Recordings_Marina_NPx.xlsx'), sheet_name=Sess)
excel_info = pd.read_excel(os.path.join(ExcelInfoPath +
'Recordings_Marina_NPx.xlsx'),
sheet_name=Sess)
# Select spikes from good clusters only
# Have to add the depth of the clusters
good_clus_info = cluster_info[cluster_info['group'] ==
'good'] # has depth info
good_clus = good_clus_info[['id', 'group']]
print("Found", len(good_clus), ' good clusters')
good_spikes_ind = [x in good_clus['id'].values for x in spike_clusters]
spike_clus_good = spike_clusters[good_spikes_ind]
spike_times_good = spike_times[good_spikes_ind]
# spike_stamps_good = spike_stamps[good_spikes_ind]
if excel_info['Area'][0] == 'V1':
good_clus_info['area'] = 'V1'
else:
good_clus_info['area'] = good_clus_info['depth'] > np.max(
good_clus_info['depth']) - 1000
good_clus_info['area'] = good_clus_info['area'].replace(True, 'V1')
good_clus_info['area'] = good_clus_info['area'].replace(False, 'HPC')
del spike_clusters, spike_times, spike_stamps, good_spikes_ind
# Now reading digitals from condInfo
# This has to be checked carefully again, especially for few stimuli in the session
# cond class contains the following:
# 'spontaneous_brightness': dict_keys(['name', 'time', 'timestamps', 'trl_list', 'conf'])
# 'natural_images': dict_keys(['name', 'time', 'timestamps', 'trl_list', 'conf', 'img_order', 'img_name'])
class condInfo:
pass
if sys.platform == 'linux':
mat = scipy.io.loadmat(
os.path.join((MatlabOutput + Sess), 'condInfo_01.mat'))
if sys.platform == 'win32':
mat = scipy.io.loadmat(os.path.join(PathToUpload, 'condInfo_01.mat'))
SC_stim_labels = mat['StimClass'][0][0][0][0]
SC_stim_present = np.where(mat['StimClass'][0][0][1][0] == 1)[0]
SC_stim_labels_present = SC_stim_labels[SC_stim_present]
cond = [condInfo() for i in range(len(SC_stim_labels_present))]
for stim in range(len(SC_stim_labels_present)):
cond[stim].name = SC_stim_labels_present[stim][0]
cond[stim].stiminfo = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][1] # image indices are here
# sorting out digitals for spontaneous activity
# Need this loop in case there are few periods of spont, recorded like separate blocks
if SC_stim_labels_present[stim][0] == 'spontaneous_brightness':
cond[stim].time = []
cond[stim].timestamps = []
for block in range(
len(mat['StimClass'][0][0][3][0, SC_stim_present[stim]][0]
[0])):
print(block)
cond[stim].time.append(mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][block][2])
cond[stim].timestamps.append(mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][block][3])
cond[stim].trl_list = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][1]
cond[stim].conf = mat['StimClass'][0][0][2][
0,
SC_stim_present[stim]] # config is very likely wrong and useless
# sorting out digitals for natural images
if SC_stim_labels_present[stim][0] == 'natural_images':
cond[stim].time = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][2]
cond[stim].timestamps = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][3]
img_order = []
for i in range(len(cond[stim].stiminfo)):
img_order.append(int(cond[stim].stiminfo[i][2]))
cond[stim].img_order = img_order
cond[stim].img_name = cond[stim].conf[0][0][0][10][
0] # currently not used but might be needed later
# sorting out digitals for drifting gratings
if SC_stim_labels_present[stim][0] == 'drifting_gratings':
cond[stim].time = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][2]
cond[stim].timestamps = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][3]
dg_orient = []
for i in range(len(cond[stim].stiminfo)):
dg_orient.append(int(cond[stim].stiminfo[i][2]))
cond[stim].dg_orient = dg_orient
# Now create NWB file
start_time = start_time # datetime(2020, 2, 27, 14, 36, 7, tzinfo=tzlocal())
nwb_subject = Subject(description="Pretty nice girl",
sex='F',
species='mouse',
subject_id=excel_info['Mouse'].values[0],
genotype=excel_info['Genotype'].values[0])
nwbfile = NWBFile(
session_description=
"NPx recording of Natural images and spontaneous activity",
session_id=Sess,
identifier='NWB123',
session_start_time=start_time,
experimenter='Marina Slashcheva',
institution='ESI, Frankfurt',
lab='Martin Vinck',
notes=' | '.join(
[x for x in list(excel_info['Note'].values) if str(x) != 'nan']),
protocol=' | '.join([
x for x in list(excel_info['experiment'].values) if str(x) != 'nan'
]),
data_collection=
'Ref: {}, Probe_angle: {}, , Depth: {}, APcoord: {}, MLcoord: {}, Recday: {}, Hemi: {}'
.format(excel_info['refCh'].values[0],
excel_info['Angle_probe'].values[0],
excel_info['Depth'].values[0],
excel_info['anteroposterior'].values[0],
excel_info['mediolateral'].values[0],
excel_info['Recday'].values[0],
excel_info['Hemisphere'].values[0]),
subject=nwb_subject)
# Did not add it for the moment, later add running as a timeseries and add to HDF5 as binary parameter
# test_ts = TimeSeries(name='test_timeseries', data=data, unit='m', timestamps=timestamps)
# Add units
nwbfile.add_unit_column(
'location',
'the anatomical location of this unit') # to be added and CHECKED
nwbfile.add_unit_column('depth', 'depth on the NPx probe')
nwbfile.add_unit_column('channel', 'channel on the NPx probe')
nwbfile.add_unit_column('fr', 'average FR according to KS')
for un in good_clus_info['id']:
info_tmp = good_clus_info[good_clus_info['id'] == un]
spike_times_tmp = spike_times_good[spike_clus_good == un]
nwbfile.add_unit(id=un,
spike_times=np.transpose(spike_times_tmp)[0],
location=info_tmp['area'].values[0],
depth=info_tmp['depth'].values[0],
channel=info_tmp['ch'].values[0],
fr=info_tmp['fr'].values[0])
del spike_times_tmp
# Add epochs
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
#if len(cond[ep].time) > 1:
for bl in range(len(cond[ep].time)):
nwbfile.add_epoch(cond[ep].time[bl][0][0],
cond[ep].time[bl][0][1], cond[ep].name)
#else:
# nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[0][1], cond[ep].name)
if cond[ep].name == 'natural_images':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[-1][1],
cond[ep].name)
if cond[ep].name == 'drifting_gratings':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[-1][1],
cond[ep].name)
# Add trials
# Images names can be also added here
nwbfile.add_trial_column(
name='start', description='start time relative to the stimulus onset')
nwbfile.add_trial_column(
name='stimset',
description='the visual stimulus type during the trial')
nwbfile.add_trial_column(name='img_id',
description='image ID for Natural Images')
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
#if len(cond[ep].time) > 1:
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0][0],
stop_time=cond[ep].time[tr][0][1],
start=cond[ep].time[tr][0][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=('gray').encode('utf8'))
# else:
# nwbfile.add_trial(start_time = cond[ep].time[0][0], stop_time = cond[ep].time[0][1],
# start = cond[ep].time[0][2],
# stimset = (cond[ep].name).encode('utf8'),
# img_id = ('gray').encode('utf8'))
if cond[ep].name == 'natural_images':
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0],
stop_time=cond[ep].time[tr][1],
start=cond[ep].time[tr][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=(str(
cond[ep].img_order[tr])).encode('utf8'))
if cond[ep].name == 'drifting_gratings':
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0],
stop_time=cond[ep].time[tr][1],
start=cond[ep].time[tr][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=(str(
cond[ep].dg_orient[tr])).encode('utf8'))
# Write NWB file
os.chdir(SaveDir)
name_to_save = Sess + '.nwb'
io = NWBHDF5IO(name_to_save, manager=get_manager(), mode='w')
io.write(nwbfile)
io.close()
del nwbfile
running_speed = TimeSeries(name='running_speed',
data=running_data,
timestamps=timestamps,
unit='cm/s')
nwbfile.add_acquisition(running_speed)
########################################
# 4) In NWB:N, an "epoch" is an interval of experiment time that can slice into a timeseries (for example running_speed,
# the one we just added). PyNWB uses an object-oriented approach to create links into these timeseries, so that data is
# not copied multiple times. Here, we extract the stimulus epochs (both fine and coarse-grained) from the Brain
# Observatory experiment using the allensdk.
for _, row in trial_table.iterrows():
nwbfile.add_epoch(start_time=row.start,
stop_time=row.end,
timeseries=[running_speed],
tags='trials')
for _, row in epoch_table.iterrows():
nwbfile.add_epoch(start_time=row.start,
stop_time=row.end,
timeseries=[running_speed],
tags='stimulus')
########################################
# 5) In the brain observatory, a two-photon microscope is used to acquire images of the calcium activity of neurons
# expressing a fluorescent protien indicator. Essentially the microscope captures picture (30 times a second) at a
# single depth in the visual cortex (the imaging plane). Let's use pynwb to store the metadata associated with this
# hardware and experimental setup:
optical_channel = OpticalChannel(
name='optical_channel',
def convert_data(self,
nwbfile: NWBFile,
metadata_dict: dict,
stub_test: bool = False,
include_spike_waveforms: bool = False):
session_path = self.input_args['folder_path']
# TODO: check/enforce format?
task_types = metadata_dict['task_types']
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
sleep_state_fpath = os.path.join(
session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(
os.path.isfile(
os.path.join(
session_path, '{}__{}.mat'.format(session_id,
task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path,
session_id + '__' + label + '.mat')
if os.path.isfile(file):
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown',
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, 'behavior',
'contains processed behavioral data').add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0],
stop_time=window[1],
label=label + '_' + str(i))
trialdata_path = os.path.join(session_path,
session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base,
'EightMazeRunInfo.mat')
trialdatainfo = [
x[0]
for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]
]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[
nwbfile.add_trial_column(x, 'description')
for x in features[4:] + ['condition']
]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6])
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states',
description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({
'start_time': row[0],
'stop_time': row[1],
'label': name
})
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x['start_time'])
]
check_module(nwbfile, 'behavior',
'contains behavioral data').add_data_interface(table)
class NWBFileTest(unittest.TestCase):
def setUp(self):
self.start = datetime(2017, 5, 1, 12, 0, 0, tzinfo=tzlocal())
self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc())
self.create = [
datetime(2017, 5, 1, 12, tzinfo=tzlocal()),
datetime(2017, 5, 2, 13, 0, 0, 1, tzinfo=tzutc()),
datetime(2017, 5, 2, 14, tzinfo=tzutc())
]
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile(
'a test session description for a test NWBFile',
'FILE123',
self.start,
file_create_date=self.create,
timestamps_reference_time=self.ref_time,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
notes='my notes',
pharmacology='drugs',
protocol='protocol',
related_publications='my pubs',
slices='my slices',
surgery='surgery',
virus='a virus',
source_script='noscript',
source_script_file_name='nofilename',
stimulus_notes='test stimulus notes',
data_collection='test data collection notes',
keywords=('these', 'are', 'keywords'))
def test_constructor(self):
self.assertEqual(self.nwbfile.session_description,
'a test session description for a test NWBFile')
self.assertEqual(self.nwbfile.identifier, 'FILE123')
self.assertEqual(self.nwbfile.session_start_time, self.start)
self.assertEqual(self.nwbfile.file_create_date, self.create)
self.assertEqual(self.nwbfile.lab, 'a test lab')
self.assertEqual(self.nwbfile.experimenter, 'A test experimenter')
self.assertEqual(self.nwbfile.institution, 'a test institution')
self.assertEqual(self.nwbfile.experiment_description,
'a test experiment description')
self.assertEqual(self.nwbfile.session_id, 'test1')
self.assertEqual(self.nwbfile.stimulus_notes, 'test stimulus notes')
self.assertEqual(self.nwbfile.data_collection,
'test data collection notes')
self.assertEqual(self.nwbfile.source_script, 'noscript')
self.assertEqual(self.nwbfile.source_script_file_name, 'nofilename')
self.assertEqual(self.nwbfile.keywords, ('these', 'are', 'keywords'))
self.assertEqual(self.nwbfile.timestamps_reference_time, self.ref_time)
def test_create_electrode_group(self):
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
d = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, d)
self.assertEqual(elecgrp.description, desc)
self.assertEqual(elecgrp.location, loc)
self.assertIs(elecgrp.device, d)
def test_create_electrode_group_invalid_index(self):
"""
Test the case where the user creates an electrode table region with
indexes that are out of range of the amount of electrodes added.
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i)
with self.assertRaises(IndexError):
nwbfile.create_electrode_table_region(list(range(6)), 'test')
def test_access_group_after_io(self):
"""
Motivated by #739
"""
nwbfile = NWBFile('a', 'b', datetime.now(tzlocal()))
device = nwbfile.create_device('a')
elecgrp = nwbfile.create_electrode_group('a',
'b',
device=device,
location='a')
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=0)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
for i in range(4):
nwbfile.add_electrode(np.nan,
np.nan,
np.nan,
np.nan,
'a',
'a',
elecgrp,
id=i + 1)
with NWBHDF5IO('electrodes_mwe.nwb', 'w') as io:
io.write(nwbfile)
with NWBHDF5IO('electrodes_mwe.nwb', 'a') as io:
nwbfile_i = io.read()
for aa, bb in zip(nwbfile_i.electrodes['group'][:],
nwbfile.electrodes['group'][:]):
self.assertEqual(aa.name, bb.name)
os.remove("electrodes_mwe.nwb")
def test_epoch_tags(self):
tags1 = ['t1', 't2']
tags2 = ['t3', 't4']
tstamps = np.arange(1.0, 100.0, 0.1, dtype=np.float)
ts = TimeSeries("test_ts",
list(range(len(tstamps))),
'unit',
timestamps=tstamps)
expected_tags = tags1 + tags2
self.nwbfile.add_epoch(0.0, 1.0, tags1, ts)
self.nwbfile.add_epoch(0.0, 1.0, tags2, ts)
tags = self.nwbfile.epoch_tags
six.assertCountEqual(self, expected_tags, tags)
def test_add_acquisition(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.acquisition), 1)
def test_add_stimulus(self):
self.nwbfile.add_stimulus(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus), 1)
def test_add_stimulus_template(self):
self.nwbfile.add_stimulus_template(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.assertEqual(len(self.nwbfile.stimulus_template), 1)
def test_add_acquisition_check_dups(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
with self.assertRaises(ValueError):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
def test_get_acquisition_empty(self):
with self.assertRaisesRegex(ValueError,
"acquisition of NWBFile 'root' is empty"):
self.nwbfile.get_acquisition()
def test_get_acquisition_multiple_elements(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
self.nwbfile.add_acquisition(
TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "more than one element in acquisition of NWBFile 'root' -- must specify a name"
with self.assertRaisesRegex(ValueError, msg):
self.nwbfile.get_acquisition()
def test_add_acquisition_invalid_name(self):
self.nwbfile.add_acquisition(
TimeSeries('test_ts', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5]))
msg = "'TEST_TS' not found in acquisition of NWBFile 'root'"
with self.assertRaisesRegex(KeyError, msg):
self.nwbfile.get_acquisition("TEST_TS")
def test_set_electrode_table(self):
table = ElectrodeTable() # noqa: F405
dev1 = self.nwbfile.create_device('dev1') # noqa: F405
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-1.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-2.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-3.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
table.add_row(x=1.0,
y=2.0,
z=3.0,
imp=-4.0,
location='CA1',
filtering='none',
group=group,
group_name='tetrode1')
self.nwbfile.set_electrode_table(table)
self.assertIs(self.nwbfile.electrodes, table)
self.assertIs(table.parent, self.nwbfile)
def test_add_unit_column(self):
self.nwbfile.add_unit_column('unit_type', 'the type of unit')
self.assertEqual(self.nwbfile.units.colnames, ('unit_type', ))
def test_add_unit(self):
self.nwbfile.add_unit(id=1)
self.assertEqual(len(self.nwbfile.units), 1)
self.nwbfile.add_unit(id=2)
self.nwbfile.add_unit(id=3)
self.assertEqual(len(self.nwbfile.units), 3)
def test_add_trial_column(self):
self.nwbfile.add_trial_column('trial_type', 'the type of trial')
self.assertEqual(self.nwbfile.trials.colnames,
('start_time', 'stop_time', 'trial_type'))
def test_add_trial(self):
self.nwbfile.add_trial(start_time=10.0, stop_time=20.0)
self.assertEqual(len(self.nwbfile.trials), 1)
self.nwbfile.add_trial(start_time=30.0, stop_time=40.0)
self.nwbfile.add_trial(start_time=50.0, stop_time=70.0)
self.assertEqual(len(self.nwbfile.trials), 3)
def test_add_invalid_times_column(self):
self.nwbfile.add_invalid_times_column(
'comments', 'description of reason for omitting time')
self.assertEqual(self.nwbfile.invalid_times.colnames,
('start_time', 'stop_time', 'comments'))
def test_add_invalid_time_interval(self):
self.nwbfile.add_invalid_time_interval(start_time=0.0, stop_time=12.0)
self.assertEqual(len(self.nwbfile.invalid_times), 1)
self.nwbfile.add_invalid_time_interval(start_time=15.0, stop_time=16.0)
self.nwbfile.add_invalid_time_interval(start_time=17.0, stop_time=20.5)
self.assertEqual(len(self.nwbfile.invalid_times), 3)
def test_add_invalid_time_w_ts(self):
ts = TimeSeries(name='name', data=[1.2], rate=1.0, unit='na')
self.nwbfile.add_invalid_time_interval(start_time=18.0,
stop_time=20.6,
timeseries=ts,
tags=('hi', 'there'))
def test_add_electrode(self):
dev1 = self.nwbfile.create_device('dev1') # noqa: F405
group = self.nwbfile.create_electrode_group('tetrode1',
'tetrode description',
'tetrode location', dev1)
self.nwbfile.add_electrode(1.0,
2.0,
3.0,
-1.0,
'CA1',
'none',
group=group,
id=1)
self.assertEqual(self.nwbfile.ec_electrodes[0][0], 1)
self.assertEqual(self.nwbfile.ec_electrodes[0][1], 1.0)
self.assertEqual(self.nwbfile.ec_electrodes[0][2], 2.0)
self.assertEqual(self.nwbfile.ec_electrodes[0][3], 3.0)
self.assertEqual(self.nwbfile.ec_electrodes[0][4], -1.0)
self.assertEqual(self.nwbfile.ec_electrodes[0][5], 'CA1')
self.assertEqual(self.nwbfile.ec_electrodes[0][6], 'none')
self.assertEqual(self.nwbfile.ec_electrodes[0][7], group)
def test_all_children(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
name = 'example_electrode_group'
desc = 'An example electrode'
loc = 'an example location'
device = self.nwbfile.create_device('a fake device')
elecgrp = self.nwbfile.create_electrode_group(name, desc, loc, device)
children = self.nwbfile.all_children()
self.assertIn(ts1, children)
self.assertIn(ts2, children)
self.assertIn(device, children)
self.assertIn(elecgrp, children)
def test_fail_if_source_script_file_name_without_source_script(self):
with self.assertRaises(ValueError):
# <-- source_script_file_name without source_script is not allowed
NWBFile('a test session description for a test NWBFile',
'FILE123',
self.start,
source_script=None,
source_script_file_name='nofilename')
def test_get_neurodata_type(self):
ts1 = TimeSeries('test_ts1', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
ts2 = TimeSeries('test_ts2', [0, 1, 2, 3, 4, 5],
'grams',
timestamps=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5])
self.nwbfile.add_acquisition(ts1)
self.nwbfile.add_acquisition(ts2)
p1 = ts1.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p1, self.nwbfile)
p2 = ts2.get_ancestor(neurodata_type='NWBFile')
self.assertIs(p2, self.nwbfile)
def chang2nwb(blockpath,
outpath=None,
session_start_time=None,
session_description=None,
identifier=None,
anin4=False,
ecog_format='auto',
external_subject=True,
include_pitch=False,
include_intensity=False,
speakers=True,
mic=False,
mini=False,
hilb=False,
verbose=False,
imaging_path=None,
parse_transcript=False,
include_cortical_surfaces=True,
include_electrodes=True,
include_ekg=True,
subject_image_list=None,
rest_period=None,
load_warped=False,
**kwargs):
"""
Parameters
----------
blockpath: str
outpath: None | str
if None, output = [blockpath]/[blockname].nwb
session_start_time: datetime.datetime
default: datetime(1900, 1, 1)
session_description: str
default: blockname
identifier: str
default: blockname
anin4: False | str
Whether or not to convert ANIN4. ANIN4 is used as an extra channel for
things like button presses, and is usually unused. If a string is
supplied, that is used as the name of the timeseries.
ecog_format: str
({'htk'}, 'mat', 'raw')
external_subject: bool (optional)
True: (default) cortical mesh is saved in an external file and a link is
provided to that file. This is useful if you have multiple sessions for a single subject.
False: cortical mesh is saved normally
include_pitch: bool (optional)
add pitch data. Default: False
include_intensity: bool (optional)
add intensity data. Default: False
speakers: bool (optional)
Default: False
mic: bool (optional)
default: False
mini: only save data stub. Used for testing
hilb: bool
include Hilbert Transform data. Default: False
verbose: bool (optional)
imaging_path: str (optional)
None: use IMAGING_DIR
'local': use subject_dir/Imaging/
else: use supplied string
parse_transcript: str (optional)
include_cortical_surfaces: bool (optional)
include_electrodes: bool (optional)
include_ekg: bool (optional)
subject_image_list: list (optional)
List of paths of images to include
rest_period: None | array-like
kwargs: dict
passed to pynwb.NWBFile
Returns
-------
"""
behav_module = None
basepath, blockname = os.path.split(blockpath)
subject_id = get_subject_id(blockname)
if identifier is None:
identifier = blockname
if session_description is None:
session_description = blockname
if outpath is None:
outpath = blockpath + '.nwb'
out_base_path = os.path.split(outpath)[0]
if session_start_time is None:
session_start_time = datetime(1900, 1, 1).astimezone(timezone('UTC'))
if imaging_path is None:
subj_imaging_path = path.join(IMAGING_PATH, subject_id)
elif imaging_path == 'local':
subj_imaging_path = path.join(basepath, 'imaging')
else:
subj_imaging_path = os.path.join(imaging_path, subject_id)
# file paths
bad_time_file = path.join(blockpath, 'Artifacts', 'badTimeSegments.mat')
ecog_path = path.join(blockpath, 'RawHTK')
ecog400_path = path.join(blockpath, 'ecog400', 'ecog.mat')
elec_metadata_file = path.join(subj_imaging_path, 'elecs',
'TDT_elecs_all.mat')
mesh_path = path.join(subj_imaging_path, 'Meshes')
pial_files = glob.glob(path.join(mesh_path, '*pial.mat'))
# Create the NWB file object
nwbfile = NWBFile(session_description,
identifier,
session_start_time,
datetime.now().astimezone(),
session_id=identifier,
institution='University of California, San Francisco',
lab='Chang Lab',
**kwargs)
nwbfile.add_electrode_column('bad', 'electrode identified as too noisy')
bad_elecs_inds = get_bad_elecs(blockpath)
if include_electrodes:
add_electrodes(nwbfile,
elec_metadata_file,
bad_elecs_inds,
load_warped=load_warped)
else:
device = nwbfile.create_device('256Grid')
electrode_group = nwbfile.create_electrode_group(
name='256Grid electrodes',
description='auto_group',
location='location',
device=device)
for elec_counter in range(256):
bad = elec_counter in bad_elecs_inds
nwbfile.add_electrode(id=elec_counter + 1,
x=np.nan,
y=np.nan,
z=np.nan,
imp=np.nan,
location=' ',
filtering='none',
group=electrode_group,
bad=bad)
ecog_elecs = list(range(len(nwbfile.electrodes)))
ecog_elecs_region = nwbfile.create_electrode_table_region(
ecog_elecs, 'ECoG electrodes on brain')
# Read electrophysiology data from HTK files and add them to NWB file
if ecog_format == 'auto':
ecog_rate, data, ecog_path = auto_ecog(blockpath,
ecog_elecs,
verbose=False)
elif ecog_format == 'htk':
if verbose:
print('reading htk acquisition...', flush=True)
ecog_rate, data = readhtks(ecog_path, ecog_elecs)
data = data.squeeze()
if verbose:
print('done', flush=True)
elif ecog_format == 'mat':
with File(ecog400_path, 'r') as f:
data = f['ecogDS']['data'][:, ecog_elecs]
ecog_rate = f['ecogDS']['sampFreq'][:].ravel()[0]
ecog_path = ecog400_path
elif ecog_format == 'raw':
ecog_path = os.path.join(tdt_data_path, subject_id, blockname,
'raw.mat')
ecog_rate, data = load_wavs(ecog_path)
else:
raise ValueError('unrecognized argument: ecog_format')
ts_desc = "all Wav data"
if mini:
data = data[:2000]
ecog_ts = ElectricalSeries(name='ElectricalSeries',
data=H5DataIO(data, compression='gzip'),
electrodes=ecog_elecs_region,
rate=ecog_rate,
description=ts_desc,
conversion=0.001)
nwbfile.add_acquisition(ecog_ts)
if include_ekg:
ekg_elecs = find_ekg_elecs(elec_metadata_file)
if len(ekg_elecs):
add_ekg(nwbfile, ecog_path, ekg_elecs)
if mic:
# Add microphone recording from room
fs, data = get_analog(blockpath, 1)
nwbfile.add_acquisition(
TimeSeries('microphone',
data,
'audio unit',
rate=fs,
description="audio recording from microphone in room"))
if speakers:
fs, data = get_analog(blockpath, 2)
# Add audio stimulus 1
nwbfile.add_stimulus(
TimeSeries('speaker 1',
data,
'NA',
rate=fs,
description="audio stimulus 1"))
# Add audio stimulus 2
fs, data = get_analog(blockpath, 3)
if fs is not None:
nwbfile.add_stimulus(
TimeSeries('speaker 2',
data,
'NA',
rate=fs,
description='the second stimulus source'))
if anin4:
fs, data = get_analog(blockpath, 4)
nwbfile.add_acquisition(
TimeSeries(anin4,
data,
'aux unit',
rate=fs,
description="aux analog recording"))
# Add bad time segments
if os.path.exists(bad_time_file) and os.stat(bad_time_file).st_size:
bad_time = sio.loadmat(bad_time_file)['badTimeSegments']
for row in bad_time:
nwbfile.add_invalid_time_interval(start_time=row[0],
stop_time=row[1],
tags=('ECoG artifact', ),
timeseries=ecog_ts)
if rest_period is not None:
nwbfile.add_epoch_column(name='label', description='label')
nwbfile.add_epoch(start_time=rest_period[0],
stop_time=rest_period[1],
label='rest_period')
if hilb:
block_hilb_path = os.path.join(hilb_dir, subject_id, blockname,
blockname + '_AA.h5')
file = File(block_hilb_path, 'r')
data = transpose_iter(
file['X']) # transposes data during iterative write
filter_center = file['filter_center'][:]
filter_sigma = file['filter_sigma'][:]
data = H5DataIO(DataChunkIterator(tqdm(data,
desc='writing hilbert data'),
buffer_size=400 * 20),
compression='gzip')
decomp_series = DecompositionSeries(
name='LFPDecompositionSeries',
description='Gaussian band Hilbert transform',
data=data,
rate=400.,
source_timeseries=ecog_ts,
metric='amplitude')
for band_mean, band_stdev in zip(filter_center, filter_sigma):
decomp_series.add_band(band_mean=band_mean, band_stdev=band_stdev)
hilb_mod = nwbfile.create_processing_module(
name='ecephys', description='holds hilbert analysis results')
hilb_mod.add_container(decomp_series)
if include_cortical_surfaces:
subject = ECoGSubject(subject_id=subject_id)
subject.cortical_surfaces = create_cortical_surfaces(
pial_files, subject_id)
else:
subject = Subject(subject_id=subject_id, species='H**o sapiens')
if subject_image_list is not None:
subject = add_images_to_subject(subject, subject_image_list)
if external_subject:
subj_fpath = path.join(out_base_path, subject_id + '.nwb')
if not os.path.isfile(subj_fpath):
subj_nwbfile = NWBFile(session_description=subject_id,
identifier=subject_id,
subject=subject,
session_start_time=datetime(
1900, 1, 1).astimezone(timezone('UTC')))
with NWBHDF5IO(subj_fpath, manager=manager, mode='w') as subj_io:
subj_io.write(subj_nwbfile)
subj_read_io = NWBHDF5IO(subj_fpath, manager=manager, mode='r')
subj_nwbfile = subj_read_io.read()
subject = subj_nwbfile.subject
nwbfile.subject = subject
if parse_transcript:
if parse_transcript == 'CV':
parseout = parse(blockpath, blockname)
df = make_df(parseout, 0, subject_id, align_pos=1)
nwbfile.add_trial_column('cv_transition_time',
'time of CV transition in seconds')
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
nwbfile.add_trial_column('condition', 'syllable spoken')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
cv_transition_time=row['align'],
speak=row['mode'] == 'speak',
condition=row['label'])
elif parse_transcript == 'singing':
parseout = parse(blockpath, blockname)
df = make_df(parseout, 0, subject_id, align_pos=0)
if not len(df):
df = pd.DataFrame(parseout)
df['mode'] = 'speak'
df = df.loc[df['label'].astype('bool'), :] # handle empty labels
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
nwbfile.add_trial_column('condition', 'syllable spoken')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
speak=row['mode'] == 'speak',
condition=row['label'])
elif parse_transcript == 'emphasis':
parseout = parse(blockpath, blockname)
try:
df = make_df(parseout, 0, subject_id, align_pos=0)
except:
df = pd.DataFrame(parseout)
if not len(df):
df = pd.DataFrame(parseout)
df = df.loc[df['label'].astype('bool'), :] # handle empty labels
nwbfile.add_trial_column('condition', 'word emphasized')
nwbfile.add_trial_column(
'speak',
'if True, subject is speaking. If False, subject is listening')
for _, row in df.iterrows():
nwbfile.add_trial(start_time=row['start'],
stop_time=row['stop'],
speak=True,
condition=row['label'])
elif parse_transcript == 'MOCHA':
nwbfile = create_transcription(nwbfile, transcript_path, blockname)
# behavior
if include_pitch:
if behav_module is None:
behav_module = nwbfile.create_processing_module(
'behavior', 'processing about behavior')
if os.path.isfile(
os.path.join(blockpath, 'pitch_' + blockname + '.mat')):
fs, data = load_pitch(blockpath)
pitch_ts = TimeSeries(
data=data,
rate=fs,
unit='Hz',
name='pitch',
description=
'Pitch as extracted from Praat. NaNs mark unvoiced regions.')
behav_module.add_container(
BehavioralTimeSeries(name='pitch', time_series=pitch_ts))
else:
print('No pitch file for ' + blockname)
if include_intensity:
if behav_module is None:
behav_module = nwbfile.create_processing_module(
'behavior', 'processing about behavior')
if os.path.isfile(
os.path.join(blockpath, 'intensity_' + blockname + '.mat')):
fs, data = load_pitch(blockpath)
intensity_ts = TimeSeries(
data=data,
rate=fs,
unit='dB',
name='intensity',
description='Intensity of speech in dB extracted from Praat.')
behav_module.add_container(
BehavioralTimeSeries(name='intensity',
time_series=intensity_ts))
else:
print('No intensity file for ' + blockname)
# Export the NWB file
with NWBHDF5IO(outpath, manager=manager, mode='w') as io:
io.write(nwbfile)
if external_subject:
subj_read_io.close()
if hilb:
file.close()
# read check
with NWBHDF5IO(outpath, manager=manager, mode='r') as io:
io.read()
def nwb_copy_file(old_file, new_file, cp_objs={}, save_to_file=True):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path, nwbfile
String or path to nwb file '/path/to/old_file.nwb'. Alternatively, the
nwbfile object.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
save_to_file: Boolean
If True, saves directly to new_file.nwb. If False, only returns nwb_new.
Returns:
--------
nwb_new : nwbfile object
"""
manager = get_manager()
# Get from nwbfile object in memory or from file
if isinstance(old_file, NWBFile):
nwb_old = old_file
io1 = False
else:
io1 = NWBHDF5IO(str(old_file),
'r',
manager=manager,
load_namespaces=True)
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(
session_description=str(nwb_old.session_description),
identifier=id_generator(),
session_start_time=nwb_old.session_start_time,
)
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs and nwb_old.electrode_groups is not None:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name))
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs and nwb_old.electrodes is not None:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux]))
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = [
'x', 'y', 'z', 'imp', 'location', 'filtering', 'group',
'group_name'
]
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [
str(elem) for elem in nwb_old.electrodes[var].data[:]
]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(
name=str(var),
description=str(nwb_old.electrodes[var].description),
data=var_data)
# If Bipolar scheme for electrodes
for v in nwb_old.lab_meta_data.values():
if isinstance(v, EcephysExt) and hasattr(
v, 'bipolar_scheme_table'):
bst_old = v.bipolar_scheme_table
bst_new = BipolarSchemeTable(
name=bst_old.name, description=bst_old.description)
ecephys_ext = EcephysExt(name=v.name)
ecephys_ext.bipolar_scheme_table = bst_new
nwb_new.add_lab_meta_data(ecephys_ext)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs and nwb_old.epochs is not None:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags', 'timeseries']
[new_vars.remove(var) for var in default_vars if var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(
name=var,
description=nwb_old.epochs[var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs and nwb_old.invalid_times is not None:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs and nwb_old.trials is not None:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(
name=var,
description=nwb_old.trials[var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs and nwb_old.intervals is not None:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
if 'behavior' in cp_objs:
interfaces = [
nwb_old.processing['behavior'].data_interfaces[key]
for key in cp_objs['behavior']
]
if 'behavior' not in nwb_new.processing:
# Add behavior module to NWB file
behavior_module = ProcessingModule(
name='behavior', description='behavioral data.')
nwb_new.add_processing_module(behavior_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
behavior_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
# Can get raw ElecetricalSeries and Mic recording
if 'acquisition' in cp_objs:
for acq_name in cp_objs['acquisition']:
obj_old = nwb_old.acquisition[acq_name]
acq = copy_obj(obj_old, nwb_old, nwb_new)
nwb_new.add_acquisition(acq)
# Surveys ---------------------------------------------------------
if 'surveys' in cp_objs and 'behavior' in nwb_old.processing:
surveys_list = [
v for v in
nwb_old.processing['behavior'].data_interfaces.values()
if v.neurodata_type == 'SurveyTable'
]
if cp_objs['surveys'] and len(surveys_list) > 0:
if 'behavior' not in nwb_new.processing:
# Add behavior module to NWB file
behavior_module = ProcessingModule(
name='behavior', description='behavioral data.')
nwb_new.add_processing_module(behavior_module)
for obj_old in surveys_list:
srv = copy_obj(obj_old, nwb_old, nwb_new)
behavior_module.add_data_interface(srv)
# Subject ---------------------------------------------------------
if nwb_old.subject is not None:
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(**nwb_old.subject.fields)
# Write new file with copied fields
if save_to_file:
io2.write(nwb_new, link_data=False)
# Close old file and return new nwbfile object
if io1:
io1.close()
return nwb_new
def run_conversion(self,
nwbfile: NWBFile,
metadata_dict: dict,
stub_test: bool = False):
session_path = Path(self.source_data["folder_path"])
session_id = session_path.stem
# Stimuli
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
# States
sleep_state_fpath = session_path / f"{session_id}.SleepState.states.mat"
# label renaming specific to Peyrache
state_label_names = dict(WAKEstate="Awake",
NREMstate="Non-REM",
REMstate="REM")
if sleep_state_fpath.is_file():
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states",
description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(start_time=row[0],
stop_time=row[1],
label=state_label_names[name]))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"Contains behavioral data.").add(table)
# Position
pos_names = ["RedLED", "BlueLED"]
pos_idx_from = [0, 2]
pos_idx_to = [2, 4]
# Raw position
whlfile_path = session_path / f"{session_id}.whl"
whl_data = np.loadtxt(whlfile_path)
for name, idx_from, idx_to in zip(pos_names, pos_idx_from, pos_idx_to):
nwbfile.add_acquisition(
peyrache_spatial_series(
name=name,
description=
"Raw sensor data. Values of -1 indicate that LED detection failed.",
data=whl_data[:, idx_from:idx_to],
conversion=np.nan, # whl file is in arbitrary grid units
))
# Processed position
posfile_path = session_path / f"{session_id}.pos"
if posfile_path.is_file(
): # at least Mouse32-140820 was missing a .pos file
try:
pos_data = np.loadtxt(posfile_path)
pos_obj = Position(name="SubjectPosition")
for name, idx_from, idx_to in zip(pos_names, pos_idx_from,
pos_idx_to):
pos_obj.add_spatial_series(
peyrache_spatial_series(
name=name,
description=
("(x,y) coordinates tracking subject movement through the maze."
"Values of -1 indicate that LED detection failed."
),
data=pos_data[:, idx_from:idx_to],
conversion=1e-2, # from cm to m
))
check_module(nwbfile, "behavior",
"Contains behavioral data.").add(pos_obj)
except ValueError: # data issue present in at least Mouse17-170201
warn(f"Skipping .pos file for session {session_id}!")
# Epochs - only available for sessions with raw data
epoch_file = session_path / "raw" / f"{session_id}-raw-info" / f"{session_id}-behaviors.txt"
if epoch_file.is_file():
epoch_data = pd.read_csv(epoch_file, header=1)[f"{session_id}:"]
epoch_dat_inds = []
epoch_names = []
for epochs in epoch_data:
inds, name = epochs.split(": ")
epoch_dat_inds.append(inds.split(" "))
epoch_names.append(name)
epoch_windows = [0]
for epoch in epoch_dat_inds:
exp_end_times = []
for dat_ind in epoch:
recording_file = session_path / "raw" / f"{session_id}{dat_ind}.dat"
info_extractor = NeuroscopeRecordingExtractor(
recording_file)
dat_end_time = info_extractor.get_num_frames(
) / info_extractor.get_sampling_frequency() # seconds
exp_end_times.extend([dat_end_time])
epoch_windows.extend([epoch_windows[-1] + sum(exp_end_times)] *
2)
epoch_windows = np.array(epoch_windows[:-1]).reshape(-1, 2)
for j, epoch_name in enumerate(epoch_names):
nwbfile.add_epoch(start_time=epoch_windows[j][0],
stop_time=epoch_windows[j][1],
tags=[epoch_name])
def convert_data(
self,
nwbfile: NWBFile,
metadata_dict: dict,
stub_test: bool = False,
include_spike_waveforms: bool = False,
):
"""Convert the behavioral portion of a particular session of the GrosmarkAD dataset."""
session_path = self.input_args["folder_path"]
subject_path, session_id = os.path.split(session_path)
# Stimuli
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
# States
sleep_state_fpath = os.path.join(session_path,
"{session_id}.SleepState.states.mat")
# label renaming specific to Watson
state_label_names = dict(WAKEstate="Awake",
NREMstate="Non-REM",
REMstate="REM")
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states",
description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(
start_time=row[0],
stop_time=row[1],
label=state_label_names[name],
))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"contains behavioral data").add_data_interface(table)
# Position
pos_filepath = Path(
session_path) / f"{session_id}.position.behavior.mat"
pos_mat = loadmat(str(pos_filepath.absolute()))
starting_time = float(
pos_mat["position"]["timestamps"][0][0]
[0]) # confirmed to be a regularly sampled series
rate = float(
pos_mat["position"]["timestamps"][0][0][1]) - starting_time
if pos_mat["position"]["units"][0][0][0] == "m":
conversion = 1.0
else:
warnings.warn(
f"Spatial units ({pos_mat['position']['units'][0][0][0]}) not listed in meters; "
"setting conversion to nan.")
conversion = np.nan
pos_data = [[x[0], y[0]] for x, y in zip(
pos_mat["position"]["position"][0][0]["x"][0][0],
pos_mat["position"]["position"][0][0]["y"][0][0],
)]
linearized_data = [[
lin[0]
] for lin in pos_mat["position"]["position"][0][0]["lin"][0][0]]
label = pos_mat["position"]["behaviorinfo"][0][0]["MazeType"][0][0][
0].replace(" ", "")
pos_obj = Position(name=f"{label}Position")
spatial_series_object = SpatialSeries(
name=f"{label}SpatialSeries",
description=
"(x,y) coordinates tracking subject movement through the maze.",
data=H5DataIO(pos_data, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
starting_time=starting_time,
rate=rate,
resolution=np.nan,
)
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, "behavior",
"contains processed behavioral data").add_data_interface(pos_obj)
lin_pos_obj = Position(name=f"{label}LinearizedPosition")
lin_spatial_series_object = SpatialSeries(
name=f"{label}LinearizedTimeSeries",
description=
"Linearized position, defined as starting at the edge of reward area, "
"and increasing clockwise, terminating at the opposing edge of the reward area.",
data=H5DataIO(linearized_data, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
starting_time=starting_time,
rate=rate,
resolution=np.nan,
)
lin_pos_obj.add_spatial_series(lin_spatial_series_object)
check_module(nwbfile, "behavior",
"contains processed behavioral data").add_data_interface(
lin_pos_obj)
# Epochs
epoch_names = list(pos_mat["position"]["Epochs"][0][0].dtype.names)
epoch_windows = [[float(start), float(stop)]
for x in pos_mat["position"]["Epochs"][0][0][0][0]
for start, stop in x]
nwbfile.add_epoch_column("label", "name of epoch")
for j, epoch_name in enumerate(epoch_names):
nwbfile.add_epoch(
start_time=epoch_windows[j][0],
stop_time=epoch_windows[j][1],
label=epoch_name,
)
def run_conversion(self,
nwbfile: NWBFile,
metadata: dict,
stub_test: bool = False):
session_path = Path(self.source_data["folder_path"])
task_types = [
dict(name="OpenFieldPosition_ExtraLarge"),
dict(name="OpenFieldPosition_New_Curtain", conversion=0.46),
dict(name="OpenFieldPosition_New", conversion=0.46),
dict(name="OpenFieldPosition_Old_Curtain", conversion=0.46),
dict(name="OpenFieldPosition_Old", conversion=0.46),
dict(name="OpenFieldPosition_Oldlast", conversion=0.46),
dict(name="EightMazePosition", conversion=0.65 / 2),
]
subject_path = session_path.parent
session_id = session_path.stem
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
sleep_state_fpath = session_path / f"{session_id}--StatePeriod.mat"
exist_pos_data = any([
(session_path / "{session_id}__{task_type['name']}.mat").is_file()
for task_type in task_types
])
if exist_pos_data:
nwbfile.add_epoch_column("label", "Name of epoch.")
# Epoch intervals
for task_type in task_types:
label = task_type["name"]
file = session_path / f"{session_id}__{label}.mat"
if file.is_file():
pos_obj = Position(name=f"{label}_position")
matin = loadmat(file)
tt = matin["twhl_norm"][:, 0]
exp_times = find_discontinuities(tt)
if "conversion" in task_type:
conversion = task_type["conversion"]
else:
conversion = np.nan
for pos_type in ("twhl_norm", "twhl_linearized"):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=f"{label}_{pos_type}_spatial_series",
data=H5DataIO(pos_data_norm, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression="gzip"),
)
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, "behavior",
"Contains processed behavioral data.").add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(
start_time=window[0],
stop_time=window[1],
tags=f"{label}_{str(i)}",
)
# Trial intervals
trialdata_path = session_path / f"{session_id}__EightMazeRun.mat"
if trialdata_path.is_file():
trials_data = loadmat(trialdata_path)["EightMazeRun"]
trialdatainfo_path = subject_path / "EightMazeRunInfo.mat"
trialdatainfo = [
x[0]
for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]
]
features = trialdatainfo[:7]
features[:2] = (
"start_time",
"stop_time",
)
[
nwbfile.add_trial_column(x, "description")
for x in features[4:] + ["condition"]
]
for trial_data in trials_data:
if trial_data[3]:
cond = "run_left"
else:
cond = "run_right"
nwbfile.add_trial(
start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6],
)
# SLeep states
if sleep_state_fpath.is_file():
matin = loadmat(sleep_state_fpath)["StatePeriod"]
table = TimeIntervals(name="states",
description="sleep states of animal")
table.add_column(name="label", description="sleep state")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(start_time=row[0], stop_time=row[1], label=name))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"Contains behavioral data.").add_data_interface(table)
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None, include_spike_waveforms=True, stub=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id, age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b)
print(lfp_channel)
custom_column = [{'name': 'theta_reference',
'description': 'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel}]
ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks)
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp',
description='lfp signal for all shank electrodes')
for name, channel in special_electrode_dict.items():
ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data',
data=all_channels_data[channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
for shankn in np.arange(1, 9, dtype=int):
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
decomp_series = DecompositionSeries(name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data, rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase', unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown', conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
custom_unit_columns = [
{
'name': 'cell_type',
'description': 'name of cell type',
'data': celltype_names},
{
'name': 'global_id',
'description': 'global id for cell for entire experiment',
'data': df_unit_features['unitID'].values},
{
'name': 'max_electrode',
'description': 'electrode that has the maximum amplitude of the waveform',
'data': get_max_electrodes(nwbfile, session_path),
'table': nwbfile.electrodes
}]
ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks)
trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond,
error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states', description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({'start_time': row[0], 'stop_time': row[1], 'label': name})
[table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])]
check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')
def convert_data(
self, nwbfile: NWBFile, metadata_dict: dict, stub_test: bool = False, include_spike_waveforms: bool = False
):
session_path = self.input_args["folder_path"]
# TODO: check/enforce format?
task_types = metadata_dict.get("task_types", [])
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
[nwbfile.add_stimulus(x) for x in get_events(session_path)]
exist_pos_data = any(
os.path.isfile(os.path.join(session_path, "{}__{}.mat".format(session_id, task_type["name"])))
for task_type in task_types
)
if exist_pos_data:
nwbfile.add_epoch_column("label", "name of epoch")
for task_type in task_types:
label = task_type["name"]
file = os.path.join(session_path, session_id + "__" + label + ".mat")
if os.path.isfile(file):
pos_obj = Position(name=label + "_position")
matin = loadmat(file)
tt = matin["twhl_norm"][:, 0]
exp_times = find_discontinuities(tt)
if "conversion" in task_type:
conversion = task_type["conversion"]
else:
conversion = np.nan
for pos_type in ("twhl_norm", "twhl_linearized"):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + "_{}_spatial_series".format(pos_type),
data=H5DataIO(pos_data_norm, compression="gzip"),
reference_frame="unknown",
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression="gzip"),
)
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, "behavior", "contains processed behavioral data").add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1], label=label + "_" + str(i))
trialdata_path = os.path.join(session_path, session_id + "__EightMazeRun.mat")
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)["EightMazeRun"]
trialdatainfo_path = os.path.join(fpath_base, "EightMazeRunInfo.mat")
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)["EightMazeRunInfo"][0]]
features = trialdatainfo[:7]
features[:2] = (
"start_time",
"stop_time",
)
[nwbfile.add_trial_column(x, "description") for x in features[4:] + ["condition"]]
for trial_data in trials_data:
if trial_data[3]:
cond = "run_left"
else:
cond = "run_right"
nwbfile.add_trial(
start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6],
)
sleep_state_fpath = os.path.join(session_path, "{}.SleepState.states.mat".format(session_id))
# label renaming specific to Watson
state_label_names = {"WAKEstate": "Awake", "NREMstate": "Non-REM", "REMstate": "REM"}
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states", description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({"start_time": row[0], "stop_time": row[1], "label": state_label_names[name]})
[table.add_row(**row) for row in sorted(data, key=lambda x: x["start_time"])]
check_module(nwbfile, "behavior", "contains behavioral data").add_data_interface(table)
def main():
import os.path
# prerequisites: start
import numpy as np
rate = 10.0
np.random.seed(1234)
data_len = 1000
ephys_data = np.random.rand(data_len)
ephys_timestamps = np.arange(data_len) / rate
spatial_timestamps = ephys_timestamps[::10]
spatial_data = np.cumsum(np.random.normal(size=(2,
len(spatial_timestamps))),
axis=-1).T
# prerequisites: end
# create-nwbfile: start
from datetime import datetime
from dateutil.tz import tzlocal
from pynwb import NWBFile
f = NWBFile(
'the PyNWB tutorial',
'my first synthetic recording',
'EXAMPLE_ID',
datetime.now(tzlocal()),
experimenter='Dr. Bilbo Baggins',
lab='Bag End Laboratory',
institution='University of Middle Earth at the Shire',
experiment_description=
'I went on an adventure with thirteen dwarves to reclaim vast treasures.',
session_id='LONELYMTN')
# create-nwbfile: end
# save-nwbfile: start
from pynwb import NWBHDF5IO
filename = "example.h5"
io = NWBHDF5IO(filename, mode='w')
io.write(f)
io.close()
# save-nwbfile: end
os.remove(filename)
# create-device: start
device = f.create_device(name='trodes_rig123', source="a source")
# create-device: end
# create-electrode-groups: start
electrode_name = 'tetrode1'
source = "an hypothetical source"
description = "an example tetrode"
location = "somewhere in the hippocampus"
electrode_group = f.create_electrode_group(electrode_name,
source=source,
description=description,
location=location,
device=device)
# create-electrode-groups: end
# create-electrode-table-region: start
for idx in [1, 2, 3, 4]:
f.add_electrode(idx,
x=1.0,
y=2.0,
z=3.0,
imp=float(-idx),
location='CA1',
filtering='none',
description='channel %s' % idx,
group=electrode_group)
electrode_table_region = f.create_electrode_table_region(
[0, 2], 'the first and third electrodes')
# create-electrode-table-region: end
# create-timeseries: start
from pynwb.ecephys import ElectricalSeries
from pynwb.behavior import SpatialSeries
ephys_ts = ElectricalSeries(
'test_ephys_data',
'an hypothetical source',
ephys_data,
electrode_table_region,
timestamps=ephys_timestamps,
# Alternatively, could specify starting_time and rate as follows
# starting_time=ephys_timestamps[0],
# rate=rate,
resolution=0.001,
comments=
"This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
f.add_acquisition(ephys_ts)
spatial_ts = SpatialSeries(
'test_spatial_timeseries',
'a stumbling rat',
spatial_data,
'origin on x,y-plane',
timestamps=spatial_timestamps,
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T"
)
f.add_acquisition(spatial_ts)
# create-timeseries: end
# create-data-interface: start
from pynwb.ecephys import LFP
from pynwb.behavior import Position
lfp = f.add_acquisition(LFP('a hypothetical source'))
ephys_ts = lfp.create_electrical_series(
'test_ephys_data',
'an hypothetical source',
ephys_data,
electrode_table_region,
timestamps=ephys_timestamps,
resolution=0.001,
comments=
"This data was randomly generated with numpy, using 1234 as the seed", # noqa: E501
description="Random numbers generated with numpy.random.rand")
pos = f.add_acquisition(Position('a hypothetical source'))
spatial_ts = pos.create_spatial_series(
'test_spatial_timeseries',
'a stumbling rat',
spatial_data,
'origin on x,y-plane',
timestamps=spatial_timestamps,
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T"
) # noqa: E501
# create-data-interface: end
# create-epochs: start
epoch_tags = ('example_epoch', )
f.add_epoch(name='epoch1',
start_time=0.0,
stop_time=1.0,
tags=epoch_tags,
description="the first test epoch",
timeseries=[ephys_ts, spatial_ts])
f.add_epoch(name='epoch2',
start_time=0.0,
stop_time=1.0,
tags=epoch_tags,
description="the second test epoch",
timeseries=[ephys_ts, spatial_ts])
# create-epochs: end
# create-compressed-timeseries: start
from pynwb.ecephys import ElectricalSeries
from pynwb.behavior import SpatialSeries
from pynwb.form.backends.hdf5 import H5DataIO
ephys_ts = ElectricalSeries(
'test_compressed_ephys_data',
'an hypothetical source',
H5DataIO(ephys_data, compress=True),
electrode_table_region,
timestamps=H5DataIO(ephys_timestamps, compress=True),
resolution=0.001,
comments=
"This data was randomly generated with numpy, using 1234 as the seed",
description="Random numbers generated with numpy.random.rand")
f.add_acquisition(ephys_ts)
spatial_ts = SpatialSeries(
'test_compressed_spatial_timeseries',
'a stumbling rat',
H5DataIO(spatial_data, compress=True),
'origin on x,y-plane',
timestamps=H5DataIO(spatial_timestamps, compress=True),
resolution=0.1,
comments="This data was generated with numpy, using 1234 as the seed",
description="This 2D Brownian process generated with "
"np.cumsum(np.random.normal(size=(2, len(spatial_timestamps))), axis=-1).T"
)
f.add_acquisition(spatial_ts)
def run_conversion(
self,
nwbfile: NWBFile,
metadata: dict,
stub_test: bool = False,
):
session_path = Path(self.source_data["folder_path"])
session_id = session_path.name
# Stimuli
[
nwbfile.add_stimulus(x) for x in get_events(
session_path=session_path,
suffixes=[".lrw.evt", ".puf.evt", ".rip.evt", ".rrw.evt"])
]
# Epochs
df = pd.read_csv(session_path / f"{session_id}.cat.evt",
sep=" ",
names=("time", "begin_or_end", "of", "epoch_name"))
epoch_starts = []
for j in range(int(len(df) / 2)):
epoch_starts.append(df["time"][2 * j])
nwbfile.add_epoch(start_time=epoch_starts[j],
stop_time=df["time"][2 * j + 1],
tags=[df["epoch_name"][2 * j][18:]])
# Trials
trialdata_path = session_path / f"{session_id}-TrackRunTimes.mat"
if trialdata_path.is_file():
trials_data = loadmat(trialdata_path)["trackruntimes"]
for trial_data in trials_data:
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1])
# .whl position
whl_files = []
for whl_file in whl_files:
add_position_data(nwbfile=nwbfile,
session_path=session_path,
whl_file_path=whl_file,
starting_time=epoch_starts[j])
# States
sleep_state_fpath = session_path / f"{session_id}.SleepState.states.mat"
# label renaming
state_label_names = dict(WAKEstate="Awake",
NREMstate="Non-REM",
REMstate="REM")
if sleep_state_fpath.is_file():
matin = loadmat(sleep_state_fpath)["SleepState"]["ints"][0][0]
table = TimeIntervals(name="states",
description="Sleep states of animal.")
table.add_column(name="label", description="Sleep state.")
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append(
dict(start_time=row[0],
stop_time=row[1],
label=state_label_names[name]))
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x["start_time"])
]
check_module(nwbfile, "behavior",
"Contains behavioral data.").add(table)
for un in good_clus_info['id']:
info_tmp = good_clus_info[good_clus_info['id'] == un]
spike_times_tmp = spike_times_good[spike_clus_good == un]
nwbfile.add_unit(id=un,
spike_times=np.transpose(spike_times_tmp)[0],
location=info_tmp['area'].values[0],
depth=info_tmp['depth'].values[0],
channel=info_tmp['ch'].values[0],
fr=info_tmp['fr'].values[0])
del spike_times_tmp
# Add epochs
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[0][1],
cond[ep].name)
if cond[ep].name == 'natural_images':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[-1][1],
cond[ep].name)
# Add trials
# Images names can be also added here
nwbfile.add_trial_column(
name='stimset', description='the visual stimulus type during the trial')
nwbfile.add_trial_column(name='img_id',
description='image ID for Natural Images')
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
nwbfile.add_trial(start_time=cond[ep].time[0][0],
stop_time=cond[ep].time[0][1],
def yuta2nwb(
session_path='D:/BuzsakiData/SenzaiY/YutaMouse41/YutaMouse41-150903',
# '/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None,
include_spike_waveforms=True,
stub=True,
cache_spec=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path,
'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls,
'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in [
'genotype', 'DOB', 'implantation', 'Probe', 'Surgery',
'virus injection', 'mouseID'
]:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id,
age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(
session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
nshanks = len(shank_channels)
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls,
hilus_csv_path,
session_start_time,
session_id,
b=b)
custom_column = [{
'name': 'theta_reference',
'description':
'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel
}]
ns.write_electrode_table(nwbfile,
session_path,
custom_columns=custom_column,
max_shanks=max_shanks)
print('reading raw electrode data...', end='', flush=True)
if stub:
# example recording extractor for fast testing
xml_filepath = os.path.join(session_path, session_id + '.xml')
xml_root = et.parse(xml_filepath).getroot()
acq_sampling_frequency = float(
xml_root.find('acquisitionSystem').find('samplingRate').text)
num_channels = 4
num_frames = 10000
X = np.random.normal(0, 1, (num_channels, num_frames))
geom = np.random.normal(0, 1, (num_channels, 2))
X = (X * 100).astype(int)
sre = se.NumpyRecordingExtractor(
timeseries=X, sampling_frequency=acq_sampling_frequency, geom=geom)
else:
nre = se.NeuroscopeRecordingExtractor('{}/{}.dat'.format(
session_path, session_id))
sre = se.SubRecordingExtractor(nre, channel_ids=all_shank_channels)
print('writing raw electrode data...', end='', flush=True)
se.NwbRecordingExtractor.add_electrical_series(sre, nwbfile)
print('done.')
print('reading spiking units...', end='', flush=True)
if stub:
spike_times = [200, 300, 400]
num_frames = 10000
allshanks = []
for k in range(nshanks):
SX = se.NumpySortingExtractor()
for j in range(len(spike_times)):
SX.add_unit(unit_id=j + 1,
times=np.sort(
np.random.uniform(0, num_frames,
spike_times[j])))
allshanks.append(SX)
se_allshanks = se.MultiSortingExtractor(allshanks)
se_allshanks.set_sampling_frequency(acq_sampling_frequency)
else:
se_allshanks = se.NeuroscopeMultiSortingExtractor(session_path,
keep_mua_units=False)
electrode_group = []
for shankn in np.arange(1, nshanks + 1, dtype=int):
for id in se_allshanks.sortings[shankn - 1].get_unit_ids():
electrode_group.append(nwbfile.electrode_groups['shank' +
str(shankn)])
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id,
session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
# Add custom column data into the SortingExtractor so it can be written by the converter
# Note there is currently a hidden assumption that the way in which the NeuroscopeSortingExtractor
# merges the cluster IDs matches one-to-one with the get_UnitFeatureCell_features extraction
property_descriptions = {
'cell_type': 'name of cell type',
'global_id': 'global id for cell for entire experiment',
'shank_id': '0-indexed id of cluster of shank',
'electrode_group': 'the electrode group that each spike unit came from'
}
property_values = {
'cell_type': celltype_names,
'global_id': df_unit_features['unitID'].values,
'shank_id': [x - 2 for x in df_unit_features['unitIDshank'].values],
# - 2 b/c the get_UnitFeatureCell_features removes 0 and 1 IDs from each shank
'electrode_group': electrode_group
}
for unit_id in se_allshanks.get_unit_ids():
for property_name in property_descriptions.keys():
se_allshanks.set_unit_property(
unit_id, property_name,
property_values[property_name][unit_id])
se.NwbSortingExtractor.write_sorting(
se_allshanks,
nwbfile=nwbfile,
property_descriptions=property_descriptions)
print('done.')
# Read and write LFP's
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_lfp_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_lfp_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile,
lfp_data,
lfp_fs,
name='lfp',
description='lfp signal for all shank electrodes')
# Read and add special environmental electrodes
for name, channel in special_electrode_dict.items():
ts = TimeSeries(
name=name,
description=
'environmental electrode recorded inline with neural data',
data=all_channels_lfp_data[:, channel],
rate=lfp_fs,
unit='V',
#conversion=np.nan,
resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(
lfp_data[:, all_shank_channels == lfp_channel].ravel(),
lfp_fs,
passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
nshanks = min((max_shanks, len(ns.get_shank_channels(session_path))))
for shankn in np.arange(nshanks, dtype=int) + 1:
# Get spike activty from .spk file on a per-shank and per-sample basis
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
# Get the LFP Decomposition Series
decomp_series = DecompositionSeries(
name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data,
rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase',
unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys',
'contains processed extracellular electrophysiology data'
).add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path,
'{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(
os.path.isfile(
os.path.join(session_path, '{}__{}.mat'.format(
session_id, task_type['name']))) for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown',
conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(
nwbfile, 'behavior',
'contains processed behavioral data').add_data_interface(
pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0],
stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
trialdata_path = os.path.join(session_path,
session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [
x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]
]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[
nwbfile.add_trial_column(x, 'description')
for x in features[4:] + ['condition']
]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0],
stop_time=trial_data[1],
condition=cond,
error_run=trial_data[4],
stim_run=trial_data[5],
both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states',
description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({
'start_time': row[0],
'stop_time': row[1],
'label': name
})
[
table.add_row(**row)
for row in sorted(data, key=lambda x: x['start_time'])
]
check_module(nwbfile, 'behavior',
'contains behavioral data').add_data_interface(table)
print('writing NWB file...', end='', flush=True)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile, cache_spec=cache_spec)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')
