def __init__(self, metadata, nwbfile=None):
"""
Parameters
----------
metadata: dict
nwbfile: pynwb.NWBFile
"""
if nwbfile is None:
self.nwbfile = self.create_NWBFile(**metadata['NWBFile'])
else:
self.nwbfile = nwbfile
if 'Subject' in metadata:
self.nwbfile.subject = Subject(**metadata['Subject'])
# add devices
self.devices = dict()
for domain in ('Icephys', 'Ecephys', 'Ophys'):
if domain in metadata and 'Device' in metadata[domain]:
self.devices.update(
self.create_devices(metadata[domain]['Device']))
if 'Icephys' in metadata and ('Electrode' in metadata['Icephys']):
self.ic_elecs = self.create_icephys_elecs(
metadata['Icephys']['Electrode'])
def subject_metadata_acquisition(self):
# Check if metadata about the subject exists and prompt the user if he wants to add some
if self.data.get('subject_metadata') is None:
print("No subject metadata found \n ")
self.data['subject_metadata'] = dict()
elif len(self.data['subject_metadata']) == 0:
print("No subject metadata found \n ")
else:
print("Found subject metadata : " +
str(list(self.data['subject_metadata'].keys())))
add_metadata = input(
"Do you want to add more subject metadata ? (yes/no) ")
# Prevent errors if other input than yes/np
while add_metadata.replace(
" ", "").lower() != "no" and add_metadata.replace(
" ", "").lower() != "yes":
add_metadata = input(
"Do you want to add more subject metadata ? (yes/no) ")
if add_metadata.replace(" ", "").lower() == "yes":
self.data['subject_metadata'] = self.add_optional_metadata(
self.data['subject_metadata'])
self.subject = Subject(
age=self.data['subject_metadata'].get("age"),
description=self.data['subject_metadata'].get("description"),
genotype=self.data['subject_metadata'].get("genotype"),
sex=self.data['subject_metadata'].get("sex"),
species=self.data['subject_metadata'].get("species"),
subject_id=self.data['subject_metadata'].get("subject_id"),
weight=self.data['subject_metadata'].get("weight"),
date_of_birth=self.data['subject_metadata'].get("date_of_birth"))
def setUp(self):
self.subject = Subject(age='12 mo',
description='An unfortunate rat',
genotype='WT',
sex='M',
species='Rattus norvegicus',
subject_id='RAT123',
weight='2 lbs',
date_of_birth=datetime(2017,
5,
1,
12,
tzinfo=tzlocal()))
self.start = datetime(2017, 5, 1, 12, tzinfo=tzlocal())
self.path = 'nwbfile_test.h5'
self.nwbfile = NWBFile(
'a test session description for a test NWBFile',
'FILE123',
self.start,
experimenter='A test experimenter',
lab='a test lab',
institution='a test institution',
experiment_description='a test experiment description',
session_id='test1',
subject=self.subject)
def setUpContainer(self):
return Subject(age='12 mo',
description='An unfortunate rat',
genotype='WT',
sex='M',
species='Rattus norvegicus',
subject_id='RAT123',
weight='2 lbs')
def create_subject(nwbfile):
if nwbfile.subject is not None:
sub = nwbfile.subject
sub_ = Subject(age=sub.age, description=sub.description, genotype=sub.genotype, sex=sub.sex,
species=sub.species, subject_id=sub.subject_id, weight=sub.weight,
date_of_birth=sub.date_of_birth)
else:
sub_ = None
return sub_
def setUpContainer(self):
return Subject(age='12 mo',
description='An unfortunate rat',
genotype='WT',
sex='M',
species='Rattus norvegicus',
subject_id='RAT123',
weight='2 lbs',
date_of_birth=datetime(1970, 1, 1, 12, tzinfo=tzutc()))
def setUpContainer(self):
""" Return the test Subject """
return Subject(age='P90D',
description='An unfortunate rat',
genotype='WT',
sex='M',
species='Rattus norvegicus',
subject_id='RAT123',
weight='2 kg',
date_of_birth=datetime(1970, 1, 1, 12, tzinfo=tzutc()),
strain='my_strain')
def create_subject(self, metadata_subject):
"""
This method is called at __init__.
This method can be overridden by child classes if necessary.
Adds information about Subject to self.nwbfile.
Parameters
----------
metadata_subject: dict
"""
self.nwbfile.subject = Subject(**metadata_subject)
def make_nwbfile_from_metadata(metadata: dict):
"""Make NWBFile from available metadata."""
metadata = dict_deep_update(get_default_nwbfile_metadata(), metadata)
nwbfile_kwargs = metadata["NWBFile"]
if "Subject" in metadata:
# convert ISO 8601 string to datetime
if "date_of_birth" in metadata["Subject"] and isinstance(metadata["Subject"]["date_of_birth"], str):
metadata["Subject"]["date_of_birth"] = datetime.fromisoformat(metadata["Subject"]["date_of_birth"])
nwbfile_kwargs.update(subject=Subject(**metadata["Subject"]))
# convert ISO 8601 string to datetime
if isinstance(nwbfile_kwargs.get("session_start_time", None), str):
nwbfile_kwargs["session_start_time"] = datetime.fromisoformat(metadata["NWBFile"]["session_start_time"])
return NWBFile(**nwbfile_kwargs)
def build(self, use_htk=False):
'''Build NWB file content.
Parameters
----------
use_htk: applicable for auditory datasets only.
if False, use HTK files. if True, use TDT files directly.
Returns:
--------
nwb_content: an NWBFile object.
'''
logger.info('Building components for NWB')
block_name = self.metadata['block_name']
nwb_content = NWBFile(
session_description=self.metadata['session_description'], # 'foo',
experimenter=self.metadata['experimenter'],
lab=self.metadata['lab'],
institution=self.metadata['institution'],
session_start_time=self.session_start_time,
file_create_date=datetime.now(),
identifier=str(uuid.uuid1()), # block_name,
session_id=block_name,
experiment_description=self.metadata['experiment_description'],
subject=Subject(
subject_id=self.metadata['subject']['subject id'],
description=self.metadata['subject']['description'],
genotype=self.metadata['subject']['genotype'],
sex=self.metadata['subject']['sex'],
species=self.metadata['subject']['species']),
notes=self.metadata.get('notes', None),
pharmacology=self.metadata.get('pharmacology', None),
surgery=self.metadata.get('surgery', None),
)
self.device_originator.make(nwb_content)
self.electrode_groups_originator.make(nwb_content)
electrode_table_regions = self.electrodes_originator.make(nwb_content)
if self.experiment_type == 'auditory':
if use_htk:
# legacy pipeline
self.htk_originator.make(nwb_content, electrode_table_regions)
else:
self.tdt_originator.make(nwb_content, electrode_table_regions)
self.stimulus_originator.make(nwb_content)
elif self.experiment_type == 'behavior':
# TODO: implement as needed
pass
return nwb_content
def run_conversion(self,
nwbfile: NWBFile,
metadata: dict = None,
overwrite: bool = False):
assert isinstance(nwbfile,
NWBFile), "'nwbfile' should be of type pynwb.NWBFile"
metadata_default = self.get_metadata()
metadata = dict_deep_update(metadata_default, metadata)
# Subject:
if nwbfile.subject is None:
nwbfile.subject = Subject(**metadata['Subject'])
# adding behavior:
start_time = 0.0
rate = 1 / self.data_frame.time.diff().mean()
beh_ts = []
for behdict in self.beh_args:
if 'cm' in behdict['unit']:
conv = 1e-2
behdict.update(unit='m')
else:
conv = 1
behdict.update(starting_time=start_time,
rate=rate,
data=self.data_frame[behdict['name']].to_numpy() *
conv)
beh_ts.append(TimeSeries(**behdict))
if 'behavior' not in nwbfile.processing:
beh_mod = nwbfile.create_processing_module(
'behavior', 'Container for behavior time series')
beh_mod.add(
BehavioralTimeSeries(time_series=beh_ts,
name='BehavioralTimeSeries'))
else:
beh_mod = nwbfile.processing['behavior']
if 'BehavioralTimeSeries' not in beh_mod.data_interfaces:
beh_mod.add(
BehavioralTimeSeries(time_series=beh_ts,
name='BehavioralTimeSeries'))
# adding stimulus:
for inp_kwargs in self.stimulus_args:
if inp_kwargs['name'] not in nwbfile.stimulus:
inp_kwargs.update(
starting_time=start_time,
rate=rate,
data=self.data_frame[inp_kwargs['name']].to_numpy())
nwbfile.add_stimulus(TimeSeries(**inp_kwargs))
def _old_method_for_creating_test_file():
nwb_content = pynwb.NWBFile(
session_description='test-session-description',
experimenter='test-experimenter-name',
lab='test-lab',
institution='test-institution',
session_start_time=datetime.now(),
timestamps_reference_time=datetime.fromtimestamp(0, pytz.utc),
identifier=str(uuid.uuid1()),
session_id='test-session-id',
notes='test-notes',
experiment_description='test-experiment-description',
subject=Subject(description='test-subject-description',
genotype='test-subject-genotype',
sex='female',
species='test-subject-species',
subject_id='test-subject-id',
weight='2 lbs'),
)
nwb_content.add_epoch(
start_time=float(0), # start time
stop_time=float(100), # end time (what are the units?)
tags='epoch1')
nwb_content.create_device(name='test-device')
nwb_content.create_electrode_group(
name='test-electrode-group',
location='test-electrode-group-location',
device=nwb_content.devices['test-device'],
description='test-electrode-group-description')
num_channels = 8
for m in range(num_channels):
location = [0, m]
grp = nwb_content.electrode_groups['test-electrode-group']
impedance = -1.0
nwb_content.add_electrode(id=m,
x=float(location[0]),
y=float(location[1]),
z=float(0),
imp=impedance,
location='test-electrode-group-location',
filtering='none',
group=grp)
nwb_fname = os.environ['NWB_DATAJOINT_BASE_DIR'] + '/test.nwb'
with pynwb.NWBHDF5IO(path=nwb_fname, mode='w') as io:
io.write(nwb_content)
io.close()
def _create_nwb_files(video_list):
base_path = video_list[0][0].parent.parent
base_nwb_path = base_path / "nwbfiles"
base_nwb_path.mkdir(parents=True, exist_ok=True)
for no, vid_loc in enumerate(video_list):
vid_1 = vid_loc[0]
vid_2 = vid_loc[1]
subject_id = f"mouse{no}"
session_id = f"sessionid{no}"
subject = Subject(
subject_id=subject_id,
species="Mus musculus",
sex="M",
description="lab mouse ",
)
device = Device(f"imaging_device_{no}")
name = f"{vid_1.stem}_{no}"
nwbfile = NWBFile(
f"{name}{no}",
"desc: contains movie for dandi .mp4 storage as external",
datetime.now(tzlocal()),
experimenter="Experimenter name",
session_id=session_id,
subject=subject,
devices=[device],
)
image_series = ImageSeries(
name=f"MouseWhiskers{no}",
format="external",
external_file=[str(vid_1), str(vid_2)],
starting_frame=[0, 2],
starting_time=0.0,
rate=150.0,
)
nwbfile.add_acquisition(image_series)
nwbfile_path = base_nwb_path / f"{name}.nwb"
with NWBHDF5IO(str(nwbfile_path), "w") as io:
io.write(nwbfile)
return base_nwb_path
def create_nwb(self,nev_fp,ncs_fps,blocks, desc=''):
if 0 in blocks:
practice_incl = True
else:
practice_incl = False
subject = Subject(sex=self.sex,subject_id=str(self.patient_id),species=self.species)
self.nwb = self.seeg.to_nwb(ncs_fps,nev_fp=nev_fp)
# if hasattr(self,'electrode_locations'):
# self.nwb = nlx_to_nwb(nev_fp=nev_fp, ncs_paths=ncs_fps,desc=desc,
# practice_incl=practice_incl,
# electrode_locations=self.electrode_locations)
# else:
# self.nwb = nlx_to_nwb(nev_fp=nev_fp, ncs_paths=ncs_fps,desc=desc,
# practice_incl=practice_incl)
self.nwb.subject = subject
self.add_behavior(nev_fp,blocks,practice_incl)
return self.nwb
def test_nwbfile_source_script(self):
''' test NWBFile construction '''
# source_script, in plain text
info = get_software_info()
source_script = (f"Created by nsds-lab-to-nwb {info['version']} "
f"({info['url']}) "
f"(git@{info['git_describe']})")
# do we also need this when source_script is used?
source_script_file_name = os.path.basename(__file__)
# print(f'source_script_file_name={source_script_file_name}')
nwb_content = NWBFile(
session_description='session description text',
experimenter='experimenter',
lab='Bouchard Lab',
institution='LBNL',
session_start_time=self.dummy_time,
file_create_date=self.current_time,
identifier=str(uuid.uuid1()),
session_id='RFLYY_BXX',
experiment_description='experiment description text',
subject=Subject(
subject_id='animal name',
description='animal description',
genotype='animal genotype',
sex='animal sex',
species='animal species',
weight='animal weight',
),
notes='note in plain text',
pharmacology='pharmacology note in plain text',
surgery='surgery note in plain text',
source_script=source_script,
source_script_file_name=source_script_file_name,
)
with NWBHDF5IO(path=self.output_file, mode='w') as nwb_fileIO:
nwb_fileIO.write(nwb_content)
def get_metadata(self):
file_path = self.source_data['file_path']
subject_id, session_id = get_track_session_info(file_path)
self.subject_id = subject_id
self.datestr = session_id[:4]
session_start_time = datetime.datetime.strptime(
year + self.datestr, '%y%m%d')
session_start_time = timezone('US/Pacific').localize(
session_start_time)
file = h5py.File(file_path, 'r')
trial_contrast = get_data(file, 'trial_contrast')
return dict(
NWBFile=dict(session_description='straight track virtual reality.',
identifier=session_id,
session_start_time=session_start_time,
lab='Giocomo',
institution='Stanford University',
experiment_description='trial contrast: {}'.format(
int(trial_contrast[0])),
subject=Subject(subject_id=subject_id)))
img_order.append(int(cond[stim].stiminfo[i][2]))
cond[stim].img_order = img_order
cond[stim].img_name = cond[stim].conf[0][0][0][10][
0] # currently not used but might be needed later
# This is how class can be turned into dict
# vars(cond[0])
a = cond[1].__dict__
a.keys()
# %% Create NWB files
start_time = datetime(2020, 2, 27, 14, 36, 7, tzinfo=tzlocal())
nwb_subject = Subject(description="Pretty nice girl",
sex='F',
species='mouse',
subject_id=excel_info['Mouse'].values[0],
genotype=excel_info['Genotype'].values[0])
nwbfile = NWBFile(
session_description=
"NPx recording of Natural images and spontaneous activity",
session_id=Sess,
identifier='NWB123',
session_start_time=start_time,
experimenter='Marina Slashcheva',
institution='ESI, Frankfurt',
lab='Martin Vinck',
notes=' | '.join(
[x for x in list(excel_info['Note'].values) if str(x) != 'nan']),
protocol=' | '.join(
def test_show_subject():
node = Subject(age="8", sex="m", species="macaque")
show_fields(node)
def convert(
input_file,
session_start_time,
subject_date_of_birth,
subject_id='I5',
subject_description='naive',
subject_genotype='wild-type',
subject_sex='M',
subject_weight='11.6g',
subject_species='Mus musculus',
subject_brain_region='Medial Entorhinal Cortex',
surgery='Probe: +/-3.3mm ML, 0.2mm A of sinus, then as deep as possible',
session_id='npI5_0417_baseline_1',
experimenter='Kei Masuda',
experiment_description='Virtual Hallway Task',
institution='Stanford University School of Medicine',
lab_name='Giocomo Lab'):
"""
Read in the .mat file specified by input_file and convert to .nwb format.
Parameters
----------
input_file : np.ndarray (..., n_channels, n_time)
the .mat file to be converted
subject_id : string
the unique subject ID number for the subject of the experiment
subject_date_of_birth : datetime ISO 8601
the date and time the subject was born
subject_description : string
important information specific to this subject that differentiates it from other members of it's species
subject_genotype : string
the genetic strain of this species.
subject_sex : string
Male or Female
subject_weight :
the weight of the subject around the time of the experiment
subject_species : string
the name of the species of the subject
subject_brain_region : basestring
the name of the brain region where the electrode probe is recording from
surgery : str
information about the subject's surgery to implant electrodes
session_id: string
human-readable ID# for the experiment session that has a one-to-one relationship with a recording session
session_start_time : datetime
date and time that the experiment started
experimenter : string
who ran the experiment, first and last name
experiment_description : string
what task was being run during the session
institution : string
what institution was the experiment performed in
lab_name : string
the lab where the experiment was performed
Returns
-------
nwbfile : NWBFile
The contents of the .mat file converted into the NWB format. The nwbfile is saved to disk using NDWHDF5
"""
# input matlab data
matfile = hdf5storage.loadmat(input_file)
# output path for nwb data
def replace_last(source_string, replace_what, replace_with):
head, _sep, tail = source_string.rpartition(replace_what)
return head + replace_with + tail
outpath = replace_last(input_file, '.mat', '.nwb')
create_date = datetime.today()
timezone_cali = pytz.timezone('US/Pacific')
create_date_tz = timezone_cali.localize(create_date)
# if loading data from config.yaml, convert string dates into datetime
if isinstance(session_start_time, str):
session_start_time = datetime.strptime(session_start_time,
'%B %d, %Y %I:%M%p')
session_start_time = timezone_cali.localize(session_start_time)
if isinstance(subject_date_of_birth, str):
subject_date_of_birth = datetime.strptime(subject_date_of_birth,
'%B %d, %Y %I:%M%p')
subject_date_of_birth = timezone_cali.localize(subject_date_of_birth)
# create unique identifier for this experimental session
uuid_identifier = uuid.uuid1()
# Create NWB file
nwbfile = NWBFile(
session_description=experiment_description, # required
identifier=uuid_identifier.hex, # required
session_id=session_id,
experiment_description=experiment_description,
experimenter=experimenter,
surgery=surgery,
institution=institution,
lab=lab_name,
session_start_time=session_start_time, # required
file_create_date=create_date_tz) # optional
# add information about the subject of the experiment
experiment_subject = Subject(subject_id=subject_id,
species=subject_species,
description=subject_description,
genotype=subject_genotype,
date_of_birth=subject_date_of_birth,
weight=subject_weight,
sex=subject_sex)
nwbfile.subject = experiment_subject
# adding constants via LabMetaData container
# constants
sample_rate = float(matfile['sp'][0]['sample_rate'][0][0][0])
n_channels_dat = int(matfile['sp'][0]['n_channels_dat'][0][0][0])
dat_path = matfile['sp'][0]['dat_path'][0][0][0]
offset = int(matfile['sp'][0]['offset'][0][0][0])
data_dtype = matfile['sp'][0]['dtype'][0][0][0]
hp_filtered = bool(matfile['sp'][0]['hp_filtered'][0][0][0])
vr_session_offset = matfile['sp'][0]['vr_session_offset'][0][0][0]
# container
lab_metadata = LabMetaData_ext(name='LabMetaData',
acquisition_sampling_rate=sample_rate,
number_of_electrodes=n_channels_dat,
file_path=dat_path,
bytes_to_skip=offset,
raw_data_dtype=data_dtype,
high_pass_filtered=hp_filtered,
movie_start_time=vr_session_offset)
nwbfile.add_lab_meta_data(lab_metadata)
# Adding trial information
nwbfile.add_trial_column(
'trial_contrast',
'visual contrast of the maze through which the mouse is running')
trial = np.ravel(matfile['trial'])
trial_nums = np.unique(trial)
position_time = np.ravel(matfile['post'])
# matlab trial numbers start at 1. To correctly index trial_contract vector,
# subtracting 1 from 'num' so index starts at 0
for num in trial_nums:
trial_times = position_time[trial == num]
nwbfile.add_trial(start_time=trial_times[0],
stop_time=trial_times[-1],
trial_contrast=matfile['trial_contrast'][num - 1][0])
# Add mouse position inside:
position = Position()
position_virtual = np.ravel(matfile['posx'])
# position inside the virtual environment
sampling_rate = 1 / (position_time[1] - position_time[0])
position.create_spatial_series(
name='Position',
data=position_virtual,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='The start of the trial, which begins at the start '
'of the virtual hallway.',
conversion=0.01,
description='Subject position in the virtual hallway.',
comments='The values should be >0 and <400cm. Values greater than '
'400cm mean that the mouse briefly exited the maze.',
)
# physical position on the mouse wheel
physical_posx = position_virtual
trial_gain = np.ravel(matfile['trial_gain'])
for num in trial_nums:
physical_posx[trial ==
num] = physical_posx[trial == num] / trial_gain[num - 1]
position.create_spatial_series(
name='PhysicalPosition',
data=physical_posx,
starting_time=position_time[0],
rate=sampling_rate,
reference_frame='Location on wheel re-referenced to zero '
'at the start of each trial.',
conversion=0.01,
description='Physical location on the wheel measured '
'since the beginning of the trial.',
comments='Physical location found by dividing the '
'virtual position by the "trial_gain"')
nwbfile.add_acquisition(position)
# Add timing of lick events, as well as mouse's virtual position during lick event
lick_events = BehavioralEvents()
lick_events.create_timeseries(
'LickEvents',
data=np.ravel(matfile['lickx']),
timestamps=np.ravel(matfile['lickt']),
unit='centimeter',
description='Subject position in virtual hallway during the lick.')
nwbfile.add_acquisition(lick_events)
# Add information on the visual stimulus that was shown to the subject
# Assumed rate=60 [Hz]. Update if necessary
# Update external_file to link to Unity environment file
visualization = ImageSeries(
name='ImageSeries',
unit='seconds',
format='external',
external_file=list(['https://unity.com/VR-and-AR-corner']),
starting_time=vr_session_offset,
starting_frame=[[0]],
rate=float(60),
description='virtual Unity environment that the mouse navigates through'
)
nwbfile.add_stimulus(visualization)
# Add the recording device, a neuropixel probe
recording_device = nwbfile.create_device(name='neuropixel_probes')
electrode_group_description = 'single neuropixels probe http://www.open-ephys.org/neuropixelscorded'
electrode_group_name = 'probe1'
electrode_group = nwbfile.create_electrode_group(
electrode_group_name,
description=electrode_group_description,
location=subject_brain_region,
device=recording_device)
# Add information about each electrode
xcoords = np.ravel(matfile['sp'][0]['xcoords'][0])
ycoords = np.ravel(matfile['sp'][0]['ycoords'][0])
data_filtered_flag = matfile['sp'][0]['hp_filtered'][0][0]
if data_filtered_flag:
filter_desc = 'The raw voltage signals from the electrodes were high-pass filtered'
else:
filter_desc = 'The raw voltage signals from the electrodes were not high-pass filtered'
num_recording_electrodes = xcoords.shape[0]
recording_electrodes = range(0, num_recording_electrodes)
# create electrode columns for the x,y location on the neuropixel probe
# the standard x,y,z locations are reserved for Allen Brain Atlas location
nwbfile.add_electrode_column('rel_x', 'electrode x-location on the probe')
nwbfile.add_electrode_column('rel_y', 'electrode y-location on the probe')
for idx in recording_electrodes:
nwbfile.add_electrode(id=idx,
x=np.nan,
y=np.nan,
z=np.nan,
rel_x=float(xcoords[idx]),
rel_y=float(ycoords[idx]),
imp=np.nan,
location='medial entorhinal cortex',
filtering=filter_desc,
group=electrode_group)
# Add information about each unit, termed 'cluster' in giocomo data
# create new columns in unit table
nwbfile.add_unit_column(
'quality',
'labels given to clusters during manual sorting in phy (1=MUA, '
'2=Good, 3=Unsorted)')
# cluster information
cluster_ids = matfile['sp'][0]['cids'][0][0]
cluster_quality = matfile['sp'][0]['cgs'][0][0]
# spikes in time
spike_times = np.ravel(matfile['sp'][0]['st'][0]) # the time of each spike
spike_cluster = np.ravel(
matfile['sp'][0]['clu'][0]) # the cluster_id that spiked at that time
for i, cluster_id in enumerate(cluster_ids):
unit_spike_times = spike_times[spike_cluster == cluster_id]
waveforms = matfile['sp'][0]['temps'][0][cluster_id]
nwbfile.add_unit(id=int(cluster_id),
spike_times=unit_spike_times,
quality=cluster_quality[i],
waveform_mean=waveforms,
electrode_group=electrode_group)
# Trying to add another Units table to hold the results of the automatic spike sorting
# create TemplateUnits units table
template_units = Units(
name='TemplateUnits',
description='units assigned during automatic spike sorting')
template_units.add_column(
'tempScalingAmps',
'scaling amplitude applied to the template when extracting spike',
index=True)
# information on extracted spike templates
spike_templates = np.ravel(matfile['sp'][0]['spikeTemplates'][0])
spike_template_ids = np.unique(spike_templates)
# template scaling amplitudes
temp_scaling_amps = np.ravel(matfile['sp'][0]['tempScalingAmps'][0])
for i, spike_template_id in enumerate(spike_template_ids):
template_spike_times = spike_times[spike_templates ==
spike_template_id]
temp_scaling_amps_per_template = temp_scaling_amps[spike_templates ==
spike_template_id]
template_units.add_unit(id=int(spike_template_id),
spike_times=template_spike_times,
electrode_group=electrode_group,
tempScalingAmps=temp_scaling_amps_per_template)
# create ecephys processing module
spike_template_module = nwbfile.create_processing_module(
name='ecephys',
description='units assigned during automatic spike sorting')
# add template_units table to processing module
spike_template_module.add(template_units)
print(nwbfile)
print('converted to NWB:N')
print('saving ...')
with NWBHDF5IO(outpath, 'w') as io:
io.write(nwbfile)
print('saved', outpath)
session_info_matin = loadmat(
'/Users/bendichter/dev/buzcode/exampleDataStructs/20170505_396um_0um_merge.sessionInfo.mat',
struct_as_record=True)
date_text = session_info_matin['sessionInfo']['Date'][0][0][0]
animal_info_matin = loadmat(os.path.join(fpath_base,
fname + '.animalMetaData.mat'),
struct_as_record=True)
keys = ('ID', 'strain', 'species', 'surgeryDate')
animal_info = {key: animal_info_matin['animal'][key][0][0][0] for key in keys}
session_start_time = dateparse(date_text, yearfirst=True)
subject = Subject(subject_id=animal_info['ID'],
strain=animal_info['strain'],
species=animal_info['species'])
if 'DOB' in animal_info and type(animal_info['DOB']) is not str:
subject.age = str(session_start_time - animal_info['DOB'])
nwbfile = NWBFile(
session_description='mouse in open exploration and theta maze',
identifier=fname,
session_start_time=session_start_time,
institution='NYU',
lab='Buzsaki',
subject=subject)
all_ts = []
def nwb_copy_file(old_file, new_file, cp_objs={}):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path
String such as '/path/to/old_file.nwb'.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
"""
manager = get_manager()
# Open original signal file
with NWBHDF5IO(old_file, 'r', manager=manager,
load_namespaces=True) as io1:
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(session_description=str(nwb_old.session_description),
identifier='',
session_start_time=datetime.now(tzlocal()))
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[
aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name)
)
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux])
)
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering',
'group', 'group_name']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [str(elem) for elem in nwb_old.electrodes[
var].data[:]]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(name=str(var),
description=
str(nwb_old.electrodes[
var].description),
data=var_data)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags',
'timeseries']
[new_vars.remove(var) for var in default_vars if
var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(name=var,
description=nwb_old.epochs[
var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(
start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(name=var,
description=nwb_old.trials[
var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
if cp_objs['ecephys'] is True:
interfaces = nwb_old.processing[
'ecephys'].data_interfaces.keys()
else: # list of items
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.'
)
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
if 'acquisition' in cp_objs:
if cp_objs['acquisition'] is True:
all_acq_names = list(nwb_old.acquisition.keys())
else: # list of items
all_acq_names = cp_objs['acquisition']
for acq_name in all_acq_names:
obj_old = nwb_old.acquisition[acq_name]
obj = copy_obj(obj_old, nwb_old, nwb_new)
if obj is not None:
nwb_new.add_acquisition(obj)
# Subject ---------------------------------------------------------
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
subject_id=nwb_old.subject.subject_id,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
# Write new file with copied fields
io2.write(nwb_new, link_data=False)
def yuta2nwb(session_path='/Users/bendichter/Desktop/Buzsaki/SenzaiBuzsaki2017/YutaMouse41/YutaMouse41-150903',
subject_xls=None, include_spike_waveforms=True, stub=True):
subject_path, session_id = os.path.split(session_path)
fpath_base = os.path.split(subject_path)[0]
identifier = session_id
mouse_number = session_id[9:11]
if '-' in session_id:
subject_id, date_text = session_id.split('-')
b = False
else:
subject_id, date_text = session_id.split('b')
b = True
if subject_xls is None:
subject_xls = os.path.join(subject_path, 'YM' + mouse_number + ' exp_sheet.xlsx')
else:
if not subject_xls[-4:] == 'xlsx':
subject_xls = os.path.join(subject_xls, 'YM' + mouse_number + ' exp_sheet.xlsx')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_xls)
subject_data = {}
for key in ['genotype', 'DOB', 'implantation', 'Probe', 'Surgery', 'virus injection', 'mouseID']:
names = df.iloc[:, 0]
if key in names.values:
subject_data[key] = df.iloc[np.argmax(names == key), 1]
if isinstance(subject_data['DOB'], datetime):
age = session_start_time - subject_data['DOB']
else:
age = None
subject = Subject(subject_id=subject_id, age=str(age),
genotype=subject_data['genotype'],
species='mouse')
nwbfile = NWBFile(session_description='mouse in open exploration and theta maze',
identifier=identifier,
session_start_time=session_start_time.astimezone(),
file_create_date=datetime.now().astimezone(),
experimenter='Yuta Senzai',
session_id=session_id,
institution='NYU',
lab='Buzsaki',
subject=subject,
related_publications='DOI:10.1016/j.neuron.2016.12.011')
print('reading and writing raw position data...', end='', flush=True)
ns.add_position_data(nwbfile, session_path)
shank_channels = ns.get_shank_channels(session_path)[:8]
all_shank_channels = np.concatenate(shank_channels)
print('setting up electrodes...', end='', flush=True)
hilus_csv_path = os.path.join(fpath_base, 'early_session_hilus_chans.csv')
lfp_channel = get_reference_elec(subject_xls, hilus_csv_path, session_start_time, session_id, b=b)
print(lfp_channel)
custom_column = [{'name': 'theta_reference',
'description': 'this electrode was used to calculate LFP canonical bands',
'data': all_shank_channels == lfp_channel}]
ns.write_electrode_table(nwbfile, session_path, custom_columns=custom_column, max_shanks=max_shanks)
print('reading LFPs...', end='', flush=True)
lfp_fs, all_channels_data = ns.read_lfp(session_path, stub=stub)
lfp_data = all_channels_data[:, all_shank_channels]
print('writing LFPs...', flush=True)
# lfp_data[:int(len(lfp_data)/4)]
lfp_ts = ns.write_lfp(nwbfile, lfp_data, lfp_fs, name='lfp',
description='lfp signal for all shank electrodes')
for name, channel in special_electrode_dict.items():
ts = TimeSeries(name=name, description='environmental electrode recorded inline with neural data',
data=all_channels_data[channel], rate=lfp_fs, unit='V', conversion=np.nan, resolution=np.nan)
nwbfile.add_acquisition(ts)
# compute filtered LFP
print('filtering LFP...', end='', flush=True)
all_lfp_phases = []
for passband in ('theta', 'gamma'):
lfp_fft = filter_lfp(lfp_data[:, all_shank_channels == lfp_channel].ravel(), lfp_fs, passband=passband)
lfp_phase, _ = hilbert_lfp(lfp_fft)
all_lfp_phases.append(lfp_phase[:, np.newaxis])
data = np.dstack(all_lfp_phases)
print('done.', flush=True)
if include_spike_waveforms:
print('writing waveforms...', end='', flush=True)
for shankn in np.arange(1, 9, dtype=int):
ns.write_spike_waveforms(nwbfile, session_path, shankn, stub=stub)
print('done.', flush=True)
decomp_series = DecompositionSeries(name='LFPDecompositionSeries',
description='Theta and Gamma phase for reference LFP',
data=data, rate=lfp_fs,
source_timeseries=lfp_ts,
metric='phase', unit='radians')
decomp_series.add_band(band_name='theta', band_limits=(4, 10))
decomp_series.add_band(band_name='gamma', band_limits=(30, 80))
check_module(nwbfile, 'ecephys', 'contains processed extracellular electrophysiology data').add_data_interface(decomp_series)
[nwbfile.add_stimulus(x) for x in ns.get_events(session_path)]
# create epochs corresponding to experiments/environments for the mouse
sleep_state_fpath = os.path.join(session_path, '{}--StatePeriod.mat'.format(session_id))
exist_pos_data = any(os.path.isfile(os.path.join(session_path, '{}__{}.mat'.format(session_id, task_type['name'])))
for task_type in task_types)
if exist_pos_data:
nwbfile.add_epoch_column('label', 'name of epoch')
for task_type in task_types:
label = task_type['name']
file = os.path.join(session_path, session_id + '__' + label + '.mat')
if os.path.isfile(file):
print('loading position for ' + label + '...', end='', flush=True)
pos_obj = Position(name=label + '_position')
matin = loadmat(file)
tt = matin['twhl_norm'][:, 0]
exp_times = find_discontinuities(tt)
if 'conversion' in task_type:
conversion = task_type['conversion']
else:
conversion = np.nan
for pos_type in ('twhl_norm', 'twhl_linearized'):
if pos_type in matin:
pos_data_norm = matin[pos_type][:, 1:]
spatial_series_object = SpatialSeries(
name=label + '_{}_spatial_series'.format(pos_type),
data=H5DataIO(pos_data_norm, compression='gzip'),
reference_frame='unknown', conversion=conversion,
resolution=np.nan,
timestamps=H5DataIO(tt, compression='gzip'))
pos_obj.add_spatial_series(spatial_series_object)
check_module(nwbfile, 'behavior', 'contains processed behavioral data').add_data_interface(pos_obj)
for i, window in enumerate(exp_times):
nwbfile.add_epoch(start_time=window[0], stop_time=window[1],
label=label + '_' + str(i))
print('done.')
# there are occasional mismatches between the matlab struct and the neuroscope files
# regions: 3: 'CA3', 4: 'DG'
df_unit_features = get_UnitFeatureCell_features(fpath_base, session_id, session_path)
celltype_names = []
for celltype_id, region_id in zip(df_unit_features['fineCellType'].values,
df_unit_features['region'].values):
if celltype_id == 1:
if region_id == 3:
celltype_names.append('pyramidal cell')
elif region_id == 4:
celltype_names.append('granule cell')
else:
raise Exception('unknown type')
elif not np.isfinite(celltype_id):
celltype_names.append('missing')
else:
celltype_names.append(celltype_dict[celltype_id])
custom_unit_columns = [
{
'name': 'cell_type',
'description': 'name of cell type',
'data': celltype_names},
{
'name': 'global_id',
'description': 'global id for cell for entire experiment',
'data': df_unit_features['unitID'].values},
{
'name': 'max_electrode',
'description': 'electrode that has the maximum amplitude of the waveform',
'data': get_max_electrodes(nwbfile, session_path),
'table': nwbfile.electrodes
}]
ns.add_units(nwbfile, session_path, custom_unit_columns, max_shanks=max_shanks)
trialdata_path = os.path.join(session_path, session_id + '__EightMazeRun.mat')
if os.path.isfile(trialdata_path):
trials_data = loadmat(trialdata_path)['EightMazeRun']
trialdatainfo_path = os.path.join(fpath_base, 'EightMazeRunInfo.mat')
trialdatainfo = [x[0] for x in loadmat(trialdatainfo_path)['EightMazeRunInfo'][0]]
features = trialdatainfo[:7]
features[:2] = 'start_time', 'stop_time',
[nwbfile.add_trial_column(x, 'description') for x in features[4:] + ['condition']]
for trial_data in trials_data:
if trial_data[3]:
cond = 'run_left'
else:
cond = 'run_right'
nwbfile.add_trial(start_time=trial_data[0], stop_time=trial_data[1], condition=cond,
error_run=trial_data[4], stim_run=trial_data[5], both_visit=trial_data[6])
"""
mono_syn_fpath = os.path.join(session_path, session_id+'-MonoSynConvClick.mat')
matin = loadmat(mono_syn_fpath)
exc = matin['FinalExcMonoSynID']
inh = matin['FinalInhMonoSynID']
#exc_obj = CatCellInfo(name='excitatory_connections',
# indices_values=[], cell_index=exc[:, 0] - 1, indices=exc[:, 1] - 1)
#module_cellular.add_container(exc_obj)
#inh_obj = CatCellInfo(name='inhibitory_connections',
# indices_values=[], cell_index=inh[:, 0] - 1, indices=inh[:, 1] - 1)
#module_cellular.add_container(inh_obj)
"""
if os.path.isfile(sleep_state_fpath):
matin = loadmat(sleep_state_fpath)['StatePeriod']
table = TimeIntervals(name='states', description='sleep states of animal')
table.add_column(name='label', description='sleep state')
data = []
for name in matin.dtype.names:
for row in matin[name][0][0]:
data.append({'start_time': row[0], 'stop_time': row[1], 'label': name})
[table.add_row(**row) for row in sorted(data, key=lambda x: x['start_time'])]
check_module(nwbfile, 'behavior', 'contains behavioral data').add_data_interface(table)
if stub:
out_fname = session_path + '_stub.nwb'
else:
out_fname = session_path + '.nwb'
print('writing NWB file...', end='', flush=True)
with NWBHDF5IO(out_fname, mode='w') as io:
io.write(nwbfile)
print('done.')
print('testing read...', end='', flush=True)
# test read
with NWBHDF5IO(out_fname, mode='r') as io:
io.read()
print('done.')
def create_nwb_file(Sess, start_time):
sr = 30000 #30kHz
if sys.platform == 'win32':
SaveDir = os.path.join(r'C:\Users\slashchevam\Desktop\NPx\Results',
Sess)
RawDataDir = r'C:\Users\slashchevam\Desktop\NPx'
ExcelInfoPath = RawDataDir
PathToUpload = os.path.join(RawDataDir, Sess)
if sys.platform == 'linux':
SaveDir = os.path.join('/mnt/gs/departmentN4/Marina/NPx/Results', Sess)
RawDataDir = '/mnt/gs/projects/OWVinckNatIm/NPx_recordings/'
PAthToAnalyzed = '/experiment1/recording1/continuous/Neuropix-PXI-100.0/'
MatlabOutput = '/mnt/gs/projects/OWVinckNatIm/NPx_processed/Lev0_condInfo/'
ExcelInfoPath = '/mnt/gs/departmentN4/Marina/'
PathToUpload = RawDataDir + Sess + PAthToAnalyzed
if not os.path.exists(SaveDir):
os.makedirs(SaveDir)
os.chdir(SaveDir)
# Upload all the data
spike_stamps = np.load(os.path.join(PathToUpload, "spike_times.npy"))
spike_times = spike_stamps / sr
spike_clusters = np.load(os.path.join(PathToUpload, "spike_clusters.npy"))
cluster_group = pd.read_csv(os.path.join(PathToUpload,
"cluster_group.tsv"),
sep="\t")
cluster_info = pd.read_csv(os.path.join(PathToUpload, "cluster_info.tsv"),
sep="\t")
if len(cluster_group) != len(cluster_info):
print('Cluster group (manual labeling) and claster info do not match!')
#excel_info = pd.read_excel((ExcelInfoPath + '\\Recordings_Marina_NPx.xlsx'), sheet_name=Sess)
excel_info = pd.read_excel(os.path.join(ExcelInfoPath +
'Recordings_Marina_NPx.xlsx'),
sheet_name=Sess)
# Select spikes from good clusters only
# Have to add the depth of the clusters
good_clus_info = cluster_info[cluster_info['group'] ==
'good'] # has depth info
good_clus = good_clus_info[['id', 'group']]
print("Found", len(good_clus), ' good clusters')
good_spikes_ind = [x in good_clus['id'].values for x in spike_clusters]
spike_clus_good = spike_clusters[good_spikes_ind]
spike_times_good = spike_times[good_spikes_ind]
# spike_stamps_good = spike_stamps[good_spikes_ind]
if excel_info['Area'][0] == 'V1':
good_clus_info['area'] = 'V1'
else:
good_clus_info['area'] = good_clus_info['depth'] > np.max(
good_clus_info['depth']) - 1000
good_clus_info['area'] = good_clus_info['area'].replace(True, 'V1')
good_clus_info['area'] = good_clus_info['area'].replace(False, 'HPC')
del spike_clusters, spike_times, spike_stamps, good_spikes_ind
# Now reading digitals from condInfo
# This has to be checked carefully again, especially for few stimuli in the session
# cond class contains the following:
# 'spontaneous_brightness': dict_keys(['name', 'time', 'timestamps', 'trl_list', 'conf'])
# 'natural_images': dict_keys(['name', 'time', 'timestamps', 'trl_list', 'conf', 'img_order', 'img_name'])
class condInfo:
pass
if sys.platform == 'linux':
mat = scipy.io.loadmat(
os.path.join((MatlabOutput + Sess), 'condInfo_01.mat'))
if sys.platform == 'win32':
mat = scipy.io.loadmat(os.path.join(PathToUpload, 'condInfo_01.mat'))
SC_stim_labels = mat['StimClass'][0][0][0][0]
SC_stim_present = np.where(mat['StimClass'][0][0][1][0] == 1)[0]
SC_stim_labels_present = SC_stim_labels[SC_stim_present]
cond = [condInfo() for i in range(len(SC_stim_labels_present))]
for stim in range(len(SC_stim_labels_present)):
cond[stim].name = SC_stim_labels_present[stim][0]
cond[stim].stiminfo = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][1] # image indices are here
# sorting out digitals for spontaneous activity
# Need this loop in case there are few periods of spont, recorded like separate blocks
if SC_stim_labels_present[stim][0] == 'spontaneous_brightness':
cond[stim].time = []
cond[stim].timestamps = []
for block in range(
len(mat['StimClass'][0][0][3][0, SC_stim_present[stim]][0]
[0])):
print(block)
cond[stim].time.append(mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][block][2])
cond[stim].timestamps.append(mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][block][3])
cond[stim].trl_list = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][1]
cond[stim].conf = mat['StimClass'][0][0][2][
0,
SC_stim_present[stim]] # config is very likely wrong and useless
# sorting out digitals for natural images
if SC_stim_labels_present[stim][0] == 'natural_images':
cond[stim].time = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][2]
cond[stim].timestamps = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][3]
img_order = []
for i in range(len(cond[stim].stiminfo)):
img_order.append(int(cond[stim].stiminfo[i][2]))
cond[stim].img_order = img_order
cond[stim].img_name = cond[stim].conf[0][0][0][10][
0] # currently not used but might be needed later
# sorting out digitals for drifting gratings
if SC_stim_labels_present[stim][0] == 'drifting_gratings':
cond[stim].time = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][2]
cond[stim].timestamps = mat['StimClass'][0][0][3][
0, SC_stim_present[stim]][0][0][0][3]
dg_orient = []
for i in range(len(cond[stim].stiminfo)):
dg_orient.append(int(cond[stim].stiminfo[i][2]))
cond[stim].dg_orient = dg_orient
# Now create NWB file
start_time = start_time # datetime(2020, 2, 27, 14, 36, 7, tzinfo=tzlocal())
nwb_subject = Subject(description="Pretty nice girl",
sex='F',
species='mouse',
subject_id=excel_info['Mouse'].values[0],
genotype=excel_info['Genotype'].values[0])
nwbfile = NWBFile(
session_description=
"NPx recording of Natural images and spontaneous activity",
session_id=Sess,
identifier='NWB123',
session_start_time=start_time,
experimenter='Marina Slashcheva',
institution='ESI, Frankfurt',
lab='Martin Vinck',
notes=' | '.join(
[x for x in list(excel_info['Note'].values) if str(x) != 'nan']),
protocol=' | '.join([
x for x in list(excel_info['experiment'].values) if str(x) != 'nan'
]),
data_collection=
'Ref: {}, Probe_angle: {}, , Depth: {}, APcoord: {}, MLcoord: {}, Recday: {}, Hemi: {}'
.format(excel_info['refCh'].values[0],
excel_info['Angle_probe'].values[0],
excel_info['Depth'].values[0],
excel_info['anteroposterior'].values[0],
excel_info['mediolateral'].values[0],
excel_info['Recday'].values[0],
excel_info['Hemisphere'].values[0]),
subject=nwb_subject)
# Did not add it for the moment, later add running as a timeseries and add to HDF5 as binary parameter
# test_ts = TimeSeries(name='test_timeseries', data=data, unit='m', timestamps=timestamps)
# Add units
nwbfile.add_unit_column(
'location',
'the anatomical location of this unit') # to be added and CHECKED
nwbfile.add_unit_column('depth', 'depth on the NPx probe')
nwbfile.add_unit_column('channel', 'channel on the NPx probe')
nwbfile.add_unit_column('fr', 'average FR according to KS')
for un in good_clus_info['id']:
info_tmp = good_clus_info[good_clus_info['id'] == un]
spike_times_tmp = spike_times_good[spike_clus_good == un]
nwbfile.add_unit(id=un,
spike_times=np.transpose(spike_times_tmp)[0],
location=info_tmp['area'].values[0],
depth=info_tmp['depth'].values[0],
channel=info_tmp['ch'].values[0],
fr=info_tmp['fr'].values[0])
del spike_times_tmp
# Add epochs
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
#if len(cond[ep].time) > 1:
for bl in range(len(cond[ep].time)):
nwbfile.add_epoch(cond[ep].time[bl][0][0],
cond[ep].time[bl][0][1], cond[ep].name)
#else:
# nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[0][1], cond[ep].name)
if cond[ep].name == 'natural_images':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[-1][1],
cond[ep].name)
if cond[ep].name == 'drifting_gratings':
nwbfile.add_epoch(cond[ep].time[0][0], cond[ep].time[-1][1],
cond[ep].name)
# Add trials
# Images names can be also added here
nwbfile.add_trial_column(
name='start', description='start time relative to the stimulus onset')
nwbfile.add_trial_column(
name='stimset',
description='the visual stimulus type during the trial')
nwbfile.add_trial_column(name='img_id',
description='image ID for Natural Images')
for ep in range(len(cond)):
if cond[ep].name == 'spontaneous_brightness':
#if len(cond[ep].time) > 1:
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0][0],
stop_time=cond[ep].time[tr][0][1],
start=cond[ep].time[tr][0][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=('gray').encode('utf8'))
# else:
# nwbfile.add_trial(start_time = cond[ep].time[0][0], stop_time = cond[ep].time[0][1],
# start = cond[ep].time[0][2],
# stimset = (cond[ep].name).encode('utf8'),
# img_id = ('gray').encode('utf8'))
if cond[ep].name == 'natural_images':
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0],
stop_time=cond[ep].time[tr][1],
start=cond[ep].time[tr][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=(str(
cond[ep].img_order[tr])).encode('utf8'))
if cond[ep].name == 'drifting_gratings':
for tr in range(len(cond[ep].time)):
nwbfile.add_trial(start_time=cond[ep].time[tr][0],
stop_time=cond[ep].time[tr][1],
start=cond[ep].time[tr][2],
stimset=(cond[ep].name).encode('utf8'),
img_id=(str(
cond[ep].dg_orient[tr])).encode('utf8'))
# Write NWB file
os.chdir(SaveDir)
name_to_save = Sess + '.nwb'
io = NWBHDF5IO(name_to_save, manager=get_manager(), mode='w')
io.write(nwbfile)
io.close()
del nwbfile
def no2nwb(NOData, session_use, subjects):
# Prepare the NO data that will be coverted to the NWB format
session = NOData.sessions[session_use]
events = NOData._get_event_data(session_use, experiment_type='All')
cell_ids = NOData.ls_cells(session_use)
experiment_id_learn = session['experiment_id_learn']
experiment_id_recog = session['experiment_id_recog']
task_descr = session['task_descr']
# Get the metadata for the subject
df_session = subjects[subjects['session_id'] == session_use]
print('session_use')
print(session_use)
print('age')
print(str(df_session['age'].values[0]))
print('epilepsy_diagnosis')
print(str(df_session['epilepsy_diagnosis'].values[0]))
nwb_subject = Subject(
age=str(df_session['age'].values[0]),
description=df_session['epilepsy_diagnosis'].values[0],
sex=df_session['sex'].values[0],
subject_id=df_session['subject_id'].values[0])
# Create the NWB file
nwbfile = NWBFile(
#source='https://datadryad.org/bitstream/handle/10255/dryad.163179/RecogMemory_MTL_release_v2.zip',
session_description='RecogMemory dataset session use 5' + session['session'],
identifier=session['session_id'],
session_start_time=datetime.datetime.now(),# TODO: need to check out the time for session start
file_create_date=datetime.datetime.now(),
experiment_description="learning: " + str(experiment_id_learn) + ", " + \
"recognition: " + \
str(experiment_id_recog),
subject=nwb_subject
)
# Add event and experiment_id acquisition
# event_ts = TimeSeries(name='events', source='NA', unit='NA', data=np.asarray(events[1].values),
# timestamps=np.asarray(events[0].values))
event_ts = TimeSeries(name='events',
unit='NA',
data=np.asarray(events[1].values),
timestamps=np.asarray(events[0].values))
# experiment_ids = TimeSeries(name='experiment_ids', source='NA', unit='NA', data=np.asarray(events[2]),
# timestamps=np.asarray(events[0].values))
experiment_ids = TimeSeries(name='experiment_ids',
unit='NA',
data=np.asarray(events[2]),
timestamps=np.asarray(events[0].values))
nwbfile.add_acquisition(event_ts)
nwbfile.add_acquisition(experiment_ids)
# Add stimuli to the NWB file2
# Get the first cell from the cell list
cell = NOData.pop_cell(session_use, NOData.ls_cells(session_use)[0])
trials = cell.trials
stimuli_recog_path = [trial.file_path_recog for trial in trials]
stimuli_learn_path = [trial.file_path_learn for trial in trials]
# Add stimuli recog
counter = 1
for path in stimuli_recog_path:
folders = path.split('\\')
path = os.path.join('./RecogMemory_MTL_release_v2', 'Stimuli',
folders[0], folders[1], folders[2])
img = cv2.imread(path)
name = 'stimuli_recog_' + str(counter)
stimulus_recog = ImageSeries(name=name,
data=img,
unit='NA',
format='',
timestamps=[0.0])
nwbfile.add_stimulus(stimulus_recog)
counter += 1
# Add stimuli learn
counter = 1
for path in stimuli_learn_path:
if path == 'NA':
continue
folders = path.split('\\')
path = os.path.join('./RecogMemory_MTL_release_v2', 'Stimuli',
folders[0], folders[1], folders[2])
img = cv2.imread(path)
name = 'stimuli_learn_' + str(counter)
stimulus_learn = ImageSeries(name=name,
data=img,
unit='NA',
format='',
timestamps=[0.0])
nwbfile.add_stimulus(stimulus_learn)
counter += 1
# Add epochs and trials: storing start and end times for a stimulus
# First extract the category ids and names that we need
# The metadata for each trials will be store in a trial table
cat_id_recog = [trial.category_recog for trial in trials]
cat_name_recog = [trial.category_name_recog for trial in trials]
cat_id_learn = [trial.category_learn for trial in trials]
cat_name_learn = [trial.category_name_learn for trial in trials]
# Extract the event timestamps
events_learn_stim_on = events[(events[2] == experiment_id_learn) &
(events[1] == NOData.markers['stimulus_on'])]
events_learn_stim_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['stimulus_off'])]
events_learn_delay1_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay1_off'])]
events_learn_delay2_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay2_off'])]
events_recog_stim_on = events[(events[2] == experiment_id_recog) &
(events[1] == NOData.markers['stimulus_on'])]
events_recog_stim_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['stimulus_off'])]
events_recog_delay1_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay1_off'])]
events_recog_delay2_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay2_off'])]
# Extract new_old label
new_old_recog = [trial.new_old_recog for trial in trials]
# Create the trial tables
nwbfile.add_trial_column('stim_on', 'the time when the stimulus is shown')
nwbfile.add_trial_column('stim_off', 'the time when the stimulus is off')
nwbfile.add_trial_column('delay1_off', 'the time when delay1 is off')
nwbfile.add_trial_column('delay2_off', 'the time when delay2 is off')
nwbfile.add_trial_column('stim_phase',
'learning/recognition phase during the trial')
nwbfile.add_trial_column('category_id', 'the category id of the stimulus')
nwbfile.add_trial_column('category_name',
'the category name of the stimulus')
nwbfile.add_trial_column('external_image_file',
'the file path to the stimulus')
nwbfile.add_trial_column('new_old_labels_recog',
'labels for new or old stimulus')
range_recog = np.amin([
len(events_recog_stim_on),
len(events_recog_stim_off),
len(events_recog_delay1_off),
len(events_recog_delay2_off)
])
range_learn = np.amin([
len(events_learn_stim_on),
len(events_learn_stim_off),
len(events_learn_delay1_off),
len(events_learn_delay2_off)
])
# Iterate the event list and add information into each epoch and trial table
for i in range(range_learn):
# nwbfile.create_epoch(start_time=events_learn_stim_on.iloc[i][0],
# stop_time=events_learn_stim_off.iloc[i][0],
# timeseries=[event_ts, experiment_ids],
# tags='stimulus_learn',
# description='learning phase stimulus')
# nwbfile.add_trial({'start': events_learn_stim_on.iloc[i][0],
# 'end': events_learn_delay2_off.iloc[i][0],
# 'stim_on': events_learn_stim_on.iloc[i][0],
# 'stim_off': events_learn_stim_off.iloc[i][0],
# 'delay1_off': events_learn_delay1_off.iloc[i][0],
# 'delay2_off': events_learn_delay2_off.iloc[i][0],
# 'stim_phase': 'learn',
# 'category_id': cat_id_learn[i],
# 'category_name': cat_name_learn[i],
# 'external_image_file': stimuli_learn_path[i],
# 'new_old_labels_recog': -1})
nwbfile.add_trial(start_time=events_learn_stim_on.iloc[i][0],
stop_time=events_learn_delay2_off.iloc[i][0],
stim_on=events_learn_stim_on.iloc[i][0],
stim_off=events_learn_stim_off.iloc[i][0],
delay1_off=events_learn_delay1_off.iloc[i][0],
delay2_off=events_learn_delay2_off.iloc[i][0],
stim_phase='learn',
category_id=cat_id_learn[i],
category_name=cat_name_learn[i],
external_image_file=stimuli_learn_path[i],
new_old_labels_recog='NA')
for i in range(range_recog):
# nwbfile.create_epoch(start_time=events_recog_stim_on.iloc[i][0],
# stop_time=events_recog_stim_off.iloc[i][0],
# timeseries=[event_ts, experiment_ids],
# tags='stimulus_recog',
# description='recognition phase stimulus')
nwbfile.add_trial(start_time=events_recog_stim_on.iloc[i][0],
stop_time=events_recog_delay2_off.iloc[i][0],
stim_on=events_recog_stim_on.iloc[i][0],
stim_off=events_recog_stim_off.iloc[i][0],
delay1_off=events_recog_delay1_off.iloc[i][0],
delay2_off=events_recog_delay2_off.iloc[i][0],
stim_phase='recog',
category_id=cat_id_recog[i],
category_name=cat_name_recog[i],
external_image_file=stimuli_recog_path[i],
new_old_labels_recog=new_old_recog[i])
# Add the waveform clustering and the spike data.
# Create necessary processing modules for different kinds of waveform data
clustering_processing_module = ProcessingModule(
'Spikes', 'The spike data contained')
clusterWaveform_learn_processing_module = ProcessingModule(
'MeanWaveforms_learn',
'The mean waveforms for the clustered raw signal for learning phase')
clusterWaveform_recog_processing_module = ProcessingModule(
'MeanWaveforms_recog',
'The mean waveforms for the clustered raw signal for recognition phase'
)
IsolDist_processing_module = ProcessingModule('IsoDist', 'The IsolDist')
SNR_processing_module = ProcessingModule('SNR', 'SNR (signal-to-noise)')
# Get the unique channel id that we will be iterate over
channel_ids = np.unique([cell_id[0] for cell_id in cell_ids])
# Interate the channel list
for channel_id in channel_ids:
cell_name = 'A' + str(channel_id) + '_cells.mat'
file_path = os.path.join('RecogMemory_MTL_release_v2', 'Data',
'sorted', session['session'], task_descr,
cell_name)
try:
cell_mat = loadmat(file_path)
except FileNotFoundError:
print("File not found")
continue
spikes = cell_mat['spikes']
meanWaveform_recog = cell_mat['meanWaveform_recog']
meanWaveform_learn = cell_mat['meanWaveform_learn']
IsolDist_SNR = cell_mat['IsolDist_SNR']
spike_id = np.asarray([spike[0] for spike in spikes])
spike_cluster_id = np.asarray([spike[1] for spike in spikes])
spike_timestamps = np.asarray([spike[2] / 1000000 for spike in spikes])
clustering = Clustering(description='Spikes of the channel detected',
num=spike_id,
peak_over_rms=np.asarray([0]),
times=spike_timestamps,
name='channel' + str(channel_id))
clustering_processing_module.add_data_interface(clustering)
for i in range(len(meanWaveform_learn[0][0][0][0])):
waveform_mean_learn = ClusterWaveforms(
clustering_interface=clustering,
waveform_filtering='NA',
waveform_sd=np.asarray([[0]]),
waveform_mean=np.asarray([meanWaveform_learn[0][0][1][i]]),
name='waveform_learn_cluster_id_' +
str(meanWaveform_learn[0][0][0][0][i]))
try:
clusterWaveform_learn_processing_module.add_data_interface(
waveform_mean_learn)
except ValueError as e:
print(
'Catch an error in adding waveform interface to the recog processing module:'
+ str(e))
continue
# Adding mean waveform recognition into the processing module
for i in range(len(meanWaveform_recog[0][0][0][0])):
waveform_mean_recog = ClusterWaveforms(
clustering_interface=clustering,
waveform_filtering='NA',
waveform_sd=np.asarray([[0]]),
waveform_mean=np.asarray([meanWaveform_recog[0][0][1][i]]),
name='waveform_recog_cluster_id_' +
str(meanWaveform_recog[0][0][0][0][i]))
try:
clusterWaveform_recog_processing_module.add_data_interface(
waveform_mean_recog)
except ValueError as e:
print(
'Catch an error in adding waveform interface to the recog processing module:'
+ str(e))
continue
# Adding IsolDist_SNR data into the processing module
# Here I use feature extraction to store the IsolDist_SNR data because
# they are extracted from the original signals.
# print(IsolDist_SNR[0][0][0])
for i in range(len(IsolDist_SNR[0][0][1][0])):
isoldist_data_interface = TimeSeries(
data=[IsolDist_SNR[0][0][1][0][i]],
unit='NA',
timestamps=[0],
name='IsolDist_' + str(IsolDist_SNR[0][0][0][0][i]))
try:
IsolDist_processing_module.add_data_interface(
isoldist_data_interface)
except ValueError as e:
print(
'Catch an error in adding IsolDist to the processing module:'
+ str(e))
continue
SNR_data_interface = TimeSeries(unit='NA',
description='The SNR data',
data=[IsolDist_SNR[0][0][2][0][i]],
timestamps=[0],
name='SNR_' +
str(IsolDist_SNR[0][0][0][0][i]))
try:
SNR_processing_module.add_data_interface(SNR_data_interface)
except ValueError as e:
print(
'Catch an error in adding SNR to the processing module:' +
str(e))
continue
nwbfile.add_processing_module(clustering_processing_module)
nwbfile.add_processing_module(clusterWaveform_learn_processing_module)
nwbfile.add_processing_module(clusterWaveform_recog_processing_module)
nwbfile.add_processing_module(IsolDist_processing_module)
nwbfile.add_processing_module(SNR_processing_module)
return nwbfile
def write_segmentation(
segext_obj: SegmentationExtractor,
save_path: PathType = None,
plane_num=0,
metadata: dict = None,
overwrite: bool = True,
buffer_size: int = 10,
nwbfile=None,
):
assert (
save_path is None or nwbfile is None
), "Either pass a save_path location, or nwbfile object, but not both!"
# parse metadata correctly:
if isinstance(segext_obj, MultiSegmentationExtractor):
segext_objs = segext_obj.segmentations
if metadata is not None:
assert isinstance(metadata, list), (
"For MultiSegmentationExtractor enter 'metadata' as a list of "
"SegmentationExtractor metadata")
assert len(metadata) == len(segext_objs), (
"The 'metadata' argument should be a list with the same "
"number of elements as the segmentations in the "
"MultiSegmentationExtractor")
else:
segext_objs = [segext_obj]
if metadata is not None and not isinstance(metadata, list):
metadata = [metadata]
metadata_base_list = [
NwbSegmentationExtractor.get_nwb_metadata(sgobj)
for sgobj in segext_objs
]
print(f"writing nwb for {segext_obj.extractor_name}\n")
# updating base metadata with new:
for num, data in enumerate(metadata_base_list):
metadata_input = metadata[num] if metadata else {}
metadata_base_list[num] = dict_recursive_update(
metadata_base_list[num], metadata_input)
metadata_base_common = metadata_base_list[0]
# build/retrieve nwbfile:
if nwbfile is not None:
assert isinstance(
nwbfile, NWBFile), "'nwbfile' should be of type pynwb.NWBFile"
write = False
else:
write = True
save_path = Path(save_path)
assert save_path.suffix == ".nwb"
if save_path.is_file() and not overwrite:
nwbfile_exist = True
file_mode = "r+"
else:
if save_path.is_file():
os.remove(save_path)
if not save_path.parent.is_dir():
save_path.parent.mkdir(parents=True)
nwbfile_exist = False
file_mode = "w"
io = NWBHDF5IO(str(save_path), file_mode)
if nwbfile_exist:
nwbfile = io.read()
else:
nwbfile = NWBFile(**metadata_base_common["NWBFile"])
# Subject:
if metadata_base_common.get("Subject") and nwbfile.subject is None:
nwbfile.subject = Subject(**metadata_base_common["Subject"])
# Processing Module:
if "ophys" not in nwbfile.processing:
ophys = nwbfile.create_processing_module(
"ophys", "contains optical physiology processed data")
else:
ophys = nwbfile.get_processing_module("ophys")
for plane_no_loop, (segext_obj, metadata) in enumerate(
zip(segext_objs, metadata_base_list)):
# Device:
if metadata["Ophys"]["Device"][0]["name"] not in nwbfile.devices:
nwbfile.create_device(**metadata["Ophys"]["Device"][0])
# ImageSegmentation:
image_segmentation_name = (
"ImageSegmentation" if plane_no_loop == 0 else
f"ImageSegmentation_Plane{plane_no_loop}")
if image_segmentation_name not in ophys.data_interfaces:
image_segmentation = ImageSegmentation(
name=image_segmentation_name)
ophys.add(image_segmentation)
else:
image_segmentation = ophys.data_interfaces.get(
image_segmentation_name)
# OpticalChannel:
optical_channels = [
OpticalChannel(**i) for i in metadata["Ophys"]["ImagingPlane"]
[0]["optical_channel"]
]
# ImagingPlane:
image_plane_name = ("ImagingPlane" if plane_no_loop == 0 else
f"ImagePlane_{plane_no_loop}")
if image_plane_name not in nwbfile.imaging_planes.keys():
input_kwargs = dict(
name=image_plane_name,
device=nwbfile.get_device(
metadata_base_common["Ophys"]["Device"][0]["name"]),
)
metadata["Ophys"]["ImagingPlane"][0][
"optical_channel"] = optical_channels
input_kwargs.update(**metadata["Ophys"]["ImagingPlane"][0])
if "imaging_rate" in input_kwargs:
input_kwargs["imaging_rate"] = float(
input_kwargs["imaging_rate"])
imaging_plane = nwbfile.create_imaging_plane(**input_kwargs)
else:
imaging_plane = nwbfile.imaging_planes[image_plane_name]
# PlaneSegmentation:
input_kwargs = dict(
description="output from segmenting imaging plane",
imaging_plane=imaging_plane,
)
ps_metadata = metadata["Ophys"]["ImageSegmentation"][
"plane_segmentations"][0]
if ps_metadata[
"name"] not in image_segmentation.plane_segmentations:
ps_exist = False
else:
ps = image_segmentation.get_plane_segmentation(
ps_metadata["name"])
ps_exist = True
roi_ids = segext_obj.get_roi_ids()
accepted_list = segext_obj.get_accepted_list()
accepted_list = [] if accepted_list is None else accepted_list
rejected_list = segext_obj.get_rejected_list()
rejected_list = [] if rejected_list is None else rejected_list
accepted_ids = [1 if k in accepted_list else 0 for k in roi_ids]
rejected_ids = [1 if k in rejected_list else 0 for k in roi_ids]
roi_locations = np.array(segext_obj.get_roi_locations()).T
def image_mask_iterator():
for id in segext_obj.get_roi_ids():
img_msks = segext_obj.get_roi_image_masks(
roi_ids=[id]).T.squeeze()
yield img_msks
if not ps_exist:
input_kwargs.update(
**ps_metadata,
columns=[
VectorData(
data=H5DataIO(
DataChunkIterator(image_mask_iterator(),
buffer_size=buffer_size),
compression=True,
compression_opts=9,
),
name="image_mask",
description="image masks",
),
VectorData(
data=roi_locations,
name="RoiCentroid",
description=
"x,y location of centroid of the roi in image_mask",
),
VectorData(
data=accepted_ids,
name="Accepted",
description=
"1 if ROi was accepted or 0 if rejected as a cell during segmentation operation",
),
VectorData(
data=rejected_ids,
name="Rejected",
description=
"1 if ROi was rejected or 0 if accepted as a cell during segmentation operation",
),
],
id=roi_ids,
)
ps = image_segmentation.create_plane_segmentation(
**input_kwargs)
# Fluorescence Traces:
if "Flourescence" not in ophys.data_interfaces:
fluorescence = Fluorescence()
ophys.add(fluorescence)
else:
fluorescence = ophys.data_interfaces["Fluorescence"]
roi_response_dict = segext_obj.get_traces_dict()
roi_table_region = ps.create_roi_table_region(
description=f"region for Imaging plane{plane_no_loop}",
region=list(range(segext_obj.get_num_rois())),
)
rate = (np.float("NaN")
if segext_obj.get_sampling_frequency() is None else
segext_obj.get_sampling_frequency())
for i, j in roi_response_dict.items():
data = getattr(segext_obj, f"_roi_response_{i}")
if data is not None:
data = np.asarray(data)
trace_name = "RoiResponseSeries" if i == "raw" else i.capitalize(
)
trace_name = (trace_name if plane_no_loop == 0 else
trace_name + f"_Plane{plane_no_loop}")
input_kwargs = dict(
name=trace_name,
data=data.T,
rois=roi_table_region,
rate=rate,
unit="n.a.",
)
if trace_name not in fluorescence.roi_response_series:
fluorescence.create_roi_response_series(**input_kwargs)
# create Two Photon Series:
if "TwoPhotonSeries" not in nwbfile.acquisition:
warn(
"could not find TwoPhotonSeries, using ImagingExtractor to create an nwbfile"
)
# adding images:
images_dict = segext_obj.get_images_dict()
if any([image is not None for image in images_dict.values()]):
images_name = ("SegmentationImages" if plane_no_loop == 0 else
f"SegmentationImages_Plane{plane_no_loop}")
if images_name not in ophys.data_interfaces:
images = Images(images_name)
for img_name, img_no in images_dict.items():
if img_no is not None:
images.add_image(
GrayscaleImage(name=img_name, data=img_no.T))
ophys.add(images)
# saving NWB file:
if write:
io.write(nwbfile)
io.close()
# test read
with NWBHDF5IO(str(save_path), "r") as io:
io.read()
def no2nwb(NOData, session_use, subjects_ini, path_to_data):
'''
Purpose:
Import the data and associated meta-data from the new/old recognition dataset into an
NWB file. Each of the features of the dataset, such as the events (i.e., TTLs) or mean waveform, are
compartmentalized to the appropriate component of the NWB file.
'''
# Time scaling (covert uS -----> S for NWB file)
TIME_SCALING = 10**6
# Prepare the NO data that will be coverted to the NWB format
session = NOData.sessions[session_use]
events = NOData._get_event_data(session_use, experiment_type='All')
cell_ids = NOData.ls_cells(session_use)
experiment_id_learn = session['experiment_id_learn']
experiment_id_recog = session['experiment_id_recog']
task_descr = session['task_descr']
# Get the metadata for the subject
# ============ Read Config File ()
# load config file (subjects == config file)
# Check config file path
filename = subjects_ini
if not os.path.exists(filename):
print('This file does not exist: {}'.format(filename))
print("Check filename/and or directory")
# Read the config file
try:
# initialze the ConfigParser() class
config = configparser.ConfigParser()
# read .ini file
config.read(filename)
except:
print('Failed to read the config file..')
print('Does this file exist: {}'.format(os.path.exists(filename)))
# Read Meta-data from INI file.
for section in config.sections():
if session_use == int(section):
session_id = int(section) # The New/Old ID for the session
#Get the session ID
for value in config[section]:
if value.lower() == 'nosessions.age':
age = int(config[section][value])
if value.lower() == 'nosessions.diagnosiscode':
epilepsyDxCode = config[section][value]
epilepsyDx = getEpilepsyDx(int(epilepsyDxCode))
if value.lower() == 'nosessions.sex':
sex = config[section][value].strip("'")
if value.lower() == 'nosessions.id':
ID = config[section][value].strip("'")
if value.lower() == 'nosessions.session':
pt_session = config[section][value].strip("'")
if value.lower() == 'nosessions.date':
unformattedDate = config[section][value].strip("'")
date = datetime.strptime(unformattedDate, '%Y-%m-%d')
finaldate = date.replace(hour=0, minute=0)
if value.lower() == 'nosessions.institution':
institution = config[section][value].strip("'")
if value.lower() == 'nosessions.la':
LA = config[section][value].strip("'").split(',')
if LA[0] == 'NaN':
LA_x = np.nan
LA_y = np.nan
LA_z = np.nan
else:
LA_x = float(LA[0])
LA_y = float(LA[1])
LA_z = float(LA[2])
if value.lower() == 'nosessions.ra':
RA = config[section][value].strip("'").split(',')
if RA[0] == 'NaN':
RA_x = np.nan
RA_y = np.nan
RA_z = np.nan
else:
RA_x = float(RA[0])
RA_y = float(RA[1])
RA_z = float(RA[2])
if value.lower() == 'nosessions.lh':
LH = config[section][value].strip("'").split(',')
if LH[0] == 'NaN':
LH_x = np.nan
LH_y = np.nan
LH_z = np.nan
else:
LH_x = float(LH[0])
LH_y = float(LH[1])
LH_z = float(LH[2])
if value.lower() == 'nosessions.rh':
RH = config[section][value].strip("'").split(',')
if RH[0] == 'NaN':
RH_x = np.nan
RH_y = np.nan
RH_z = np.nan
else:
RH_x = float(RH[0])
RH_y = float(RH[1])
RH_z = float(RH[2])
if value.lower() == 'nosessions.system':
signalSystem = config[section][value].strip("'")
# =================================================================
print(
'======================================================================='
)
print('session use: {}'.format(session_id))
print('age: {}'.format(age))
print('epilepsy_diagnosis: {}'.format(epilepsyDx))
nwb_subject = Subject(age=str(age),
description=epilepsyDx,
sex=sex,
species='Human',
subject_id=pt_session[:pt_session.find('_')])
# Create the NWB file
nwbfile = NWBFile(
#source='https://datadryad.org/bitstream/handle/10255/dryad.163179/RecogMemory_MTL_release_v2.zip',
session_description='New/Old recognition task for ID: {}. '.format(
session_id),
identifier='{}_{}'.format(ID, session_use),
session_start_time=finaldate, #default session start time
file_create_date=datetime.now(),
experiment_description=
'The data contained within this file describes a new/old recogntion task performed in '
'patients with intractable epilepsy implanted with depth electrodes and Behnke-Fried '
'microwires in the human Medical Temporal Lobe (MTL).',
institution=institution,
keywords=[
'Intracranial Recordings', 'Intractable Epilepsy',
'Single-Unit Recordings', 'Cognitive Neuroscience', 'Learning',
'Memory', 'Neurosurgery'
],
related_publications=
'Faraut et al. 2018, Scientific Data; Rutishauser et al. 2015, Nat Neurosci;',
lab='Rutishauser',
subject=nwb_subject,
data_collection='learning: {}, recognition: {}'.format(
session['experiment_id_learn'], session['experiment_id_recog']))
# Add events and experiment_id acquisition
events_description = (
""" The events coorespond to the TTL markers for each trial. For the learning trials, the TTL markers
are the following: 55 = start of the experiment, 1 = stimulus ON, 2 = stimulus OFF, 3 = Question Screen Onset [“Is this an animal?”],
20 = Yes (21 = NO) during learning, 6 = End of Delay after Response, 66 = End of Experiment. For the recognition trials,
the TTL markers are the following: 55 = start of experiment, 1 = stimulus ON, 2 = stimulus OFF, 3 = Question Screen Onset [“Have you seen this image before?”],
31:36 = Confidence (Yes vs. No) response [31 (new, confident), 32 (new, probably), 33 (new, guess), 34 (old, guess),
35 (old, probably), 36 (old, confident)], 66 = End of Experiment"""
)
event_ts = AnnotationSeries(name='events',
data=np.asarray(events[1].values).astype(str),
timestamps=np.asarray(events[0].values) /
TIME_SCALING,
description=events_description)
experiment_ids_description = (
"""The experiment_ids coorespond to the encoding (i.e., learning) or recogniton trials. The learning trials are demarcated by: {}. The recognition trials are demarcated by: {}. """
.format(experiment_id_learn, experiment_id_recog))
experiment_ids = TimeSeries(name='experiment_ids',
unit='NA',
data=np.asarray(events[2]),
timestamps=np.asarray(events[0].values) /
TIME_SCALING,
description=experiment_ids_description)
nwbfile.add_acquisition(event_ts)
nwbfile.add_acquisition(experiment_ids)
# Add stimuli to the NWB file
# Get the first cell from the cell list
cell = NOData.pop_cell(session_use,
NOData.ls_cells(session_use)[0], path_to_data)
trials = cell.trials
stimuli_recog_path = [trial.file_path_recog for trial in trials]
stimuli_learn_path = [trial.file_path_learn for trial in trials]
# Add epochs and trials: storing start and end times for a stimulus
# First extract the category ids and names that we need
# The metadata for each trials will be store in a trial table
cat_id_recog = [trial.category_recog for trial in trials]
cat_name_recog = [trial.category_name_recog for trial in trials]
cat_id_learn = [trial.category_learn for trial in trials]
cat_name_learn = [trial.category_name_learn for trial in trials]
# Extract the event timestamps
events_learn_stim_on = events[(events[2] == experiment_id_learn) &
(events[1] == NOData.markers['stimulus_on'])]
events_learn_stim_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['stimulus_off'])]
events_learn_delay1_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay1_off'])]
events_learn_delay2_off = events[(events[2] == experiment_id_learn) & (
events[1] == NOData.markers['delay2_off'])]
events_learn = events[(events[2] == experiment_id_learn)]
events_learn_response = []
events_learn_response_time = []
for i in range(len(events_learn[0])):
if (events_learn.iloc[i, 1]
== NOData.markers['response_learning_animal']) or (
events_learn.iloc[i, 1]
== NOData.markers['response_learning_non_animal']):
events_learn_response.append(events_learn.iloc[i, 1] - 20)
events_learn_response_time.append(events_learn.iloc[i, 0])
events_recog_stim_on = events[(events[2] == experiment_id_recog) &
(events[1] == NOData.markers['stimulus_on'])]
events_recog_stim_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['stimulus_off'])]
events_recog_delay1_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay1_off'])]
events_recog_delay2_off = events[(events[2] == experiment_id_recog) & (
events[1] == NOData.markers['delay2_off'])]
events_recog = events[(events[2] == experiment_id_recog)]
events_recog_response = []
events_recog_response_time = []
for i in range(len(events_recog[0])):
if ((events_recog.iloc[i, 1] == NOData.markers['response_1'])
or (events_recog.iloc[i, 1] == NOData.markers['response_2'])
or (events_recog.iloc[i, 1] == NOData.markers['response_3'])
or (events_recog.iloc[i, 1] == NOData.markers['response_4'])
or (events_recog.iloc[i, 1] == NOData.markers['response_5'])
or (events_recog.iloc[i, 1] == NOData.markers['response_6'])):
events_recog_response.append(events_recog.iloc[i, 1])
events_recog_response_time.append(events_recog.iloc[i, 0])
# Extract new_old label
new_old_recog = [trial.new_old_recog for trial in trials]
# Create the trial tables
nwbfile.add_trial_column('stim_on_time',
'The Time when the Stimulus is Shown')
nwbfile.add_trial_column('stim_off_time',
'The Time when the Stimulus is Off')
nwbfile.add_trial_column('delay1_time', 'The Time when Delay1 is Off')
nwbfile.add_trial_column('delay2_time', 'The Time when Delay2 is Off')
nwbfile.add_trial_column('stim_phase',
'Learning/Recognition Phase During the Trial')
nwbfile.add_trial_column('stimCategory', 'The Category ID of the Stimulus')
nwbfile.add_trial_column('category_name',
'The Category Name of the Stimulus')
nwbfile.add_trial_column('external_image_file',
'The File Path to the Stimulus')
nwbfile.add_trial_column(
'new_old_labels_recog',
'''The Ground truth Labels for New or Old Stimulus. 0 == Old Stimuli
(presented during the learning phase), 1 = New Stimuli (not seen )'during learning phase'''
)
nwbfile.add_trial_column('response_value',
'The Response for Each Stimulus')
nwbfile.add_trial_column('response_time',
'The Response Time for each Stimulus')
range_recog = np.amin([
len(events_recog_stim_on),
len(events_recog_stim_off),
len(events_recog_delay1_off),
len(events_recog_delay2_off)
])
range_learn = np.amin([
len(events_learn_stim_on),
len(events_learn_stim_off),
len(events_learn_delay1_off),
len(events_learn_delay2_off)
])
# Iterate the event list and add information into each epoch and trial table
for i in range(range_learn):
nwbfile.add_trial(
start_time=(events_learn_stim_on.iloc[i][0]) / (TIME_SCALING),
stop_time=(events_learn_delay2_off.iloc[i][0]) / (TIME_SCALING),
stim_on_time=(events_learn_stim_on.iloc[i][0]) / (TIME_SCALING),
stim_off_time=(events_learn_stim_off.iloc[i][0]) / (TIME_SCALING),
delay1_time=(events_learn_delay1_off.iloc[i][0]) / (TIME_SCALING),
delay2_time=(events_learn_delay2_off.iloc[i][0]) / (TIME_SCALING),
stim_phase='learn',
stimCategory=cat_id_learn[i],
category_name=cat_name_learn[i],
external_image_file=stimuli_learn_path[i],
new_old_labels_recog='NA',
response_value=events_learn_response[i],
response_time=(events_learn_response_time[i]) / (TIME_SCALING))
for i in range(range_recog):
nwbfile.add_trial(
start_time=events_recog_stim_on.iloc[i][0] / (TIME_SCALING),
stop_time=events_recog_delay2_off.iloc[i][0] / (TIME_SCALING),
stim_on_time=events_recog_stim_on.iloc[i][0] / (TIME_SCALING),
stim_off_time=events_recog_stim_off.iloc[i][0] / (TIME_SCALING),
delay1_time=events_recog_delay1_off.iloc[i][0] / (TIME_SCALING),
delay2_time=events_recog_delay2_off.iloc[i][0] / (TIME_SCALING),
stim_phase='recog',
stimCategory=cat_id_recog[i],
category_name=cat_name_recog[i],
external_image_file=stimuli_recog_path[i],
new_old_labels_recog=new_old_recog[i],
response_value=events_recog_response[i],
response_time=events_recog_response_time[i] / (TIME_SCALING))
# Add the waveform clustering and the spike data.
# Get the unique channel id that we will be iterate over
channel_ids = np.unique([cell_id[0] for cell_id in cell_ids])
# unique unit id
unit_id = 0
# Create unit columns
nwbfile.add_unit_column('origClusterID', 'The original cluster id')
nwbfile.add_unit_column('waveform_mean_encoding',
'The mean waveform for encoding phase.')
nwbfile.add_unit_column('waveform_mean_recognition',
'The mean waveform for the recognition phase.')
nwbfile.add_unit_column('IsolationDist', 'IsolDist')
nwbfile.add_unit_column('SNR', 'SNR')
nwbfile.add_unit_column('waveform_mean_sampling_rate',
'The Sampling Rate of Waveform')
#Add Stimuli
stimuli_presentation = []
# Add stimuli learn
counter = 1
for path in stimuli_learn_path:
if path == 'NA':
continue
folders = path.split('\\')
path = os.path.join(path_to_data, 'Stimuli', folders[0], folders[1],
folders[2])
img = cv2.imread(path)
resized_image = cv2.resize(img, (300, 400))
stimuli_presentation.append(resized_image)
# Add stimuli recog
counter = 1
for path in stimuli_recog_path:
folders = path.split('\\')
path = os.path.join(path_to_data, 'Stimuli', folders[0], folders[1],
folders[2])
img = cv2.imread(path)
resized_image = cv2.resize(img, (300, 400))
stimuli_presentation.append(resized_image)
name = 'stimuli_recog_' + str(counter)
# Add stimuli to OpticalSeries
stimulus_presentation_on_time = []
for n in range(0, len(events_learn_stim_on)):
stimulus_presentation_on_time.append(events_learn_stim_on.iloc[n][0] /
(TIME_SCALING))
for n in range(0, len(events_recog_stim_on)):
stimulus_presentation_on_time.append(events_recog_stim_on.iloc[n][0] /
(TIME_SCALING))
name = 'StimulusPresentation'
stimulus = OpticalSeries(name=name,
data=stimuli_presentation,
timestamps=stimulus_presentation_on_time[:],
orientation='lower left',
format='raw',
unit='meters',
field_of_view=[.2, .3, .7],
distance=0.7,
dimension=[300, 400, 3])
nwbfile.add_stimulus(stimulus)
# Get Unit data
all_spike_cluster_ids = []
all_selected_time_stamps = []
all_IsolDist = []
all_SNR = []
all_selected_mean_waveform_learn = []
all_selected_mean_waveform_recog = []
all_mean_waveform = []
all_channel_id = []
all_oriClusterIDs = []
all_channel_numbers = []
all_brain_area = []
# Iterate the channel list
# load brain area file
brain_area_file_path = os.path.join(path_to_data, 'Data', 'events',
session['session'], task_descr,
'brainArea.mat')
try:
brain_area_mat = loadmat(brain_area_file_path)
except FileNotFoundError:
print("brain_area_mat file not found")
for channel_id in channel_ids:
cell_name = 'A' + str(channel_id) + '_cells.mat'
cell_file_path = os.path.join(path_to_data, 'Data', 'sorted',
session['session'], task_descr,
cell_name)
try:
cell_mat = loadmat(cell_file_path)
except FileNotFoundError:
print("cell mat file not found")
continue
spikes = cell_mat['spikes']
meanWaveform_recog = cell_mat['meanWaveform_recog']
meanWaveform_learn = cell_mat['meanWaveform_learn']
IsolDist_SNR = cell_mat['IsolDist_SNR']
spike_cluster_id = np.asarray([spike[1] for spike in spikes
]) # Each Cluster ID of the spike
spike_timestamps = (np.asarray([spike[2] for spike in spikes])) / (
TIME_SCALING) # Timestamps of spikes for each ClusterID
unique_cluster_ids = np.unique(spike_cluster_id)
# If there are more than one cluster.
for id in unique_cluster_ids:
# Grab brain area
brain_area = extra_brain_area(brain_area_mat, channel_id)
selected_spike_timestamps = spike_timestamps[spike_cluster_id ==
id]
IsolDist, SNR = extract_IsolDist_SNR_by_cluster_id(
IsolDist_SNR, id)
selected_mean_waveform_learn = extra_mean_waveform(
meanWaveform_learn, id)
selected_mean_waveform_recog = extra_mean_waveform(
meanWaveform_recog, id)
# If the mean waveform does not have 256 elements, we set the mean wave form to all 0
if len(selected_mean_waveform_learn) != 256:
selected_mean_waveform_learn = np.zeros(256)
if len(selected_mean_waveform_recog) != 256:
selected_mean_waveform_recog = np.zeros(256)
mean_waveform = np.hstack(
[selected_mean_waveform_learn, selected_mean_waveform_recog])
# Append unit data
all_spike_cluster_ids.append(id)
all_selected_time_stamps.append(selected_spike_timestamps)
all_IsolDist.append(IsolDist)
all_SNR.append(SNR)
all_selected_mean_waveform_learn.append(
selected_mean_waveform_learn)
all_selected_mean_waveform_recog.append(
selected_mean_waveform_recog)
all_mean_waveform.append(mean_waveform)
all_channel_id.append(channel_id)
all_oriClusterIDs.append(int(id))
all_channel_numbers.append(channel_id)
all_brain_area.append(brain_area)
unit_id += 1
nwbfile.add_electrode_column(
name='origChannel',
description='The original channel ID for the channel')
#Add Device
device = nwbfile.create_device(name=signalSystem)
# Add Electrodes (brain Area Locations, MNI coordinates for microwires)
length_all_spike_cluster_ids = len(all_spike_cluster_ids)
for electrodeNumber in range(0, len(channel_ids)):
brainArea_location = extra_brain_area(brain_area_mat,
channel_ids[electrodeNumber])
if brainArea_location == 'RH': # Right Hippocampus
full_brainArea_Location = 'Right Hippocampus'
electrode_name = '{}-microwires-{}'.format(
signalSystem, channel_ids[electrodeNumber])
description = "Behnke Fried/Micro Inner Wire Bundle (Behnke-Fried BF08R-SP05X-000 and WB09R-SP00X-0B6; Ad-Tech Medical)"
location = full_brainArea_Location
# Add electrode group
electrode_group = nwbfile.create_electrode_group(
electrode_name,
description=description,
location=location,
device=device)
#Add Electrode
nwbfile.add_electrode([channel_ids[electrodeNumber]],
x=RH_x,
y=RH_y,
z=RH_z,
imp=np.nan,
location=full_brainArea_Location,
filtering='300-3000Hz',
group=electrode_group,
origChannel=channel_ids[electrodeNumber])
if brainArea_location == 'LH':
full_brainArea_Location = 'Left Hippocampus'
electrode_name = '{}-microwires-{}'.format(
signalSystem, channel_ids[electrodeNumber])
description = "Behnke Fried/Micro Inner Wire Bundle (Behnke-Fried BF08R-SP05X-000 and WB09R-SP00X-0B6; Ad-Tech Medical)"
location = full_brainArea_Location
# Add electrode group
electrode_group = nwbfile.create_electrode_group(
electrode_name,
description=description,
location=location,
device=device)
nwbfile.add_electrode([all_channel_id[electrodeNumber]],
x=LH_x,
y=LH_y,
z=LH_z,
imp=np.nan,
location=full_brainArea_Location,
filtering='300-3000Hz',
group=electrode_group,
origChannel=channel_ids[electrodeNumber])
if brainArea_location == 'RA':
full_brainArea_Location = 'Right Amygdala'
electrode_name = '{}-microwires-{}'.format(
signalSystem, channel_ids[electrodeNumber])
description = "Behnke Fried/Micro Inner Wire Bundle (Behnke-Fried BF08R-SP05X-000 and WB09R-SP00X-0B6; Ad-Tech Medical)"
location = full_brainArea_Location
# Add electrode group
electrode_group = nwbfile.create_electrode_group(
electrode_name,
description=description,
location=location,
device=device)
nwbfile.add_electrode([all_channel_id[electrodeNumber]],
x=RA_x,
y=RA_y,
z=RA_z,
imp=np.nan,
location=full_brainArea_Location,
filtering='300-3000Hz',
group=electrode_group,
origChannel=channel_ids[electrodeNumber])
if brainArea_location == 'LA':
full_brainArea_Location = 'Left Amygdala'
electrode_name = '{}-microwires-{}'.format(
signalSystem, channel_ids[electrodeNumber])
description = "Behnke Fried/Micro Inner Wire Bundle (Behnke-Fried BF08R-SP05X-000 and WB09R-SP00X-0B6; Ad-Tech Medical)"
location = full_brainArea_Location
# Add electrode group
electrode_group = nwbfile.create_electrode_group(
electrode_name,
description=description,
location=location,
device=device)
nwbfile.add_electrode([all_channel_id[electrodeNumber]],
x=LA_x,
y=LA_y,
z=LA_z,
imp=np.nan,
location=full_brainArea_Location,
filtering='300-3000Hz',
group=electrode_group,
origChannel=channel_ids[electrodeNumber])
# Create Channel list index
channel_list = list(range(0, length_all_spike_cluster_ids))
unique_channel_ids = np.unique(all_channel_id)
length_ChannelIds = len(np.unique(all_channel_id))
for yy in range(0, length_ChannelIds):
a = np.array(np.where(unique_channel_ids[yy] == all_channel_id))
b = a[0]
c = b.tolist()
for i in c:
channel_list[i] = yy
#Add WAVEFORM Sampling RATE
waveform_mean_sampling_rate = [98.4 * 10**3]
waveform_mean_sampling_rate_matrix = [waveform_mean_sampling_rate
] * (length_all_spike_cluster_ids)
# Add Units to NWB file
for index_id in range(0, length_all_spike_cluster_ids):
nwbfile.add_unit(
id=index_id,
spike_times=all_selected_time_stamps[index_id],
origClusterID=all_oriClusterIDs[index_id],
IsolationDist=all_IsolDist[index_id],
SNR=all_SNR[index_id],
waveform_mean_encoding=all_selected_mean_waveform_learn[index_id],
waveform_mean_recognition=all_selected_mean_waveform_recog[
index_id],
electrodes=[channel_list[index_id]],
waveform_mean_sampling_rate=waveform_mean_sampling_rate_matrix[
index_id])
return nwbfile
def create_nwb_file(subject_data_yaml_file, session_data_yaml_file):
"""
Create an NWB file object using all metadata containing in YAML file
Args:
subject_data_yaml_file (str): Absolute path to YAML file containing the subject metadata
session_data_yaml_file (str): Absolute path to YAML file containing the session metadata
"""
subject_data_yaml = None
with open(subject_data_yaml_file, 'r') as stream:
subject_data_yaml = yaml.safe_load(stream)
if subject_data_yaml is None:
print(f"Issue while reading the file {subject_data_yaml}")
return None
session_data_yaml = None
with open(session_data_yaml_file, 'r') as stream:
session_data_yaml = yaml.safe_load(stream)
if session_data_yaml is None:
print(f"Issue while reading the file {session_data_yaml}")
return None
keys_kwargs_subject = [
"age", "weight", "genotype", "subject_id", "species", "sex",
"date_of_birth"
]
kwargs_subject = dict()
for key in keys_kwargs_subject:
kwargs_subject[key] = subject_data_yaml.get(key)
if kwargs_subject[key] is not None:
kwargs_subject[key] = str(kwargs_subject[key])
if "date_of_birth" in kwargs_subject:
kwargs_subject["date_of_birth"] = datetime.strptime(
kwargs_subject["date_of_birth"], '%m/%d/%Y')
print(f'kwargs_subject {kwargs_subject}')
subject = Subject(**kwargs_subject)
#####################################
# ### creating the NWB file ###
#####################################
keys_kwargs_nwb_file = [
"session_description", "identifier", "session_id",
"session_start_time", "experimenter", "experiment_description",
"institution", "keywords", "notes", "pharmacology", "protocol",
"related_publications", "source_script", "source_script_file_name",
"surgery", "virus", "stimulus_notes", "slices", "lab_meta_data"
]
kwargs_nwb_file = dict()
for key in keys_kwargs_nwb_file:
kwargs_nwb_file[key] = session_data_yaml.get(key)
if kwargs_nwb_file[key] is not None:
kwargs_nwb_file[key] = str(kwargs_nwb_file[key])
if "session_description" not in kwargs_nwb_file:
print(
f"session_description is needed in the file {session_data_yaml_file}"
)
return
if "identifier" not in kwargs_nwb_file:
print(f"identifier is needed in the file {session_data_yaml_file}")
return
if "session_start_time" not in kwargs_nwb_file:
print(
f"session_start_time is needed in the file {session_data_yaml_file}"
)
return
else:
kwargs_nwb_file["session_start_time"] = datetime.strptime(
kwargs_nwb_file["session_start_time"], '%m/%d/%Y %H:%M:%S')
print(
f"kwargs_nwb_file['session_start_time'] {kwargs_nwb_file['session_start_time']}"
)
if "session_id" not in kwargs_nwb_file:
kwargs_nwb_file["session_id"] = kwargs_nwb_file["identifier"]
# #### arguments that are not in the yaml file (yet ?)
# file_create_date, timestamps_reference_time=None, acquisition=None, analysis=None, stimulus=None,
# stimulus_template=None, epochs=None, epoch_tags=set(), trials=None, invalid_times=None,
# time_intervals=None, units=None, modules=None, electrodes=None,
# electrode_groups=None, ic_electrodes=None, sweep_table=None, imaging_planes=None,
# ogen_sites=None, devices=None
kwargs_nwb_file["subject"] = subject
kwargs_nwb_file["file_create_date"] = datetime.now(tzlocal())
# TODO: See how to load invalid_times, from yaml file ?
# kwargs_nwb_file["invalid_times"] = invalid_times
print(f'kwargs_nwb_file {kwargs_nwb_file}')
nwb_file = NWBFile(**kwargs_nwb_file)
# nwb_file.invalid_times = TimeIntervals(name="invalid_times",
# description="Time intervals to be removed from analysis'")
return nwb_file
def build(self):
"""Build NWBFile
Returns:
NWBFile: Return NWBFile content
"""
logger.info('Building components for NWB')
nwb_content = NWBFile(
session_description=self.metadata['session description'],
experimenter=self.metadata['experimenter name'],
lab=self.metadata['lab'],
institution=self.metadata['institution'],
session_start_time=self.session_time_extractor.
get_session_start_time(),
timestamps_reference_time=datetime.fromtimestamp(0, pytz.utc),
identifier=str(uuid.uuid1()),
session_id=self.metadata['session_id'],
notes=self.link_to_notes,
experiment_description=self.metadata['experiment description'],
subject=Subject(
description=self.metadata['subject']['description'],
genotype=self.metadata['subject']['genotype'],
sex=self.metadata['subject']['sex'],
species=self.metadata['subject']['species'],
subject_id=self.metadata['subject']['subject id'],
weight=str(self.metadata['subject']['weight']),
),
)
self.processing_module_originator.make(nwb_content)
if 'associated_files' in self.metadata:
self.associated_files_originator.make(nwb_content)
self.position_originator.make(nwb_content)
valid_map_dict = self.__build_corrupted_data_manager()
shanks_electrodes_dict = self.shanks_electrode_originator.make()
shanks_dict = self.shanks_originator.make(shanks_electrodes_dict)
probes = self.probes_originator.make(nwb_content, shanks_dict,
valid_map_dict['probes'])
self.data_acq_device_originator.make(nwb_content)
self.header_device_originator.make(nwb_content)
self.camera_device_originator.make(nwb_content)
self.video_files_originator.make(nwb_content)
electrode_groups = self.electrode_group_originator.make(
nwb_content, probes, valid_map_dict['electrode_groups'])
self.electrodes_originator.make(nwb_content, electrode_groups,
valid_map_dict['electrodes'],
valid_map_dict['electrode_groups'])
self.electrodes_extension_originator.make(nwb_content,
valid_map_dict['electrodes'])
self.epochs_originator.make(nwb_content)
self.sample_count_timestamp_corespondence_originator.make(nwb_content)
self.task_originator.make(nwb_content)
self.camera_sample_frame_counts_originator.make(nwb_content)
if self.process_dio:
self.dio_originator.make(nwb_content)
if self.process_mda:
self.mda_originator.make(nwb_content)
if self.process_analog:
self.analog_originator.make(nwb_content)
return nwb_content
def setUpContainer(self):
return Subject()
lab = 'Buzsaki'
subject_id, date_text = fname.split('-')
session_start_time = dateparse(date_text, yearfirst=True)
df = pd.read_excel(subject_fpath)
subject_data = {}
for key in ['genotype', 'DOB', 'implantation', 'Probe']:
subject_data[key] = df.ix[df.ix[:, 0] == key, 1].values[0]
age = session_start_time - subject_data['DOB']
subject = Subject(subject_id=subject_id,
age=str(age),
genotype=subject_data['genotype'],
species='mouse',
source='source')
source = fname
nwbfile = NWBFile(source,
session_description,
identifier,
session_start_time,
datetime.now(),
institution=institution,
lab=lab,
subject=subject)
all_ts = []