class FRETTest(unittest.TestCase):
def setUp(self):
self.nwbfile = NWBFile('description', 'id',
datetime.now().astimezone())
def test_add_fretseries(self):
# Create and add device
device = Device(name='Device')
self.nwbfile.add_device(device)
# Create optical channels
opt_ch_d = OpticalChannel(name='optical_channel',
description='optical_channel_description',
emission_lambda=529.)
opt_ch_a = OpticalChannel(name='optical_channel',
description='optical_channel_description',
emission_lambda=633.)
# Create FRET
fs_d = FRETSeries(name='donor',
fluorophore='mCitrine',
optical_channel=opt_ch_d,
device=device,
description='description of donor series',
data=np.random.randn(100, 10, 10),
rate=200.,
unit='no unit')
fs_a = FRETSeries(name='acceptor',
fluorophore='mKate2',
optical_channel=opt_ch_a,
device=device,
description='description of acceptor series',
data=np.random.randn(100, 10, 10),
rate=200.,
unit='no unit')
fret = FRET(name='FRET',
excitation_lambda=482.,
donor=fs_d,
acceptor=fs_a)
self.nwbfile.add_acquisition(fret)
filename = 'test_fret.nwb'
with NWBHDF5IO(filename, 'w') as io:
io.write(self.nwbfile)
with NWBHDF5IO(filename, mode='r', load_namespaces=True) as io:
io.read()
os.remove(filename)
def test_nwbfileio(self):
testdir = tempfile.mkdtemp()
nwbfile = NWBFile(**temp_session_nwbfile)
nwbfile.add_lab_meta_data(IblSessionData(**temp_sessions))
nwbfile.subject = IblSubject(**temp_subject)
for i, name in zip(temp_probes, probe_names):
nwbfile.add_device(IblProbes(name, **i))
saveloc = os.path.join(testdir, 'test.nwb')
with NWBHDF5IO(saveloc, mode='w') as io:
io.write(nwbfile)
with NWBHDF5IO(saveloc, mode='r', load_namespaces=True) as io:
read_nwbfile = io.read()
for i, j in temp_sessions.items():
attr_loop = getattr(read_nwbfile.lab_meta_data['Ibl_session_data'], i, None)
if attr_loop:
if isinstance(attr_loop, h5py._hl.dataset.Dataset):
assert all(getattr(read_nwbfile.lab_meta_data['Ibl_session_data'], i).value == j)
else:
assert getattr(read_nwbfile.lab_meta_data['Ibl_session_data'], i) == j
for i, j in temp_subject.items():
attr_loop = getattr(read_nwbfile.subject, i, None)
if attr_loop:
if isinstance(attr_loop, h5py._hl.dataset.Dataset):
assert all(getattr(read_nwbfile.subject, i).value == j)
else:
assert getattr(read_nwbfile.subject, i) == j
for no,probe_name in enumerate(probe_names):
for i, j in temp_probes[no].items():
attr_loop = getattr(read_nwbfile.devices[probe_name], i, None)
if attr_loop:
if isinstance(attr_loop, h5py._hl.dataset.Dataset):
assert all(getattr(read_nwbfile.devices[probe_name], i).value == j)
else:
assert getattr(read_nwbfile.devices[probe_name], i) == j
shutil.rmtree(testdir)
class Preprocessing:
def load_yaml(self):
# Open YAML file with metadata if existing then dump all data in a dict
if os.path.isfile("data.yaml"):
with open("data.yaml", 'r') as stream:
self.data = yaml.safe_load(stream)
# Same but with .yml extension
elif os.path.isfile("data.yml"):
with open("data.yml", 'r') as stream:
self.data = yaml.safe_load(stream)
else:
self.data = dict()
if self.data is None:
self.data = dict()
# Dump the dict in the YAML file to save what the user inputted for future use
with open('data.yaml', 'w') as outfile:
yaml.dump(self.data,
outfile,
default_flow_style=False,
allow_unicode=True)
def add_required_metadata(self, data, key, metadata_type):
# Prompt user to give required metadata
# Need to be wary of the type (put " " for string and datetime.datetime(%Y, %m, %d) for datetime)
print("Missing required " + metadata_type + " metadata : " + key +
"\n")
metadata_value = input("Type the value (with respect of the type) : ")
data[key] = eval(metadata_value)
# Dump the dict in the YAML file to save what the user inputted for future use
with open('data.yaml', 'w') as outfile:
yaml.dump(self.data,
outfile,
default_flow_style=False,
allow_unicode=True)
return data
def add_optional_metadata(self, data):
# Allow user to add as much metadata as he wants with the key he wants
# Need to refer to documentation if he wants to fill existing attributes but he can create new ones and use
# them in his own code
keyboard_input = False
while not keyboard_input:
# Prompt user to give optional metadata
# Need to be wary of the type (put " " for string and datetime.datetime(%Y, %m, %d) for datetime)
metadata_key = input(
"Type the name of the metadata you want to add ? ")
metadata_key.replace(" ", "").lower().replace("", "_")
metadata_value = input(
"Type the value (with respect of the type) : ")
data[metadata_key] = eval(metadata_value)
go = input("Continue ? (yes/no)")
# Prevent errors if other input than yes/np
while go.replace(" ", "").lower() != "no" and go.replace(
" ", "").lower() != "yes":
go = input("Continue ? (yes/no)")
if go.replace(" ", "").lower() == "no":
keyboard_input = True
# Dump the dict in the YAML file to save what the user inputted for future use
with open('data.yaml', 'w') as outfile:
yaml.dump(self.data,
outfile,
default_flow_style=False,
allow_unicode=True)
return data
def ophys_metadata_acquisition(self):
# TODO: Peut être faire un dictionnaire de sous-dictionnaires correspondant à chaque classe à remplir
# Points positifs : Plus grand lisibilité dans le YAML, gestion simple des ambiguités de nom
# tout en gardant la notation de NWB.
# Points négatifs : Plus dur à rentrer en input pour l'utilisateur (il faut lui demander à quelle classe
# correspond sa valeur).
# List of the required metadata
# TODO : maybe better implementation is possible ?
required_metadata = [
"session_description", "identifier", "session_start_time"
]
if self.data.get('ophys_metadata') is None:
self.data['ophys_metadata'] = dict()
for i in required_metadata:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
else:
# Check if YAML file doesn't have all the required attributes and ask them the missing ones
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
print("Found ophys metadata : " +
str(list(self.data['ophys_metadata'].keys())))
add_metadata = input(
"Do you want to add more ophys metadata ? (yes/no) ")
# Prevent errors if other input than yes/np
while add_metadata.replace(
" ", "").lower() != "no" and add_metadata.replace(
" ", "").lower() != "yes":
add_metadata = input(
"Do you want to add more ophys metadata ? (yes/no) ")
if add_metadata.replace(" ", "").lower() == "yes":
self.data["ophys_metadata"] = self.add_optional_metadata(
self.data["ophys_metadata"])
# Create new NWB file with all known attributes
self.nwbfile = NWBFile(
session_description=self.data['ophys_metadata'].get(
"session_description"),
identifier=self.data['ophys_metadata'].get("identifier"),
session_start_time=self.data['ophys_metadata'].get(
"session_start_time"),
file_create_date=self.data['ophys_metadata'].get(
"file_create_date"),
timestamps_reference_time=self.data['ophys_metadata'].get(
"timestamps_reference_time"),
experimenter=self.data['ophys_metadata'].get("experimenter"),
experiment_description=self.data['ophys_metadata'].get(
"experiment_description"),
session_id=self.data['ophys_metadata'].get("session_id"),
institution=self.data['ophys_metadata'].get("institution"),
keywords=self.data['ophys_metadata'].get("keywords"),
notes=self.data['ophys_metadata'].get("notes"),
pharmacology=self.data['ophys_metadata'].get("pharmacology"),
protocol=self.data['ophys_metadata'].get("protocol"),
related_publications=self.data['ophys_metadata'].get(
"related_publications"),
slices=self.data['ophys_metadata'].get("slices"),
source_script=self.data['ophys_metadata'].get("source_script"),
source_script_file_name=self.data['ophys_metadata'].get(
"source_script_file_name"),
data_collection=self.data['ophys_metadata'].get(
"self.data['ophys_metadata']_collection"),
surgery=self.data['ophys_metadata'].get("surgery"),
virus=self.data['ophys_metadata'].get("virus"),
stimulus_notes=self.data['ophys_metadata'].get("stimulus_notes"),
lab=self.data['ophys_metadata'].get("lab"),
subject=self.subject)
def subject_metadata_acquisition(self):
# Check if metadata about the subject exists and prompt the user if he wants to add some
if self.data.get('subject_metadata') is None:
print("No subject metadata found \n ")
self.data['subject_metadata'] = dict()
elif len(self.data['subject_metadata']) == 0:
print("No subject metadata found \n ")
else:
print("Found subject metadata : " +
str(list(self.data['subject_metadata'].keys())))
add_metadata = input(
"Do you want to add more subject metadata ? (yes/no) ")
# Prevent errors if other input than yes/np
while add_metadata.replace(
" ", "").lower() != "no" and add_metadata.replace(
" ", "").lower() != "yes":
add_metadata = input(
"Do you want to add more subject metadata ? (yes/no) ")
if add_metadata.replace(" ", "").lower() == "yes":
self.data['subject_metadata'] = self.add_optional_metadata(
self.data['subject_metadata'])
self.subject = Subject(
age=self.data['subject_metadata'].get("age"),
description=self.data['subject_metadata'].get("description"),
genotype=self.data['subject_metadata'].get("genotype"),
sex=self.data['subject_metadata'].get("sex"),
species=self.data['subject_metadata'].get("species"),
subject_id=self.data['subject_metadata'].get("subject_id"),
weight=self.data['subject_metadata'].get("weight"),
date_of_birth=self.data['subject_metadata'].get("date_of_birth"))
def cicada_create_device(self):
"""
class pynwb.device.Device(name, parent=None)
"""
required_metadata = ["device_name"]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
self.device = Device(
name=self.data['ophys_metadata'].get("device_name"))
self.nwbfile.add_device(self.device)
def cicada_create_optical_channel(self):
required_metadata = [
"optical_channel_name", "optical_channel_description",
"optical_channel_emission_lambda"
]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
self.optical_channel = OpticalChannel(
name=self.data['ophys_metadata'].get("optical_channel_name"),
description=self.data['ophys_metadata'].get(
"optical_channel_description"),
emission_lambda=self.data['ophys_metadata'].get(
"optical_channel_emission_lambda"))
def cicada_create_module(self):
required_metadata = [
"processing_module_name", "processing_module_description"
]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
self.mod = self.nwbfile.create_processing_module(
name=self.data['ophys_metadata'].get("processing_module_name"),
description=self.data['ophys_metadata'].get(
"processing_module_description"))
def cicada_create_imaging_plane(self):
""" class pynwb.ophys.ImagingPlane(name, optical_channel, description, device, excitation_lambda,
imaging_rate, indicator, location,
manifold=None, conversion=None, unit=None, reference_frame=None, parent=None)
"""
required_metadata = [
"imaging_plane_name", "imaging_plane_description",
"imaging_plane_excitation_lambda", "imaging_plane_imaging_rate",
"imaging_plane_indicator", "imaging_plane_location"
]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
# Nom du module où récupérer les infos de métadonnée
name_module = "imaging_plane_"
self.imaging_plane = self.nwbfile.create_imaging_plane(
name=self.data['ophys_metadata'].get(name_module + "name"),
optical_channel=self.optical_channel,
description=self.data['ophys_metadata'].get(name_module +
"description"),
device=self.device,
excitation_lambda=self.data['ophys_metadata'].get(
name_module + "excitation_lambda"),
imaging_rate=self.data['ophys_metadata'].get(name_module +
"imaging_rate"),
indicator=self.data['ophys_metadata'].get(name_module +
"indicator"),
location=self.data['ophys_metadata'].get(name_module + "location"),
manifold=self.data['ophys_metadata'].get(name_module + "manifold"),
conversion=self.data['ophys_metadata'].get(name_module +
"conversion"),
unit=self.data['ophys_metadata'].get(name_module + "unit"),
reference_frame=self.data['ophys_metadata'].get(name_module +
"reference_frame"))
def cicada_create_two_photon_series(self,
data_to_store=None,
external_file=None):
""" class pynwb.ophys.TwoPhotonSeries(name, imaging_plane,
data=None, unit=None, format=None, field_of_view=None, pmt_gain=None, scan_line_rate=None,
external_file=None, starting_frame=None, bits_per_pixel=None, dimension=[nan], resolution=0.0,
conversion=1.0, timestamps=None, starting_time=None, rate=None, comments='no comments',
description='no description', control=None, control_description=None, parent=None)
"""
required_metadata = ["two_photon_name"]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
# Nom du module où récupérer les infos de métadonnée
name_module = "two_photon_"
self.movie_two_photon = TwoPhotonSeries(
name=self.data['ophys_metadata'].get(name_module + "name"),
imaging_plane=self.imaging_plane,
data=data_to_store,
unit=self.data['ophys_metadata'].get(name_module + "unit"),
format=self.data['ophys_metadata'].get(name_module + "format"),
field_of_view=self.data['ophys_metadata'].get(name_module +
"field_of_view"),
pmt_gain=self.data['ophys_metadata'].get(name_module + "pmt_gain"),
scan_line_rate=self.data['ophys_metadata'].get(name_module +
"scan_line_rate"),
external_file=external_file,
starting_frame=self.data['ophys_metadata'].get(name_module +
"starting_frame"),
bits_per_pixel=self.data['ophys_metadata'].get(name_module +
"bits_per_pixel"),
dimension=data_to_store.shape[1:],
resolution=0.0,
conversion=1.0,
timestamps=self.data['ophys_metadata'].get(name_module +
"timestamps"),
starting_time=self.data['ophys_metadata'].get(name_module +
"starting_time"),
rate=1.0,
comments="no comments",
description="no description",
control=self.data['ophys_metadata'].get(name_module + "control"),
control_description=self.data['ophys_metadata'].get(
name_module + "control_description"),
parent=self.data['ophys_metadata'].get(name_module + "parent"))
self.nwbfile.add_acquisition(self.movie_two_photon)
def cicada_create_motion_correction(self):
""" class pynwb.ophys.MotionCorrection(corrected_images_stacks={}, name='MotionCorrection')
"""
# Nom du module où récupérer les infos de métadonnée
name_module = "motion_correction_"
corrected_images_stacks = {}
self.motion_correction = MotionCorrection(
corrected_images_stacks=corrected_images_stacks,
name="MotionCorrection")
self.mod.add_data_interface(self.motion_correction)
def cicada_add_corrected_image_stack(self,
corrected=None,
original=None,
xy_translation=None):
""" class pynwb.ophys.CorrectedImageStack(corrected, original, xy_translation, name='CorrectedImageStack')
"""
# Nom du module où récupérer les infos de métadonnée
name_module = "corrected_image_stack_"
self.corrected_image_stack = CorrectedImageStack(
corrected=corrected,
original=original,
xy_translation=xy_translation,
name="CorrectedImageStack")
self.motion_correction.add_corrected_image_stack(
self.corrected_image_stack)
def cicada_add_plane_segmentation(self):
""" class pynwb.ophys.PlaneSegmentation(description, imaging_plane,
name=None, reference_images=None, id=None, columns=None, colnames=None)
"""
required_metadata = ["plane_segmentation_description"]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
# Nom du module où récupérer les infos de métadonnée
name_module = "plane_segmentation_"
self.plane_segmentation = PlaneSegmentation(
description=self.data['ophys_metadata'].get(name_module +
"description"),
imaging_plane=self.imaging_plane,
name=self.data['ophys_metadata'].get(name_module + "name"),
reference_images=self.data['ophys_metadata'].get(
name_module + "reference_image"),
id=self.data['ophys_metadata'].get(name_module + "id"),
columns=self.data['ophys_metadata'].get(name_module + "columns"),
colnames=self.data['ophys_metadata'].get(name_module + "colnames"))
self.image_segmentation.add_plane_segmentation(self.plane_segmentation)
def cicada_add_roi_in_plane_segmentation(self,
pixel_mask=None,
voxel_mask=None,
image_mask=None,
id_roi=None):
"""add_roi(pixel_mask=None, voxel_mask=None, image_mask=None, id=None)
"""
self.plane_segmentation.add_roi(pixel_mask=pixel_mask,
voxel_mask=voxel_mask,
image_mask=image_mask,
id=id_roi)
def cicada_create_roi_table_region_in_plane_segmentation(
self, region=slice(None, None, None)):
"""create_roi_table_region(description, region=slice(None, None, None), name='rois')"""
required_metadata = ["roi_table_region_description"]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
# Nom du module où récupérer les infos de métadonnée
name_module = "roi_table_region_"
self.table_region = self.plane_segmentation.create_roi_table_region(
description=self.data['ophys_metadata'].get(name_module +
"description"),
region=region,
name="rois")
def cicada_create_image_segmentation(self):
""" class pynwb.ophys.ImageSegmentation(plane_segmentations={}, name='ImageSegmentation')
"""
# Nom du module où récupérer les infos de métadonnée
name_module = "image_segmentation_"
plane_segmentations = {}
self.image_segmentation = ImageSegmentation(
plane_segmentations=plane_segmentations, name="ImageSegmentation")
self.mod.add_data_interface(self.image_segmentation)
def cicada_create_fluorescence(self):
""" class pynwb.ophys.Fluorescence(roi_response_series={}, name='Fluorescence')
"""
# Nom du module où récupérer les infos de métadonnée
name_module = "fluorescence_"
roi_response_series = {}
self.fluorescence = Fluorescence(
roi_response_series=roi_response_series, name="Fluorescence")
self.mod.add_data_interface(self.fluorescence)
def cicada_create_DfOverF(self):
""" class pynwb.ophys.DfOverF(roi_response_series={}, name='DfOverF')
"""
# Nom du module où récupérer les infos de métadonnée
name_module = "DfOverF_"
roi_response_series = {}
self.DfOverF = DfOverF(roi_response_series=roi_response_series,
name="DfOverF")
self.mod.add_data_interface(self.DfOverF)
def cicada_add_roi_response_series(self,
module,
traces_data=None,
rois=None):
""" class pynwb.ophys.RoiResponseSeries(name, data, unit, rois,
resolution=0.0, conversion=1.0, timestamps=None, starting_time=None, rate=None, comments='no comments',
description='no description', control=None, control_description=None, parent=None)
"""
required_metadata = [
"roi_response_series_name", "roi_response_series_unit"
]
metadata_to_add = list(
set(required_metadata) -
set(list(self.data['ophys_metadata'].keys())))
for i in metadata_to_add:
self.data["ophys_metadata"] = self.add_required_metadata(
self.data["ophys_metadata"], i, "ophys")
# Nom du module où récupérer les infos de métadonnée
name_module = "roi_response_series_"
roi_response_series = RoiResponseSeries(
name=self.data['ophys_metadata'].get(name_module + "name"),
data=traces_data,
unit=self.data['ophys_metadata'].get(name_module + "unit"),
rois=self.table_region,
resolution=0.0,
conversion=1.0,
timestamps=self.data['ophys_metadata'].get(name_module +
"timestamp"),
starting_time=self.data['ophys_metadata'].get(name_module +
"starting_time"),
rate=1.0,
comments="no comments",
description="no description",
control=self.data['ophys_metadata'].get(name_module + "control"),
control_description=self.data['ophys_metadata'].get(
name_module + "control_description"),
parent=self.data['ophys_metadata'].get(name_module + "parent"))
if module == "DfOverF":
self.DfOverF.add_roi_response_series(roi_response_series)
elif module == "fluorescence":
self.fluorescence.add_roi_response_series(roi_response_series)
else:
print(
f"erreur : le nom du module doit être 'DfOverF' ou 'fluorescence', et non {module} !"
)
def find_roi(self):
# Chemin du dossier suite2p
data_path = "C:/Users/François/Documents/dossier François/Stage INMED/" \
"Programmes/Godly Ultimate Interface/NWB/exp2nwb-master/src/suite2p"
self.suite2p_data = dict()
# Ouverture des fichiers stat et is_cell
f = np.load(data_path + "/F.npy", allow_pickle=True)
self.suite2p_data["F"] = f
f_neu = np.load(data_path + "/Fneu.npy", allow_pickle=True)
self.suite2p_data["Fneu"] = f_neu
spks = np.load(data_path + "/spks.npy", allow_pickle=True)
self.suite2p_data["spks"] = spks
stat = np.load(data_path + "/stat.npy", allow_pickle=True)
self.suite2p_data["stat"] = stat
is_cell = np.load(data_path + "/iscell.npy", allow_pickle=True)
self.suite2p_data["is_cell"] = is_cell
# Trouve les coordonnées de chaque ROI (cellule ici)
coord = []
for cell in np.arange(len(stat)):
if is_cell[cell][0] == 0:
continue
print(is_cell[cell][0])
list_points_coord = [
(x, y, 1)
for x, y in zip(stat[cell]["xpix"], stat[cell]["ypix"])
]
# coord.append(np.array(list_points_coord).transpose())
# La suite permet d'avoir uniquement les contours (sans les pixels intérieurs)
"""
# ATTENTION ! Il faut : from shapely.geometry import MultiPoint, LineString
convex_hull = MultiPoint(list_points_coord).convex_hull
if isinstance(convex_hull, LineString):
coord_shapely = MultiPoint(list_points_coord).convex_hull.coords
else:
coord_shapely = MultiPoint(list_points_coord).convex_hull.exterior.coords
coord.append(np.array(coord_shapely).transpose())
"""
self.suite2p_data[
"coord"] = coord # Contient la liste des pixels inclus dans chaque ROI
experimenter='Dr. Bilbo Baggins',
lab='Bag End Laboratory',
institution='University of Middle Earth at the Shire',
experiment_description=('I went on an adventure with thirteen '
'dwarves to reclaim vast treasures.'),
session_id='LONELYMTN')
####################
# Adding metadata about acquisition
# ---------------------------------
#
# Before you can add your data, you will need to provide some information about how that data was generated.
# This amounts describing the device, imaging plane and the optical channel used.
device = Device('imaging_device_1')
nwbfile.add_device(device)
optical_channel = OpticalChannel('my_optchan', 'description', 500.)
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln', optical_channel, 'a very interesting part of the brain',
device, 600., 300., 'GFP', 'my favorite brain location', np.ones(
(5, 5, 3)), 4.0, 'manifold unit', 'A frame to refer to')
####################
# Adding two-photon image data
# ----------------------------
#
# Now that you have your :py:class:`~pynwb.ophys.ImagingPlane`, you can create a
# :py:class:`~pynwb.ophys.TwoPhotonSeries` - the class representing two photon imaging data.
#
# From here you have two options. The first option is to supply the image data to PyNWB, using the `data` argument.
# The other option is the provide a path the images. These two options have trade-offs, so it is worth spending time
def nwb_copy_file(old_file, new_file, cp_objs={}):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path
String such as '/path/to/old_file.nwb'.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
"""
manager = get_manager()
# Open original signal file
with NWBHDF5IO(old_file, 'r', manager=manager,
load_namespaces=True) as io1:
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(session_description=str(nwb_old.session_description),
identifier='',
session_start_time=datetime.now(tzlocal()))
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[
aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name)
)
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux])
)
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = ['x', 'y', 'z', 'imp', 'location', 'filtering',
'group', 'group_name']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [str(elem) for elem in nwb_old.electrodes[
var].data[:]]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(name=str(var),
description=
str(nwb_old.electrodes[
var].description),
data=var_data)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags',
'timeseries']
[new_vars.remove(var) for var in default_vars if
var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(name=var,
description=nwb_old.epochs[
var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(
start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(name=var,
description=nwb_old.trials[
var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
if cp_objs['ecephys'] is True:
interfaces = nwb_old.processing[
'ecephys'].data_interfaces.keys()
else: # list of items
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.'
)
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
if 'acquisition' in cp_objs:
if cp_objs['acquisition'] is True:
all_acq_names = list(nwb_old.acquisition.keys())
else: # list of items
all_acq_names = cp_objs['acquisition']
for acq_name in all_acq_names:
obj_old = nwb_old.acquisition[acq_name]
obj = copy_obj(obj_old, nwb_old, nwb_new)
if obj is not None:
nwb_new.add_acquisition(obj)
# Subject ---------------------------------------------------------
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
subject_id=nwb_old.subject.subject_id,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
# Write new file with copied fields
io2.write(nwb_new, link_data=False)
def run_conversion(self, nwbfile: NWBFile, metadata: dict):
"""
Run conversionfor this data interface.
Reads optophysiology raw data from .rsd files and adds it to nwbfile.
XXXXXXX_A.rsd - Raw data from donor
XXXXXXX_B.rsd - Raw data from acceptor
XXXXXXXXX.rsh - Header data
Parameters
----------
nwbfile : NWBFile
metadata : dict
"""
dir_cortical_imaging = self.source_data['dir_cortical_imaging']
ophys_list = [i.name for i in Path(dir_cortical_imaging).glob('*.rsh')]
if len(ophys_list) > 0:
fname_prefix = ophys_list[0].split('-')[0]
else:
raise OSError(f"No .rsd file found in directory: {dir_cortical_imaging}.\n"
"Did you choose the correct path for source data?")
def data_gen(channel, trial):
"""channel = 'A' or 'B'"""
# Read trial-specific metadata file .rsh
trial_meta = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{trial}_{channel}.rsh")
file_rsm, files_raw, acquisition_date, sample_rate, n_frames = self.read_trial_meta(trial_meta=trial_meta)
# Iterates over all files within the same trial
for fn, fraw in enumerate(files_raw):
print('adding channel ' + channel + ', trial: ', trial, ': ', 100 * fn / len(files_raw), '%')
fpath = os.path.join(dir_cortical_imaging, fraw)
# Open file as a byte array
with open(fpath, "rb") as f:
byte = f.read(1000000000)
# Data as word array: 'h' signed, 'H' unsigned
words = np.array(struct.unpack('h' * (len(byte) // 2), byte))
# Iterates over frames within the same file (n_frames, 100, 100)
n_frames = int(len(words) / 12800)
words_reshaped = words.reshape(12800, n_frames, order='F')
frames = np.zeros((n_frames, 100, 100))
excess_frames = np.zeros((n_frames, 20, 100))
for ifr in range(n_frames):
iframe = -words_reshaped[:, ifr].reshape(128, 100, order='F').astype('int16')
frames[ifr, :, :] = iframe[20:120, :]
excess_frames[ifr, :, :] = iframe[0:20, :]
yield iframe[20:120, :]
# # Analog signals are taken from excess data variable
# analog_1 = np.squeeze(np.squeeze(excess_frames[:, 12, 0:80:4]).reshape(20*256, 1))
# analog_2 = np.squeeze(np.squeeze(excess_frames[:, 14, 0:80:4]).reshape(20*256, 1))
# stim_trg = np.squeeze(np.squeeze(excess_frames[:, 8, 0:80:4]).reshape(20*256, 1))
# Get session_start_time from first header file
all_files = os.listdir(dir_cortical_imaging)
all_headers = [f for f in all_files if ('.rsh' in f) and ('_A' not in f) and ('_B' not in f)]
all_headers.sort()
_, _, acquisition_date, _, _ = self.read_trial_meta(trial_meta=Path(dir_cortical_imaging) / all_headers[0])
session_start_time = datetime.strptime(acquisition_date, '%Y/%m/%d %H:%M:%S')
session_start_time_tzaware = pytz.timezone('EST').localize(session_start_time)
if session_start_time_tzaware != nwbfile.session_start_time:
print("Session start time in current nwbfile does not match the start time from rsd files.")
print("Ophys data conversion aborted.")
return
# Create and add device
device = Device(name=metadata['Ophys']['Device']['name'])
nwbfile.add_device(device)
# Get FRETSeries metadata
meta_donor = metadata['Ophys']['FRET']['donor'][0]
meta_acceptor = metadata['Ophys']['FRET']['acceptor'][0]
# OpticalChannels
opt_ch_donor = OpticalChannel(
name=meta_donor['optical_channel'][0]['name'],
description=meta_donor['optical_channel'][0]['description'],
emission_lambda=float(meta_donor['optical_channel'][0]['emission_lambda'])
)
opt_ch_acceptor = OpticalChannel(
name=meta_acceptor['optical_channel'][0]['name'],
description=meta_acceptor['optical_channel'][0]['description'],
emission_lambda=float(meta_acceptor['optical_channel'][0]['emission_lambda'])
)
# Add trials intervals values only if no trials data exists in nwbfile
if nwbfile.trials is not None:
add_trials = False
print('Trials already exist in current nwb file. Ophys trials intervals not added.')
else:
add_trials = True
# Iterate over trials, creates a FRET group per trial
trials_numbers = [f.split('-')[1].replace('.rsh', '') for f in all_headers]
for tr in trials_numbers:
# Read trial-specific metadata file .rsh
trial_meta_A = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_A.rsh")
trial_meta_B = os.path.join(dir_cortical_imaging, f"{fname_prefix}-{tr}_B.rsh")
file_rsm_A, files_raw_A, acquisition_date_A, sample_rate_A, n_frames_A = self.read_trial_meta(trial_meta=trial_meta_A)
file_rsm_B, files_raw_B, acquisition_date_B, sample_rate_B, n_frames_B = self.read_trial_meta(trial_meta=trial_meta_B)
absolute_start_time = datetime.strptime(acquisition_date_A, '%Y/%m/%d %H:%M:%S')
relative_start_time = float((absolute_start_time - nwbfile.session_start_time.replace(tzinfo=None)).seconds)
# Checks if Acceptor and Donor channels have the same basic parameters
assert acquisition_date_A == acquisition_date_B, \
"Acquisition date of channels do not match. Trial=" + str(tr)
assert sample_rate_A == sample_rate_B, \
"Sample rate of channels do not match. Trial=" + str(tr)
assert n_frames_A == n_frames_B, \
"Number of frames of channels do not match. Trial=" + str(tr)
assert relative_start_time >= 0., \
"Starting time is negative. Trial=" + str(tr)
# Create iterator
data_donor = DataChunkIterator(
data=data_gen(channel='A', trial=tr),
iter_axis=0,
buffer_size=10000,
maxshape=(None, 100, 100)
)
data_acceptor = DataChunkIterator(
data=data_gen(channel='B', trial=tr),
iter_axis=0,
buffer_size=10000,
maxshape=(None, 100, 100)
)
# FRETSeries
frets_donor = FRETSeries(
name='donor',
fluorophore=meta_donor['fluorophore'],
optical_channel=opt_ch_donor,
device=device,
description=meta_donor['description'],
data=data_donor,
starting_time=relative_start_time,
rate=sample_rate_A,
unit=meta_donor['unit'],
)
frets_acceptor = FRETSeries(
name='acceptor',
fluorophore=meta_acceptor['fluorophore'],
optical_channel=opt_ch_acceptor,
device=device,
description=meta_acceptor['description'],
data=data_acceptor,
starting_time=relative_start_time,
rate=sample_rate_B,
unit=meta_acceptor['unit']
)
# Add FRET to acquisition
meta_fret = metadata['Ophys']['FRET']
fret = FRET(
name=meta_fret['name'] + '_' + str(tr),
excitation_lambda=float(meta_fret['excitation_lambda']),
donor=frets_donor,
acceptor=frets_acceptor
)
nwbfile.add_acquisition(fret)
# Add trial
if add_trials:
tr_stop = relative_start_time + n_frames_A / sample_rate_A
nwbfile.add_trial(
start_time=relative_start_time,
stop_time=tr_stop
)
# DEVICES & ELECTRODE GROUPS
"""
Add in probes as devices and electrode groups.
"""
probe_descriptions = pd.read_csv('probes.description.tsv', sep='\t')
probe_descriptions = list(probe_descriptions['description'])
electrode_groups = list()
for i in range(len(probe_descriptions)):
probe_device = Device(name=str(i))
probe_electrode_group = ElectrodeGroup(
name='Probe' + str(i + 1),
description='Neuropixels Phase3A opt3',
device=probe_device,
location='')
nwb_file.add_device(probe_device)
electrode_groups.append(probe_electrode_group)
nwb_file.add_electrode_group(probe_electrode_group)
# CHANNELS
"""
Add channel information into the Electrode Table.
"""
# Read data
insertion_df = pd.read_csv('probes.insertion.tsv', sep='\t')
insertion_df['group_name'] = insertion_df.index.values
channel_site = read_npy_file('channels.site.npy')
channel_brain = pd.read_csv('channels.brainLocation.tsv', sep='\t')
channel_probes = read_npy_file('channels.probe.npy')
from pynwb import NWBFile, NWBHDF5IO
from pynwb.device import Device
from pynwb.ophys import OpticalChannel
from ndx_fret import FRET, FRETSeries
from datetime import datetime
import numpy as np
nwb = NWBFile('session_description', 'identifier', datetime.now().astimezone())
# Create and add device
device = Device(name='Device')
nwb.add_device(device)
# Create optical channels
opt_ch_d = OpticalChannel(name='optical_channel',
description='optical_channel_description',
emission_lambda=529.)
opt_ch_a = OpticalChannel(name='optical_channel',
description='optical_channel_description',
emission_lambda=633.)
# Create FRET
fs_d = FRETSeries(name='donor',
fluorophore='mCitrine',
optical_channel=opt_ch_d,
device=device,
description='description of donor series',
data=np.random.randn(100, 10, 10),
rate=200.,
unit='no unit')
data_dir = '/Volumes/black_backup/data/Soltesz/example_miniscope'
settings_and_notes_file = os.path.join(data_dir, 'settings_and_notes.dat')
df = pd.read_csv(settings_and_notes_file, sep='\t').loc[0]
session_start_time = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
nwb = NWBFile('session_description', 'identifier', session_start_time)
miniscope = Miniscope(name='Miniscope',
excitation=int(df['excitation']),
msCamExposure=int(df['msCamExposure']),
recordLength=int(df['recordLength']))
nwb.add_device(miniscope)
ms_files = [
os.path.split(x)[1]
for x in natsorted(glob(os.path.join(data_dir, 'msCam*.avi')))
]
behav_files = [
os.path.split(x)[1]
for x in natsorted(glob(os.path.join(data_dir, 'behavCam*.avi')))
]
nwb.add_acquisition(
ImageSeries(name='OnePhotonSeries',
format='external',
external_file=ms_files,
session_start_time=session_start_time,
experimenter='Joseph E. ODoherty',
lab='Sabes lab',
institution='University of California, San Francisco',
experiment_description=experiment_description,
session_id='indy_20160407_02')
# Create Device and ElectrodeGroup and adding electrode information to nwb.
# M1
#Create device
device_M1 = Device('Recording_Device_M1')
nwb.add_device(device_M1)
# Create electrode group
electrode_group_M1 = nwb.create_electrode_group(name='ElectrodeArrayM1', description="96 Channels Electrode Array",
location="Motor Cortex",
device=device_M1)
# Add metadata about each electrode in the group
for idx in np.arange(96):
nwb.add_electrode(x=np.nan, y=np.nan, z=np.nan,
imp=np.nan,
location='M1', filtering='none',
group=electrode_group_M1)
# S1
class TestNWBFileReading(TestCase):
def setUp(self):
start_time = datetime(2017, 4, 3, 11, tzinfo=tzlocal())
create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
self.nwb_file_content = NWBFile(
session_description='demonstrate NWBFile basics',
identifier='nwb_file',
session_start_time=start_time,
file_create_date=create_date)
def test_read_nwb_header_device_successfully(self):
header_device = HeaderDevice(
name='header_device',
headstage_serial='Sample headstage_serial',
headstage_smart_ref_on='Sample headstage_smart_ref_on',
realtime_mode='Sample realtime_mode',
headstage_auto_settle_on='Sample headstage_auto_settle_on',
timestamp_at_creation='Sample timestamp_at_creation',
controller_firmware_version='Sample controller_firmware_version',
controller_serial='Sample controller_serial',
save_displayed_chan_only='Sample save_displayed_chan_only',
headstage_firmware_version='Sample headstage_firmware_version',
qt_version='Sample qt_version',
compile_date='Sample compile_date',
compile_time='Sample compile_time',
file_prefix='Sample file_prefix',
headstage_gyro_sensor_on='Sample headstage_gyro_sensor_on',
headstage_mag_sensor_on='Sample headstage_mag_sensor_on',
trodes_version='Sample trodes_version',
headstage_accel_sensor_on='Sample headstage_accel_sensor_on',
commit_head='Sample commit_head',
system_time_at_creation='Sample system_time_at_creation',
file_path='Sample file_path',
)
self.nwb_file_content.add_device(header_device)
nwb_file_handler = NWBHDF5IO('header_device.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('header_device.nwb'))
with pynwb.NWBHDF5IO('header_device.nwb', 'r') as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertEqual(nwb_file.devices['header_device'].commit_head,
header_device.commit_head)
self.delete_nwb('header_device')
def test_read_nwb_probe_successfully(self):
shanks_electrode = ShanksElectrode(name='electrode_shank',
rel_x=1.0,
rel_y=2.0,
rel_z=3.0)
shank = Shank(name='shank')
shank.add_shanks_electrode(shanks_electrode)
probe = Probe(name='probe',
units='mm',
id=1,
probe_type='type_1',
probe_description='2',
contact_size=1.0,
contact_side_numbering=False)
probe.add_shank(shank)
self.nwb_file_content.add_device(probe)
nwb_file_handler = NWBHDF5IO('probe.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('probe.nwb'))
with pynwb.NWBHDF5IO('probe.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertContainerEqual(nwb_file.devices['probe'], probe)
self.delete_nwb('probe')
def test_read_nwb_data_acq_device_successfully(self):
data_acq_device = DataAcqDevice(name='DataAcqDevice1',
system='System1',
amplifier='Amplifier1',
adc_circuit='adc_circuit1')
self.nwb_file_content.add_device(data_acq_device)
nwb_file_handler = NWBHDF5IO('data_acq_device.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('data_acq_device.nwb'))
with pynwb.NWBHDF5IO('data_acq_device.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertContainerEqual(nwb_file.devices['DataAcqDevice1'],
data_acq_device)
self.delete_nwb('data_acq_device')
def test_read_nwb_camera_device_successfully(self):
camera_device = CameraDevice(name='CameraDevice1',
meters_per_pixel=0.20,
camera_name='test name',
model='ndx2000',
lens='500dpt',
manufacturer='sony')
self.nwb_file_content.add_device(camera_device)
nwb_file_handler = NWBHDF5IO('camera_device.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('camera_device.nwb'))
with pynwb.NWBHDF5IO('camera_device.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertContainerEqual(nwb_file.devices['CameraDevice1'],
camera_device)
self.delete_nwb('camera_device')
def test_read_nwb_nwb_image_series_successfully(self):
device_1 = Device('device1')
device_2 = Device('device2')
mock_timestamps = [1, 2, 3]
mock_external_file = ['some file']
nwb_image_series = NwbImageSeries(name='NwbImageSeries1',
timestamps=mock_timestamps,
external_file=mock_external_file,
devices=[device_1, device_2])
behavioral_time_series = BehavioralEvents(name="BehavioralTimeSeries")
behavioral_time_series.add_timeseries(nwb_image_series)
processing_module = ProcessingModule(name='ProcessingModule',
description='')
processing_module.add_data_interface(behavioral_time_series)
self.nwb_file_content.add_processing_module(processing_module)
self.nwb_file_content.add_stimulus_template(nwb_image_series)
nwb_file_handler = NWBHDF5IO('nwb_image_series.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('nwb_image_series.nwb'))
with pynwb.NWBHDF5IO('nwb_image_series.nwb', 'r',
load_namespaces=True) as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertContainerEqual(
nwb_file.stimulus_template['NwbImageSeries1'],
nwb_image_series)
self.assertContainerEqual(
nwb_file.processing['ProcessingModule'].data_interfaces[
'BehavioralTimeSeries'].time_series['NwbImageSeries1'],
nwb_image_series)
self.delete_nwb('nwb_image_series')
def test_read_nwb_nwb_electrode_group_successfully(self):
device = Device('device_0')
self.nwb_file_content.add_device(device)
nwb_electrode_group = NwbElectrodeGroup(
name='nwb_electrode_group_0',
description='Sample description',
location='Sample location',
device=device,
targeted_location='predicted location',
targeted_x=1.0,
targeted_y=2.0,
targeted_z=3.0,
units='um')
self.nwb_file_content.add_electrode_group(nwb_electrode_group)
nwb_file_handler = NWBHDF5IO('nwb_electrode_group.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('nwb_electrode_group.nwb'))
with pynwb.NWBHDF5IO('nwb_electrode_group.nwb',
'r') as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertEqual(
nwb_file.electrode_groups['nwb_electrode_group_0'].name,
nwb_electrode_group.name)
self.assertEqual(
nwb_file.electrode_groups['nwb_electrode_group_0'].
targeted_location, nwb_electrode_group.targeted_location)
self.delete_nwb('nwb_electrode_group')
def test_read_nwb_associated_files_successfully(self):
associated_files = AssociatedFiles(
name='file1',
description='description of file1',
content='1 2 3 content of file test',
task_epochs='1, 2')
self.nwb_file_content.add_processing_module(
ProcessingModule('associated_files',
'description_of_associaed_files'))
self.nwb_file_content.processing['associated_files'].add(
associated_files)
nwb_file_handler = NWBHDF5IO('associated_files.nwb', mode='w')
nwb_file_handler.write(self.nwb_file_content)
nwb_file_handler.close()
self.assertTrue(os.path.exists('associated_files.nwb'))
with pynwb.NWBHDF5IO('associated_files.nwb', 'r') as nwb_file_handler:
nwb_file = nwb_file_handler.read()
self.assertIsInstance(
nwb_file.processing['associated_files']['file1'],
AssociatedFiles)
self.assertEqual(
'file1', nwb_file.processing['associated_files']['file1'].name)
self.assertEqual(
'description of file1', nwb_file.processing['associated_files']
['file1'].fields['description'])
self.assertEqual(
'1 2 3 content of file test',
nwb_file.processing['associated_files']
['file1'].fields['content'])
self.assertEqual(
'1, 2', nwb_file.processing['associated_files']
['file1'].fields['task_epochs'])
self.delete_nwb('associated_files')
@staticmethod
def delete_nwb(filename):
os.remove(filename + '.nwb')
def setUp(self):
nwbfile = NWBFile(
'my first synthetic recording',
'EXAMPLE_ID',
datetime.now(tzlocal()),
experimenter='Dr. Bilbo Baggins',
lab='Bag End Laboratory',
institution='University of Middle Earth at the Shire',
experiment_description=('I went on an adventure with thirteen '
'dwarves to reclaim vast treasures.'),
session_id='LONELYMTN')
device = Device('imaging_device_1')
nwbfile.add_device(device)
optical_channel = OpticalChannel('my_optchan', 'description', 500.)
imaging_plane = nwbfile.create_imaging_plane(
'my_imgpln', optical_channel,
'a very interesting part of the brain',
device, 600., 300., 'GFP', 'my favorite brain location',
np.ones((5, 5, 3)), 4.0, 'manifold unit', 'A frame to refer to')
self.image_series = TwoPhotonSeries(name='test_iS',
dimension=[2],
data=np.random.rand(10, 5, 5, 3),
external_file=['images.tiff'],
imaging_plane=imaging_plane,
starting_frame=[0],
format='tiff',
starting_time=0.0,
rate=1.0)
nwbfile.add_acquisition(self.image_series)
mod = nwbfile.create_processing_module(
'ophys', 'contains optical physiology processed data')
img_seg = ImageSegmentation()
mod.add(img_seg)
ps = img_seg.create_plane_segmentation(
'output from segmenting my favorite imaging plane', imaging_plane,
'my_planeseg', self.image_series)
w, h = 3, 3
pix_mask1 = [(0, 0, 1.1), (1, 1, 1.2), (2, 2, 1.3)]
vox_mask1 = [(0, 0, 0, 1.1), (1, 1, 1, 1.2), (2, 2, 2, 1.3)]
img_mask1 = [[0.0 for x in range(w)] for y in range(h)]
img_mask1[0][0] = 1.1
img_mask1[1][1] = 1.2
img_mask1[2][2] = 1.3
ps.add_roi(pixel_mask=pix_mask1,
image_mask=img_mask1,
voxel_mask=vox_mask1)
pix_mask2 = [(0, 0, 2.1), (1, 1, 2.2)]
vox_mask2 = [(0, 0, 0, 2.1), (1, 1, 1, 2.2)]
img_mask2 = [[0.0 for x in range(w)] for y in range(h)]
img_mask2[0][0] = 2.1
img_mask2[1][1] = 2.2
ps.add_roi(pixel_mask=pix_mask2,
image_mask=img_mask2,
voxel_mask=vox_mask2)
fl = Fluorescence()
mod.add(fl)
rt_region = ps.create_roi_table_region('the first of two ROIs',
region=[0])
data = np.array([0., 1., 2., 3., 4., 5., 6., 7., 8.,
9.]).reshape(10, 1)
timestamps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
rrs = fl.create_roi_response_series('my_rrs',
data,
rt_region,
unit='lumens',
timestamps=timestamps)
self.df_over_f = DfOverF(rrs)
def test_device():
nwbfile = NWBFile('session description',
'session id',
datetime.now(tzlocal()),
experimenter='experimenter name',
lab='lab name',
institution='institution name',
experiment_description=('experiment description'),
session_id='sessionid')
laserline_device = LaserLine(name='mylaserline1',
reference='test_ref_laserline',
analog_modulation_frequency=Measurement(
name='None',
description='None',
unit='Hz',
data=[100]),
power=Measurement(name='None',
description='None',
unit='uW',
data=[100]))
laserline_devices_ = LaserLineDevices()
laserline_devices_.add_laserline(laserline_device)
laserline_devices_.create_laserline(
name='mylaserline2',
reference='test_ref_laserline',
analog_modulation_frequency=Measurement(name='None',
description='None',
unit='Hz',
data=[100]),
power=Measurement(name='None',
description='None',
unit='uW',
data=[100]))
photodetector_device = PhotoDetector(name='myphotodetector1',
reference='test_ref_photodetector',
gain=Measurement(name='None',
description='None',
unit='Hz',
data=[1]),
bandwidth=Measurement(
name='None',
description='None',
unit='uW',
data=[50]))
photodetector_devices_ = PhotoDetectorDevices()
photodetector_devices_.add_photodetector(photodetector_device)
lockinamp_device = LockInAmplifier(name='mylockinamp',
demodulation_filter_order=10.0,
reference='test_ref',
demod_bandwidth=Measurement(
name='None',
description='None',
unit='Hz',
data=[150]),
columns=[
VectorData(name='channel_name',
description='None',
data=['name1', 'name2']),
Measurement(name='offset',
description='None',
unit='mV',
data=[140, 260]),
VectorData(name='gain',
description='None',
data=[250, 250])
])
lockinamp_devices_ = LockInAmplifierDevices()
lockinamp_devices_.add_lockinamp(lockinamp_device)
nwbfile.add_device(
TEMPO(name='tempo_test',
laserline_devices=laserline_devices_,
photodetector_devices=photodetector_devices_,
lockinamp_devices=lockinamp_devices_))
# testing laserline_device
assert_array_equal(
np.array(nwbfile.devices['tempo_test'].laserline_devices.children[0].
analog_modulation_frequency.data), np.array([100]))
# testing photodetector_device
assert_array_equal(
np.array(nwbfile.devices['tempo_test'].photodetector_devices.
children[0].bandwidth.data), np.array([50]))
# testing lockinamp_device
assert_array_equal(
np.array(nwbfile.devices['tempo_test'].lockinamp_devices.children[0].
columns[1].data), np.array([140, 260]))
with NWBHDF5IO('test_ndx-tempo.nwb', 'w') as io:
io.write(nwbfile)
del nwbfile
with NWBHDF5IO('test_ndx-tempo.nwb', 'r', load_namespaces=True) as io:
nwb = io.read()
# testing laserline_device
assert_array_equal(
np.array(nwb.devices['tempo_test'].laserline_devices.children[0].
analog_modulation_frequency.data), np.array([100]))
# testing photodetector_device
assert_array_equal(
np.array(nwb.devices['tempo_test'].photodetector_devices.
children[0].bandwidth.data), np.array([50]))
# testing lockinamp_device
assert_array_equal(
np.array(nwb.devices['tempo_test'].lockinamp_device.children[0].
columns[1].data), np.array([140, 260]))
class Alyx2NWBConverter:
def __init__(self,
saveloc=None,
nwb_metadata_file=None,
metadata_obj: Alyx2NWBMetadata = None,
one_object: ONE = None,
save_raw=False,
save_camera_raw=False,
complevel=4,
shuffle=False,
buffer_size=1):
"""
Retrieve all Alyx session, subject metadata, raw data for eid using the one apis load method
Map that to nwb supported datatypes and create an nwb file.
Parameters
----------
saveloc: str, Path
save location of nwbfile
nwb_metadata_file: [dict, str]
output of Alyx2NWBMetadata as a dict/json location str
metadata_obj: Alyx2NWBMetadata
one_object: ONE()
save_raw: bool
will load and save large raw files: ecephys.raw.ap/lf.cbin to nwb
save_camera_raw: bool
will load and save mice camera movie .mp4: _iblrig_Camera.raw
complevel: int
level of compression to apply to raw datasets
(0-9)>(low,high). https://docs.h5py.org/en/latest/high/dataset.html
shuffle: bool
Enable shuffle I/O filter. http://docs.h5py.org/en/latest/high/dataset.html#dataset-shuffle
"""
self.buffer_size = buffer_size
self.complevel = complevel
self.shuffle = shuffle
if nwb_metadata_file is not None:
if isinstance(nwb_metadata_file, dict):
self.nwb_metadata = nwb_metadata_file
elif isinstance(nwb_metadata_file, str):
with open(nwb_metadata_file, 'r') as f:
self.nwb_metadata = json.load(f)
elif metadata_obj is not None:
self.nwb_metadata = metadata_obj.complete_metadata
else:
raise Exception(
'required one of argument: nwb_metadata_file OR metadata_obj')
if one_object is not None:
self.one_object = one_object
elif metadata_obj is not None:
self.one_object = metadata_obj.one_obj
else:
Warning('creating a ONE object and continuing')
self.one_object = ONE()
if saveloc is None:
Warning('saving nwb file in current working directory')
self.saveloc = str(Path.cwd())
else:
self.saveloc = str(saveloc)
self.eid = self.nwb_metadata["eid"]
if not isinstance(self.nwb_metadata['NWBFile']['session_start_time'],
datetime):
self.nwb_metadata['NWBFile']['session_start_time'] = \
datetime.strptime(self.nwb_metadata['NWBFile']['session_start_time'], '%Y-%m-%dT%X').replace(
tzinfo=pytz.utc)
self.nwb_metadata['IBLSubject']['date_of_birth'] = \
datetime.strptime(self.nwb_metadata['IBLSubject']['date_of_birth'], '%Y-%m-%dT%X').replace(
tzinfo=pytz.utc)
# create nwbfile:
self.initialize_nwbfile()
self.no_probes = len(self.nwb_metadata['Probes'])
if self.no_probes == 0:
warnings.warn(
'could not find probe information, will create trials, behavior, acquisition'
)
self.electrode_table_exist = False
self._one_data = _OneData(self.one_object,
self.eid,
self.no_probes,
self.nwb_metadata,
save_raw=save_raw,
save_camera_raw=save_camera_raw)
def initialize_nwbfile(self):
"""
Creates self.nwbfile, devices and electrode group of nwb file.
"""
nwbfile_args = dict(identifier=str(uuid.uuid4()), )
nwbfile_args.update(**self.nwb_metadata['NWBFile'])
self.nwbfile = NWBFile(**nwbfile_args)
# create devices
[
self.nwbfile.create_device(**idevice_meta)
for idevice_meta in self.nwb_metadata['Ecephys']['Device']
]
if 'ElectrodeGroup' in self.nwb_metadata['Ecephys']:
self.create_electrode_groups(self.nwb_metadata['Ecephys'])
def create_electrode_groups(self, metadata_ecephys):
"""
This method is called at __init__.
Use metadata to create ElectrodeGroup object(s) in the NWBFile
Parameters
----------
metadata_ecephys : dict
Dict with key:value pairs for defining the Ecephys group from where this
ElectrodeGroup belongs. This should contain keys for required groups
such as 'Device', 'ElectrodeGroup', etc.
"""
for metadata_elec_group in metadata_ecephys['ElectrodeGroup']:
eg_name = metadata_elec_group['name']
# Tests if ElectrodeGroup already exists
aux = [i.name == eg_name for i in self.nwbfile.children]
if any(aux):
print(eg_name + ' already exists in current NWBFile.')
else:
device_name = metadata_elec_group['device']
if device_name in self.nwbfile.devices:
device = self.nwbfile.devices[device_name]
else:
print('Device ', device_name, ' for ElectrodeGroup ',
eg_name, ' does not exist.')
print('Make sure ', device_name,
' is defined in metadata.')
eg_description = metadata_elec_group['description']
eg_location = metadata_elec_group['location']
self.nwbfile.create_electrode_group(name=eg_name,
location=eg_location,
device=device,
description=eg_description)
def check_module(self, name, description=None):
"""
Check if processing module exists. If not, create it. Then return module
Parameters
----------
name: str
description: str | None (optional)
Returns
-------
pynwb.module
"""
if name in self.nwbfile.processing:
return self.nwbfile.processing[name]
else:
if description is None:
description = name
return self.nwbfile.create_processing_module(name, description)
def create_stimulus(self):
"""
Creates stimulus data in nwbfile
"""
stimulus_list = self._get_data(
self.nwb_metadata['Stimulus'].get('time_series'))
for i in stimulus_list:
self.nwbfile.add_stimulus(pynwb.TimeSeries(**i))
def create_units(self):
"""
Units table in nwbfile
"""
if self.no_probes == 0:
return
if not self.electrode_table_exist:
self.create_electrode_table_ecephys()
unit_table_list = self._get_data(self.nwb_metadata['Units'])
# no required arguments for units table. Below are default columns in the table.
default_args = [
'id', 'waveform_mean', 'electrodes', 'electrode_group',
'spike_times', 'obs_intervals'
]
default_ids = _get_default_column_ids(
default_args, [i['name'] for i in unit_table_list])
if len(default_ids) != len(default_args):
warnings.warn(f'could not find all of {default_args} clusters')
non_default_ids = list(
set(range(len(unit_table_list))).difference(set(default_ids)))
default_dict = {
unit_table_list[id]['name']: unit_table_list[id]['data']
for id in default_ids
}
for cluster_no in range(len(unit_table_list[0]['data'])):
add_dict = dict()
for ibl_dataset_name in default_dict:
if ibl_dataset_name == 'electrodes':
add_dict.update({
ibl_dataset_name:
[default_dict[ibl_dataset_name][cluster_no]]
})
if ibl_dataset_name == 'spike_times':
add_dict.update({
ibl_dataset_name:
default_dict[ibl_dataset_name][cluster_no]
})
elif ibl_dataset_name == 'obs_intervals': # common across all clusters
add_dict.update(
{ibl_dataset_name: default_dict[ibl_dataset_name]})
elif ibl_dataset_name == 'electrode_group':
add_dict.update({
ibl_dataset_name:
self.nwbfile.electrode_groups[self.nwb_metadata[
'Probes'][default_dict[ibl_dataset_name]
[cluster_no]]['name']]
})
elif ibl_dataset_name == 'id':
if cluster_no >= self._one_data.data_attrs_dump[
'unit_table_length'][0]:
add_dict.update({
ibl_dataset_name:
default_dict[ibl_dataset_name][cluster_no] +
self._one_data.data_attrs_dump['unit_table_length']
[0]
})
else:
add_dict.update({
ibl_dataset_name:
default_dict[ibl_dataset_name][cluster_no]
})
elif ibl_dataset_name == 'waveform_mean':
add_dict.update({
ibl_dataset_name:
np.mean(default_dict[ibl_dataset_name][cluster_no],
axis=1)
}) # finding the mean along all the channels of the sluter
self.nwbfile.add_unit(**add_dict)
for id in non_default_ids:
if isinstance(unit_table_list[id]['data'], object):
unit_table_list[id]['data'] = unit_table_list[id][
'data'].tolist() # convert string numpy
self.nwbfile.add_unit_column(
name=unit_table_list[id]['name'],
description=unit_table_list[id]['description'],
data=unit_table_list[id]['data'])
def create_electrode_table_ecephys(self):
"""
Creates electrode table
"""
if self.no_probes == 0:
return
if self.electrode_table_exist:
pass
electrode_table_list = self._get_data(
self.nwb_metadata['ElectrodeTable'])
# electrode table has required arguments:
required_args = ['group', 'x', 'y']
default_ids = _get_default_column_ids(
required_args, [i['name'] for i in electrode_table_list])
non_default_ids = list(
set(range(len(electrode_table_list))).difference(set(default_ids)))
default_dict = {
electrode_table_list[id]['name']: electrode_table_list[id]['data']
for id in default_ids
}
if 'group' in default_dict:
group_labels = default_dict['group']
else: # else fill with probe zero data.
group_labels = np.concatenate([
np.ones(self._one_data.
data_attrs_dump['electrode_table_length'][i],
dtype=int) * i for i in range(self.no_probes)
])
for electrode_no in range(len(electrode_table_list[0]['data'])):
if 'x' in default_dict:
x = default_dict['x'][electrode_no][0]
y = default_dict['y'][electrode_no][1]
else:
x = float('NaN')
y = float('NaN')
group_data = self.nwbfile.electrode_groups[self.nwb_metadata[
'Probes'][group_labels[electrode_no]]['name']]
self.nwbfile.add_electrode(x=x,
y=y,
z=float('NaN'),
imp=float('NaN'),
location='None',
group=group_data,
filtering='none')
for id in non_default_ids:
self.nwbfile.add_electrode_column(
name=electrode_table_list[id]['name'],
description=electrode_table_list[id]['description'],
data=electrode_table_list[id]['data'])
# create probes specific DynamicTableRegion:
self.probe_dt_region = [
self.nwbfile.create_electrode_table_region(region=list(
range(self._one_data.data_attrs_dump['electrode_table_length']
[j])),
description=i['name'])
for j, i in enumerate(self.nwb_metadata['Probes'])
]
self.probe_dt_region_all = self.nwbfile.create_electrode_table_region(
region=list(
range(
sum(self._one_data.
data_attrs_dump['electrode_table_length']))),
description='AllProbes')
self.electrode_table_exist = True
def create_timeseries_ecephys(self):
"""
create SpikeEventSeries, ElectricalSeries, Spectrum datatypes within nwbfile>processing>ecephys
"""
if self.no_probes == 0:
return
if not self.electrode_table_exist:
self.create_electrode_table_ecephys()
if 'ecephys' not in self.nwbfile.processing:
mod = self.nwbfile.create_processing_module(
'ecephys', 'Processed electrophysiology data of IBL')
else:
mod = self.nwbfile.get_processing_module('ecephys')
for neurodata_type_name, neurodata_type_args_list in self.nwb_metadata[
'Ecephys']['Ecephys'].items():
data_retrieved_args_list = self._get_data(
neurodata_type_args_list
) # list of dicts with keys as argument names
for no, neurodata_type_args in enumerate(data_retrieved_args_list):
ibl_dataset_name = neurodata_type_args_list[no]['data']
if 'ElectricalSeries' in neurodata_type_name:
timestamps_names = self._one_data.data_attrs_dump[
'_iblqc_ephysTimeRms.timestamps']
data_names = self._one_data.data_attrs_dump[
'_iblqc_ephysTimeRms.rms']
for data_idx, data in enumerate(
neurodata_type_args['data']):
probe_no = [
j for j in range(self.no_probes)
if self.nwb_metadata['Probes'][j]['name'] in
data_names[data_idx]
][0]
if data.shape[1] > self._one_data.data_attrs_dump[
'electrode_table_length'][probe_no]:
if 'channels.rawInd' in self._one_data.loaded_datasets:
channel_idx = self._one_data.loaded_datasets[
'channels.rawInd'][probe_no].data.astype(
'int')
else:
warnings.warn('could not find channels.rawInd')
break
else:
channel_idx = slice(None)
mod.add(
ElectricalSeries(
name=data_names[data_idx],
description=neurodata_type_args['description'],
timestamps=neurodata_type_args['timestamps']
[timestamps_names.index(data_names[data_idx])],
data=data[:, channel_idx],
electrodes=self.probe_dt_region[probe_no]))
elif 'Spectrum' in neurodata_type_name:
if ibl_dataset_name in '_iblqc_ephysSpectralDensity.power':
freqs_names = self._one_data.data_attrs_dump[
'_iblqc_ephysSpectralDensity.freqs']
data_names = self._one_data.data_attrs_dump[
'_iblqc_ephysSpectralDensity.power']
for data_idx, data in enumerate(
neurodata_type_args['data']):
mod.add(
Spectrum(name=data_names[data_idx],
frequencies=neurodata_type_args[
'frequencies'][freqs_names.index(
data_names[data_idx])],
power=data))
elif 'SpikeEventSeries' in neurodata_type_name:
neurodata_type_args.update(
dict(electrodes=self.probe_dt_region_all))
mod.add(
pynwb.ecephys.SpikeEventSeries(**neurodata_type_args))
def create_behavior(self):
"""
Create behavior processing module
"""
self.check_module('behavior')
for behavior_datatype in self.nwb_metadata['Behavior']:
if behavior_datatype == 'Position':
position_cont = pynwb.behavior.Position()
time_series_list_details = self._get_data(
self.nwb_metadata['Behavior'][behavior_datatype]
['spatial_series'])
if len(time_series_list_details) == 0:
continue
# rate_list = [150.0,60.0,60.0] # based on the google doc for _iblrig_body/left/rightCamera.raw,
dataname_list = self._one_data.data_attrs_dump['camera.dlc']
data_list = time_series_list_details[0]['data']
timestamps_list = time_series_list_details[0]['timestamps']
for dataname, data, timestamps in zip(dataname_list, data_list,
timestamps_list):
colnames = data.columns
data_np = data.to_numpy()
x_column_ids = [
n for n, k in enumerate(colnames) if 'x' in k
]
for x_column_id in x_column_ids:
data_loop = data_np[:, x_column_id:x_column_id + 2]
position_cont.create_spatial_series(
name=dataname + colnames[x_column_id][:-2],
data=data_loop,
reference_frame='none',
timestamps=timestamps,
conversion=1e-3)
self.nwbfile.processing['behavior'].add(position_cont)
elif not (behavior_datatype == 'BehavioralEpochs'):
time_series_func = pynwb.TimeSeries
time_series_list_details = self._get_data(
self.nwb_metadata['Behavior'][behavior_datatype]
['time_series'])
if len(time_series_list_details) == 0:
continue
time_series_list_obj = []
for i in time_series_list_details:
unit = 'radians/sec' if 'velocity' in i[
'name'] else 'radians'
time_series_list_obj.append(
time_series_func(**i, unit=unit))
func = getattr(pynwb.behavior, behavior_datatype)
self.nwbfile.processing['behavior'].add(
func(time_series=time_series_list_obj))
else:
time_series_func = pynwb.epoch.TimeIntervals
time_series_list_details = self._get_data(
self.nwb_metadata['Behavior'][behavior_datatype]
['time_intervals'])
if len(time_series_list_details) == 0:
continue
for k in time_series_list_details:
time_intervals = time_series_func('BehavioralEpochs')
for time_interval in k['timestamps']:
time_intervals.add_interval(
start_time=time_interval[0],
stop_time=time_interval[1])
time_intervals.add_column(k['name'],
k['description'],
data=k['data'])
self.nwbfile.processing['behavior'].add(time_intervals)
def create_acquisition(self):
"""
Acquisition data like audiospectrogram(raw beh data), nidq(raw ephys data), raw camera data.
These are independent of probe type.
"""
for neurodata_type_name, neurodata_type_args_list in self.nwb_metadata[
'Acquisition'].items():
data_retrieved_args_list = self._get_data(neurodata_type_args_list)
for neurodata_type_args in data_retrieved_args_list:
if neurodata_type_name == 'ImageSeries':
for types, times in zip(neurodata_type_args['data'],
neurodata_type_args['timestamps']):
customargs = dict(name='camera_raw',
external_file=[str(types)],
format='external',
timestamps=times,
unit='n.a.')
self.nwbfile.add_acquisition(ImageSeries(**customargs))
elif neurodata_type_name == 'DecompositionSeries':
neurodata_type_args['bands'] = np.squeeze(
neurodata_type_args['bands'])
freqs = DynamicTable(
'bands',
'spectogram frequencies',
id=np.arange(neurodata_type_args['bands'].shape[0]))
freqs.add_column('freq',
'frequency value',
data=neurodata_type_args['bands'])
neurodata_type_args.update(dict(bands=freqs))
temp = neurodata_type_args['data'][:, :, np.newaxis]
neurodata_type_args['data'] = np.moveaxis(
temp, [0, 1, 2], [0, 2, 1])
ts = neurodata_type_args.pop('timestamps')
starting_time = ts[0][0] if isinstance(
ts[0], np.ndarray) else ts[0]
neurodata_type_args.update(
dict(starting_time=np.float64(starting_time),
rate=1 / np.mean(np.diff(ts.squeeze())),
unit='sec'))
self.nwbfile.add_acquisition(
DecompositionSeries(**neurodata_type_args))
elif neurodata_type_name == 'ElectricalSeries':
if not self.electrode_table_exist:
self.create_electrode_table_ecephys()
if neurodata_type_args['name'] in ['raw.lf', 'raw.ap']:
for probe_no in range(self.no_probes):
if neurodata_type_args['data'][probe_no].shape[
1] > self._one_data.data_attrs_dump[
'electrode_table_length'][probe_no]:
if 'channels.rawInd' in self._one_data.loaded_datasets:
channel_idx = self._one_data.loaded_datasets[
'channels.rawInd'][
probe_no].data.astype('int')
else:
warnings.warn(
'could not find channels.rawInd')
break
else:
channel_idx = slice(None)
self.nwbfile.add_acquisition(
ElectricalSeries(
name=neurodata_type_args['name'] + '_' +
self.nwb_metadata['Probes'][probe_no]
['name'],
starting_time=np.abs(
np.round(
neurodata_type_args['timestamps']
[probe_no][0, 1], 2)
), # round starting times of the order of 1e-5
rate=neurodata_type_args['data']
[probe_no].fs,
data=H5DataIO(
DataChunkIterator(
_iter_datasetview(
neurodata_type_args['data']
[probe_no],
channel_ids=channel_idx),
buffer_size=self.buffer_size),
compression=True,
shuffle=self.shuffle,
compression_opts=self.complevel),
electrodes=self.probe_dt_region[probe_no],
channel_conversion=neurodata_type_args[
'data']
[probe_no].channel_conversion_sample2v[
neurodata_type_args['data']
[probe_no].type][channel_idx]))
elif neurodata_type_args['name'] in ['raw.nidq']:
self.nwbfile.add_acquisition(
ElectricalSeries(**neurodata_type_args))
def create_probes(self):
"""
Fills in all the probes metadata into the custom NeuroPixels extension.
"""
for i in self.nwb_metadata['Probes']:
self.nwbfile.add_device(IblProbes(**i))
def create_iblsubject(self):
"""
Populates the custom subject extension for IBL mice daata
"""
self.nwbfile.subject = IblSubject(**self.nwb_metadata['IBLSubject'])
def create_lab_meta_data(self):
"""
Populates the custom lab_meta_data extension for IBL sessions data
"""
self.nwbfile.add_lab_meta_data(
IblSessionData(**self.nwb_metadata['IBLSessionsData']))
def create_trials(self):
table_data = self._get_data(self.nwb_metadata['Trials'])
required_fields = ['start_time', 'stop_time']
required_data = [i for i in table_data if i['name'] in required_fields]
optional_data = [
i for i in table_data if i['name'] not in required_fields
]
if len(required_fields) != len(required_data):
warnings.warn(
'could not find required datasets: trials.start_time, trials.stop_time, '
'skipping trials table')
return
for start_time, stop_time in zip(required_data[0]['data'][:, 0],
required_data[1]['data'][:, 1]):
self.nwbfile.add_trial(start_time=start_time, stop_time=stop_time)
for op_data in optional_data:
if op_data['data'].shape[0] == required_data[0]['data'].shape[0]:
self.nwbfile.add_trial_column(
name=op_data['name'],
description=op_data['description'],
data=op_data['data'])
else:
warnings.warn(
f'shape of trials.{op_data["name"]} does not match other trials.* datasets'
)
def _get_data(self, sub_metadata):
"""
Uses OneData class to query ONE datasets on server and download them locally
Parameters
----------
sub_metadata: [list, dict]
list of metadata dicts containing a data key with a dataset type string as value to retrieve data from(npy, tsv etc)
Returns
-------
out_dict: dict
dictionary with actual data loaded in the data field
"""
include_idx = []
out_dict_trim = []
alt_datatypes = ['bands', 'power', 'frequencies', 'timestamps']
if isinstance(sub_metadata, list):
out_dict = deepcopy(sub_metadata)
elif isinstance(sub_metadata, dict):
out_dict = deepcopy(list(sub_metadata))
else:
return []
req_datatypes = ['data']
for count, neurodata_type_args in enumerate(out_dict):
for alt_names in alt_datatypes:
if neurodata_type_args.get(
alt_names
): # in case of Decomposotion series, Spectrum
neurodata_type_args[
alt_names] = self._one_data.download_dataset(
neurodata_type_args[alt_names],
neurodata_type_args['name'])
req_datatypes.append(alt_names)
if neurodata_type_args[
'name'] == 'id': # valid in case of units table.
neurodata_type_args['data'] = self._one_data.download_dataset(
neurodata_type_args['data'], 'cluster_id')
else:
out_dict[count]['data'] = self._one_data.download_dataset(
neurodata_type_args['data'], neurodata_type_args['name'])
if all([out_dict[count][i] is not None for i in req_datatypes]):
include_idx.extend([count])
out_dict_trim.extend([out_dict[j0] for j0 in include_idx])
return out_dict_trim
def run_conversion(self):
"""
Single method to create all datasets and metadata in nwbfile in one go
Returns
-------
"""
execute_list = [
self.create_stimulus, self.create_trials,
self.create_electrode_table_ecephys,
self.create_timeseries_ecephys, self.create_units,
self.create_behavior, self.create_probes, self.create_iblsubject,
self.create_lab_meta_data, self.create_acquisition
]
t = tqdm(execute_list)
for i in t:
t.set_postfix(current=f'creating nwb ' + i.__name__.split('_')[-1])
i()
print('done converting')
def write_nwb(self, read_check=True):
"""
After run_conversion(), write nwbfile to disk with the loaded nwbfile
Parameters
----------
read_check: bool
Round trip verification
"""
print('Saving to file, please wait...')
with NWBHDF5IO(self.saveloc, 'w') as io:
io.write(self.nwbfile)
print('File successfully saved at: ', str(self.saveloc))
if read_check:
with NWBHDF5IO(self.saveloc, 'r') as io:
io.read()
print('Read check: OK')
def conversion_function(source_paths,
f_nwb,
metadata,
add_raw=False,
add_processed=True,
add_behavior=True,
plot_rois=False):
"""
Copy data stored in a set of .npz files to a single NWB file.
Parameters
----------
source_paths : dict
Dictionary with paths to source files/directories. e.g.:
{'raw_data': {'type': 'file', 'path': ''},
'raw_info': {'type': 'file', 'path': ''}
'processed_data': {'type': 'file', 'path': ''},
'sparse_matrix': {'type': 'file', 'path': ''},
'ref_image',: {'type': 'file', 'path': ''}}
f_nwb : str
Path to output NWB file, e.g. 'my_file.nwb'.
metadata : dict
Metadata dictionary
add_raw : bool
Whether to convert raw data or not.
add_processed : bool
Whether to convert processed data or not.
add_behavior : bool
Whether to convert behavior data or not.
plot_rois : bool
Plot ROIs
"""
# Source files
file_raw = None
file_info = None
file_processed = None
file_sparse_matrix = None
file_reference_image = None
for k, v in source_paths.items():
if source_paths[k]['path'] != '':
fname = source_paths[k]['path']
if k == 'raw_data':
file_raw = h5py.File(fname, 'r')
if k == 'raw_info':
file_info = scipy.io.loadmat(fname,
struct_as_record=False,
squeeze_me=True)
if k == 'processed_data':
file_processed = np.load(fname)
if k == 'sparse_matrix':
file_sparse_matrix = np.load(fname)
if k == 'ref_image':
file_reference_image = np.load(fname)
# Initialize a NWB object
nwb = NWBFile(**metadata['NWBFile'])
# Create and add device
device = Device(name=metadata['Ophys']['Device'][0]['name'])
nwb.add_device(device)
# Creates one Imaging Plane for each channel
fs = 1. / (file_processed['time'][0][1] - file_processed['time'][0][0])
for meta_ip in metadata['Ophys']['ImagingPlane']:
# Optical channel
opt_ch = OpticalChannel(
name=meta_ip['optical_channel'][0]['name'],
description=meta_ip['optical_channel'][0]['description'],
emission_lambda=meta_ip['optical_channel'][0]['emission_lambda'])
nwb.create_imaging_plane(
name=meta_ip['name'],
optical_channel=opt_ch,
description=meta_ip['description'],
device=device,
excitation_lambda=meta_ip['excitation_lambda'],
imaging_rate=fs,
indicator=meta_ip['indicator'],
location=meta_ip['location'],
)
# Raw optical data
if add_raw:
print('Adding raw data...')
for meta_tps in metadata['Ophys']['TwoPhotonSeries']:
if meta_tps['name'][-1] == 'R':
raw_data = file_raw['R']
else:
raw_data = file_raw['Y']
def data_gen(data):
xl, yl, zl, tl = data.shape
chunk = 0
while chunk < tl:
val = data[:, :, :, chunk]
chunk += 1
print('adding data chunk: ', chunk)
yield val
xl, yl, zl, tl = raw_data.shape
tps_data = DataChunkIterator(data=data_gen(data=raw_data),
iter_axis=0,
maxshape=(tl, xl, yl, zl))
# Change dimensions from (X,Y,Z,T) in mat file to (T,X,Y,Z) nwb standard
#raw_data = np.moveaxis(raw_data, -1, 0)
tps = TwoPhotonSeries(
name=meta_tps['name'],
imaging_plane=nwb.imaging_planes[meta_tps['imaging_plane']],
data=tps_data,
rate=file_info['info'].daq.scanRate)
nwb.add_acquisition(tps)
# Processed data
if add_processed:
print('Adding processed data...')
ophys_module = ProcessingModule(
name='Ophys',
description='contains optical physiology processed data.',
)
nwb.add_processing_module(ophys_module)
# Create Image Segmentation compartment
img_seg = ImageSegmentation(
name=metadata['Ophys']['ImageSegmentation']['name'])
ophys_module.add(img_seg)
# Create plane segmentation and add ROIs
meta_ps = metadata['Ophys']['ImageSegmentation'][
'plane_segmentations'][0]
ps = img_seg.create_plane_segmentation(
name=meta_ps['name'],
description=meta_ps['description'],
imaging_plane=nwb.imaging_planes[meta_ps['imaging_plane']],
)
# Add ROIs
indices = file_sparse_matrix['indices']
indptr = file_sparse_matrix['indptr']
dims = np.squeeze(file_processed['dims'])
for start, stop in zip(indptr, indptr[1:]):
voxel_mask = make_voxel_mask(indices[start:stop], dims)
ps.add_roi(voxel_mask=voxel_mask)
# Visualize 3D voxel masks
if plot_rois:
plot_rois_function(plane_segmentation=ps, indptr=indptr)
# DFF measures
dff = DfOverF(name=metadata['Ophys']['DfOverF']['name'])
ophys_module.add(dff)
# create ROI regions
n_cells = file_processed['dFF'].shape[0]
roi_region = ps.create_roi_table_region(description='RoiTableRegion',
region=list(range(n_cells)))
# create ROI response series
dff_data = file_processed['dFF']
tt = file_processed['time'].ravel()
meta_rrs = metadata['Ophys']['DfOverF']['roi_response_series'][0]
meta_rrs['data'] = dff_data.T
meta_rrs['rois'] = roi_region
meta_rrs['timestamps'] = tt
dff.create_roi_response_series(**meta_rrs)
# Creates GrayscaleVolume containers and add a reference image
grayscale_volume = GrayscaleVolume(
name=metadata['Ophys']['GrayscaleVolume']['name'],
data=file_reference_image['im'])
ophys_module.add(grayscale_volume)
# Behavior data
if add_behavior:
print('Adding behavior data...')
# Ball motion
behavior_mod = nwb.create_processing_module(
name='Behavior',
description='holds processed behavior data',
)
meta_ts = metadata['Behavior']['TimeSeries'][0]
meta_ts['data'] = file_processed['ball'].ravel()
tt = file_processed['time'].ravel()
meta_ts['timestamps'] = tt
behavior_ts = TimeSeries(**meta_ts)
behavior_mod.add(behavior_ts)
# Re-arranges spatial data of body-points positions tracking
pos = file_processed['dlc']
n_points = 8
pos_reshaped = pos.reshape(
(-1, n_points, 3)) # dims=(nSamples,n_points,3)
# Creates a Position object and add one SpatialSeries for each body-point position
position = Position()
for i in range(n_points):
position.create_spatial_series(
name='SpatialSeries_' + str(i),
data=pos_reshaped[:, i, :],
timestamps=tt,
reference_frame=
'Description defining what the zero-position is.',
conversion=np.nan)
behavior_mod.add(position)
# Trial times
trialFlag = file_processed['trialFlag'].ravel()
trial_inds = np.hstack(
(0, np.where(np.diff(trialFlag))[0], trialFlag.shape[0] - 1))
trial_times = tt[trial_inds]
for start, stop in zip(trial_times, trial_times[1:]):
nwb.add_trial(start_time=start, stop_time=stop)
# Saves to NWB file
with NWBHDF5IO(f_nwb, mode='w') as io:
io.write(nwb)
print('NWB file saved with size: ', os.stat(f_nwb).st_size / 1e6, ' mb')
def save(self,
file_name,
to32=True,
order='F',
imagej=False,
bigtiff=True,
excitation_lambda=488.0,
compress=0,
q_max=99.75,
q_min=1,
var_name_hdf5='mov',
sess_desc='some_description',
identifier='some identifier',
imaging_plane_description='some imaging plane description',
emission_lambda=520.0,
indicator='OGB-1',
location='brain',
starting_time=0.,
experimenter='Dr Who',
lab_name=None,
institution=None,
experiment_description='Experiment Description',
session_id='Session ID'):
"""
Save the timeseries in single precision. Supported formats include
TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB
Args:
file_name: str
name of file. Possible formats are tif, avi, npz, mmap and hdf5
to32: Bool
whether to transform to 32 bits
order: 'F' or 'C'
C or Fortran order
var_name_hdf5: str
Name of hdf5 file subdirectory
q_max, q_min: float in [0, 100]
percentile for maximum/minimum clipping value if saving as avi
(If set to None, no automatic scaling to the dynamic range [0, 255] is performed)
Raises:
Exception 'Extension Unknown'
"""
name, extension = os.path.splitext(file_name)[:2]
extension = extension.lower()
logging.debug("Parsing extension " + str(extension))
if extension in ['.tif', '.tiff', '.btf']:
with tifffile.TiffWriter(file_name, bigtiff=bigtiff,
imagej=imagej) as tif:
for i in range(self.shape[0]):
if i % 200 == 0:
logging.debug(str(i) + ' frames saved')
curfr = self[i].copy()
if to32 and not ('float32' in str(self.dtype)):
curfr = curfr.astype(np.float32)
tif.save(curfr, compress=compress)
elif extension == '.npz':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
np.savez(file_name,
input_arr=input_arr,
start_time=self.start_time,
fr=self.fr,
meta_data=self.meta_data,
file_name=self.file_name)
elif extension == '.avi':
codec = None
try:
codec = cv2.FOURCC('I', 'Y', 'U', 'V')
except AttributeError:
codec = cv2.VideoWriter_fourcc(*'IYUV')
if q_max is None or q_min is None:
data = self.astype(np.uint8)
else:
if q_max < 100:
maxmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_max)
else:
maxmov = np.nanmax(self)
if q_min > 0:
minmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_min)
else:
minmov = np.nanmin(self)
data = 255 * (self - minmov) / (maxmov - minmov)
np.clip(data, 0, 255, data)
data = data.astype(np.uint8)
y, x = data[0].shape
vw = cv2.VideoWriter(file_name,
codec,
self.fr, (x, y),
isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.file_name[0] is not None:
f_name = self.file_name
else:
f_name = ''
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
if self.meta_data[0] is None:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': [],
'file_name': f_name
})
else:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': self.meta_data,
'file_name': f_name
})
elif extension in ('.hdf5', '.h5'):
with h5py.File(file_name, "w") as f:
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
dset = f.create_dataset(var_name_hdf5, data=input_arr)
dset.attrs["fr"] = self.fr
dset.attrs["start_time"] = self.start_time
try:
dset.attrs["file_name"] = [
a.encode('utf8') for a in self.file_name
]
except:
logging.warning('No file saved')
if self.meta_data[0] is not None:
logging.debug("Metadata for saved file: " +
str(self.meta_data))
dset.attrs["meta_data"] = cpk.dumps(self.meta_data)
elif extension == '.mmap':
base_name = name
T = self.shape[0]
dims = self.shape[1:]
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
input_arr = np.transpose(input_arr,
list(range(1,
len(dims) + 1)) + [0])
input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F')
fname_tot = memmap_frames_filename(base_name, dims, T, order)
fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(np.uint64(np.prod(dims)), np.uint64(T)),
order=order)
big_mov[:] = np.asarray(input_arr, dtype=np.float32)
big_mov.flush()
del big_mov, input_arr
return fname_tot
elif extension == '.nwb':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
# Create NWB file
nwbfile = NWBFile(sess_desc,
identifier,
datetime.now(tzlocal()),
experimenter=experimenter,
lab=lab_name,
institution=institution,
experiment_description=experiment_description,
session_id=session_id)
# Get the device
device = Device('imaging_device')
nwbfile.add_device(device)
# OpticalChannel
optical_channel = OpticalChannel('OpticalChannel',
'main optical channel',
emission_lambda=emission_lambda)
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=self.fr,
indicator=indicator,
location=location)
# Images
image_series = TwoPhotonSeries(name=var_name_hdf5,
dimension=self.shape[1:],
data=input_arr,
imaging_plane=imaging_plane,
starting_frame=[0],
starting_time=starting_time,
rate=self.fr)
nwbfile.add_acquisition(image_series)
with NWBHDF5IO(file_name, 'w') as io:
io.write(nwbfile)
return file_name
else:
logging.error("Extension " + str(extension) + " unknown")
raise Exception('Extension Unknown')
def nwb_copy_file(old_file, new_file, cp_objs={}, save_to_file=True):
"""
Copy fields defined in 'obj', from existing NWB file to new NWB file.
Parameters
----------
old_file : str, path, nwbfile
String or path to nwb file '/path/to/old_file.nwb'. Alternatively, the
nwbfile object.
new_file : str, path
String such as '/path/to/new_file.nwb'.
cp_objs : dict
Name:Value pairs (Group:Children) listing the groups and respective
children from the current NWB file to be copied. Children can be:
- Boolean, indicating an attribute (e.g. for institution, lab)
- List of strings, containing several children names
Example:
{'institution':True,
'lab':True,
'acquisition':['microphone'],
'ecephys':['LFP','DecompositionSeries']}
save_to_file: Boolean
If True, saves directly to new_file.nwb. If False, only returns nwb_new.
Returns:
--------
nwb_new : nwbfile object
"""
manager = get_manager()
# Get from nwbfile object in memory or from file
if isinstance(old_file, NWBFile):
nwb_old = old_file
io1 = False
else:
io1 = NWBHDF5IO(str(old_file),
'r',
manager=manager,
load_namespaces=True)
nwb_old = io1.read()
# Creates new file
nwb_new = NWBFile(
session_description=str(nwb_old.session_description),
identifier=id_generator(),
session_start_time=nwb_old.session_start_time,
)
with NWBHDF5IO(new_file, mode='w', manager=manager,
load_namespaces=False) as io2:
# Institution name ------------------------------------------------
if 'institution' in cp_objs:
nwb_new.institution = str(nwb_old.institution)
# Lab name --------------------------------------------------------
if 'lab' in cp_objs:
nwb_new.lab = str(nwb_old.lab)
# Session id ------------------------------------------------------
if 'session' in cp_objs:
nwb_new.session_id = nwb_old.session_id
# Devices ---------------------------------------------------------
if 'devices' in cp_objs:
for aux in list(nwb_old.devices.keys()):
dev = Device(nwb_old.devices[aux].name)
nwb_new.add_device(dev)
# Electrode groups ------------------------------------------------
if 'electrode_groups' in cp_objs and nwb_old.electrode_groups is not None:
for aux in list(nwb_old.electrode_groups.keys()):
nwb_new.create_electrode_group(
name=str(nwb_old.electrode_groups[aux].name),
description=str(nwb_old.electrode_groups[aux].description),
location=str(nwb_old.electrode_groups[aux].location),
device=nwb_new.get_device(
nwb_old.electrode_groups[aux].device.name))
# Electrodes ------------------------------------------------------
if 'electrodes' in cp_objs and nwb_old.electrodes is not None:
nElec = len(nwb_old.electrodes['x'].data[:])
for aux in np.arange(nElec):
nwb_new.add_electrode(
x=nwb_old.electrodes['x'][aux],
y=nwb_old.electrodes['y'][aux],
z=nwb_old.electrodes['z'][aux],
imp=nwb_old.electrodes['imp'][aux],
location=str(nwb_old.electrodes['location'][aux]),
filtering=str(nwb_old.electrodes['filtering'][aux]),
group=nwb_new.get_electrode_group(
nwb_old.electrodes['group'][aux].name),
group_name=str(nwb_old.electrodes['group_name'][aux]))
# if there are custom variables
new_vars = list(nwb_old.electrodes.colnames)
default_vars = [
'x', 'y', 'z', 'imp', 'location', 'filtering', 'group',
'group_name'
]
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
if var == 'label':
var_data = [
str(elem) for elem in nwb_old.electrodes[var].data[:]
]
else:
var_data = np.array(nwb_old.electrodes[var].data[:])
nwb_new.add_electrode_column(
name=str(var),
description=str(nwb_old.electrodes[var].description),
data=var_data)
# If Bipolar scheme for electrodes
for v in nwb_old.lab_meta_data.values():
if isinstance(v, EcephysExt) and hasattr(
v, 'bipolar_scheme_table'):
bst_old = v.bipolar_scheme_table
bst_new = BipolarSchemeTable(
name=bst_old.name, description=bst_old.description)
ecephys_ext = EcephysExt(name=v.name)
ecephys_ext.bipolar_scheme_table = bst_new
nwb_new.add_lab_meta_data(ecephys_ext)
# Epochs ----------------------------------------------------------
if 'epochs' in cp_objs and nwb_old.epochs is not None:
nEpochs = len(nwb_old.epochs['start_time'].data[:])
for i in np.arange(nEpochs):
nwb_new.add_epoch(
start_time=nwb_old.epochs['start_time'].data[i],
stop_time=nwb_old.epochs['stop_time'].data[i])
# if there are custom variables
new_vars = list(nwb_old.epochs.colnames)
default_vars = ['start_time', 'stop_time', 'tags', 'timeseries']
[new_vars.remove(var) for var in default_vars if var in new_vars]
for var in new_vars:
nwb_new.add_epoch_column(
name=var,
description=nwb_old.epochs[var].description,
data=nwb_old.epochs[var].data[:])
# Invalid times ---------------------------------------------------
if 'invalid_times' in cp_objs and nwb_old.invalid_times is not None:
nInvalid = len(nwb_old.invalid_times['start_time'][:])
for aux in np.arange(nInvalid):
nwb_new.add_invalid_time_interval(
start_time=nwb_old.invalid_times['start_time'][aux],
stop_time=nwb_old.invalid_times['stop_time'][aux])
# Trials ----------------------------------------------------------
if 'trials' in cp_objs and nwb_old.trials is not None:
nTrials = len(nwb_old.trials['start_time'])
for aux in np.arange(nTrials):
nwb_new.add_trial(start_time=nwb_old.trials['start_time'][aux],
stop_time=nwb_old.trials['stop_time'][aux])
# if there are custom variables
new_vars = list(nwb_old.trials.colnames)
default_vars = ['start_time', 'stop_time']
[new_vars.remove(var) for var in default_vars]
for var in new_vars:
nwb_new.add_trial_column(
name=var,
description=nwb_old.trials[var].description,
data=nwb_old.trials[var].data[:])
# Intervals -------------------------------------------------------
if 'intervals' in cp_objs and nwb_old.intervals is not None:
all_objs_names = list(nwb_old.intervals.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.intervals[obj_name]
# create and add TimeIntervals
obj = TimeIntervals(name=obj_old.name,
description=obj_old.description)
nInt = len(obj_old['start_time'])
for ind in np.arange(nInt):
obj.add_interval(start_time=obj_old['start_time'][ind],
stop_time=obj_old['stop_time'][ind])
# Add to file
nwb_new.add_time_intervals(obj)
# Stimulus --------------------------------------------------------
if 'stimulus' in cp_objs:
all_objs_names = list(nwb_old.stimulus.keys())
for obj_name in all_objs_names:
obj_old = nwb_old.stimulus[obj_name]
obj = TimeSeries(name=obj_old.name,
description=obj_old.description,
data=obj_old.data[:],
rate=obj_old.rate,
resolution=obj_old.resolution,
conversion=obj_old.conversion,
starting_time=obj_old.starting_time,
unit=obj_old.unit)
nwb_new.add_stimulus(obj)
# Processing modules ----------------------------------------------
if 'ecephys' in cp_objs:
interfaces = [
nwb_old.processing['ecephys'].data_interfaces[key]
for key in cp_objs['ecephys']
]
# Add ecephys module to NWB file
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
nwb_new.add_processing_module(ecephys_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
ecephys_module.add_data_interface(obj)
if 'behavior' in cp_objs:
interfaces = [
nwb_old.processing['behavior'].data_interfaces[key]
for key in cp_objs['behavior']
]
if 'behavior' not in nwb_new.processing:
# Add behavior module to NWB file
behavior_module = ProcessingModule(
name='behavior', description='behavioral data.')
nwb_new.add_processing_module(behavior_module)
for interface_old in interfaces:
obj = copy_obj(interface_old, nwb_old, nwb_new)
if obj is not None:
behavior_module.add_data_interface(obj)
# Acquisition -----------------------------------------------------
# Can get raw ElecetricalSeries and Mic recording
if 'acquisition' in cp_objs:
for acq_name in cp_objs['acquisition']:
obj_old = nwb_old.acquisition[acq_name]
acq = copy_obj(obj_old, nwb_old, nwb_new)
nwb_new.add_acquisition(acq)
# Surveys ---------------------------------------------------------
if 'surveys' in cp_objs and 'behavior' in nwb_old.processing:
surveys_list = [
v for v in
nwb_old.processing['behavior'].data_interfaces.values()
if v.neurodata_type == 'SurveyTable'
]
if cp_objs['surveys'] and len(surveys_list) > 0:
if 'behavior' not in nwb_new.processing:
# Add behavior module to NWB file
behavior_module = ProcessingModule(
name='behavior', description='behavioral data.')
nwb_new.add_processing_module(behavior_module)
for obj_old in surveys_list:
srv = copy_obj(obj_old, nwb_old, nwb_new)
behavior_module.add_data_interface(srv)
# Subject ---------------------------------------------------------
if nwb_old.subject is not None:
if 'subject' in cp_objs:
try:
cortical_surfaces = CorticalSurfaces()
surfaces = nwb_old.subject.cortical_surfaces.surfaces
for sfc in list(surfaces.keys()):
cortical_surfaces.create_surface(
name=surfaces[sfc].name,
faces=surfaces[sfc].faces,
vertices=surfaces[sfc].vertices)
nwb_new.subject = ECoGSubject(
cortical_surfaces=cortical_surfaces,
subject_id=nwb_old.subject.subject_id,
age=nwb_old.subject.age,
description=nwb_old.subject.description,
genotype=nwb_old.subject.genotype,
sex=nwb_old.subject.sex,
species=nwb_old.subject.species,
weight=nwb_old.subject.weight,
date_of_birth=nwb_old.subject.date_of_birth)
except:
nwb_new.subject = Subject(**nwb_old.subject.fields)
# Write new file with copied fields
if save_to_file:
io2.write(nwb_new, link_data=False)
# Close old file and return new nwbfile object
if io1:
io1.close()
return nwb_new
