def find_protocol_details(FILES):
"""
"""
# protocol based on the first one:
data0 = Data(FILES[0])
protocols = list(data0.protocols)
STIM = {}
for key in data0.nwbfile.stimulus.keys():
STIM[key] = []
subjects = ['' for i in range(len(FILES))]
sessions, sessions_per_subject = np.arange(len(FILES)), np.zeros(
len(FILES), dtype=int)
for i, f in enumerate(FILES):
data = Data(f)
if not list(data.protocols) == protocols:
print('/!\ not a set of consistent files /!\ ')
break
subjects[i] = data.metadata['subject_ID']
sessions_per_subject[i] = np.sum(
data.metadata['subject_ID'] == np.array(subjects))
data.close()
filename = '%s-%isubjects-%isessions.nwb' % (
data0.metadata['protocol'], len(np.unique(subjects)), len(FILES))
data0.close()
return protocols, subjects, sessions, sessions_per_subject, filename, STIM
def preload_datafolder(self, fn):
data = Data(fn, metadata_only=True, with_tlim=False)
if data.nwbfile is not None:
return {
'display_name': data.df_name,
'subject': data.nwbfile.subject.description
}
else:
return {'display_name': '', 'subject': ''}
def find_matching_CaImaging_data(
filename,
CaImaging_root_folder,
min_protocol_duration=10, # seconds
verbose=True):
success, folder = False, ''
CA_FILES = build_Ca_filelist(CaImaging_root_folder)
data = Data(filename, metadata_only=True, with_tlim=True)
Tstart = data.metadata['NIdaq_Tstart']
st = datetime.datetime.fromtimestamp(Tstart).strftime('%H:%M:%S.%f')
true_tstart = StartTime_to_day_seconds(st)
true_duration = data.tlim[1] - data.tlim[0]
true_tstop = true_tstart + true_duration
times = np.arange(int(true_tstart), int(true_tstop))
day = datetime.datetime.fromtimestamp(Tstart).strftime('%Y_%m_%d')
# first insuring the good day in the CA FOLDERS
day_cond = (np.array(CA_FILES['date']) == day)
if len(times) > min_protocol_duration and (np.sum(day_cond) > 0):
# then we loop over Ca-imaging files to find the overlap
for ica in np.arange(len(CA_FILES['StartTime']))[day_cond]:
times2 = np.arange(int(CA_FILES['StartTime'][ica]),
int(CA_FILES['EndTime'][ica]))
if (len(np.intersect1d(times, times2)) > min_protocol_duration):
success, folder = True, CA_FILES['Bruker_folder'][ica]
percent_overlap = 100. * len(np.intersect1d(
times, times2)) / len(times)
print(50 * '-')
print(' => matched to %s with %.1f %% overlap' %
(folder, percent_overlap))
print(50 * '-')
data.close()
return success, folder
def __init__(self, filename):
data = Data(filename, metadata_only=True, verbose=False)
# computing episodes
self.episode_static_patch = EpisodeResponse(
filename,
protocol_id=data.get_protocol_id('static-patch'),
quantities=['dFoF'],
prestim_duration=3,
verbose=False)
self.episode_moving_dots = EpisodeResponse(
filename,
protocol_id=data.get_protocol_id('moving-dots'),
quantities=['dFoF'],
prestim_duration=3,
verbose=False)
self.episode_mixed = EpisodeResponse(
filename,
protocol_id=data.get_protocol_id('mixed-moving-dots-static-patch'),
quantities=['dFoF'],
prestim_duration=3,
verbose=False)
self.nROIs = self.episode_mixed.data.nROIs
self.episode_random_dots, self.episode_mixed_random_dots = None, None
if 'random-line-dots' in data.protocols:
self.episode_random_dots = EpisodeResponse(
filename,
protocol_id=data.get_protocol_id('random-line-dots'),
quantities=['dFoF'],
prestim_duration=3,
verbose=False)
else:
self.episode_random_dots = None
if 'random-mixed-moving-dots-static-patch' in data.protocols:
self.episode_mixed_random_dots = EpisodeResponse(
filename,
protocol_id=data.get_protocol_id(
'random-mixed-moving-dots-static-patch'),
quantities=['dFoF'],
prestim_duration=3,
verbose=False)
else:
self.episode_mixed_random_dots = None
else:
return True
elif (self.pvalue is not None):
return False
else:
print(' /!\ no valid p-value for significance test !! /!\ ')
return False
def pval_annot(self, size=5):
"""
uses the
"""
if self.positive and not self.sign:
return 'n.s.', size
elif self.pvalue < 1e-3:
return '***', size + 1
elif self.pvalue < 1e-2:
return '**', size + 1
elif self.pvalue < 0.05:
return '*', size + 1
else:
return 'n.s.', size
if __name__ == '__main__':
# filename = os.path.join(os.path.expanduser('~'), 'DATA', 'CaImaging', 'Wild_Type_GCamp6f', '2021_03_23-11-26-36.nwb')
filename = sys.argv[-1]
FullData = Data(filename)
def __init__(self, filename=''):
super().__init__()
self.app = QtWidgets.QApplication(sys.argv)
self.data = Data(filename)
self.CaImaging_key = 'Fluorescence'
np.mean((stats.y - stats.x) / stats.x))
for key in summary_data:
summary_data[key] = np.array(summary_data[key])
return summary_data
if __name__ == '__main__':
from analysis.read_NWB import Data
filename = sys.argv[-1]
if '.nwb' in sys.argv[-1]:
data = Data(filename)
data.build_dFoF()
episode = EpisodeResponse(data,
quantities=[
'Photodiode-Signal', 'pupil', 'gaze',
'facemotion', 'dFoF', 'rawFluo',
'Running-Speed'
])
print(episode.quantities)
# from datavyz import ge
# ge.plot(episode.t, episode.PhotodiodeSignal.mean(axis=0), sy=episode.PhotodiodeSignal.std(axis=0))
# ge.show()
# episode = EpisodeResponse(data,
# quantities=['Pupil', 'CaImaging', 'CaImaging'],
# quantities_args=[{}, {'subquantity':'Fluorescence'}, {'subquantity':'dFoF', 'roiIndices':np.arange(10)}])
class MainWindow(guiparts.NewWindow):
def __init__(self,
app,
args=None,
parent=None,
raw_data_visualization=False,
df_width=600,
selector_height=30,
win1_Wmax=1200,
win1_Wmin=300,
win1_Hmax=300,
win2_Wmax=500,
fullscreen=False):
self.app = app
self.settings = settings
self.raw_data_visualization = raw_data_visualization
self.no_subsampling = False
super(MainWindow, self).__init__(i=0, title='Data Visualization')
# play button
self.updateTimer = QtCore.QTimer()
self.updateTimer.timeout.connect(self.next_frame)
self.cwidget = QtWidgets.QWidget(self)
self.setCentralWidget(self.cwidget)
self.statusBar.showMessage(
'open file [O], refresh plot [R], play/pause [Space], initial-view [I], max-window [M] '
)
mainLayout = QtWidgets.QVBoxLayout()
Layout1 = QtWidgets.QHBoxLayout()
mainLayout.addLayout(Layout1)
Layout11 = QtWidgets.QVBoxLayout()
Layout1.addLayout(Layout11)
guiparts.create_calendar(self, Layout11)
self.cal.setMaximumHeight(150)
# folder box
self.fbox = QtWidgets.QComboBox(self)
self.fbox.setFont(guiparts.smallfont)
self.fbox.activated.connect(self.scan_folder)
self.fbox.setMaximumHeight(selector_height)
self.folder_default_key = ' [root datafolder]'
self.fbox.addItem(self.folder_default_key)
self.fbox.setCurrentIndex(0)
Layout11.addWidget(self.fbox)
# subject box
self.sbox = QtWidgets.QComboBox(self)
self.sbox.setFont(guiparts.smallfont)
self.sbox.activated.connect(self.pick_subject) # To be written !!
self.sbox.setMaximumHeight(selector_height)
self.subject_default_key = ' [subject] '
self.sbox.addItem(self.subject_default_key)
self.sbox.setCurrentIndex(0)
Layout11.addWidget(self.sbox)
# notes
self.notes = QtWidgets.QLabel('\n[exp info]' + 5 * '\n', self)
self.notes.setFont(guiparts.smallfont)
self.notes.setMaximumHeight(60)
Layout11.addWidget(self.notes)
self.pbox = QtWidgets.QComboBox(self)
self.pbox.setFont(guiparts.smallfont)
self.pbox.activated.connect(self.display_quantities)
self.pbox.setMaximumHeight(selector_height)
self.pbox.addItem('[visualization/analysis]')
self.pbox.addItem('-> Show Raw Data')
self.pbox.addItem('-> Trial-average')
self.pbox.addItem('-> draw figures')
self.pbox.addItem('-> build PDF summary')
self.pbox.addItem('-> open PDF summary')
self.pbox.setCurrentIndex(0)
Layout11.addWidget(self.pbox)
Layout113 = QtWidgets.QHBoxLayout()
Layout11.addLayout(Layout113)
add_buttons(self, Layout113)
Layout12 = QtWidgets.QVBoxLayout()
Layout1.addLayout(Layout12)
self.dbox = QtWidgets.QComboBox(self)
self.dbox.setMinimumWidth(df_width)
self.dbox.setMaximumWidth(win1_Wmax)
self.dbox.setMinimumHeight(selector_height)
self.dbox.setMaximumHeight(selector_height)
self.dbox.activated.connect(self.pick_datafile)
Layout12.addWidget(self.dbox)
self.win1 = pg.GraphicsLayoutWidget()
self.win1.setMaximumHeight(int(win1_Hmax - 1.5 * selector_height))
Layout12.addWidget(self.win1)
self.winTrace = pg.GraphicsLayoutWidget()
mainLayout.addWidget(self.winTrace)
guiparts.build_slider(self, mainLayout)
self.init_panels()
self.plot = self.winTrace.addPlot()
self.plot.hideAxis('left')
self.plot.setMouseEnabled(x=True, y=False)
# self.plot.setMenuEnabled(False)
self.plot.setLabel('bottom', 'time (s)')
self.xaxis = self.plot.getAxis('bottom')
self.scatter = pg.ScatterPlotItem()
self.plot.addItem(self.scatter)
Layout122 = QtWidgets.QHBoxLayout()
Layout12.addLayout(Layout122)
self.stimSelect = QtWidgets.QCheckBox("vis. stim")
self.stimSelect.clicked.connect(self.select_stim)
self.stimSelect.setStyleSheet('color: grey;')
self.pupilSelect = QtWidgets.QCheckBox("pupil")
self.pupilSelect.setStyleSheet('color: red;')
self.gazeSelect = QtWidgets.QCheckBox("gaze")
self.gazeSelect.setStyleSheet('color: orange;')
self.faceMtnSelect = QtWidgets.QCheckBox("whisk.")
self.faceMtnSelect.setStyleSheet('color: magenta;')
self.runSelect = QtWidgets.QCheckBox("run")
self.photodiodeSelect = QtWidgets.QCheckBox("photodiode")
self.photodiodeSelect.setStyleSheet('color: grey;')
self.ephysSelect = QtWidgets.QCheckBox("ephys")
self.ephysSelect.setStyleSheet('color: blue;')
self.ophysSelect = QtWidgets.QCheckBox("ophys")
self.ophysSelect.setStyleSheet('color: green;')
for x in [
self.stimSelect, self.pupilSelect, self.gazeSelect,
self.faceMtnSelect, self.runSelect, self.photodiodeSelect,
self.ephysSelect, self.ophysSelect
]:
x.setFont(guiparts.smallfont)
Layout122.addWidget(x)
self.roiPick = QtWidgets.QLineEdit()
self.roiPick.setText(' [...] ')
self.roiPick.setMinimumWidth(50)
self.roiPick.setMaximumWidth(250)
self.roiPick.returnPressed.connect(self.select_ROI)
self.roiPick.setFont(guiparts.smallfont)
self.ephysPick = QtWidgets.QLineEdit()
self.ephysPick.setText(' ')
# self.ephysPick.returnPressed.connect(self.select_ROI)
self.ephysPick.setFont(guiparts.smallfont)
self.guiKeywords = QtWidgets.QLineEdit()
self.guiKeywords.setText(' [GUI keywords] ')
# self.guiKeywords.setFixedWidth(200)
self.guiKeywords.returnPressed.connect(self.keyword_update)
self.guiKeywords.setFont(guiparts.smallfont)
Layout122.addWidget(self.guiKeywords)
# Layout122.addWidget(self.ephysPick)
Layout122.addWidget(self.roiPick)
self.subsamplingSelect = QtWidgets.QCheckBox("subsampl.")
self.subsamplingSelect.setStyleSheet('color: grey;')
self.subsamplingSelect.setFont(guiparts.smallfont)
Layout122.addWidget(self.subsamplingSelect)
self.annotSelect = QtWidgets.QCheckBox("annot.")
self.annotSelect.setStyleSheet('color: grey;')
self.annotSelect.setFont(guiparts.smallfont)
Layout122.addWidget(self.annotSelect)
self.imgSelect = QtWidgets.QCheckBox("img")
self.imgSelect.setStyleSheet('color: grey;')
self.imgSelect.setFont(guiparts.smallfont)
self.imgSelect.setChecked(True)
self.imgSelect.clicked.connect(self.remove_img)
Layout122.addWidget(self.imgSelect)
self.cwidget.setLayout(mainLayout)
self.show()
self.fbox.addItems(FOLDERS.keys())
self.windowTA, self.windowBM = None, None # sub-windows
if args is not None:
self.root_datafolder = args.root_datafolder
else:
self.root_datafolder = os.path.join(os.path.expanduser('~'),
'DATA')
self.time, self.data, self.roiIndices, self.tzoom = 0, None, [], [
0, 50
]
self.CaImaging_bg_key, self.planeID = 'meanImg', 0
self.CaImaging_key = 'Fluorescence'
self.FILES_PER_DAY, self.FILES_PER_SUBJECT, self.SUBJECTS = {}, {}, {}
self.minView = False
self.showwindow()
if (args is not None) and hasattr(args, 'datafile') and os.path.isfile(
args.datafile):
self.datafile = args.datafile
self.load_file(self.datafile)
plots.raw_data_plot(self, self.tzoom)
def init_panel_imgs(self):
self.pScreenimg.setImage(np.ones((10, 12)) * 50)
self.pFaceimg.setImage(np.ones((10, 12)) * 50)
self.pPupilimg.setImage(np.ones((10, 12)) * 50)
self.pFacemotionimg.setImage(np.ones((10, 12)) * 50)
self.pCaimg.setImage(np.ones((50, 50)) * 100)
self.pupilContour.setData([0], [0], size=1, brush=pg.mkBrush(0, 0, 0))
self.faceMotionContour.setData(
[0], [0],
size=2,
brush=pg.mkBrush(*settings['colors']['FaceMotion'][:3]))
self.facePupilContour.setData(
[0], [0],
size=2,
brush=pg.mkBrush(*settings['colors']['Pupil'][:3]))
def remove_img(self):
if not self.imgSelect.isChecked():
self.init_panel_imgs()
def init_panels(self):
# screen panel
self.pScreen = self.win1.addViewBox(lockAspect=True,
invertY=False,
border=[1, 1, 1],
colspan=2)
self.pScreenimg = pg.ImageItem(np.ones((10, 12)) * 50)
# FaceCamera panel
self.pFace = self.win1.addViewBox(lockAspect=True,
invertY=True,
border=[1, 1, 1],
colspan=2)
self.faceMotionContour = pg.ScatterPlotItem()
self.facePupilContour = pg.ScatterPlotItem()
self.pFaceimg = pg.ImageItem(np.ones((10, 12)) * 50)
# Pupil panel
self.pPupil = self.win1.addViewBox(lockAspect=True,
invertY=True,
border=[1, 1, 1])
self.pupilContour = pg.ScatterPlotItem()
self.pPupilimg = pg.ImageItem(np.ones((10, 12)) * 50)
# Facemotion panel
self.pFacemotion = self.win1.addViewBox(lockAspect=True,
invertY=True,
border=[1, 1, 1])
self.facemotionROI = pg.ScatterPlotItem()
self.pFacemotionimg = pg.ImageItem(np.ones((10, 12)) * 50)
# Ca-Imaging panel
self.pCa = self.win1.addViewBox(lockAspect=True,
invertY=True,
border=[1, 1, 1])
self.pCaimg = pg.ImageItem(np.ones((50, 50)) * 100)
for x, y in zip([
self.pScreen, self.pFace, self.pPupil, self.pPupil,
self.pFacemotion, self.pFacemotion, self.pCa, self.pFace,
self.pFace
], [
self.pScreenimg, self.pFaceimg, self.pPupilimg,
self.pupilContour, self.pFacemotionimg, self.facemotionROI,
self.pCaimg, self.faceMotionContour, self.facePupilContour
]):
x.addItem(y)
def open_file(self):
filename, _ = QtWidgets.QFileDialog.getOpenFileName(
self,
"Open Multimodal Experimental Recording (NWB file) ",
(FOLDERS[self.fbox.currentText()] if self.fbox.currentText()
in FOLDERS else os.path.join(os.path.expanduser('~'), 'DATA')),
filter="*.nwb")
# filename = '/home/yann/UNPROCESSED/2021_06_17/2021_06_17-12-57-44.nwb'
# filename = '/home/yann/DATA/CaImaging/NDNFcre_GCamp6s/Batch-2_September_2021/2021_09_10/2021_09_10-14-55-23.nwb'
if filename != '':
self.datafile = filename
self.load_file(self.datafile)
plots.raw_data_plot(self, self.tzoom)
else:
print('"%s" filename not loaded/recognized ! ' % filename)
def save(self):
pass
def reset(self):
self.windowTA, self.windowBM = None, None # sub-windows
self.notes.clear()
self.subsamplingSelect.setChecked(True)
self.annotSelect.setChecked(False)
self.stimSelect.setChecked(False)
self.init_panel_imgs()
self.roiIndices = None
def select_stim(self):
if self.stimSelect.isChecked():
if self.data.visual_stim is None:
self.load_VisualStim()
self.keyword_update(string='stim')
else:
self.keyword_update(string='no_stim')
self.data.visual_stim = None
def select_ROI(self):
""" see select ROI above """
self.roiIndices = self.select_ROI_from_pick(self.data)
plots.raw_data_plot(self, self.tzoom, with_roi=True)
def load_file(self, filename):
""" should be a minimal processing so that the loading is fast"""
self.reset()
self.data = Data(filename)
self.tzoom = self.data.tlim
self.notes.setText(self.data.description)
self.cal.setSelectedDate(self.data.nwbfile.session_start_time.date())
if self.dbox.currentIndex() < 1:
self.dbox.clear()
self.dbox.addItem(self.data.df_name)
self.dbox.setCurrentIndex(0)
if len(self.SUBJECTS.keys()) == 0:
self.sbox.clear()
self.sbox.addItem(self.subject_default_key)
self.sbox.addItem(self.data.nwbfile.subject.description)
self.sbox.setCurrentIndex(1)
self.pbox.setCurrentIndex(1)
if 'ophys' in self.data.nwbfile.processing:
self.roiPick.setText(' [select ROI: %i-%i]' %
(0, len(self.data.valid_roiIndices) - 1))
self.ophysSelect.setChecked(True)
if ('Electrophysiological-Signal' in self.data.nwbfile.acquisition
) or ('Vm' in self.data.nwbfile.acquisition) or (
'LFP' in self.data.nwbfile.acquisition):
self.ephysSelect.setChecked(True)
if 'Photodiode-Signal' in self.data.nwbfile.acquisition:
self.photodiodeSelect.setChecked(True)
if 'Running-Speed' in self.data.nwbfile.acquisition:
self.runSelect.setChecked(True)
self.runSelect.isChecked()
if 'FaceMotion' in self.data.nwbfile.processing:
coords = self.data.nwbfile.processing[
'FaceMotion'].description.split(
'facemotion ROI: (x0,dx,y0,dy)=(')[1].split(
')\n')[0].split(',')
coords = [int(c) for c in coords]
self.faceMotionContour.setData(
np.concatenate([
np.linspace(x1, x2, 20) for x1, x2 in zip([
coords[1], coords[1], coords[1] +
coords[3], coords[1] + coords[3], coords[1]
], [
coords[1], coords[1] + coords[3], coords[1] +
coords[3], coords[1], coords[1]
])
]),
np.concatenate([
np.linspace(y1, y2, 20) for y1, y2 in zip([
coords[0], coords[0] + coords[2], coords[0] +
coords[2], coords[0], coords[0]
], [
coords[0] + coords[2], coords[0] +
coords[2], coords[0], coords[0], coords[0]
])
]))
self.faceMtnSelect.setChecked(True)
if 'Pupil' in self.data.nwbfile.processing:
self.pupil_mm_to_pix = 1. / float(
self.data.nwbfile.processing['Pupil'].description.split(
'pix_to_mm=')[1].split('\n')[0])
coords = self.data.nwbfile.processing['Pupil'].description.split(
'pupil ROI: (xmin,xmax,ymin,ymax)=(')[1].split(')\n')[0].split(
',')
if len(coords) == 3: # bug (fixed), typo in previous datafiles
coords.append(coords[2][3:])
coords[2] = coords[2][:3]
coords = [int(c) for c in coords]
self.facePupilContour.setData(
np.concatenate([
np.linspace(x1, x2, 10) for x1, x2 in
zip([coords[2], coords[2], coords[3], coords[3]],
[coords[2], coords[3], coords[3], coords[2]])
]),
np.concatenate([
np.linspace(y1, y2, 10) for y1, y2 in
zip([coords[0], coords[1], coords[1], coords[0]],
[coords[1], coords[1], coords[0], coords[0]])
]))
self.pupilSelect.setChecked(True)
if 'Pupil' in self.data.nwbfile.processing:
self.gaze_center = [
np.mean(self.data.nwbfile.processing['Pupil'].
data_interfaces['cx'].data[:]),
np.mean(self.data.nwbfile.processing['Pupil'].
data_interfaces['cy'].data[:])
]
self.gazeSelect.setChecked(True)
def load_VisualStim(self):
# load visual stimulation
if self.data.metadata['VisualStim']:
self.data.init_visual_stim()
else:
self.data.visual_stim = None
print(' /!\ No stimulation in this recording /!\ ')
def scan_folder(self):
print('inspecting the folder "%s" [...]' %
FOLDERS[self.fbox.currentText()])
FILES0 = get_files_with_extension(FOLDERS[self.fbox.currentText()],
extension='.nwb',
recursive=True)
TIMES, DATES, FILES = [], [], []
for f in FILES0:
Time = f.split(os.path.sep)[-1].replace('.nwb', '').split('-')
if len(Time) >= 4:
TIMES.append(3600 * int(Time[0]) + 60 * int(Time[1]) +
int(Time[2]))
DATES.append(f.split(os.path.sep)[-1].split('-')[0])
FILES.append(f)
TIMES, DATES, FILES = np.array(TIMES), np.array(DATES), np.array(FILES)
NDATES = np.array([
datetime.date(*[int(dd) for dd in date.split('_')]).toordinal()
for date in DATES
])
self.FILES_PER_DAY = {}
guiparts.reinit_calendar(
self,
min_date=tuple(
int(dd) for dd in DATES[np.argmin(NDATES)].split('_')),
max_date=tuple(
int(dd) for dd in DATES[np.argmax(NDATES)].split('_')))
for d in np.unique(DATES):
try:
self.cal.setDateTextFormat(
QtCore.QDate(
datetime.date(*[int(dd) for dd in d.split('_')])),
self.highlight_format)
day_cond = (DATES == d)
time_sorted = np.argsort(TIMES[day_cond])
self.FILES_PER_DAY[d] = [os.path.join(FOLDERS[self.fbox.currentText()], f)\
for f in np.array(FILES)[day_cond][time_sorted]]
except BaseException as be:
print(be)
print('error for date %s' % d)
print(' -> found n=%i datafiles ' % len(FILES))
def compute_subjects(self):
FILES = get_files_with_extension(FOLDERS[self.fbox.currentText()],
extension='.nwb',
recursive=True)
print(
' looping over n=%i datafiles to fetch "subjects" metadata [...]' %
len(FILES))
DATES = np.array(
[f.split(os.path.sep)[-1].split('-')[0] for f in FILES])
SUBJECTS, DISPLAY_NAMES, SDATES, NDATES = [], [], [], []
for fn, date in zip(FILES, DATES):
infos = self.preload_datafolder(fn)
SDATES.append(date)
SUBJECTS.append(infos['subject'])
DISPLAY_NAMES.append(infos['display_name'])
NDATES.append(
datetime.date(*[int(dd)
for dd in date.split('_')]).toordinal())
self.SUBJECTS = {}
for s in np.unique(SUBJECTS):
cond = (np.array(SUBJECTS) == s)
self.SUBJECTS[s] = {
'display_names': np.array(DISPLAY_NAMES)[cond],
'datafiles': np.array(FILES)[cond],
'dates': np.array(SDATES)[cond],
'dates_num': np.array(NDATES)[cond]
}
print(' -> found n=%i subjects ' % len(self.SUBJECTS.keys()))
self.sbox.clear()
self.sbox.addItems([self.subject_default_key]+\
list(self.SUBJECTS.keys()))
self.sbox.setCurrentIndex(0)
def pick_subject(self):
self.plot.clear()
if self.sbox.currentText() == self.subject_default_key:
self.compute_subjects()
elif self.sbox.currentText() in self.SUBJECTS:
self.FILES_PER_DAY = {} # re-init
date_min = self.SUBJECTS[self.sbox.currentText()]['dates'][
np.argmin(self.SUBJECTS[self.sbox.currentText()]['dates_num'])]
date_max = self.SUBJECTS[self.sbox.currentText()]['dates'][
np.argmax(self.SUBJECTS[self.sbox.currentText()]['dates_num'])]
guiparts.reinit_calendar(
self,
min_date=tuple(int(dd) for dd in date_min.split('_')),
max_date=tuple(int(dd) for dd in date_max.split('_')))
for d in np.unique(
self.SUBJECTS[self.sbox.currentText()]['dates']):
self.cal.setDateTextFormat(
QtCore.QDate(
datetime.date(*[int(dd) for dd in d.split('_')])),
self.highlight_format)
self.FILES_PER_DAY[d] = [
f for f in np.array(self.SUBJECTS[
self.sbox.currentText()]['datafiles'])[self.SUBJECTS[
self.sbox.currentText()]['dates'] == d]
]
else:
print(' /!\ subject not recognized /!\ ')
def pick_date(self):
date = self.cal.selectedDate()
self.day = '%s_%02d_%02d' % (date.year(), date.month(), date.day())
for i in string.digits:
self.day = self.day.replace('_%s_' % i, '_0%s_' % i)
if self.day in self.FILES_PER_DAY:
self.list_protocol = self.FILES_PER_DAY[self.day]
self.update_df_names()
def pick_datafile(self):
self.plot.clear()
i = self.dbox.currentIndex()
if i > 0:
self.datafile = self.list_protocol[i - 1]
self.load_file(self.datafile)
plots.raw_data_plot(self, self.tzoom)
else:
# self.data.metadata = None
self.notes.setText(20 * '-' + 5 * '\n')
def update_df_names(self):
self.dbox.clear()
# self.pbox.clear()
# self.sbox.clear()
self.plot.clear()
self.pScreenimg.setImage(np.ones((10, 12)) * 50)
self.pFaceimg.setImage(np.ones((10, 12)) * 50)
self.pPupilimg.setImage(np.ones((10, 12)) * 50)
self.pCaimg.setImage(np.ones((50, 50)) * 100)
if len(self.list_protocol) > 0:
self.dbox.addItem(' ...' + 70 * ' ' + '(select a data-folder) ')
for fn in self.list_protocol:
self.dbox.addItem(self.preload_datafolder(fn)['display_name'])
def preload_datafolder(self, fn):
data = Data(fn, metadata_only=True, with_tlim=False)
if data.nwbfile is not None:
return {
'display_name': data.df_name,
'subject': data.nwbfile.subject.description
}
else:
return {'display_name': '', 'subject': ''}
def add_datafolder_annotation(self):
info = 20 * '-' + '\n'
if self.data.metadata['protocol'] == 'None':
self.notes.setText('\nNo visual stimulation')
else:
for key in self.data.metadata:
if (key[:2] == 'N-') and (key != 'N-repeat') and (
self.data.metadata[key] > 1): # meaning it was varied
info += '%s=%i (%.1f to %.1f)\n' % (
key, self.data.metadata[key],
self.data.metadata[key[2:] + '-1'],
self.data.metadata[key[2:] + '-2'])
info += '%s=%i' % ('N-repeat', self.data.metadata['N-repeat'])
self.notes.setText(info)
def display_quantities(self,
force=False,
plot_update=True,
with_images=False,
with_scatter=False):
"""
# IMPLEMENT OTHER ANALYSIS HERE
"""
if self.pbox.currentText() == '-> Trial-average':
self.windowTA = TrialAverageWindow(parent=self)
self.windowTA.show()
elif self.pbox.currentText() == '-> draw figures':
self.windowFG = FiguresWindow(self.datafile, parent=self)
self.windowFG.show()
elif self.pbox.currentText() == '-> build PDF summary':
cmd = '%s %s %s --verbose' % (
python_path,
os.path.join(str(pathlib.Path(__file__).resolve().parents[1]),
'analysis', 'summary_pdf.py'), self.datafile)
p = subprocess.Popen(cmd, shell=True)
print('"%s" launched as a subprocess' % cmd)
elif self.pbox.currentText() == '-> open PDF summary':
pdf_folder = summary_pdf_folder(self.datafile)
if os.path.isdir(pdf_folder):
PDFS = os.listdir(pdf_folder)
for pdf in PDFS:
print(' - opening: "%s"' % pdf)
os.system(
'$(basename $(xdg-mime query default application/pdf) .desktop) %s & '
% os.path.join(pdf_folder, pdf))
else:
print('no PDF summary files found !')
else:
self.plot.clear()
plots.raw_data_plot(self,
self.tzoom,
plot_update=plot_update,
with_images=with_images,
with_scatter=with_scatter)
self.pbox.setCurrentIndex(1)
def back_to_initial_view(self):
self.time = 0
self.tzoom = self.data.tlim
self.display_quantities(force=True)
def hitting_space(self):
if self.pauseButton.isEnabled():
self.pause()
else:
self.play()
def launch_movie(
self,
movie_refresh_time=0.2, # all in seconds
time_for_full_window=25):
self.tzoom = self.plot.getAxis('bottom').range
self.view = (self.tzoom[1] - self.tzoom[0]) / 2. * np.array([-1, 1])
# forcing time at 10% of the window
self.time = self.tzoom[0] + .1 * (self.tzoom[1] - self.tzoom[0])
self.display_quantities()
# then preparing the update rule
N_updates = int(time_for_full_window / movie_refresh_time) + 1
self.dt_movie = (self.tzoom[1] - self.tzoom[0]) / N_updates
self.updateTimer.start(int(1e3 * movie_refresh_time))
def play(self):
print('playing')
self.pauseButton.setEnabled(True)
self.playButton.setEnabled(False)
self.playButton.setChecked(True)
self.launch_movie()
def next_frame(self):
print(self.time, self.dt_movie)
self.time = self.time + self.dt_movie
self.prev_time = self.time
if self.time > self.tzoom[0] + .8 * (self.tzoom[1] - self.tzoom[0]):
print('block')
self.tzoom = [self.view[0] + self.time, self.view[1] + self.time]
if self.time == self.prev_time:
self.time = self.time + 2. * self.dt_movie
self.display_quantities()
def pause(self):
print('stopping')
self.updateTimer.stop()
self.playButton.setEnabled(True)
self.pauseButton.setEnabled(False)
self.pauseButton.setChecked(True)
def refresh(self):
self.plot.clear()
self.tzoom = self.plot.getAxis('bottom').range
self.display_quantities()
def update_frame(self, from_slider=True):
self.tzoom = self.plot.getAxis('bottom').range
if from_slider:
# update time based on slider
self.time = self.tzoom[0]+(self.tzoom[1]-self.tzoom[0])*\
float(self.frameSlider.value())/self.settings['Npoints']
else:
self.time = self.xaxis.range[0]
plots.raw_data_plot(self,
self.tzoom,
with_images=True,
with_scatter=True,
plot_update=False)
def showwindow(self):
if self.minView:
self.minView = self.maxview()
else:
self.minView = self.minview()
def maxview(self):
self.showFullScreen()
return False
def minview(self):
self.showNormal()
return True
def see_metadata(self):
for key, val in self.data.metadata.items():
print('- %s : ' % key, val)
def change_settings(self):
pass
def quit(self):
if self.data is not None:
self.data.io.close()
sys.exit()
def load_file(self, filename):
""" should be a minimal processing so that the loading is fast"""
self.reset()
self.data = Data(filename)
self.tzoom = self.data.tlim
self.notes.setText(self.data.description)
self.cal.setSelectedDate(self.data.nwbfile.session_start_time.date())
if self.dbox.currentIndex() < 1:
self.dbox.clear()
self.dbox.addItem(self.data.df_name)
self.dbox.setCurrentIndex(0)
if len(self.SUBJECTS.keys()) == 0:
self.sbox.clear()
self.sbox.addItem(self.subject_default_key)
self.sbox.addItem(self.data.nwbfile.subject.description)
self.sbox.setCurrentIndex(1)
self.pbox.setCurrentIndex(1)
if 'ophys' in self.data.nwbfile.processing:
self.roiPick.setText(' [select ROI: %i-%i]' %
(0, len(self.data.valid_roiIndices) - 1))
self.ophysSelect.setChecked(True)
if ('Electrophysiological-Signal' in self.data.nwbfile.acquisition
) or ('Vm' in self.data.nwbfile.acquisition) or (
'LFP' in self.data.nwbfile.acquisition):
self.ephysSelect.setChecked(True)
if 'Photodiode-Signal' in self.data.nwbfile.acquisition:
self.photodiodeSelect.setChecked(True)
if 'Running-Speed' in self.data.nwbfile.acquisition:
self.runSelect.setChecked(True)
self.runSelect.isChecked()
if 'FaceMotion' in self.data.nwbfile.processing:
coords = self.data.nwbfile.processing[
'FaceMotion'].description.split(
'facemotion ROI: (x0,dx,y0,dy)=(')[1].split(
')\n')[0].split(',')
coords = [int(c) for c in coords]
self.faceMotionContour.setData(
np.concatenate([
np.linspace(x1, x2, 20) for x1, x2 in zip([
coords[1], coords[1], coords[1] +
coords[3], coords[1] + coords[3], coords[1]
], [
coords[1], coords[1] + coords[3], coords[1] +
coords[3], coords[1], coords[1]
])
]),
np.concatenate([
np.linspace(y1, y2, 20) for y1, y2 in zip([
coords[0], coords[0] + coords[2], coords[0] +
coords[2], coords[0], coords[0]
], [
coords[0] + coords[2], coords[0] +
coords[2], coords[0], coords[0], coords[0]
])
]))
self.faceMtnSelect.setChecked(True)
if 'Pupil' in self.data.nwbfile.processing:
self.pupil_mm_to_pix = 1. / float(
self.data.nwbfile.processing['Pupil'].description.split(
'pix_to_mm=')[1].split('\n')[0])
coords = self.data.nwbfile.processing['Pupil'].description.split(
'pupil ROI: (xmin,xmax,ymin,ymax)=(')[1].split(')\n')[0].split(
',')
if len(coords) == 3: # bug (fixed), typo in previous datafiles
coords.append(coords[2][3:])
coords[2] = coords[2][:3]
coords = [int(c) for c in coords]
self.facePupilContour.setData(
np.concatenate([
np.linspace(x1, x2, 10) for x1, x2 in
zip([coords[2], coords[2], coords[3], coords[3]],
[coords[2], coords[3], coords[3], coords[2]])
]),
np.concatenate([
np.linspace(y1, y2, 10) for y1, y2 in
zip([coords[0], coords[1], coords[1], coords[0]],
[coords[1], coords[1], coords[0], coords[0]])
]))
self.pupilSelect.setChecked(True)
if 'Pupil' in self.data.nwbfile.processing:
self.gaze_center = [
np.mean(self.data.nwbfile.processing['Pupil'].
data_interfaces['cx'].data[:]),
np.mean(self.data.nwbfile.processing['Pupil'].
data_interfaces['cy'].data[:])
]
self.gazeSelect.setChecked(True)
def build_summary_episodes(
FILES,
roi_prefix=10000, # to have unique roi per session, session X has roi IDs: X*roi_prefix+i_ROI
prestim_duration=2,
modalities=['pupil', 'facemotion', 'running-speed'],
dt_sampling=20, # ms
Nmax=100000):
"""
"""
protocols, subjects, sessions, sessions_per_subject, filename, STIM = find_protocol_details(
FILES)
FULL_EPISODE_ARRAY, QUANT = [], {
'subject': [],
'session_per_subject': [],
'session': [],
'roi': [],
'pupil': [],
'running-speed': [],
'facemotion': []
}
print('- building "%s" by concatenating episodes from n=%i files [...]' %
(filename, len(FILES)))
for session, f in enumerate(FILES):
print(' -> session #%i: %s' % (session + 1, f))
data = Data(f)
for ip, p in enumerate(protocols):
if len(protocols) > 1:
duration = data.metadata['Protocol-%i-presentation-duration' %
(ip + 1)]
else:
duration = data.metadata['presentation-duration']
# build episodes of other modalities (running, ...)
if ('Pupil' in data.nwbfile.processing) and ('pupil'
in modalities):
Pupil_episodes = EpisodeResponse(
data,
protocol_id=ip,
prestim_duration=prestim_duration,
dt_sampling=dt_sampling, # ms
quantity='Pupil')
t_pupil_cond = (Pupil_episodes.t > 0) & (Pupil_episodes.t <
duration)
else:
Pupil_episodes = None
if ('Running-Speed'
in data.nwbfile.acquisition) and ('running-speed'
in modalities):
Running_episodes = EpisodeResponse(
data,
protocol_id=ip,
prestim_duration=prestim_duration,
dt_sampling=dt_sampling, # ms
quantity='Running-Speed')
t_running_cond = (Running_episodes.t > 0) & (Running_episodes.t
< duration)
else:
Running_episodes = None
if ('FaceMotion' in data.nwbfile.processing) and ('facemotion'
in modalities):
FaceMotion_episodes = EpisodeResponse(
data,
protocol_id=ip,
prestim_duration=prestim_duration,
dt_sampling=dt_sampling, # ms
quantity='FaceMotion')
t_facemotion_cond = (FaceMotion_episodes.t >
0) & (FaceMotion_episodes.t < duration)
else:
FaceMotion_episodes = None
for roi in range(np.sum(data.iscell))[:Nmax]:
roiID = roi_prefix * session + roi
EPISODES = EpisodeResponse(
data,
protocol_id=ip,
quantity='CaImaging',
subquantity='dF/F',
dt_sampling=dt_sampling, # ms
roiIndex=roi,
prestim_duration=prestim_duration)
for iEp in range(EPISODES.resp.shape[0]):
FULL_EPISODE_ARRAY.append(EPISODES.resp[iEp, :])
for key in data.nwbfile.stimulus.keys():
STIM[key].append(data.nwbfile.stimulus[key].data[iEp])
QUANT['roi'].append(roiID)
QUANT['session'].append(session)
QUANT['subject'].append(data.metadata['subject_ID'])
QUANT['session_per_subject'].append(
sessions_per_subject[session])
if Running_episodes is not None:
QUANT['running-speed'].append(
Running_episodes.resp[iEp, :][t_running_cond])
else:
QUANT['running-speed'].append(666.) # flag for None
if Pupil_episodes is not None:
QUANT['pupil'].append(
Pupil_episodes.resp[iEp, :][t_pupil_cond])
else:
QUANT['pupil'].append(666.) # flag for None
if FaceMotion_episodes is not None:
QUANT['facemotion'].append(FaceMotion_episodes.resp[
iEp, :][t_facemotion_cond])
else:
QUANT['facemotion'].append(666.) # flag for None
# set up the NWBFile
description = 'Summary data concatenating episodes from the datafiles:\n'
for f in FILES:
description += '- %s\n' % f
nwbfile = pynwb.NWBFile(
session_description=description,
identifier=filename,
session_start_time=datetime.datetime.now().astimezone())
episode_waveforms = pynwb.TimeSeries(name='episode_waveforms',
data=np.array(FULL_EPISODE_ARRAY),
unit='dF/F',
timestamps=EPISODES.t)
nwbfile.add_acquisition(episode_waveforms)
for key in STIM:
stim = pynwb.TimeSeries(name=key,
data=np.array(STIM[key]),
unit='None',
rate=1.)
nwbfile.add_stimulus(stim)
for key in QUANT:
stim = pynwb.TimeSeries(name=key,
data=np.array(QUANT[key]),
unit='None',
rate=1.)
nwbfile.add_acquisition(stim)
io = pynwb.NWBHDF5IO(filename, mode='w')
io.write(nwbfile)
io.close()
return mean_q1, var_q1, mean_q2, var_q2
if __name__ == '__main__':
"""
some common tools for analysis
"""
import matplotlib.pylab as plt
import sys, os, pathlib
sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
from analysis.read_NWB import Data
filename = sys.argv[-1]
data = Data(filename)
# CCF, tshift = crosscorrel_on_NWB_quantity(data.nwbfile.acquisition['Running-Speed'],
# data.nwbfile.acquisition['Running-Speed'],
# # t_q2=data.Fluorescence.timestamps[:],
# # q2=data.Fluorescence.data[data.iscell[0],:],
# tmax=100)
# CCF, tshift = crosscorrel_on_NWB_quantity(t_q1=data.Fluorescence.timestamps[:],
# q1=data.Fluorescence.data[data.iscell[0],:],
# t_q2=data.Fluorescence.timestamps[:],
# q2=data.Fluorescence.data[data.iscell[0],:],
# tmax=20)
CCF, tshift = autocorrel_on_NWB_quantity(
t_q1=data.Fluorescence.timestamps[:],
q1=data.Fluorescence.data[data.iscell[0], :],
tmax=1000)