def exportHtml(self, fileName=False):
if fileName is False:
self.fileDialog = FileDialog(self, "Save HTML as...",
"htmltemp.log")
#self.fileDialog.setFileMode(QtGui.QFileDialog.AnyFile)
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.exportHtml)
return
if fileName[-5:] != '.html':
fileName += '.html'
#doc = self.ui.output.document().toHtml('utf-8')
#for e in self.displayedEntries:
#if e.has_key('tracebackHtml'):
#doc = re.sub(r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'%(str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
doc = self.pageTemplate
for e in self.displayedEntries:
doc += self.cache[id(e)]
for e in self.displayedEntries:
if 'tracebackHtml' in e:
doc = re.sub(
r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'
% (str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
f = open(fileName, 'w')
f.write(doc)
f.close()
def openClicked(self):
startDir = self.chart.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Load Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.chart.loadFile)
def get_yaml_config_params():
F = FileDialog()
fname = F.getOpenFileName(caption='Select a Config File')[0]
#load yaml params files
with open(fname, 'r') as f:
params = yaml.load(f)
return params
def addFile(self):
file_filters = self.source_type.currentText()
if file_filters == "hdf5":
file_filters = "*.h5 *.hdf5"
elif file_filters == "psana":
file_filters = "*.xtc2"
self.fileDialog = FileDialog(None, "Load Data", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path)
for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def saveFile(self,
fileName=None,
startDir=None,
suggestedFileName='flowchart.fc'):
"""
Save this flowchart to a .fc file
"""
if fileName is None:
if startDir is None:
startDir = self.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Save Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
return
if not fileName.endswith('.fc'):
fileName += ".fc"
state = self.saveState()
state = json.dumps(state,
indent=2,
separators=(',', ': '),
sort_keys=True,
cls=amitypes.TypeEncoder)
with open(fileName, 'w') as f:
f.write(state)
f.write('\n')
ctrl = self.widget()
ctrl.graph_info.labels(self.hutch,
ctrl.graph_name).info({'graph': state})
ctrl.chartWidget.updateStatus(f"Saved graph to: {fileName}")
self.sigFileSaved.emit(fileName)
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path) for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def export(self):
from tifffile import imsave
from pyqtgraph import FileDialog
start_doc = getattr(self.catalog, self.stream).metadata['start']
if 'FileName' in start_doc:
current_dir = str(Path(start_doc['FileName']).parent)
current_name = Path(start_doc['FileName']).stem
elif 'sample_name' in start_doc:
current_dir = str(
Path.home()) # sample name is usually just te file stem
current_name = Path(start_doc['sample_name']).stem
else:
current_dir = Path.home()
current_name = 'xicamNCEM_output'
# Get a file path to save to in current directory
fd = FileDialog()
fd.setNameFilter("TIF (*.tif)")
fd.setDirectory(current_dir)
fd.selectFile(current_name)
fd.setFileMode(FileDialog.AnyFile)
fd.setAcceptMode(FileDialog.AcceptSave)
if fd.exec_():
file_names = fd.selectedFiles()[0]
outPath = Path(file_names)
else:
return
if outPath.suffix != '.tif':
outPath = outPath.with_suffix('.tif')
scale0, units0 = self._get_physical_size()
# Avoid overflow in field of view later
if units0[0] == 'm':
units0 = ['um', 'um']
scale0 = [ii * 1e6 for ii in scale0]
# Change scale from pixel to 1/pixel
FOV = [1 / ii for ii in scale0]
# Add units to metadata for Imagej type output
metadata = {'unit': units0[0]}
# Get the data and change to float
image = self.xarray[self.currentIndex, :, :].astype('f')
imsave(outPath, image, imagej=True, resolution=FOV, metadata=metadata)
def run_gui(path=None, data=None):
"""
Loads PCF or CNMF data object from a provided path and loads the CaImAn GUI for component inspection.
This GUI was tweaked to not require any storage-intensive mmap files, but can therefore not show individual frames.
:param path: optional str, directory of the PCF or CNMF object from which component data should be loaded. If None
and data=None, a window prompt will open to select a directory where to look for a CNMF/PCF file.
:param data: optional, data in form of an already loaded cnm object can be provided directly
:return:
"""
try:
cv2.setNumThreads(1)
except:
print('Open CV is naturally single threaded')
try:
if __IPYTHON__:
# print(1)
# this is used for debugging purposes only. allows to reload classes
# when changed
get_ipython().magic('load_ext autoreload')
get_ipython().magic('autoreload 2')
except NameError:
print('Not launched under iPython')
def make_color_img(img, gain=255, min_max=None, out_type=np.uint8):
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img - min_) / (max_ - min_) * gain
img = img.astype(out_type)
img = np.dstack([img] * 3)
return img
### FIND DATA ###
if data is None: # different conditions on file loading (not necessary if data was already provided)
if path is None: # if no path has been given, open a window prompt to select a directory
F = FileDialog()
# load object saved by CNMF
path = F.getExistingDirectory(
caption='Select folder from which to load a PCF or CNMF file')
try: # first try to get CNMF data from a PCF object (should be most up-to-date)
cnm_obj = pipe.load_pcf(path).cnmf
except FileNotFoundError:
try:
cnm_obj = pipe.load_cnmf(path)
except FileNotFoundError:
raise FileNotFoundError(
f'Could not find data to load in {path}!')
else:
cnm_obj = data
# movie NOT NEEDED IN VERSION WITHOUT MMAP FILE
# if not os.path.exists(cnm_obj.mmap_file):
# M = FileDialog()
# cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0]
#
# if fpath[-3:] == 'nwb':
# mov = cm.load(cnm_obj.mmap_file, var_name_hdf5='acquisition/TwoPhotonSeries')
# else:
# mov = cm.load(cnm_obj.mmap_file)
estimates = cnm_obj.estimates
params_obj = cnm_obj.params
# min_mov = np.min(mov)
# max_mov = np.max(mov)
if not hasattr(estimates, 'Cn'):
if not os.path.exists(cnm_obj.mmap_file):
M = FileDialog()
cnm_obj.mmap_file = M.getOpenFileName(
caption='Load memory mapped file', filter='*.mmap')[0]
mov = cm.load(cnm_obj.mmap_file)
estimates.Cn = cm.local_correlations(mov, swap_dim=False)
Cn = estimates.Cn
# min_mov_denoise = np.min(estimates.A)*estimates.C.min()
# max_mov_denoise = np.max(estimates.A)*estimates.C.max()
background_num = -1
neuron_selected = False
nr_index = 0
min_background = np.min(estimates.b, axis=0) * np.min(estimates.f, axis=1)
max_background = np.max(estimates.b, axis=0) * np.max(estimates.f, axis=1)
if not hasattr(estimates, 'accepted_list'):
# if estimates.discarded_components.A.shape[-1] > 0:
# estimates.restore_discarded_components()
estimates.accepted_list = np.array([], dtype=np.int)
estimates.rejected_list = np.array([], dtype=np.int)
estimates.img_components = estimates.A.toarray().reshape(
(estimates.dims[0], estimates.dims[1], -1),
order='F').transpose([2, 0, 1])
estimates.cms = np.array([
scipy.ndimage.measurements.center_of_mass(comp)
for comp in estimates.img_components
])
estimates.idx_components = np.arange(estimates.nr)
estimates.idx_components_bad = np.array([])
estimates.background_image = make_color_img(estimates.Cn)
# Generate image data
estimates.img_components /= estimates.img_components.max(
axis=(1, 2))[:, None, None]
estimates.img_components *= 255
estimates.img_components = estimates.img_components.astype(np.uint8)
def draw_contours_overall(md):
if md is "reset":
draw_contours()
elif md is "neurons":
if neuron_selected is True:
#if a specific neuron has been selected, only one contour should be changed while thrshcomp_line is changing
if nr_index is 0:
#if user does not start to move through the frames
draw_contours_update(estimates.background_image, img)
draw_contours_update(comp2_scaled, img2)
else:
# NEVER CALLED IN THIS VERSION WITHOUT MMAP SINCE NR_INDEX NEVER CHANGES (NO NR_VLINE)
draw_contours_update(raw_mov_scaled, img)
draw_contours_update(frame_denoise_scaled, img2)
else:
#if no specific neuron has been selected, all the contours are changing
draw_contours()
else:
#md is "background":
return
def draw_contours():
global thrshcomp_line, estimates, img
bkgr_contours = estimates.background_image.copy()
if len(estimates.idx_components) > 0:
contours = [
cv2.findContours(
cv2.threshold(img, np.int(thrshcomp_line.value()), 255,
0)[1], cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)[0]
for img in estimates.img_components[estimates.idx_components]
]
SNRs = np.array(estimates.r_values)
iidd = np.array(estimates.idx_components)
idx1 = np.where(SNRs[iidd] < 0.1)[0]
idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0]
idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0]
idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0]
idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0]
idx6 = np.where(SNRs[iidd] >= 0.9)[0]
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx1], []), -1,
(255, 0, 0), 1)
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx2], []), -1,
(0, 255, 0), 1)
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx3], []), -1,
(0, 0, 255), 1)
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx4], []), -1,
(255, 255, 0), 1)
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx5], []), -1,
(255, 0, 255), 1)
cv2.drawContours(bkgr_contours,
sum([contours[jj] for jj in idx6], []), -1,
(0, 255, 255), 1)
img.setImage(bkgr_contours, autoLevels=False)
# pg.setConfigOptions(imageAxisOrder='row-major')
def draw_contours_update(cf, im):
global thrshcomp_line, estimates
curFrame = cf.copy()
if len(estimates.idx_components) > 0:
contours = [
cv2.findContours(
cv2.threshold(img, np.int(thrshcomp_line.value()), 255,
0)[1], cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)[0]
for img in estimates.img_components[estimates.idx_components]
]
SNRs = np.array(estimates.r_values)
iidd = np.array(estimates.idx_components)
idx1 = np.where(SNRs[iidd] < 0.1)[0]
idx2 = np.where((SNRs[iidd] >= 0.1) & (SNRs[iidd] < 0.25))[0]
idx3 = np.where((SNRs[iidd] >= 0.25) & (SNRs[iidd] < 0.5))[0]
idx4 = np.where((SNRs[iidd] >= 0.5) & (SNRs[iidd] < 0.75))[0]
idx5 = np.where((SNRs[iidd] >= 0.75) & (SNRs[iidd] < 0.9))[0]
idx6 = np.where(SNRs[iidd] >= 0.9)[0]
if min_dist_comp in idx1:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(255, 0, 0), 1)
if min_dist_comp in idx2:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(0, 255, 0), 1)
if min_dist_comp in idx3:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(0, 0, 255), 1)
if min_dist_comp in idx4:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(255, 255, 0), 1)
if min_dist_comp in idx5:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(255, 0, 255), 1)
if min_dist_comp in idx6:
cv2.drawContours(curFrame, contours[min_dist_comp], -1,
(0, 255, 255), 1)
im.setImage(curFrame, autoLevels=False)
#%% START BUILDING THE APPLICATION WINDOW
# Always start by initializing Qt (only once per application)
app = QtGui.QApplication([])
# Define a top-level widget to hold everything
w = QtGui.QWidget()
# Create some widgets to be placed inside
btn = QtGui.QPushButton('press me')
text = QtGui.QLineEdit('enter text')
# Histogram controller (win)
win = pg.GraphicsLayoutWidget()
win.setMaximumWidth(300)
win.setMinimumWidth(200)
hist = pg.HistogramLUTItem() # Contrast/color control
win.addItem(hist)
# Plotting windows
p1 = pg.PlotWidget(
) # raw movie window (top-mid), all contours are drawn here
p2 = pg.PlotWidget(
) # trace window (bottom-mid), calcium trace of the selected component
p3 = pg.PlotWidget(
) # denoised movie window (top-right), only selected contour is drawn here
# parameter table for online evaluation and mode change
t = ParameterTree()
# parameter table for neuron selection/sorting
t_action = ParameterTree()
action_layout = QtGui.QGridLayout()
## Create a grid layout to manage the widgets size and position
layout = QtGui.QGridLayout()
w.setLayout(layout)
# A plot area (ViewBox + axes) for displaying the image
#p1 = win.addPlot(title="Image here")
# Item for displaying image data
img = pg.ImageItem()
p1.addItem(img)
img2 = pg.ImageItem()
p3.addItem(img2)
hist.setImageItem(img)
# Draggable line for setting isocurve level
thrshcomp_line = pg.InfiniteLine(angle=0, movable=True, pen='g')
hist.vb.addItem(thrshcomp_line)
hist.vb.setMouseEnabled(y=False) # makes user interaction a little easier
thrshcomp_line.setValue(100)
thrshcomp_line.setZValue(1000) # bring iso line above contrast controls
## Add widgets to the layout in their proper positions
layout.addWidget(win, 1, 0) # histogram
layout.addWidget(p3, 0, 2) # denoised movie
layout.addWidget(t, 0, 0) # upper-right table
layout.addWidget(t_action, 1, 2) # bottom-right table
layout.addWidget(p1, 0, 1) # raw movie
layout.addWidget(p2, 1, 1) # calcium trace window
#enable only horizontal zoom for the traces component
p2.setMouseEnabled(x=True, y=False)
## Display the widget as a new window
w.show()
## Start the Qt event loop
app.exec_()
draw_contours()
hist.setLevels(estimates.background_image.min(),
estimates.background_image.max())
# Another plot area for displaying ROI data
#win.nextRow()
#p2 = win.addPlot(colspan=2)
p2.setMaximumHeight(250)
#win.resize(800, 800)
#win.show()
# set position and scale of image
img.scale(1, 1)
# img.translate(-50, 0)
# zoom to fit image
p1.autoRange()
mode = "reset"
p2.setTitle("mode: %s" % (mode))
thrshcomp_line.sigDragged.connect(lambda: draw_contours_overall(mode))
def imageHoverEvent(event):
#Show the position, pixel, and value under the mouse cursor.
global x, y, i, j, val
pos = event.pos()
i, j = pos.y(), pos.x()
i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1))
j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1))
val = estimates.background_image[i, j, 0]
ppos = img.mapToParent(pos)
x, y = ppos.x(), ppos.y()
# Monkey-patch the image to use our custom hover function.
# This is generally discouraged (you should subclass ImageItem instead),
# but it works for a very simple use like this.
img.hoverEvent = imageHoverEvent
def mouseClickEvent(event):
global mode
global x, y, i, j, val
pos = img.mapFromScene(event.pos())
x = int(pos.x())
y = int(pos.y())
if x < 0 or x > mov.shape[1] or y < 0 or y > mov.shape[2]:
# if the user click outside of the movie, do nothing and jump out of the function
return
i, j = pos.y(), pos.x()
i = int(np.clip(i, 0, estimates.background_image.shape[0] - 1))
j = int(np.clip(j, 0, estimates.background_image.shape[1] - 1))
val = estimates.background_image[i, j, 0]
if mode is "neurons":
show_neurons_clicked()
p1.mousePressEvent = mouseClickEvent
#A general rule in Qt is that if you override one mouse event handler, you must override all of them.
def release(event):
pass
p1.mouseReleaseEvent = release
def move(event):
pass
p1.mouseMoveEvent = move
## PARAMS
params = [{
'name': 'min_cnn_thr',
'type': 'float',
'value': 0.99,
'limits': (0, 1),
'step': 0.01
}, {
'name': 'cnn_lowest',
'type': 'float',
'value': 0.1,
'limits': (0, 1),
'step': 0.01
}, {
'name': 'rval_thr',
'type': 'float',
'value': 0.85,
'limits': (-1, 1),
'step': 0.01
}, {
'name': 'rval_lowest',
'type': 'float',
'value': -1,
'limits': (-1, 1),
'step': 0.01
}, {
'name': 'min_SNR',
'type': 'float',
'value': 2,
'limits': (0, 20),
'step': 0.1
}, {
'name': 'SNR_lowest',
'type': 'float',
'value': 0,
'limits': (0, 20),
'step': 0.1
}, {
'name': 'RESET',
'type': 'action'
}, {
'name': 'SHOW BACKGROUND',
'type': 'action'
}, {
'name': 'SHOW NEURONS',
'type': 'action'
}]
## Create tree of Parameter objects
pars = Parameter.create(name='params', type='group', children=params)
params_action = [{
'name': 'Filter components',
'type': 'bool',
'value': True,
'tip': "Filter components"
}, {
'name': 'View components',
'type': 'list',
'values': ['All', 'Accepted', 'Rejected', 'Unassigned'],
'value': 'All'
}, {
'name': 'ADD GROUP',
'type': 'action'
}, {
'name': 'REMOVE GROUP',
'type': 'action'
}, {
'name': 'ADD SINGLE',
'type': 'action'
}, {
'name': 'REMOVE SINGLE',
'type': 'action'
}, {
'name': 'SAVE OBJECT',
'type': 'action'
}]
pars_action = Parameter.create(name='params_action',
type='group',
children=params_action)
t_action.setParameters(pars_action, showTop=False)
t_action.setWindowTitle('Parameter Action')
def reset_button():
global mode
mode = "reset"
p2.setTitle("mode: %s" % (mode))
#clear the upper right image
zeros = np.asarray([[0] * 80 for _ in range(60)])
img2.setImage(make_color_img(zeros), autoLevels=False)
draw_contours()
pars.param('RESET').sigActivated.connect(reset_button)
def show_background_button():
global bg_vline, min_background, max_background, background_num
global mode, background_first_frame_scaled
#clear thhe upper right image
zeros = np.asarray([[0] * 80 for _ in range(60)])
img2.setImage(make_color_img(zeros), autoLevels=False)
background_num = (background_num + 1) % estimates.f.shape[0]
mode = "background"
p2.setTitle("mode: %s %d" % (mode, background_num))
# display the first frame of the background
background_first_frame = estimates.b[:, background_num].reshape(
estimates.dims, order='F')
min_background_first_frame = np.min(background_first_frame)
max_background_first_frame = np.max(background_first_frame)
background_first_frame_scaled = make_color_img(
background_first_frame,
min_max=(min_background_first_frame, max_background_first_frame))
img.setImage(background_first_frame_scaled, autoLevels=False)
# draw the trace and the infinite line
trace_background = estimates.f[background_num]
p2.plot(trace_background, clear=True)
bg_vline = pg.InfiniteLine(angle=90, movable=True)
p2.addItem(bg_vline, ignoreBounds=True)
bg_vline.setValue(0)
bg_vline.sigPositionChanged.connect(show_background_update)
def show_background_update():
global bg_index, min_background, max_background, background_scaled
bg_index = int(bg_vline.value())
if bg_index > -1 and bg_index < estimates.f.shape[-1]:
# upper left component scrolls through the frames of the background
background = estimates.b[:, background_num].dot(
estimates.f[background_num, bg_index]).reshape(estimates.dims,
order='F')
background_scaled = make_color_img(
background,
min_max=(min_background[background_num],
max_background[background_num]))
img.setImage(background_scaled, autoLevels=False)
pars.param('SHOW BACKGROUND').sigActivated.connect(show_background_button)
def show_neurons_button():
global mode, neuron_selected
mode = "neurons"
neuron_selected = False
p2.setTitle("mode: %s" % (mode))
#clear the upper right image
zeros = np.asarray([[0] * 80 for _ in range(60)])
img2.setImage(make_color_img(zeros), autoLevels=False)
def show_neurons_clicked():
global nr_index
global x, y, i, j, val, min_dist_comp, contour_single, neuron_selected, comp2_scaled
neuron_selected = True
distances = np.sum(
((x, y) - estimates.cms[estimates.idx_components])**2, axis=1)**0.5
min_dist_comp = np.argmin(distances)
contour_all = [
cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1]
for img in estimates.img_components[estimates.idx_components]
]
contour_single = contour_all[min_dist_comp]
# draw the traces (lower left component)
estimates.components_to_plot = estimates.idx_components[min_dist_comp]
p2.plot(estimates.C[estimates.components_to_plot] +
estimates.YrA[estimates.components_to_plot],
clear=True)
# plot img (upper left component)
img.setImage(estimates.background_image, autoLevels=False)
draw_contours_update(estimates.background_image, img)
# plot img2 (upper right component)
comp2 = np.multiply(estimates.Cn, contour_single > 0)
comp2_scaled = make_color_img(comp2,
min_max=(np.min(comp2), np.max(comp2)))
img2.setImage(comp2_scaled, autoLevels=False)
draw_contours_update(comp2_scaled, img2)
# set title for the upper two components
p3.setTitle("pos: (%0.1f, %0.1f) component: %d value: %g dist:%f" %
(x, y, estimates.components_to_plot, val,
distances[min_dist_comp]))
p1.setTitle("pos: (%0.1f, %0.1f) component: %d value: %g dist:%f" %
(x, y, estimates.components_to_plot, val,
distances[min_dist_comp]))
# draw the infinite line (INACTIVE IN THIS VERSION WITHOUT MMAP FILES)
# nr_vline = pg.InfiniteLine(angle=90, movable=True)
# p2.addItem(nr_vline, ignoreBounds=True)
# nr_vline.setValue(0)
# nr_vline.sigPositionChanged.connect(show_neurons_update)
nr_index = 0
def show_neurons_update(): # NOT CALLED IN THIS VERSION
global nr_index, frame_denoise_scaled, estimates, raw_mov_scaled
global min_mov, max_mov, min_mov_denoise, max_mov_denoise
if neuron_selected is False:
return
nr_index = int(nr_vline.value())
if nr_index > 0 and nr_index < mov[:, 0, 0].shape[0]:
# upper left component scrolls through the raw movie
raw_mov = mov[nr_index, :, :]
raw_mov_scaled = make_color_img(raw_mov,
min_max=(min_mov, max_mov))
img.setImage(raw_mov_scaled, autoLevels=False)
draw_contours_update(raw_mov_scaled, img)
# upper right component scrolls through the denoised movie
frame_denoise = estimates.A[:, estimates.idx_components].dot(
estimates.C[estimates.idx_components,
nr_index]).reshape(estimates.dims, order='F')
frame_denoise_scaled = make_color_img(frame_denoise,
min_max=(min_mov_denoise,
max_mov_denoise))
img2.setImage(frame_denoise_scaled, autoLevels=False)
draw_contours_update(frame_denoise_scaled, img2)
pars.param('SHOW NEURONS').sigActivated.connect(show_neurons_button)
def add_group():
estimates.accepted_list = np.union1d(estimates.accepted_list,
estimates.idx_components)
estimates.rejected_list = np.setdiff1d(estimates.rejected_list,
estimates.idx_components)
change(None, None)
pars_action.param('ADD GROUP').sigActivated.connect(add_group)
def remove_group():
estimates.rejected_list = np.union1d(estimates.rejected_list,
estimates.idx_components)
estimates.accepted_list = np.setdiff1d(estimates.accepted_list,
estimates.idx_components)
change(None, None)
pars_action.param('REMOVE GROUP').sigActivated.connect(remove_group)
def add_single():
estimates.accepted_list = np.union1d(estimates.accepted_list,
estimates.components_to_plot)
estimates.rejected_list = np.setdiff1d(estimates.rejected_list,
estimates.components_to_plot)
change(None, None)
pars_action.param('ADD SINGLE').sigActivated.connect(add_single)
def remove_single():
estimates.rejected_list = np.union1d(estimates.rejected_list,
estimates.components_to_plot)
estimates.accepted_list = np.setdiff1d(estimates.accepted_list,
estimates.components_to_plot)
change(None, None)
pars_action.param('REMOVE SINGLE').sigActivated.connect(remove_single)
def save_object():
print('Saving')
ffll = F.getSaveFileName(filter='*.hdf5')
print(ffll[0])
cnm_obj.estimates = estimates
cnm_obj.save(ffll[0])
pars_action.param('SAVE OBJECT').sigActivated.connect(save_object)
def action_pars_activated(param, changes):
change(None, None)
pars_action.sigTreeStateChanged.connect(action_pars_activated)
## If anything changes in the tree, print a message
def change(param, changes):
global estimates, pars, pars_action
set_par = pars.getValues()
if pars_action.param('Filter components').value():
for keyy in set_par.keys():
params_obj.quality.update({keyy: set_par[keyy][0]})
else:
params_obj.quality.update({
'cnn_lowest': .1,
'min_cnn_thr': 0.99,
'rval_thr': 0.85,
'rval_lowest': -1,
'min_SNR': 2,
'SNR_lowest': 0
})
estimates.filter_components(
mov,
params_obj,
dview=None,
select_mode=pars_action.param('View components').value())
if mode is "background":
return
else:
draw_contours()
pars.sigTreeStateChanged.connect(change)
change(None, None) # set params to default
t.setParameters(pars, showTop=False)
t.setWindowTitle('Parameter Quality')
## END PARAMS
## Display the widget as a new window
w.show()
## Start the Qt event loop
app.exit(app.exec_())
class SourceConfiguration(QtGui.QWidget):
sigApply = QtCore.Signal(object) # src_cfg dict
def __init__(self, parent=None):
super().__init__(parent)
self.setWindowTitle("Configure")
self.formLayout = QtWidgets.QFormLayout(self)
self.interval = QtGui.QDoubleSpinBox(self)
self.interval.setValue(0.01)
self.formLayout.addRow("Interval", self.interval)
self.init_time = QtGui.QDoubleSpinBox(self)
self.init_time.setValue(0.5)
self.formLayout.addRow("Init Time", self.init_time)
self.hb_period = QtGui.QSpinBox(self)
self.hb_period.setValue(10)
self.formLayout.addRow("Heartbeat Period", self.hb_period)
self.source_type = QtGui.QComboBox(self)
self.source_type.addItem("hdf5")
self.source_type.addItem("psana")
self.formLayout.addRow("Source Type", self.source_type)
self.repeat = QtWidgets.QCheckBox(self)
self.repeat.setChecked(True)
self.formLayout.addRow("Repeat", self.repeat)
self.files = []
self.fileListView = QtWidgets.QListView(self)
self.fileListView.setSelectionMode(
QtWidgets.QAbstractItemView.ExtendedSelection)
self.fileListModel = QtCore.QStringListModel(self.files)
self.fileListView.setModel(self.fileListModel)
self.formLayout.addRow(self.fileListView)
self.horizontalLayout = QtWidgets.QHBoxLayout()
self.addBtn = QtWidgets.QPushButton("Add", parent=self)
self.addBtn.clicked.connect(self.addFile)
self.horizontalLayout.addWidget(self.addBtn)
self.removeBtn = QtWidgets.QPushButton("Remove", parent=self)
self.removeBtn.clicked.connect(self.removeFiles)
self.horizontalLayout.addWidget(self.removeBtn)
self.applyBtn = QtWidgets.QPushButton("Apply", parent=self)
self.applyBtn.clicked.connect(self.applyClicked)
self.horizontalLayout.addWidget(self.applyBtn)
self.formLayout.addRow(self.horizontalLayout)
def addFile(self):
file_filters = self.source_type.currentText()
if file_filters == "hdf5":
file_filters = "*.h5 *.hdf5"
elif file_filters == "psana":
file_filters = "*.xtc2"
self.fileDialog = FileDialog(None, "Load Data", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def removeFiles(self):
selectionModel = self.fileListView.selectionModel()
for pth in selectionModel.selection().indexes():
pth = pth.data()
self.files.remove(pth)
self.fileListModel.setStringList(self.files)
def fileDialogFilesSelected(self, pths):
self.files.extend(pths)
self.fileListModel.setStringList(self.files)
def saveState(self):
cfg = {}
cfg['type'] = self.source_type.currentText()
cfg['interval'] = self.interval.value()
cfg['init_time'] = self.init_time.value()
cfg['hb_period'] = self.hb_period.value()
cfg['files'] = self.files
cfg['repeat'] = self.repeat.isChecked()
return cfg
def restoreState(self, state):
self.source_type.setCurrentText(state['type'])
self.interval.setValue(state['interval'])
self.init_time.setValue(state['init_time'])
self.hb_period.setValue(state['hb_period'])
self.files = state['files']
self.fileListModel.setStringList(self.files)
self.repeat.setChecked(state['repeat'])
def applyClicked(self):
cfg = self.saveState()
self.sigApply.emit(cfg)
def loadFile(self):
file_filters = "*.py"
self.fileDialog = FileDialog(None, "Load Nodes", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def main():
# Prompt user for directory containing files to be analyzed
F = FileDialog() # Calls Qt backend script to create a file dialog object
mcdir = F.getExistingDirectory(
caption='Select Motion Corrected Video Directory')
fvids = []
for file in os.listdir(mcdir):
if file.endswith("_mc.tif"):
fvids.append(os.path.join(mcdir, file))
# Set up a variable to determine which sections are lightsheet and which
# are epi. This is a horrible way to handle it - need to write new code to
# either automatically determine or prompt user for input.
# Use 1 for lightsheet and 0 for epi.
lsepi = [1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0]
# Threshold for non-binary masks to convert to binary
th = 0.05
# Load masks, videos, and dF/F curves
lsmasks = None
epimasks = None
lsunion = None
epiunion = None
lsvidconcat = None
epividconcat = None
lsvidconcattimes = [0]
epividconcattimes = [0]
for i, file in enumerate(fvids):
vid = cm.load(file)
mask = load_masks(file)
if lsepi[i]:
if lsmasks is None:
lsmasks = np.empty((0, mask.shape[1], mask.shape[2]))
lsunion = mask > th
lsvidconcat = np.empty((0, vid.shape[1], vid.shape[2]))
lsvidconcat = cm.concatenate([lsvidconcat, vid], axis=0)
lsmasks = np.concatenate((lsmasks, mask))
lsunion = mask_union(lsunion, mask > th, 10)
lsvidconcattimes.append(lsvidconcat.shape[0])
else:
if epimasks is None:
epimasks = np.empty((0, mask.shape[1], mask.shape[2]))
epiunion = mask > th
epividconcat = np.empty((0, vid.shape[1], vid.shape[2]))
epividconcat = cm.concatenate([epividconcat, vid], axis=0)
epimasks = np.concatenate((epimasks, mask))
epiunion = mask_union(epiunion, mask > th, 10)
epividconcattimes.append(epividconcat.shape[0])
print(epividconcattimes)
print(epividconcat.shape)
# Plot out the flattened masks for light-sheet and epi, and count the
# number of unique detected neurons
flFig, flAx = plt.subplots(1, 2)
unFig, unAx = plt.subplots(1, 2)
flsunion = np.zeros((lsmasks.shape[1], lsmasks.shape[2]))
ff = np.zeros((lsmasks.shape[1], lsmasks.shape[2]))
for lsmask in lsmasks:
ff = np.add(ff, lsmask > th)
for unionmask in lsunion:
flsunion = np.add(flsunion, unionmask)
flAx[0].imshow(ff)
unAx[0].imshow(flsunion)
print('Number of ls neurons: ' + str(lsunion.shape[0]))
ff = np.zeros((lsmasks.shape[1], lsmasks.shape[2]))
fepiunion = np.zeros((lsmasks.shape[1], lsmasks.shape[2]))
for epimask in epimasks:
ff = np.add(ff, epimask > th)
for unionmask in epiunion:
fepiunion = np.add(fepiunion, unionmask)
flAx[1].imshow(ff)
unAx[1].imshow(fepiunion)
print('Number of epi neurons: ' + str(epiunion.shape[0]))
# Mask operations to create the various sets and then plot them all out
sharedneurons = mask_joint(lsunion, epiunion, 10)
lsunique = mask_disjoint(sharedneurons, lsunion, 10)
epunique = mask_disjoint(sharedneurons, epiunion, 10)
allFig, allAx = plt.subplots(1, 3)
allAx[0].imshow(np.sum(lsunique, axis=0))
allAx[1].imshow(np.sum(sharedneurons, axis=0))
allAx[2].imshow(np.sum(epunique, axis=0))
print('Number of unique-to-ls neurons: ' + str(lsunique.shape[0]))
print('Number of unique-to-epi neurons: ' + str(epunique.shape[0]))
print('Number of shared neurons: ' + str(sharedneurons.shape[0]))
lsallmasks = mask_union(sharedneurons, lsunique, 10)
epallmasks = mask_union(sharedneurons, epunique, 10)
# Plot out df/F traces, custom calculated, for 'zz' number of elements
#zz=-1
#lslsdff = calculate_dff_set(lsvidconcat, lsallmasks)
#lsepdff = calculate_dff_set(lsvidconcat, epunique[0:zz])
#epepdff = calculate_dff_set(epvidconcat, epallmasks)
#eplsdff = calculate_dff_set(epvidconcat, lsunique[0:zz])
lslsdff = np.empty((lsallmasks.shape[0], 0))
for i, el in enumerate(lsvidconcattimes):
if not i == 0:
start = lsvidconcattimes[i - 1] + 1
end = lsvidconcattimes[i]
lslsdff = np.concatenate(
(lslsdff, calculate_dff_set(lsvidconcat[start:end],
lsallmasks)),
axis=1)
print(lslsdff.shape)
lslsdff = np.clip(lslsdff, 0, None)
epepdff = np.empty((epallmasks.shape[0], 0))
for i, el in enumerate(epividconcattimes):
if not i == 0:
start = epividconcattimes[i - 1] + 1
end = epividconcattimes[i]
epepdff = np.concatenate(
(epepdff, calculate_dff_set(epividconcat[start:end],
epallmasks)),
axis=1)
print(epepdff.shape)
epepdff = np.clip(epepdff, 0, None)
lspeakmax = np.zeros(lslsdff.shape[0])
lspeakcount = np.zeros(lslsdff.shape[0])
for i, lsdff in enumerate(lslsdff):
peaks, props = scipy.signal.find_peaks(lsdff,
distance=10,
prominence=(0.05, None),
width=(3, None),
height=(0.1, None))
lspeakmax[i] = max(lsdff)
if peaks.size > 0:
lspeakcount[i] = peaks.size
eppeakmax = np.zeros(epepdff.shape[0])
eppeakcount = np.zeros(epepdff.shape[0])
for i, epdff in enumerate(epepdff):
peaks, props = scipy.signal.find_peaks(epdff,
distance=10,
prominence=(0.05, None),
width=(3, None),
height=(0.1, None))
eppeakmax[i] = max(epdff)
if peaks.size > 0:
eppeakcount[i] = peaks.size
toplscount_idxs = np.argsort(lspeakcount)
toplspeak_idxs = np.argsort(lspeakmax)
topls_idxs = np.concatenate((toplscount_idxs[-4:], toplspeak_idxs[-2:]))
topsixlscount = lslsdff[topls_idxs]
topepcount_idxs = np.argsort(eppeakcount)
topeppeak_idxs = np.argsort(eppeakmax)
topep_idxs = np.concatenate((topepcount_idxs[-4:], topeppeak_idxs[-2:]))
topsixepcount = epepdff[topep_idxs]
lsmaxheatmap = np.zeros(lsallmasks[0].shape)
lscountheatmap = np.zeros(lsallmasks[0].shape)
for i, mask in enumerate(lsallmasks):
lsmaxheatmap = np.add(lsmaxheatmap, mask * lspeakmax[i])
lscountheatmap = np.add(lscountheatmap, mask * lspeakcount[i])
epmaxheatmap = np.zeros(epallmasks[0].shape)
epcountheatmap = np.zeros(epallmasks[0].shape)
for i, mask in enumerate(epallmasks):
epmaxheatmap = np.add(epmaxheatmap, mask * eppeakmax[i])
epcountheatmap = np.add(epcountheatmap, mask * eppeakcount[i])
lsrankedheatmap = np.zeros(lsallmasks[0].shape)
for i, idx in enumerate(topls_idxs):
lsrankedheatmap = np.add(lsrankedheatmap, lsallmasks[idx] * 2)
eprankedheatmap = np.zeros(epallmasks[0].shape)
for i, idx in enumerate(topep_idxs):
eprankedheatmap = np.add(eprankedheatmap, epallmasks[idx] * 2)
imageio.imwrite('ls_topmasks.png', lsrankedheatmap)
imageio.imwrite('epi_topmasks.png', eprankedheatmap)
ff, ax = plt.subplots()
ax.imshow(lsrankedheatmap)
ff, ax = plt.subplots()
ax.imshow(eprankedheatmap)
# Setting up plot information
cmap = plt.get_cmap('jet')
# Light-sheet, maximum dF/F figure
ff, ax = plt.subplots()
ax.imshow(lsmaxheatmap, cmap='jet')
vmin = math.floor(
np.min(lsmaxheatmap[np.nonzero(lsmaxheatmap)]) * 100) / 100
vmax = math.ceil(
np.max(lsmaxheatmap[np.nonzero(lsmaxheatmap)]) * 100) / 100
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
colors = cmap(np.linspace(1. - (vmax - vmin) / float(vmax), 1, cmap.N))
color_map = matplotlib.colors.LinearSegmentedColormap.from_list(
'cut_jet', colors)
cax, _ = matplotlib.colorbar.make_axes(plt.gca())
cbar = matplotlib.colorbar.ColorbarBase(
cax,
cmap=color_map,
norm=norm,
)
cbar.set_ticks([vmin, (vmax + vmin) / 2, vmax])
cbar.set_ticklabels([vmin, (vmax + vmin) / 2, vmax])
#cax.setlabel('Max. dF/F')
ax.axis('off')
ff.suptitle('Heatmap of light-sheet neurons by maximum dF/F')
plt.show()
# Light-sheet, spike count figure
ff, ax = plt.subplots()
ax.imshow(lscountheatmap, cmap='jet')
vmin = np.min(lscountheatmap[np.nonzero(lscountheatmap)])
vmax = np.max(lscountheatmap[np.nonzero(lscountheatmap)])
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
colors = cmap(np.linspace(1. - (vmax - vmin) / float(vmax), 1, cmap.N))
color_map = matplotlib.colors.LinearSegmentedColormap.from_list(
'cut_jet', colors)
cax, _ = matplotlib.colorbar.make_axes(plt.gca())
cbar = matplotlib.colorbar.ColorbarBase(
cax,
cmap=color_map,
norm=norm,
)
cbar.set_ticks([vmin, math.floor((vmax + vmin) / 2), vmax])
cbar.set_ticklabels([vmin, math.floor((vmax + vmin) / 2), vmax])
#cax.setlabel('Event Count')
ax.axis('off')
ff.suptitle('Heatmap of light-sheet neurons by spike count')
ff.show()
# Epi-illumination, maximum dF/F figure
ff, ax = plt.subplots()
ax.imshow(epmaxheatmap, cmap='jet')
vmin = math.floor(
np.min(epmaxheatmap[np.nonzero(epmaxheatmap)]) * 100) / 100
vmax = math.ceil(
np.max(epmaxheatmap[np.nonzero(epmaxheatmap)]) * 100) / 100
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
colors = cmap(np.linspace(1. - (vmax - vmin) / float(vmax), 1, cmap.N))
color_map = matplotlib.colors.LinearSegmentedColormap.from_list(
'cut_jet', colors)
cax, _ = matplotlib.colorbar.make_axes(plt.gca())
cbar = matplotlib.colorbar.ColorbarBase(
cax,
cmap=color_map,
norm=norm,
)
cbar.set_ticks([vmin, (vmax + vmin) / 2, vmax])
cbar.set_ticklabels([vmin, (vmax + vmin) / 2, vmax])
#cbar.ax.setlabel('Max. dF/F')
ax.axis('off')
ff.suptitle('Heatmap of epi-illuminated neurons by maximum dF/F')
ff.show()
# Epi-illumination, spike count figure
ff, ax = plt.subplots()
ax.imshow(epcountheatmap, cmap='jet')
vmin = np.min(epcountheatmap[np.nonzero(epcountheatmap)])
vmax = np.max(epcountheatmap[np.nonzero(epcountheatmap)])
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
colors = cmap(np.linspace(1. - (vmax - vmin) / float(vmax), 1, cmap.N))
color_map = matplotlib.colors.LinearSegmentedColormap.from_list(
'cut_jet', colors)
cax, _ = matplotlib.colorbar.make_axes(plt.gca())
cbar = matplotlib.colorbar.ColorbarBase(
cax,
cmap=color_map,
norm=norm,
)
cbar.set_ticks([vmin, math.floor((vmax + vmin) / 2), vmax])
cbar.set_ticklabels([vmin, math.floor((vmax + vmin) / 2), vmax])
#cax.setlabel('Event Count')
ax.axis('off')
ff.suptitle('Heatmap of epi-illuminated neurons by spike count')
ff.show()
ffls, axls = plt.subplots(6, 1)
ffep, axep = plt.subplots(6, 1)
for i in range(6):
axls[i].plot(topsixlscount[i])
axep[i].plot(topsixepcount[i])
ffls.suptitle('Top 6 Lightsheet neurons')
ffep.suptitle('Top 6 Epi neurons')
# export data (df/f traces and concat times) to be plotted in matlab
scipy.io.savemat(os.path.join(mcdir, 'lightsheet_dff_data.mat'), {
'trace': lslsdff,
'timebreaks': lsvidconcattimes
})
scipy.io.savemat(os.path.join(mcdir, 'epi_dff_data.mat'), {
'trace': epepdff,
'timebreaks': epividconcattimes
})
scipy.io.savemat(os.path.join(mcdir, 'lightsheet_dff_datatops.mat'), {
'trace': topsixlscount,
'timebreaks': lsvidconcattimes,
'idxs': topls_idxs
})
scipy.io.savemat(
os.path.join(mcdir, 'epi_dff_datatops.mat'), {
'trace': topsixepcount,
'timebreaks': epividconcattimes,
'idxs': topep_idxs
})
scipy.io.savemat(os.path.join(mcdir, 'epi_masks.mat'),
{'mask': epallmasks})
scipy.io.savemat(os.path.join(mcdir, 'ls_masks.mat'), {'mask': lsallmasks})
except NameError:
print('Not launched under iPython')
def make_color_img(img, gain=255, min_max=None,out_type=np.uint8):
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img-min_)/(max_-min_)*gain
img = img.astype(out_type)
img = np.dstack([img]*3)
return img
F = FileDialog()
# load object saved by CNMF
# cnm_obj = load_CNMF('/Users/agiovann/caiman_data/example_movies/memmap__d1_60_d2_80_d3_1_order_C_frames_2000_save.hdf5')
cnm_obj = load_CNMF(F.getOpenFileName(caption='Load CNMF Object',filter='*.hdf5')[0])
# movie
# mov = cm.load('/Users/agiovann/caiman_data/example_movies/memmap__d1_60_d2_80_d3_1_order_C_frames_2000_.mmap')
mov = cm.load(cnm_obj.mmap_file)
min_mov = np.min(mov)
max_mov = np.max(mov)
# load summary image
# Cn = cm.load('/Users/agiovann/caiman_data/example_movies/memmap__d1_60_d2_80_d3_1_order_C_frames_2000__Cn.tif')
Cn = cnm_obj.estimates.Cn
if data == 'CORR':
cur_img = summary_images[-1]
else:
cur_img = summary_images[0]
img.setImage(cur_img)
print(param)
print(change)
print(data)
pars_action.sigTreeStateChanged.connect(change)
def adjust_contrast(img, min_value, max_value):
img[img < min_value] = min_value
img[img > max_value] = max_value
return img
F = FileDialog()
all_pts = {}
all_ROIs = {}
pts = []
pen = pg.mkPen(color=(255, 255, 0), width=4) #, style=Qt.DashDotLine)
## Display the widget as a new window
w.show()
## Start the Qt event loop
app.exec_()
class DataViz(QtGui.QMainWindow):
"""The main application window for dataviz.
This is an MDI application with dock displaying open HDF5 files on
the left. Attributes of the selected item at left bottom.
Signals
-------
sigOpen: Emitted when a set of files have been selected in the open
files dialog. Sends out the list of file paths selected and the
mode string.
sigCloseFiles: Emitted when the user triggers closeFilesAction. This
is passed on to the HDFTreeWidget which decides which
files to close based on selection.
sigShowAttributes: Emitted when showAttributesAction is
triggered. Connected to
HDFTreeWidget.showAttributes function which
creates a widget for displaying attributes of the
HDF5 node of its current
item. HDFTreeWidget.showAttributes sends a return
signal `attributeWidgetCreated` with the created
widget so that the DataViz widget can incorporate
it as an mdi child window.
sigShowDataset: Emitted when showDatasetAction is
triggered. Connected to HDFTreeWidget's showDataset
function which creates a widget for displaying the
contents of the HDF5 node if it is a dataset.
HDFTreeWidget.showDataset sends a return signal
`sigDatasetWidgetCreated` with the created widget so
that the DataViz widget can incorporate it as an mdi
child window.
sigPlotDataset: Emitted when plotDatasetAction is
triggered. Connected to HDFTreeWidget's plotDataset
function which creates a widget for displaying teh
contents of the HDF5 node if it is a datset.
"""
sigOpen = QtCore.pyqtSignal(list, str)
sigCloseFiles = QtCore.pyqtSignal()
sigShowAttributes = QtCore.pyqtSignal()
sigShowDataset = QtCore.pyqtSignal()
sigPlotDataset = QtCore.pyqtSignal()
def __init__(self, parent=None, flags=QtCore.Qt.WindowFlags(0)):
super(DataViz, self).__init__(parent=parent, flags=flags)
self.readSettings()
self.mdiArea = QtGui.QMdiArea()
self.mdiArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.subWindowActivated.connect(self.switchPlotParamPanel)
self.setCentralWidget(self.mdiArea)
self.createTreeDock()
self.createActions()
self.createMenus()
def closeEvent(self, event):
self.writeSettings()
event.accept()
def readSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
self.lastDir = settings.value('lastDir', '.', str)
pos = settings.value('pos', QtCore.QPoint(200, 200))
if isinstance(pos, QtCore.QVariant):
pos = pos.toPyObject()
self.move(pos)
size = settings.value('size', QtCore.QSize(400, 400))
if isinstance(size, QtCore.QVariant):
size = size.toPyObject()
self.resize(size)
def writeSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
settings.setValue('lastDir', self.lastDir)
settings.setValue('pos', self.pos())
settings.setValue('size', self.size())
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path)
for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def openFilesReadWrite(self, filePaths=None):
# print(filePaths)
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.filesSelected.connect(self.openFilesReadWrite)
self.fileDialog.show()
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path)
for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r+')
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def createActions(self):
self.openFileReadOnlyAction = QtGui.QAction(
QtGui.QIcon(),
'Open file(s) readonly',
self,
shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for reading',
triggered=self.openFilesReadOnly)
self.openFileReadWriteAction = QtGui.QAction(
QtGui.QIcon(),
'&Open file(s) read/write',
self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for editing',
triggered=self.openFilesReadWrite)
self.openFileOverwriteAction = QtGui.QAction(
QtGui.QIcon(),
'Overwrite file',
self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for writing (overwrite existing)',
triggered=self.openFileOverwrite)
self.createFileAction = QtGui.QAction(
QtGui.QIcon(),
'&New file',
self,
shortcut=QtGui.QKeySequence.New,
statusTip='Create a new HDF5 file',
triggered=self.createFile)
self.closeFileAction = QtGui.QAction(
QtGui.QIcon(),
'&Close file(s)',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_K),
statusTip='Close selected files',
triggered=self.sigCloseFiles)
self.quitAction = QtGui.QAction(QtGui.QIcon(),
'&Quit',
self,
shortcut=QtGui.QKeySequence.Quit,
statusTip='Quit dataviz',
triggered=self.doQuit)
self.showAttributesAction = QtGui.QAction(
QtGui.QIcon(),
'Show attributes',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.Key_Return),
statusTip='Show attributes',
triggered=self.sigShowAttributes)
self.showDatasetAction = QtGui.QAction(
QtGui.QIcon(),
'Show dataset',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_Return),
statusTip='Show dataset',
triggered=self.sigShowDataset)
self.plotDatasetAction = QtGui.QAction(
QtGui.QIcon(),
'Plot dataset',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.ALT + QtCore.Qt.Key_P),
statusTip='Plot dataset',
triggered=self.sigPlotDataset)
def createMenus(self):
self.menuBar().setVisible(True)
self.fileMenu = self.menuBar().addMenu('&File')
self.fileMenu.addAction(self.openFileReadWriteAction)
self.fileMenu.addAction(self.openFileReadOnlyAction)
self.fileMenu.addAction(self.openFileOverwriteAction)
self.fileMenu.addAction(self.createFileAction)
self.fileMenu.addAction(self.closeFileAction)
self.fileMenu.addAction(self.quitAction)
self.editMenu = self.menuBar().addMenu('&Edit')
self.editMenu.addAction(self.tree.insertDatasetAction)
self.editMenu.addAction(self.tree.insertGroupAction)
self.editMenu.addAction(self.tree.deleteNodeAction)
self.viewMenu = self.menuBar().addMenu('&View')
self.viewMenu.addAction(self.treeDock.toggleViewAction())
self.dataMenu = self.menuBar().addMenu('&Data')
self.dataMenu.addAction(self.showAttributesAction)
self.dataMenu.addAction(self.showDatasetAction)
self.dataMenu.addAction(self.plotDatasetAction)
def createTreeDock(self):
self.treeDock = QtGui.QDockWidget('File tree', self)
self.tree = HDFTreeWidget(parent=self.treeDock)
self.sigOpen.connect(self.tree.openFiles)
self.tree.doubleClicked.connect(self.tree.createDatasetWidget)
self.tree.sigDatasetWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigDatasetWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigAttributeWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigAttributeWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigPlotWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotParamTreeCreated.connect(self.addPanelBelow)
self.tree.sigDataWidgetActivated.connect(self.activateDataWindow)
# pipe signals of dataviz to those of hdftree widget
self.sigShowAttributes.connect(self.tree.showAttributes)
self.sigShowDataset.connect(self.tree.showDataset)
self.sigPlotDataset.connect(self.tree.plotDataset)
self.sigCloseFiles.connect(self.tree.closeFiles)
self.treeDock.setWidget(self.tree)
self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.treeDock)
def addMdiChildWindow(self, widget):
if widget is not None:
subwin = self.mdiArea.addSubWindow(widget)
subwin.setWindowTitle(widget.name)
widget.show()
return subwin
def activateDataWindow(self, widget):
if widget is not None:
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
self.mdiArea.setActiveSubWindow(window)
def closeMdiChildWindow(self, widget):
if widget is not None:
self.tree.removeBufferedWidget(widget)
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
window.deleteLater()
def addPanelBelow(self, widget):
dockWidget = QtGui.QDockWidget(widget.name)
dockWidget.setWidget(widget)
self.addDockWidget(QtCore.Qt.BottomDockWidgetArea, dockWidget)
dockWidget.show()
def switchPlotParamPanel(self, subwin):
"""Make plot param tree panel visible if active subwindow has a
plotwidget. All other plot param trees will be invisible. Qt
does not provide out-of-focus signal for mdi subwindows. So
there is no counterpart of subWindowActivated that can allow
us to hide paramtrees for inactive plot widgets. Hence this
approach.
"""
if subwin is None:
return
for dockWidget in self.findChildren(QtGui.QDockWidget):
# All dockwidgets that contain paramtrees must be checked
if isinstance(dockWidget.widget(), DatasetPlotParamTree):
if isinstance(subwin.widget(), DatasetPlot) and \
dockWidget.widget() in subwin.widget().paramsToPlots:
dockWidget.setVisible(True)
else:
dockWidget.setVisible(False)
def doQuit(self):
self.writeSettings()
QtGui.QApplication.instance().closeAllWindows()
class ExportWidget(QtWidgets.QWidget):
def __init__(self, node, text):
super().__init__()
self.node = node
self.text = text
self.setWindowTitle("Export")
self.layout = QtWidgets.QFormLayout(self)
self.setLayout(self.layout)
self.name = QtWidgets.QLineEdit(parent=self)
self.docstring = QtWidgets.QTextEdit(parent=self)
self.ok = QtWidgets.QPushButton("Ok", parent=self)
self.ok.clicked.connect(self.ok_clicked)
self.layout.addRow("Name:", self.name)
self.layout.addRow("Docstring:", self.docstring)
self.layout.addWidget(self.ok)
def ok_clicked(self):
self.fileDialog = FileDialog(None, "Save File..", '.', "Python (*.py)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
def saveFile(self, fileName):
node_name = self.name.text()
docstring = self.docstring.toPlainText()
terminals = {}
for name, term in self.node.terminals.items():
state = term.saveState()
state['ttype'] = fullname(term._type)
terminals[name] = state
template = self.export(node_name, docstring, terminals, self.text)
if not fileName.endswith('.py'):
fileName += '.py'
with open(fileName, 'w') as f:
f.write(template)
def export(self, name, docstring, terminals, text):
template = f"""
from typing import Any
from amitypes import Array1d, Array2d, Array3d
from ami.flowchart.Node import Node
import ami.graph_nodes as gn
{text}
class {name}(Node):
\"""
{docstring}
\"""
nodeName = "{name}"
def __init__(self, name):
super().__init__(name, terminals={terminals})
def to_operation(self, **kwargs):
proc = EventProcessor()
return gn.Map(name=self.name()+"_operation", **kwargs,
func=proc.on_event,
begin_run=proc.begin_run,
end_run=proc.end_run,
begin_step=proc.begin_step,
end_step=proc.end_step)
"""
return template
class LogWidget(QtGui.QWidget):
"""A widget to show log entries and filter them.
"""
sigDisplayEntry = QtCore.Signal(object) ## for thread-safetyness
sigAddEntry = QtCore.Signal(object) ## for thread-safetyness
sigScrollToAnchor = QtCore.Signal(object) # for internal use.
Stylesheet = """
body {color: #000; font-family: sans;}
.entry {}
.error .message {color: #900}
.warning .message {color: #740}
.user .message {color: #009}
.status .message {color: #090}
.logExtra {margin-left: 40px;}
.traceback {color: #555; height: 0px;}
.timestamp {color: #000;}
"""
pageTemplate = """
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<style type="text/css">
%s </style>
<script type="text/javascript">
function showDiv(id) {
div = document.getElementById(id);
div.style.visibility = "visible";
div.style.height = "auto";
}
</script>
</head>
<body>
</body>
</html> """ % Stylesheet
def __init__(self, parent):
"""Creates the log widget.
@param object parent: Qt parent object for log widet
"""
QtGui.QWidget.__init__(self, parent)
self.ui = LogWidgetTemplate.Ui_Form()
self.ui.setupUi(self)
#self.ui.input.hide()
self.ui.filterTree.topLevelItem(1).setExpanded(True)
self.entries = [] ## stores all log entries in memory
self.cache = {
} ## for storing html strings of entries that have already been processed
self.displayedEntries = []
self.typeFilters = []
self.importanceFilter = 0
self.dirFilter = False
self.entryArrayBuffer = np.zeros(
1000,
dtype=[ ### a record array for quick filtering of entries
('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')
])
self.entryArray = self.entryArrayBuffer[:0]
self.filtersChanged()
self.sigDisplayEntry.connect(self.displayEntry,
QtCore.Qt.QueuedConnection)
self.sigAddEntry.connect(self.addEntry, QtCore.Qt.QueuedConnection)
self.ui.exportHtmlBtn.clicked.connect(self.exportHtml)
self.ui.filterTree.itemChanged.connect(self.setCheckStates)
self.ui.importanceSlider.valueChanged.connect(self.filtersChanged)
self.ui.output.anchorClicked.connect(self.linkClicked)
self.sigScrollToAnchor.connect(self.scrollToAnchor,
QtCore.Qt.QueuedConnection)
def loadFile(self, f):
"""Load a log file for display.
@param str f: path to file that should be laoded.
f must be able to be read by pyqtgraph configfile.py
"""
log = configfile.readConfigFile(f)
self.entries = []
self.entryArrayBuffer = np.zeros(len(log),
dtype=[('index', 'int32'),
('importance', 'int32'),
('msgType', '|S10'),
('directory', '|S100'),
('entryId', 'int32')])
self.entryArray = self.entryArrayBuffer[:]
i = 0
for k, v in log.items():
v['id'] = k[
9:] ## record unique ID to facilitate HTML generation (javascript needs this ID)
self.entries.append(v)
self.entryArray[i] = np.array(
[(i, v.get('importance', 5), v.get('msgType', 'status'),
v.get('currentDir', ''), v.get('entryId', v['id']))],
dtype=[('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')])
i += 1
self.filterEntries(
) ## puts all entries through current filters and displays the ones that pass
def addEntry(self, entry):
"""Add a log entry to the list.
"""
## All incoming messages begin here
## for thread-safetyness:
isGuiThread = QtCore.QThread.currentThread(
) == QtCore.QCoreApplication.instance().thread()
if not isGuiThread:
self.sigAddEntry.emit(entry)
return
self.entries.append(entry)
i = len(self.entryArray)
entryDir = entry.get('currentDir', None)
if entryDir is None:
entryDir = ''
arr = np.array([
(i, entry['importance'], entry['msgType'], entryDir, entry['id'])
],
dtype=[('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')])
## make more room if needed
if len(self.entryArrayBuffer) == len(self.entryArray):
newArray = np.empty(
len(self.entryArrayBuffer) + 1000, self.entryArrayBuffer.dtype)
newArray[:len(self.entryArray)] = self.entryArray
self.entryArrayBuffer = newArray
self.entryArray = self.entryArrayBuffer[:len(self.entryArray) + 1]
self.entryArray[i] = arr
self.checkDisplay(
entry) ## displays the entry if it passes the current filters
def setCheckStates(self, item, column):
if item == self.ui.filterTree.topLevelItem(1):
if item.checkState(0):
for i in range(item.childCount()):
item.child(i).setCheckState(0, QtCore.Qt.Checked)
elif item.parent() == self.ui.filterTree.topLevelItem(1):
if not item.checkState(0):
self.ui.filterTree.topLevelItem(1).setCheckState(
0, QtCore.Qt.Unchecked)
self.filtersChanged()
def filtersChanged(self):
### Update self.typeFilters, self.importanceFilter, and self.dirFilter to reflect changes.
tree = self.ui.filterTree
self.typeFilters = []
for i in range(tree.topLevelItem(1).childCount()):
child = tree.topLevelItem(1).child(i)
if tree.topLevelItem(1).checkState(0) or child.checkState(0):
text = child.text(0)
self.typeFilters.append(str(text))
self.importanceFilter = self.ui.importanceSlider.value()
# self.updateDirFilter()
self.filterEntries()
# def updateDirFilter(self, dh=None):
# if self.ui.filterTree.topLevelItem(0).checkState(0):
# if dh==None:
# self.dirFilter = self.manager.getDirOfSelectedFile().name()
# else:
# self.dirFilter = dh.name()
# else:
# self.dirFilter = False
def filterEntries(self):
"""Runs each entry in self.entries through the filters and displays if it makes it through."""
### make self.entries a record array, then filtering will be much faster (to OR true/false arrays, + them)
typeMask = self.entryArray['msgType'] == b''
for t in self.typeFilters:
typeMask += self.entryArray['msgType'] == t.encode('ascii')
mask = (self.entryArray['importance'] >
self.importanceFilter) * typeMask
#if self.dirFilter != False:
# d = np.ascontiguousarray(self.entryArray['directory'])
# j = len(self.dirFilter)
# i = len(d)
# d = d.view(np.byte).reshape(i, 100)[:, :j]
# d = d.reshape(i*j).view('|S%d' % j)
# mask *= (d == self.dirFilter)
self.ui.output.clear()
self.ui.output.document().setDefaultStyleSheet(self.Stylesheet)
indices = list(self.entryArray[mask]['index'])
self.displayEntry([self.entries[i] for i in indices])
def checkDisplay(self, entry):
### checks whether entry passes the current filters and displays it if it does.
if entry['msgType'] not in self.typeFilters:
return
elif entry['importance'] < self.importanceFilter:
return
elif self.dirFilter is not False:
if entry['currentDir'][:len(self.dirFilter)] != self.dirFilter:
return
else:
self.displayEntry([entry])
def displayEntry(self, entries):
## entries should be a list of log entries
## for thread-safetyness:
isGuiThread = QtCore.QThread.currentThread(
) == QtCore.QCoreApplication.instance().thread()
if not isGuiThread:
self.sigDisplayEntry.emit(entries)
return
for entry in entries:
if id(entry) not in self.cache:
self.cache[id(entry)] = self.generateEntryHtml(entry)
html = self.cache[id(entry)]
sb = self.ui.output.verticalScrollBar()
isMax = sb.value() == sb.maximum()
self.ui.output.append(html)
self.displayedEntries.append(entry)
if isMax:
## can't scroll to end until the web frame has processed the html change
#frame.setScrollBarValue(QtCore.Qt.Vertical, frame.scrollBarMaximum(QtCore.Qt.Vertical))
## Calling processEvents anywhere inside an error handler is forbidden
## because this can lead to Qt complaining about paint() recursion.
self.sigScrollToAnchor.emit(str(
entry['id'])) ## queued connection
def scrollToAnchor(self, anchor):
self.ui.output.scrollToAnchor(anchor)
def generateEntryHtml(self, entry):
msg = self.cleanText(entry['message'])
reasons = ""
docs = ""
exc = ""
if 'reasons' in entry:
reasons = self.formatReasonStrForHTML(entry['reasons'])
if 'docs' in entry:
docs = self.formatDocsStrForHTML(entry['docs'])
if entry.get('exception', None) is not None:
exc = self.formatExceptionForHTML(entry, entryId=entry['id'])
extra = reasons + docs + exc
if extra != "":
#extra = "<div class='logExtra'>" + extra + "</div>"
extra = "<table class='logExtra'><tr><td>" + extra + "</td></tr></table>"
#return """
#<div class='entry'>
#<div class='%s'>
#<span class='timestamp'>%s</span>
#<span class='message'>%s</span>
#%s
#</div>
#</div>
#""" % (entry['msgType'], entry['timestamp'], msg, extra)
return """
<a name="%s"/><table class='entry'><tr><td>
<table class='%s'><tr><td>
<span class='timestamp'>%s</span>
<span class='message'>%s</span>
%s
</td></tr></table>
</td></tr></table>
""" % (str(
entry['id']), entry['msgType'], entry['timestamp'], msg, extra)
@staticmethod
def cleanText(text):
text = re.sub(r'&', '&', text)
text = re.sub(r'>', '>', text)
text = re.sub(r'<', '<', text)
text = re.sub(r'\n', '<br/>\n', text)
return text
def formatExceptionForHTML(self,
entry,
exception=None,
count=1,
entryId=None):
### Here, exception is a dict that holds the message, reasons, docs, traceback and oldExceptions (which are also dicts, with the same entries)
## the count and tracebacks keywords are for calling recursively
if exception is None:
exception = entry['exception']
#if tracebacks is None:
#tracebacks = []
indent = 10
text = self.cleanText(exception['message'])
text = re.sub(r'^HelpfulException: ', '', text)
messages = [text]
if 'reasons' in exception:
reasons = self.formatReasonsStrForHTML(exception['reasons'])
text += reasons
#self.displayText(reasons, entry, color, clean=False)
if 'docs' in exception:
docs = self.formatDocsStrForHTML(exception['docs'])
#self.displayText(docs, entry, color, clean=False)
text += docs
traceback = [
self.formatTracebackForHTML(exception['traceback'], count)
]
text = [text]
if 'oldExc' in exception:
exc, tb, msgs = self.formatExceptionForHTML(entry,
exception['oldExc'],
count=count + 1)
text.extend(exc)
messages.extend(msgs)
traceback.extend(tb)
#else:
#if len(tracebacks)==count+1:
#n=0
#else:
#n=1
#for i, tb in enumerate(tracebacks):
#self.displayTraceback(tb, entry, number=i+n)
if count == 1:
exc = "<div class=\"exception\"><ol>" + "\n".join(
["<li>%s</li>" % ex for ex in text]) + "</ol></div>"
tbStr = "\n".join([
"<li><b>%s</b><br/><span class='traceback'>%s</span></li>" %
(messages[i], tb) for i, tb in enumerate(traceback)
])
#traceback = "<div class=\"traceback\" id=\"%s\"><ol>"%str(entryId) + tbStr + "</ol></div>"
entry['tracebackHtml'] = tbStr
#return exc + '<a href="#" onclick="showDiv(\'%s\')">Show traceback</a>'%str(entryId) + traceback
return exc + '<a href="exc:%s">Show traceback %s</a>' % (
str(entryId), str(entryId))
else:
return text, traceback, messages
def formatTracebackForHTML(self, tb, number):
try:
tb = [
line for line in tb
if not line.startswith("Traceback (most recent call last)")
]
except:
print("\n" + str(tb) + "\n")
raise
return re.sub(" ", " ", ("").join(map(self.cleanText, tb)))[:-1]
#tb = [self.cleanText(strip(x)) for x in tb]
#lines = []
#prefix = ''
#for l in ''.join(tb).split('\n'):
#if l == '':
#continue
#if l[:9] == "Traceback":
#prefix = ' ' + str(number) + '. '
#continue
#spaceCount = 0
#while l[spaceCount] == ' ':
#spaceCount += 1
#if prefix is not '':
#spaceCount -= 1
#lines.append(" "*(spaceCount*4) + prefix + l)
#prefix = ''
#return '<div class="traceback">' + '<br />'.join(lines) + '</div>'
#self.displayText('<br />'.join(lines), entry, color, clean=False)
def formatReasonsStrForHTML(self, reasons):
#indent = 6
reasonStr = "<table class='reasons'><tr><td>Possible reasons include:\n<ul>\n"
for r in reasons:
r = self.cleanText(r)
reasonStr += "<li>" + r + "</li>\n"
#reasonStr += " "*22 + chr(97+i) + ". " + r + "<br>"
reasonStr += "</ul></td></tr></table>\n"
return reasonStr
def formatDocsStrForHTML(self, docs):
#indent = 6
docStr = "<div class='docRefs'>Relevant documentation:\n<ul>\n"
for d in docs:
d = self.cleanText(d)
docStr += "<li><a href=\"doc:%s\">%s</a></li>\n" % (d, d)
docStr += "</ul></div>\n"
return docStr
def exportHtml(self, fileName=False):
if fileName is False:
self.fileDialog = FileDialog(self, "Save HTML as...",
"htmltemp.log")
#self.fileDialog.setFileMode(QtGui.QFileDialog.AnyFile)
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.exportHtml)
return
if fileName[-5:] != '.html':
fileName += '.html'
#doc = self.ui.output.document().toHtml('utf-8')
#for e in self.displayedEntries:
#if e.has_key('tracebackHtml'):
#doc = re.sub(r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'%(str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
doc = self.pageTemplate
for e in self.displayedEntries:
doc += self.cache[id(e)]
for e in self.displayedEntries:
if 'tracebackHtml' in e:
doc = re.sub(
r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'
% (str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
f = open(fileName, 'w')
f.write(doc)
f.close()
def linkClicked(self, url):
url = url.toString()
if url[:4] == 'doc:':
#self.manager.showDocumentation(url[4:])
print("Not implemented")
elif url[:4] == 'exc:':
cursor = self.ui.output.document().find('Show traceback %s' %
url[4:])
try:
tb = self.entries[int(url[4:]) - 1]['tracebackHtml']
except IndexError:
try:
tb = self.entries[self.entryArray[
self.entryArray['entryId'] == (
int(url[4:]))]['index']]['tracebackHtml']
except:
print("requested index %d, but only %d entries exist." %
(int(url[4:]) - 1, len(self.entries)))
raise
cursor.insertHtml(tb)
def clear(self):
self.ui.output.clear()
self.displayedEntryies = []
def ok_clicked(self):
self.fileDialog = FileDialog(None, "Save File..", '.', "Python (*.py)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
class DataViz(QtGui.QMainWindow):
"""The main application window for dataviz.
This is an MDI application with dock displaying open HDF5 files on
the left. Attributes of the selected item at left bottom.
Signals
-------
sigOpen: Emitted when a set of files have been selected in the open
files dialog. Sends out the list of file paths selected and the
mode string.
sigCloseFiles: Emitted when the user triggers closeFilesAction. This
is passed on to the HDFTreeWidget which decides which
files to close based on selection.
sigShowAttributes: Emitted when showAttributesAction is
triggered. Connected to
HDFTreeWidget.showAttributes function which
creates a widget for displaying attributes of the
HDF5 node of its current
item. HDFTreeWidget.showAttributes sends a return
signal `attributeWidgetCreated` with the created
widget so that the DataViz widget can incorporate
it as an mdi child window.
sigShowDataset: Emitted when showDatasetAction is
triggered. Connected to HDFTreeWidget's showDataset
function which creates a widget for displaying the
contents of the HDF5 node if it is a dataset.
HDFTreeWidget.showDataset sends a return signal
`sigDatasetWidgetCreated` with the created widget so
that the DataViz widget can incorporate it as an mdi
child window.
sigPlotDataset: Emitted when plotDatasetAction is
triggered. Connected to HDFTreeWidget's plotDataset
function which creates a widget for displaying teh
contents of the HDF5 node if it is a datset.
"""
sigOpen = QtCore.pyqtSignal(list, str)
sigCloseFiles = QtCore.pyqtSignal()
sigShowAttributes = QtCore.pyqtSignal()
sigShowDataset = QtCore.pyqtSignal()
sigPlotDataset = QtCore.pyqtSignal()
def __init__(self, parent=None, flags=QtCore.Qt.WindowFlags(0)):
super(DataViz, self).__init__(parent=parent, flags=flags)
self.readSettings()
self.mdiArea = QtGui.QMdiArea()
self.mdiArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.subWindowActivated.connect(self.switchPlotParamPanel)
self.setCentralWidget(self.mdiArea)
self.createTreeDock()
self.createActions()
self.createMenus()
def closeEvent(self, event):
self.writeSettings()
event.accept()
def readSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
self.lastDir = settings.value('lastDir', '.', str)
pos = settings.value('pos', QtCore.QPoint(200, 200))
if isinstance(pos, QtCore.QVariant):
pos = pos.toPyObject()
self.move(pos)
size = settings.value('size', QtCore.QSize(400, 400))
if isinstance(size, QtCore.QVariant):
size = size.toPyObject()
self.resize(size)
def writeSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
settings.setValue('lastDir', self.lastDir)
settings.setValue('pos', self.pos())
settings.setValue('size', self.size())
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path) for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def openFilesReadWrite(self, filePaths=None):
# print(filePaths)
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.filesSelected.connect(self.openFilesReadWrite)
self.fileDialog.show()
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path) for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r+')
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def createActions(self):
self.openFileReadOnlyAction = QtGui.QAction(QtGui.QIcon(), 'Open file(s) readonly', self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for reading',
triggered=self.openFilesReadOnly)
self.openFileReadWriteAction = QtGui.QAction(QtGui.QIcon(), '&Open file(s) read/write', self,
shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for editing',
triggered=self.openFilesReadWrite)
self.openFileOverwriteAction = QtGui.QAction(QtGui.QIcon(), 'Overwrite file', self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for writing (overwrite existing)',
triggered=self.openFileOverwrite)
self.createFileAction = QtGui.QAction(QtGui.QIcon(), '&New file', self,
shortcut=QtGui.QKeySequence.New,
statusTip='Create a new HDF5 file',
triggered=self.createFile)
self.closeFileAction = QtGui.QAction(QtGui.QIcon(), '&Close file(s)',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL+QtCore.Qt.Key_K),
statusTip='Close selected files',
triggered=self.sigCloseFiles)
self.quitAction = QtGui.QAction(QtGui.QIcon(), '&Quit', self,
shortcut=QtGui.QKeySequence.Quit,
statusTip='Quit dataviz',
triggered=self.doQuit)
self.showAttributesAction = QtGui.QAction(QtGui.QIcon(), 'Show attributes', self,
shortcut=QtGui.QKeySequence.InsertParagraphSeparator,
statusTip='Show attributes',
triggered=self.sigShowAttributes)
self.showDatasetAction = QtGui.QAction(QtGui.QIcon(), 'Show dataset', self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL+QtCore.Qt.Key_Return),
statusTip='Show dataset',
triggered=self.sigShowDataset)
self.plotDatasetAction = QtGui.QAction(QtGui.QIcon(), 'Plot dataset', self,
shortcut=QtGui.QKeySequence(QtCore.Qt.ALT + QtCore.Qt.Key_P),
statusTip='Plot dataset',
triggered=self.sigPlotDataset)
def createMenus(self):
self.menuBar().setVisible(True)
self.fileMenu = self.menuBar().addMenu('&File')
self.fileMenu.addAction(self.openFileReadWriteAction)
self.fileMenu.addAction(self.openFileReadOnlyAction)
self.fileMenu.addAction(self.openFileOverwriteAction)
self.fileMenu.addAction(self.createFileAction)
self.fileMenu.addAction(self.closeFileAction)
self.fileMenu.addAction(self.quitAction)
self.editMenu = self.menuBar().addMenu('&Edit')
self.editMenu.addAction(self.tree.insertDatasetAction)
self.editMenu.addAction(self.tree.insertGroupAction)
self.editMenu.addAction(self.tree.deleteNodeAction)
self.viewMenu = self.menuBar().addMenu('&View')
self.viewMenu.addAction(self.treeDock.toggleViewAction())
self.dataMenu = self.menuBar().addMenu('&Data')
self.dataMenu.addAction(self.showAttributesAction)
self.dataMenu.addAction(self.showDatasetAction)
self.dataMenu.addAction(self.plotDatasetAction)
def createTreeDock(self):
self.treeDock = QtGui.QDockWidget('File tree', self)
self.tree = HDFTreeWidget(parent=self.treeDock)
self.sigOpen.connect(self.tree.openFiles)
self.tree.doubleClicked.connect(self.tree.createDatasetWidget)
self.tree.sigDatasetWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigDatasetWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigAttributeWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigAttributeWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigPlotWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotParamTreeCreated.connect(self.addPanelBelow)
# pipe signals of dataviz to those of hdftree widget
self.sigShowAttributes.connect(self.tree.showAttributes)
self.sigShowDataset.connect(self.tree.showDataset)
self.sigPlotDataset.connect(self.tree.plotDataset)
self.sigCloseFiles.connect(self.tree.closeFiles)
self.treeDock.setWidget(self.tree)
self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.treeDock)
def addMdiChildWindow(self, widget):
if widget is not None:
subwin = self.mdiArea.addSubWindow(widget)
subwin.setWindowTitle(widget.name)
widget.show()
return subwin
def closeMdiChildWindow(self, widget):
if widget is not None:
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
window.deleteLater()
def addPanelBelow(self, widget):
dockWidget = QtGui.QDockWidget(widget.name)
dockWidget.setWidget(widget)
self.addDockWidget(QtCore.Qt.BottomDockWidgetArea, dockWidget)
dockWidget.show()
def switchPlotParamPanel(self, subwin):
"""Make plot param tree panel visible if active subwindow has a
plotwidget. All other plot param trees will be invisible. Qt
does not provide out-of-focus signal for mdi subwindows. So
there is no counterpart of subWindowActivated that can allow
us to hide paramtrees for inactive plot widgets. Hence this
approach.
"""
if subwin is None:
return
for dockWidget in self.findChildren(QtGui.QDockWidget):
# All dockwidgets that contain paramtrees must be checked
if isinstance(dockWidget.widget(), DatasetPlotParamTree):
if isinstance(subwin.widget(), DatasetPlot) and \
dockWidget.widget() in subwin.widget().paramsToPlots:
dockWidget.setVisible(True)
else:
dockWidget.setVisible(False)
def doQuit(self):
self.writeSettings()
QtGui.QApplication.instance().closeAllWindows()
class FlowchartCtrlWidget(QtGui.QWidget):
"""
The widget that contains the list of all the nodes in a flowchart and their controls,
as well as buttons for loading/saving flowcharts.
"""
def __init__(self, chart, graphmgr_addr):
super().__init__()
self.graphCommHandler = AsyncGraphCommHandler(graphmgr_addr.name,
graphmgr_addr.comm,
ctx=chart.ctx)
self.graph_name = graphmgr_addr.name
self.metadata = None
self.currentFileName = None
self.chart = chart
self.chartWidget = FlowchartWidget(chart, self)
self.ui = EditorTemplate.Ui_Toolbar()
self.ui.setupUi(parent=self, chart=self.chartWidget)
self.ui.create_model(self.ui.node_tree,
self.chart.library.getLabelTree())
self.ui.create_model(self.ui.source_tree,
self.chart.source_library.getLabelTree())
self.chart.sigNodeChanged.connect(self.ui.setPending)
self.features = Features(self.graphCommHandler)
self.ui.actionNew.triggered.connect(self.clear)
self.ui.actionOpen.triggered.connect(self.openClicked)
self.ui.actionSave.triggered.connect(self.saveClicked)
self.ui.actionSaveAs.triggered.connect(self.saveAsClicked)
self.ui.actionConfigure.triggered.connect(self.configureClicked)
self.ui.actionApply.triggered.connect(self.applyClicked)
self.ui.actionReset.triggered.connect(self.resetClicked)
# self.ui.actionProfiler.triggered.connect(self.profilerClicked)
self.ui.actionHome.triggered.connect(self.homeClicked)
self.ui.navGroup.triggered.connect(self.navClicked)
self.chart.sigFileLoaded.connect(self.setCurrentFile)
self.chart.sigFileSaved.connect(self.setCurrentFile)
self.sourceConfigure = SourceConfiguration()
self.sourceConfigure.sigApply.connect(self.configureApply)
self.libraryEditor = EditorTemplate.LibraryEditor(self, chart.library)
self.libraryEditor.sigApplyClicked.connect(self.libraryUpdated)
self.libraryEditor.sigReloadClicked.connect(self.libraryReloaded)
self.ui.libraryConfigure.clicked.connect(self.libraryEditor.show)
self.graph_info = pc.Info('ami_graph', 'AMI Client graph',
['hutch', 'name'])
@asyncSlot()
async def applyClicked(self, build_views=True):
graph_nodes = []
disconnectedNodes = []
displays = set()
msg = QtWidgets.QMessageBox(parent=self)
msg.setText("Failed to submit graph! See status.")
if self.chart.deleted_nodes:
await self.graphCommHandler.remove(self.chart.deleted_nodes)
self.chart.deleted_nodes = []
# detect if the manager has no graph (e.g. from a purge on failure)
if await self.graphCommHandler.graphVersion == 0:
# mark all the nodes as changed to force a resubmit of the whole graph
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
gnode.changed = True
# reset reference counting on views
await self.features.reset()
outputs = [n for n, d in self.chart._graph.out_degree() if d == 0]
changed_nodes = set()
failed_nodes = set()
seen = set()
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
if not gnode.enabled():
continue
if not gnode.hasInput():
disconnectedNodes.append(gnode)
continue
elif gnode.exception:
gnode.clearException()
gnode.recolor()
if gnode.changed and gnode not in changed_nodes:
changed_nodes.add(gnode)
if not hasattr(gnode, 'to_operation'):
if gnode.isSource() and gnode.viewable() and gnode.viewed:
displays.add(gnode)
elif gnode.exportable():
displays.add(gnode)
continue
for output in outputs:
paths = list(
nx.algorithms.all_simple_paths(self.chart._graph, name,
output))
if not paths:
paths = [[output]]
for path in paths:
for gnode in path:
gnode = self.chart._graph.nodes[gnode]
node = gnode['node']
if hasattr(node,
'to_operation') and node not in seen:
try:
nodes = node.to_operation(
inputs=node.input_vars(),
conditions=node.condition_vars())
except Exception:
self.chartWidget.updateStatus(
f"{node.name()} error!", color='red')
printExc(
f"{node.name()} raised exception! See console for stacktrace."
)
node.setException(True)
failed_nodes.add(node)
continue
seen.add(node)
if type(nodes) is list:
graph_nodes.extend(nodes)
else:
graph_nodes.append(nodes)
if (node.viewable()
or node.buffered()) and node.viewed:
displays.add(node)
if disconnectedNodes:
for node in disconnectedNodes:
self.chartWidget.updateStatus(f"{node.name()} disconnected!",
color='red')
node.setException(True)
msg.exec()
return
if failed_nodes:
self.chartWidget.updateStatus("failed to submit graph",
color='red')
msg.exec()
return
if graph_nodes:
await self.graphCommHandler.add(graph_nodes)
node_names = ', '.join(
set(map(lambda node: node.parent, graph_nodes)))
self.chartWidget.updateStatus(f"Submitted {node_names}")
node_names = ', '.join(set(map(lambda node: node.name(), displays)))
if displays and build_views:
self.chartWidget.updateStatus(f"Redisplaying {node_names}")
await self.chartWidget.build_views(displays,
export=True,
redisplay=True)
for node in changed_nodes:
node.changed = False
self.metadata = await self.graphCommHandler.metadata
self.ui.setPendingClear()
version = str(await self.graphCommHandler.graphVersion)
state = self.chart.saveState()
state = json.dumps(state,
indent=2,
separators=(',', ': '),
sort_keys=True,
cls=amitypes.TypeEncoder)
self.graph_info.labels(self.chart.hutch, self.graph_name).info({
'graph':
state,
'version':
version
})
def openClicked(self):
startDir = self.chart.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Load Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.chart.loadFile)
def saveClicked(self):
if self.currentFileName is None:
self.saveAsClicked()
else:
try:
self.chart.saveFile(self.currentFileName)
except Exception as e:
raise e
def saveAsClicked(self):
try:
if self.currentFileName is None:
self.chart.saveFile()
else:
self.chart.saveFile(suggestedFileName=self.currentFileName)
except Exception as e:
raise e
def setCurrentFile(self, fileName):
self.currentFileName = fileName
def homeClicked(self):
children = self.viewBox().allChildren()
self.viewBox().autoRange(items=children)
def navClicked(self, action):
if action == self.ui.actionPan:
self.viewBox().setMouseMode("Pan")
elif action == self.ui.actionSelect:
self.viewBox().setMouseMode("Select")
elif action == self.ui.actionComment:
self.viewBox().setMouseMode("Comment")
@asyncSlot()
async def resetClicked(self):
await self.graphCommHandler.destroy()
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
gnode.changed = True
await self.applyClicked()
def scene(self):
# returns the GraphicsScene object
return self.chartWidget.scene()
def viewBox(self):
return self.chartWidget.viewBox()
def chartWidget(self):
return self.chartWidget
@asyncSlot()
async def clear(self):
await self.graphCommHandler.destroy()
await self.chart.clear()
self.chartWidget.clear()
self.setCurrentFile(None)
self.chart.sigFileLoaded.emit('')
self.features = Features(self.graphCommHandler)
def configureClicked(self):
self.sourceConfigure.show()
@asyncSlot(object)
async def configureApply(self, src_cfg):
missing = []
if 'files' in src_cfg:
for f in src_cfg['files']:
if not os.path.exists(f):
missing.append(f)
if not missing:
await self.graphCommHandler.updateSources(src_cfg)
else:
missing = ' '.join(missing)
self.chartWidget.updateStatus(f"Missing {missing}!", color='red')
@asyncSlot()
async def profilerClicked(self):
await self.chart.broker.send_string("profiler", zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.Profiler(name=self.graph_name, command="show"))
@asyncSlot()
async def libraryUpdated(self):
await self.chart.broker.send_string("library", zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.Library(name=self.graph_name,
paths=self.libraryEditor.paths))
dirs = set(map(os.path.dirname, self.libraryEditor.paths))
await self.graphCommHandler.updatePath(dirs)
self.chartWidget.updateStatus("Loaded modules.")
@asyncSlot(object)
async def libraryReloaded(self, mods):
smods = set(map(lambda mod: mod.__name__, mods))
for name, gnode in self.chart._graph.nodes().items():
node = gnode['node']
if node.__module__ in smods:
await self.chart.broker.send_string(node.name(), zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.ReloadLibrary(name=node.name(), mods=smods))
self.chartWidget.updateStatus(f"Reloaded {node.name()}.")
except NameError:
print('Not launched under iPython')
def make_color_img(img, gain=255, min_max=None,out_type=np.uint8):
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img-min_)/(max_-min_)*gain
img = img.astype(out_type)
img = np.dstack([img]*3)
return img
F = FileDialog()
# load object saved by CNMF
CNMF_filename = F.getOpenFileName(caption='Load CNMF Object',filter='*.*5')[0]
cnm_obj = load_CNMF(CNMF_filename)
# load movie
if not os.path.exists(cnm_obj.mmap_file):
M = FileDialog()
cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file', filter='*.mmap')[0]
mov = cm.load(cnm_obj.mmap_file)
min_mov = np.min(mov)
max_mov = np.max(mov)
# load summary image
class Flowchart(Node):
sigFileLoaded = QtCore.Signal(object)
sigFileSaved = QtCore.Signal(object)
sigNodeCreated = QtCore.Signal(object)
sigNodeChanged = QtCore.Signal(object)
# called when output is expected to have changed
def __init__(self,
name=None,
filePath=None,
library=None,
broker_addr="",
graphmgr_addr="",
checkpoint_addr="",
prometheus_dir=None,
hutch=None):
super().__init__(name)
self.socks = []
self.library = library or LIBRARY
self.graphmgr_addr = graphmgr_addr
self.source_library = None
self.ctx = zmq.asyncio.Context()
self.broker = self.ctx.socket(
zmq.PUB) # used to create new node processes
self.broker.connect(broker_addr)
self.socks.append(self.broker)
self.graphinfo = self.ctx.socket(zmq.SUB)
self.graphinfo.setsockopt_string(zmq.SUBSCRIBE, '')
self.graphinfo.connect(graphmgr_addr.info)
self.socks.append(self.graphinfo)
self.checkpoint = self.ctx.socket(
zmq.SUB) # used to receive ctrlnode updates from processes
self.checkpoint.setsockopt_string(zmq.SUBSCRIBE, '')
self.checkpoint.connect(checkpoint_addr)
self.socks.append(self.checkpoint)
self.filePath = filePath
self._graph = nx.MultiDiGraph()
self.nextZVal = 10
self._widget = None
self._scene = None
self.deleted_nodes = []
self.prometheus_dir = prometheus_dir
self.hutch = hutch
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def close(self):
for sock in self.socks:
sock.close(linger=0)
if self._widget is not None:
self._widget.graphCommHandler.close()
self.ctx.term()
def start_prometheus(self):
port = Ports.Prometheus
while True:
try:
pc.start_http_server(port)
break
except OSError:
port += 1
if self.prometheus_dir:
if not os.path.exists(self.prometheus_dir):
os.makedirs(self.prometheus_dir)
pth = f"drpami_{socket.gethostname()}_client.json"
pth = os.path.join(self.prometheus_dir, pth)
conf = [{"targets": [f"{socket.gethostname()}:{port}"]}]
try:
with open(pth, 'w') as f:
json.dump(conf, f)
except PermissionError:
logging.error("Permission denied: %s", pth)
pass
def setLibrary(self, lib):
self.library = lib
self.widget().chartWidget.buildMenu()
def nodes(self, **kwargs):
return self._graph.nodes(**kwargs)
def createNode(self, nodeType=None, name=None, pos=None):
"""Create a new Node and add it to this flowchart.
"""
if name is None:
n = 0
while True:
name = "%s.%d" % (nodeType, n)
if name not in self._graph.nodes():
break
n += 1
# create an instance of the node
node = self.library.getNodeType(nodeType)(name)
self.addNode(node, pos)
return node
def addNode(self, node, pos=None):
"""Add an existing Node to this flowchart.
See also: createNode()
"""
if pos is None:
pos = [0, 0]
if type(pos) in [QtCore.QPoint, QtCore.QPointF]:
pos = [pos.x(), pos.y()]
item = node.graphicsItem()
item.setZValue(self.nextZVal * 2)
self.nextZVal += 1
self.viewBox.addItem(item)
pos = (find_nearest(pos[0]), find_nearest(pos[1]))
item.moveBy(*pos)
self._graph.add_node(node.name(), node=node)
node.sigClosed.connect(self.nodeClosed)
node.sigTerminalConnected.connect(self.nodeConnected)
node.sigTerminalDisconnected.connect(self.nodeDisconnected)
node.sigNodeEnabled.connect(self.nodeEnabled)
self.sigNodeCreated.emit(node)
if node.isChanged(True, True):
self.sigNodeChanged.emit(node)
@asyncSlot(object, object)
async def nodeClosed(self, node, input_vars):
self._graph.remove_node(node.name())
await self.broker.send_string(node.name(), zmq.SNDMORE)
await self.broker.send_pyobj(fcMsgs.CloseNode())
ctrl = self.widget()
name = node.name()
if hasattr(node, 'to_operation'):
self.deleted_nodes.append(name)
self.sigNodeChanged.emit(node)
elif isinstance(node, SourceNode):
await ctrl.features.discard(name, name)
await ctrl.graphCommHandler.unview(name)
elif node.viewable():
views = []
for term, in_var in input_vars.items():
discarded = await ctrl.features.discard(name, in_var)
if discarded:
views.append(in_var)
if views:
await ctrl.graphCommHandler.unview(views)
def nodeConnected(self, localTerm, remoteTerm):
if remoteTerm.isOutput() or localTerm.isCondition():
t = remoteTerm
remoteTerm = localTerm
localTerm = t
localNode = localTerm.node().name()
remoteNode = remoteTerm.node().name()
key = localNode + '.' + localTerm.name(
) + '->' + remoteNode + '.' + remoteTerm.name()
if not self._graph.has_edge(localNode, remoteNode, key=key):
self._graph.add_edge(localNode,
remoteNode,
key=key,
from_term=localTerm.name(),
to_term=remoteTerm.name())
self.sigNodeChanged.emit(localTerm.node())
def nodeDisconnected(self, localTerm, remoteTerm):
if remoteTerm.isOutput() or localTerm.isCondition():
t = remoteTerm
remoteTerm = localTerm
localTerm = t
localNode = localTerm.node().name()
remoteNode = remoteTerm.node().name()
key = localNode + '.' + localTerm.name(
) + '->' + remoteNode + '.' + remoteTerm.name()
if self._graph.has_edge(localNode, remoteNode, key=key):
self._graph.remove_edge(localNode, remoteNode, key=key)
self.sigNodeChanged.emit(localTerm.node())
@asyncSlot(object)
async def nodeEnabled(self, root):
enabled = root._enabled
outputs = [n for n, d in self._graph.out_degree() if d == 0]
sources_targets = list(it.product([root.name()], outputs))
ctrl = self.widget()
views = []
for s, t in sources_targets:
paths = list(nx.algorithms.all_simple_paths(self._graph, s, t))
for path in paths:
for node in path:
node = self._graph.nodes[node]['node']
name = node.name()
node.nodeEnabled(enabled)
if not enabled:
if hasattr(node, 'to_operation'):
self.deleted_nodes.append(name)
elif node.viewable():
for term, in_var in node.input_vars().items():
discarded = await ctrl.features.discard(
name, in_var)
if discarded:
views.append(in_var)
else:
node.changed = True
if node.conditions():
preds = self._graph.predecessors(node.name())
preds = filter(lambda n: n.startswith("Filter"), preds)
for filt in preds:
node = self._graph.nodes[filt]['node']
node.nodeEnabled(enabled)
if views:
await ctrl.graphCommHandler.unview(views)
await ctrl.applyClicked()
def connectTerminals(self, term1, term2, type_file=None):
"""Connect two terminals together within this flowchart."""
term1.connectTo(term2, type_file=type_file)
def chartGraphicsItem(self):
"""
Return the graphicsItem that displays the internal nodes and
connections of this flowchart.
Note that the similar method `graphicsItem()` is inherited from Node
and returns the *external* graphical representation of this flowchart."""
return self.viewBox
def widget(self):
"""
Return the control widget for this flowchart.
This widget provides GUI access to the parameters for each node and a
graphical representation of the flowchart.
"""
if self._widget is None:
self._widget = FlowchartCtrlWidget(self, self.graphmgr_addr)
self.scene = self._widget.scene()
self.viewBox = self._widget.viewBox()
return self._widget
def saveState(self):
"""
Return a serializable data structure representing the current state of this flowchart.
"""
state = {}
state['nodes'] = []
state['connects'] = []
state['viewbox'] = self.viewBox.saveState()
for name, node in self.nodes(data='node'):
cls = type(node)
clsName = cls.__name__
ns = {'class': clsName, 'name': name, 'state': node.saveState()}
state['nodes'].append(ns)
for from_node, to_node, data in self._graph.edges(data=True):
from_term = data['from_term']
to_term = data['to_term']
state['connects'].append((from_node, from_term, to_node, to_term))
state['source_configuration'] = self.widget(
).sourceConfigure.saveState()
state['library'] = self.widget().libraryEditor.saveState()
return state
def restoreState(self, state):
"""
Restore the state of this flowchart from a previous call to `saveState()`.
"""
self.blockSignals(True)
try:
if 'source_configuration' in state:
src_cfg = state['source_configuration']
self.widget().sourceConfigure.restoreState(src_cfg)
if src_cfg['files']:
self.widget().sourceConfigure.applyClicked()
if 'library' in state:
lib_cfg = state['library']
self.widget().libraryEditor.restoreState(lib_cfg)
self.widget().libraryEditor.applyClicked()
if 'viewbox' in state:
self.viewBox.restoreState(state['viewbox'])
nodes = state['nodes']
nodes.sort(key=lambda a: a['state']['pos'][0])
for n in nodes:
if n['class'] == 'SourceNode':
try:
ttype = eval(n['state']['terminals']['Out']['ttype'])
n['state']['terminals']['Out']['ttype'] = ttype
node = SourceNode(name=n['name'],
terminals=n['state']['terminals'])
self.addNode(node=node)
except Exception:
printExc(
"Error creating node %s: (continuing anyway)" %
n['name'])
else:
try:
node = self.createNode(n['class'], name=n['name'])
except Exception:
printExc(
"Error creating node %s: (continuing anyway)" %
n['name'])
node.restoreState(n['state'])
if hasattr(node, "display"):
node.display(topics=None, terms=None, addr=None, win=None)
if hasattr(node.widget,
'restoreState') and 'widget' in n['state']:
node.widget.restoreState(n['state']['widget'])
connections = {}
with tempfile.NamedTemporaryFile(mode='w') as type_file:
type_file.write("from mypy_extensions import TypedDict\n")
type_file.write("from typing import *\n")
type_file.write("import numbers\n")
type_file.write("import amitypes\n")
type_file.write("T = TypeVar('T')\n\n")
nodes = self.nodes(data='node')
for n1, t1, n2, t2 in state['connects']:
try:
node1 = nodes[n1]
term1 = node1[t1]
node2 = nodes[n2]
term2 = node2[t2]
self.connectTerminals(term1, term2, type_file)
if term1.isInput() or term1.isCondition:
in_name = node1.name() + '_' + term1.name()
in_name = in_name.replace('.', '_')
out_name = node2.name() + '_' + term2.name()
out_name = out_name.replace('.', '_')
else:
in_name = node2.name() + '_' + term2.name()
in_name = in_name.replace('.', '_')
out_name = node1.name() + '_' + term1.name()
out_name = out_name.replace('.', '_')
connections[(in_name, out_name)] = (term1, term2)
except Exception:
print(node1.terminals)
print(node2.terminals)
printExc("Error connecting terminals %s.%s - %s.%s:" %
(n1, t1, n2, t2))
type_file.flush()
status = subprocess.run(
["mypy", "--follow-imports", "silent", type_file.name],
capture_output=True,
text=True)
if status.returncode != 0:
lines = status.stdout.split('\n')[:-1]
for line in lines:
m = re.search(r"\"+(\w+)\"+", line)
if m:
m = m.group().replace('"', '')
for i in connections:
if i[0] == m:
term1, term2 = connections[i]
term1.disconnectFrom(term2)
break
elif i[1] == m:
term1, term2 = connections[i]
term1.disconnectFrom(term2)
break
finally:
self.blockSignals(False)
for name, node in self.nodes(data='node'):
self.sigNodeChanged.emit(node)
@asyncSlot(str)
async def loadFile(self, fileName=None):
"""
Load a flowchart (*.fc) file.
"""
with open(fileName, 'r') as f:
state = json.load(f)
ctrl = self.widget()
await ctrl.clear()
self.restoreState(state)
self.viewBox.autoRange()
self.sigFileLoaded.emit(fileName)
await ctrl.applyClicked(build_views=False)
nodes = []
for name, node in self.nodes(data='node'):
if node.viewed:
nodes.append(node)
await ctrl.chartWidget.build_views(nodes, ctrl=True, export=True)
def saveFile(self,
fileName=None,
startDir=None,
suggestedFileName='flowchart.fc'):
"""
Save this flowchart to a .fc file
"""
if fileName is None:
if startDir is None:
startDir = self.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Save Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
return
if not fileName.endswith('.fc'):
fileName += ".fc"
state = self.saveState()
state = json.dumps(state,
indent=2,
separators=(',', ': '),
sort_keys=True,
cls=amitypes.TypeEncoder)
with open(fileName, 'w') as f:
f.write(state)
f.write('\n')
ctrl = self.widget()
ctrl.graph_info.labels(self.hutch,
ctrl.graph_name).info({'graph': state})
ctrl.chartWidget.updateStatus(f"Saved graph to: {fileName}")
self.sigFileSaved.emit(fileName)
async def clear(self):
"""
Remove all nodes from this flowchart except the original input/output nodes.
"""
for name, gnode in self._graph.nodes().items():
node = gnode['node']
await self.broker.send_string(name, zmq.SNDMORE)
await self.broker.send_pyobj(fcMsgs.CloseNode())
node.close(emit=False)
self._graph = nx.MultiDiGraph()
async def updateState(self):
while True:
node_name = await self.checkpoint.recv_string()
# there is something wrong with this
# in ami.client.flowchart.NodeProcess.send_checkpoint we send a
# fcMsgs.NodeCheckPoint but we are only ever receiving the state
new_node_state = await self.checkpoint.recv_pyobj()
node = self._graph.nodes[node_name]['node']
current_node_state = node.saveState()
restore_ctrl = False
restore_widget = False
if 'ctrl' in new_node_state:
if current_node_state['ctrl'] != new_node_state['ctrl']:
current_node_state['ctrl'] = new_node_state['ctrl']
restore_ctrl = True
if 'widget' in new_node_state:
if current_node_state['widget'] != new_node_state['widget']:
restore_widget = True
current_node_state['widget'] = new_node_state['widget']
if 'geometry' in new_node_state:
node.geometry = QtCore.QByteArray.fromHex(
bytes(new_node_state['geometry'], 'ascii'))
if restore_ctrl or restore_widget:
node.restoreState(current_node_state)
node.changed = node.isChanged(restore_ctrl, restore_widget)
if node.changed:
self.sigNodeChanged.emit(node)
node.viewed = new_node_state['viewed']
async def updateSources(self, init=False):
num_workers = None
while True:
topic = await self.graphinfo.recv_string()
source = await self.graphinfo.recv_string()
msg = await self.graphinfo.recv_pyobj()
if topic == 'sources':
source_library = SourceLibrary()
for source, node_type in msg.items():
pth = []
for part in source.split(':')[:-1]:
if pth:
part = ":".join((pth[-1], part))
pth.append(part)
source_library.addNodeType(source,
amitypes.loads(node_type),
[pth])
self.source_library = source_library
if init:
break
ctrl = self.widget()
tree = ctrl.ui.source_tree
ctrl.ui.clear_model(tree)
ctrl.ui.create_model(ctrl.ui.source_tree,
self.source_library.getLabelTree())
ctrl.chartWidget.updateStatus("Updated sources.")
elif topic == 'event_rate':
if num_workers is None:
ctrl = self.widget()
compiler_args = await ctrl.graphCommHandler.compilerArgs
num_workers = compiler_args['num_workers']
events_per_second = [None] * num_workers
total_events = [None] * num_workers
time_per_event = msg[ctrl.graph_name]
worker = int(re.search(r'(\d)+', source).group())
events_per_second[worker] = len(time_per_event) / (
time_per_event[-1][1] - time_per_event[0][0])
total_events[worker] = msg['num_events']
if all(events_per_second):
events_per_second = int(np.sum(events_per_second))
total_num_events = int(np.sum(total_events))
ctrl = self.widget()
ctrl.ui.rateLbl.setText(
f"Num Events: {total_num_events} Events/Sec: {events_per_second}"
)
events_per_second = [None] * num_workers
total_events = [None] * num_workers
elif topic == 'error':
ctrl = self.widget()
if hasattr(msg, 'node_name'):
node_name = ctrl.metadata[msg.node_name]['parent']
node = self.nodes(data='node')[node_name]
node.setException(msg)
ctrl.chartWidget.updateStatus(
f"{source} {node.name()}: {msg}", color='red')
else:
ctrl.chartWidget.updateStatus(f"{source}: {msg}",
color='red')
async def run(self):
await asyncio.gather(self.updateState(), self.updateSources())
def main():
# Prompt user for directory containing files to be analyzed
F = FileDialog() # Calls Qt backend script to create a file dialog object
mcdir = F.getExistingDirectory(caption='Select Motion Corrected Video Directory')
fvids=[]
for file in os.listdir(mcdir):
if file.endswith("_mc.tif"):
fvids.append(os.path.join(mcdir, file))
# Set up a variable to determine which sections are lightsheet and which
# are epi. This is a horrible way to handle it - need to write new code to
# either automatically determine or prompt user for input.
# Use 1 for lightsheet and 0 for epi.
lsepi = [1, 0, 1, 0, 1, 0, 0, 1, 0]
# Grab the first two masks and assume that they are representative of every
# lightsheet and epi neuron. Also generate an overlap mask by logical ANDing
# the two masks together.
maskLSin = np.transpose(np.load(fvids[0]+'.npy'), axes=(2,1,0))
maskEpiin = np.transpose(np.load(fvids[1]+'.npy'), axes=(2,1,0))
maskLSt = []
maskEpit = []
for mask in maskLSin:
if (np.argwhere(mask).size > 0):
maskLSt.append(mask)
for mask in maskEpiin:
if (np.argwhere(mask).size > 0):
maskEpit.append(mask)
maskLS = np.asarray(maskLSt)
maskEpi = np.asarray(maskEpit)
# Take the first lightsheet vid, find the top N neurons, do the same for
# the first epi vid. Take the 2N masks corresponding to this, find the set
# of unique ones, and then use the remaining masks to go through all of the
# other vids.
N = 14
# Lightsheet:
vid = cm.load(fvids[0])
LSdff = calculate_dff_set(vid, maskLS)
# Epi:
vid = cm.load(fvids[1])
EPdff = calculate_dff_set(vid, maskEpi)
# Sort by top, get the overlap:
threshold = 10
topsLS = argsort_traces(LSdff)
topsEpi = argsort_traces(EPdff)
masks = mask_union(maskLS[topsLS[-N:]], maskEpi[topsEpi[-N:]], threshold)
masksTopLS = mask_disjoint(maskLS[topsLS[-N:]], masks, threshold)
masksTopEpi = mask_disjoint(maskEpi[topsEpi[-N:]], masks, threshold)
maskov = mask_joint(maskLS[topsLS[-N:]], maskEpi[topsEpi[-N:]], threshold)
print(masksTopLS.shape)
print(masksTopEpi.shape)
print(maskov.shape)
# The variable tops now contains the non-overlapping union of the top N
# neurons from epi and from light sheet. Now run through the rest of the
# analyses using only these masks.
# Can grab the top epi and top lightsheet values for each neuron. Can also
# grab on a neuron-by-neuron basis whether the peak dF/F was lightsheet or
# epi.
dff = np.empty((masks.shape[0], 0))
divs = np.zeros(len(fvids))
max_idx = np.zeros((masks.shape[0], 1))
max_val = np.zeros((masks.shape[0], 1))
maxepi_idx = np.zeros((masks.shape[0], 1))
maxepi_val = np.zeros((masks.shape[0], 1))
maxls_idx = np.zeros((masks.shape[0], 1))
maxls_val = np.zeros((masks.shape[0], 1))
flatmask = flatten_masks(masks)
lspeaks = [[] for k in range(masks.shape[0])]
lspeakvals = np.empty((0))
epipeaks = [[] for k in range(masks.shape[0])]
epipeakvals = np.empty((0))
rawtraces = np.empty((masks.shape[0], 0))
rawbckgnd = np.empty((masks.shape[0], 0))
for i, fvid in enumerate(fvids):
vid = cm.load(fvid)
traces = np.empty((masks.shape[0], vid.shape[0]))
bval = np.empty((masks.shape[0], vid.shape[0]))
dff_i = np.empty((masks.shape[0], vid.shape[0]))
for j, mask in enumerate(masks):
traces[j], bval[j] = extract_neuron_trace_uniform(vid, mask, flatmask, 2)
dff_i[j] = trace_df_f(traces[j], bval[j])
peaks, props = scipy.signal.find_peaks(dff_i[j], distance=10, prominence=(0.1, None))
if (peaks.size > 0):
if lsepi[i]:
lspeaks[j].append(peaks)
lspeakvals = np.append(lspeakvals, dff_i[j][peaks])
else:
epipeaks[j].append(peaks)
epipeakvals = np.append(epipeakvals, dff_i[j][peaks])
if (max(dff_i[j]) > max_val[j]):
max_idx[j] = i
max_val[j] = max(dff_i[j])
if lsepi[i]:
maxls_idx[j] = i
maxls_val[j] = max_val[j]
else:
maxepi_idx[j] = i
maxepi_val[j] = max_val[j]
dff = np.concatenate([dff, dff_i], axis = 1)
rawtraces = np.concatenate([rawtraces, traces], axis = 1)
rawbckgnd = np.concatenate([rawbckgnd, bval], axis = 1)
divs[i] = dff.shape[1]
# Save generated values for post-post-processing
masksout = np.transpose(masks, axes=(2,1,0))
np.save(os.path.join(mcdir, 'top_masks_out.npy'), masksout)
np.save(os.path.join(mcdir, 'top_epi_sections.npy'), maxepi_idx)
np.save(os.path.join(mcdir, 'top_epi_values.npy'), maxepi_val)
np.save(os.path.join(mcdir, 'top_ls_sections.npy'), maxls_idx)
np.save(os.path.join(mcdir, 'top_ls_values.npy'), maxls_val)
np.save(os.path.join(mcdir, 'ls_peak_vals.npy'), lspeakvals)
np.save(os.path.join(mcdir, 'epi_peak_vals.npy'), epipeakvals)
np.save(os.path.join(mcdir, 'dff_traces.npy'), dff)
np.save(os.path.join(mcdir, 'dff_div_points.npy'), divs)
# Plot out the dF/F traces, and put vertical markers at the dividers
# between video segments. User would have to manually label them as there's
# no real way to determine what segments are what.
nrow = 6
ncol = 1
dffig, ax = plt.subplots(nrow, ncol, sharex = True, sharey = True)
ll = dff[0].shape[0]
axrange = np.linspace(0, (ll-1)/10, num=ll)
for i, dfft in enumerate(dff):
if i == nrow:
ax[i-1].set_xlabel('Time (seconds)')
break
ax[i].plot(axrange, dfft)
ax[i].set_ylabel(str(i+1))
for div in divs:
ax[i].axvline(div/10, color='r', linestyle='--')
dffig.suptitle('Neuron dF/F Curves')
plt.show()
tfig, ax = plt.subplots(nrow, ncol, sharex = True, sharey = True)
ll = rawtraces[0].shape[0]
axrange = np.linspace(0, (ll-1)/10, num=ll)
for i, tr in enumerate(rawtraces):
if i >= nrow:
ax[i-1].set_xlabel('Time (seconds)')
break
ax[i].plot(axrange, np.add(tr, rawbckgnd[i]))
ax[i].plot(axrange, rawbckgnd[i])
ax[i].set_ylabel(str(i+1))
for div in divs:
ax[i].axvline(div/10, color='r', linestyle='--')
tfig.suptitle('Neuron Raw Traces + Background')
plt.show()
"""
# Next do line plot with averages + error bars.
# Set up lines for a given neuron, showing increase or decrease of max
# intensity on that neuron between lightsheet and epi.
#
intensity_lineplot = np.concatenate([maxepi_val, maxls_val], axis=1).T
avg_ls = np.mean(maxls_val)
std_ls = np.std(maxls_val)/math.sqrt(maxls_val.shape[0])
avg_epi = np.mean(maxepi_val)
std_epi = np.std(maxepi_val)/math.sqrt(maxepi_val.shape[0])
binplot, ax = plt.subplots()
plt.plot(intensity_lineplot)
ax.bar([0, 1], [avg_epi, avg_ls], yerr=[std_epi, std_ls], align='center', capsize=10, alpha=0.5)
ax.set_ylabel('Peak dF/F')
ax.set_xticks([0, 1])
ax.set_xticklabels(['Epi', 'Light-sheet'])
ax.set_title('Contrast change, Epi vs. LS')
plt.show()
"""
# Histogram of spike intensities.
histfig, ax = plt.subplots()
if not (lspeakvals.size>0):
lspeakvals = np.zeros(1)
if not (epipeakvals.size>0):
epipeakvals = np.zeros(1)
binrange = np.amax(np.concatenate([lspeakvals, epipeakvals]))
binrange = 1.5 if binrange >1.5 else math.ceil(binrange*10)/10
binset = np.linspace(0, binrange, num=int(binrange*10+1))
nls = lspeakvals.shape[0]
nepi = epipeakvals.shape[0]
epi_n, epi_bins, epi_patches = ax.hist(epipeakvals, bins=binset, alpha=0.5, label='Epi-illumination', histtype='barstacked', ec='black', lw=0, color='#7f86c1')
ls_n, ls_bins, ls_patches = ax.hist(lspeakvals, bins=binset, alpha=0.5, label='Light-sheet', histtype='barstacked', ec='black', lw=0, color='#f48466')
plt.legend(loc='upper right')
histfig.suptitle('Lightsheet vs. Epi-illumination dF/F')
plt.xlabel('dF/F')
plt.ylabel('Spike Count')
plt.show()
# Plot the image with the contours (outlines of neurons), labeled
Asparse = scipy.sparse.csc_matrix(masksout.reshape((masksout.shape[1]*masksout.shape[0], masksout.shape[2])))
lstop = np.transpose(masksTopLS, axes=(2,1,0))
epitop = np.transpose(masksTopEpi, axes=(2,1,0))
ovtop = np.transpose(maskov, axes=(2,1,0))
AsparseLS = scipy.sparse.csc_matrix(lstop.reshape((lstop.shape[1]*lstop.shape[0], lstop.shape[2])))
AsparseEpi = scipy.sparse.csc_matrix(epitop.reshape((epitop.shape[1]*epitop.shape[0], epitop.shape[2])))
AsparseOv = scipy.sparse.csc_matrix(ovtop.reshape((ovtop.shape[1]*ovtop.shape[0], ovtop.shape[2])))
vid = cm.load(fvids[0])
#Cn = cm.local_correlations(vid.transpose(1,2,0))
Cn = np.zeros((vid.shape[1], vid.shape[2]))
Cn[np.isnan(Cn)] = 0
out=plt.figure()
cm.utils.visualization.plot_contours(Asparse, Cn)
out=plt.figure()
cm.utils.visualization.plot_contours(AsparseLS, Cn)
out=plt.figure()
cm.utils.visualization.plot_contours(AsparseEpi, Cn)
out=plt.figure()
cm.utils.visualization.plot_contours(AsparseOv, Cn)
scipy.io.savemat(os.path.join(mcdir, 'epi_histogram.mat'), {'n':epi_n, 'bins':epi_bins, 'patches':epi_patches})
scipy.io.savemat(os.path.join(mcdir, 'ls_histogram.mat'), {'n':ls_n, 'bins':ls_bins, 'patches':ls_patches})
scipy.io.savemat(os.path.join(mcdir, 'epi_spike_values.mat'), {'epispikes':epipeakvals})
scipy.io.savemat(os.path.join(mcdir, 'ls_spike_values.mat'), {'lsspikes':lspeakvals})
scipy.io.savemat(os.path.join(mcdir, 'df_over_f.mat'), {'data':dff, 'indices_between_ls_or_epi':divs})
scipy.io.savemat(os.path.join(mcdir, 'rawtraces.mat'), {'data':rawtraces, 'indices_between_ls_or_epi':divs})
scipy.io.savemat(os.path.join(mcdir, 'rawbackground.mat'), {'data':rawbckgnd, 'indices_between_ls_or_epi':divs})
class LibraryEditor(QtWidgets.QWidget):
sigApplyClicked = QtCore.Signal()
sigReloadClicked = QtCore.Signal(object)
def __init__(self, ctrlWidget, library):
super().__init__()
self.setWindowTitle("Manage Library")
self.modules = {} # {mod : [nodes]}
self.paths = set()
self.ctrl = ctrlWidget
self.library = library
self.layout = QtWidgets.QGridLayout(self)
self.loadBtn = QtWidgets.QPushButton("Load Modules", parent=self)
self.loadBtn.clicked.connect(self.loadFile)
# self.reloadBtn = QtWidgets.QPushButton("Reload Selected Modules", parent=self)
# self.reloadBtn.clicked.connect(self.reloadFile)
self.tree = QtWidgets.QTreeWidget(parent=self)
self.tree.setHeaderHidden(True)
self.applyBtn = QtWidgets.QPushButton("Apply", parent=self)
self.applyBtn.clicked.connect(self.applyClicked)
self.layout.addWidget(self.loadBtn, 1, 1, 1, -1)
# self.layout.addWidget(self.reloadBtn, 1, 2, 1, 1)
self.layout.addWidget(self.tree, 2, 1, 1, -1)
self.layout.addWidget(self.applyBtn, 3, 1, 1, -1)
def loadFile(self):
file_filters = "*.py"
self.fileDialog = FileDialog(None, "Load Nodes", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def fileDialogFilesSelected(self, pths):
dirs = set(map(os.path.dirname, pths))
for pth in dirs:
if pth not in sys.path:
sys.path.append(pth)
self.paths.update(pths)
for mod in pths:
mod = os.path.basename(mod)
mod = os.path.splitext(mod)[0]
mod = importlib.import_module(mod)
if mod in self.modules:
continue
nodes = [getattr(mod, name) for name in dir(mod) if isNodeClass(getattr(mod, name))]
if not nodes:
continue
self.modules[mod] = nodes
parent = QtWidgets.QTreeWidgetItem(self.tree, [mod.__name__])
parent.mod = mod
for node in nodes:
child = QtWidgets.QTreeWidgetItem(parent, [node.__name__])
child.mod = mod
self.tree.expandAll()
def reloadFile(self):
mods = set()
for item in self.tree.selectedItems():
mods.add(item.mod)
for mod in mods:
pg.reload.reload(mod)
self.sigReloadClicked.emit(mods)
def applyClicked(self):
loaded = False
for mod, nodes in self.modules.items():
for node in nodes:
try:
self.library.addNodeType(node, [(mod.__name__, )])
loaded = True
except Exception as e:
printExc(e)
if not loaded:
return
self.ctrl.ui.clear_model(self.ctrl.ui.node_tree)
self.ctrl.ui.create_model(self.ctrl.ui.node_tree, self.library.getLabelTree(rebuild=True))
self.sigApplyClicked.emit()
def saveState(self):
return {'paths': list(self.paths)}
def restoreState(self, state):
self.fileDialogFilesSelected(state['paths'])
def make_color_img(img, gain=255, min_max=None, out_type=np.uint8):
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img - min_) / (max_ - min_) * gain
img = img.astype(out_type)
img = np.dstack([img] * 3)
return img
F = FileDialog()
# load object saved by CNMF
fpath = F.getOpenFileName(caption='Load CNMF Object',
filter='HDF5 (*.h5 *.hdf5);;NWB (*.nwb)')[0]
cnm_obj = load_CNMF(fpath)
# movie
if not os.path.exists(cnm_obj.mmap_file):
M = FileDialog()
cnm_obj.mmap_file = M.getOpenFileName(caption='Load memory mapped file',
filter='*.mmap')[0]
if fpath[-3:] == 'nwb':
mov = cm.load(cnm_obj.mmap_file,
var_name_hdf5='acquisition/TwoPhotonSeries')
def test():
def make_color_img(img, gain=255, min_max=None,out_type=np.uint8):
if min_max is None:
min_ = img.min()
max_ = img.max()
else:
min_, max_ = min_max
img = (img-min_)/(max_-min_)*gain
img = img.astype(out_type)
img = np.dstack([img]*3)
return img
data=None
path=r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M18\20191121b\N4'
### FIND DATA ###
if data is None: # different conditions on file loading (not necessary if data was already provided)
if path is None: # if no path has been given, open a window prompt to select a directory
F = FileDialog()
# load object saved by CNMF
path = F.getExistingDirectory(caption='Select folder from which to load a PCF or CNMF file')
try: # first try to get CNMF data from a PCF object (should be most up-to-date)
cnm_obj = pipe.load_pcf(path).cnmf
except FileNotFoundError:
try:
cnm_obj = pipe.load_cnmf(path)
except FileNotFoundError:
raise FileNotFoundError(f'Could not find data to load in {path}!')
else:
cnm_obj = data
def draw_contours_overall(md):
if md is "reset":
draw_contours()
elif md is "neurons":
if neuron_selected is True:
#if a specific neuron has been selected, only one contour should be changed while thrshcomp_line is changing
if nr_index is 0:
#if user does not start to move through the frames
draw_contours_update(estimates.background_image, img)
draw_contours_update(comp2_scaled, img2)
else:
# NEVER CALLED IN THIS VERSION WITHOUT MMAP SINCE NR_INDEX NEVER CHANGES (NO NR_VLINE)
draw_contours_update(raw_mov_scaled, img)
draw_contours_update(frame_denoise_scaled, img2)
else:
#if no specific neuron has been selected, all the contours are changing
draw_contours()
else:
#md is "background":
return
def draw_contours():
global thrshcomp_line, estimates, img
bkgr_contours = estimates.background_image.copy()
if len(estimates.idx_components) > 0:
contours = [cv2.findContours(cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1], cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)[0] for img in estimates.img_components[estimates.idx_components]]
SNRs = np.array(estimates.r_values)
iidd = np.array(estimates.idx_components)
idx1 = np.where(SNRs[iidd] < 0.1)[0]
idx2 = np.where((SNRs[iidd] >= 0.1) &
(SNRs[iidd] < 0.25))[0]
idx3 = np.where((SNRs[iidd] >= 0.25) &
(SNRs[iidd] < 0.5))[0]
idx4 = np.where((SNRs[iidd] >= 0.5) &
(SNRs[iidd] < 0.75))[0]
idx5 = np.where((SNRs[iidd] >= 0.75) &
(SNRs[iidd] < 0.9))[0]
idx6 = np.where(SNRs[iidd] >= 0.9)[0]
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx1], []), -1, (255, 0, 0), 1)
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx2], []), -1, (0, 255, 0), 1)
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx3], []), -1, (0, 0, 255), 1)
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx4], []), -1, (255, 255, 0), 1)
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx5], []), -1, (255, 0, 255), 1)
cv2.drawContours(bkgr_contours, sum([contours[jj] for jj in idx6], []), -1, (0, 255, 255), 1)
img.setImage(bkgr_contours, autoLevels=False)
# pg.setConfigOptions(imageAxisOrder='row-major')
def draw_contours_update(cf, im):
global thrshcomp_line, estimates
curFrame = cf.copy()
if len(estimates.idx_components) > 0:
contours = [cv2.findContours(cv2.threshold(img, np.int(thrshcomp_line.value()), 255, 0)[1], cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)[0] for img in estimates.img_components[estimates.idx_components]]
SNRs = np.array(estimates.r_values)
iidd = np.array(estimates.idx_components)
idx1 = np.where(SNRs[iidd] < 0.1)[0]
idx2 = np.where((SNRs[iidd] >= 0.1) &
(SNRs[iidd] < 0.25))[0]
idx3 = np.where((SNRs[iidd] >= 0.25) &
(SNRs[iidd] < 0.5))[0]
idx4 = np.where((SNRs[iidd] >= 0.5) &
(SNRs[iidd] < 0.75))[0]
idx5 = np.where((SNRs[iidd] >= 0.75) &
(SNRs[iidd] < 0.9))[0]
idx6 = np.where(SNRs[iidd] >= 0.9)[0]
if min_dist_comp in idx1:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 0, 0), 1)
if min_dist_comp in idx2:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 255, 0), 1)
if min_dist_comp in idx3:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 0, 255), 1)
if min_dist_comp in idx4:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 255, 0), 1)
if min_dist_comp in idx5:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (255, 0, 255), 1)
if min_dist_comp in idx6:
cv2.drawContours(curFrame, contours[min_dist_comp], -1, (0, 255, 255), 1)
im.setImage(curFrame, autoLevels=False)
# Always start by initializing Qt (only once per application)
app = QtGui.QApplication([])
try:
cv2.setNumThreads(1)
except:
print('Open CV is naturally single threaded')
## Define a top-level widget to hold everything
w = QtGui.QWidget()
## Create some widgets to be placed inside
btn = QtGui.QPushButton('press me')
text = QtGui.QLineEdit('enter text')
win = pg.GraphicsLayoutWidget()
win.setMaximumWidth(300)
win.setMinimumWidth(200)
hist = pg.HistogramLUTItem() # Contrast/color control
win.addItem(hist)
p1 = pg.PlotWidget()
p2 = pg.PlotWidget()
p3 = pg.PlotWidget()
t = ParameterTree()
t_action = ParameterTree()
action_layout = QtGui.QGridLayout()
## Create a grid layout to manage the widgets size and position
layout = QtGui.QGridLayout()
w.setLayout(layout)
# A plot area (ViewBox + axes) for displaying the image
#p1 = win.addPlot(title="Image here")
# Item for displaying image data
img = pg.ImageItem()
p1.addItem(img)
img2 = pg.ImageItem()
p3.addItem(img2)
hist.setImageItem(img)
# Draggable line for setting isocurve level (for setting contour threshold)
thrshcomp_line = pg.InfiniteLine(angle=0, movable=True, pen='g')
hist.vb.addItem(thrshcomp_line)
hist.vb.setMouseEnabled(y=False) # makes user interaction a little easier
thrshcomp_line.setValue(100)
thrshcomp_line.setZValue(1000) # bring iso line above contrast controls
## Add widgets to the layout in their proper positions
layout.addWidget(win, 1, 0) # histogram
layout.addWidget(p3, 0, 2) # denoised movie
layout.addWidget(t, 0, 0) # upper-right table
layout.addWidget(t_action, 1, 2) # bottom-right table
layout.addWidget(p1, 0, 1) # raw movie
layout.addWidget(p2, 1, 1) # calcium trace window
#enable only horizontal zoom for the traces component
p2.setMouseEnabled(x=True, y=False)
# draw something in the raw-movie field and set the histogram borders correspondingly
test_img_file = r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M18\20191121b\N4\local_correlation_image.png'
test_img = plt.imread(test_img_file)
img.setImage(np.rot90(test_img[:, :, 0], 3))
hist.setLevels(test_img[:, :, 0].min(), test_img[:, :, 0].max())
p2.setMouseEnabled(x=True, y=False)
# Another plot area for displaying ROI data
p2.setMaximumHeight(250)
# set position and scale of image
img.scale(1, 1)
# zoom to fit image
p1.autoRange()
mode = "reset"
p2.setTitle("mode: %s" % (mode))
## Display the widget as a new window
w.show()
## Start the Qt event loop
app.exec_()
