def drift_correct(pipeline): global id_filter import numpy as np import matplotlib.pyplot as plt #pipeline=visgui.pipeline filters = {'error_x': [0, 10]} if 'sig' in pipeline.keys(): dialog = wx.TextEntryDialog(None, 'Diameter (nm): ', 'Enter Fiducial Size', str(pipeline.mdh.getOrDefault('Analysis.FiducialSize', 1000))) sig = [330., 370.] if dialog.ShowModal() == wx.ID_OK: size = float(dialog.GetValue()) sigE = float(np.sqrt((size/(np.sqrt(2)*2.35))**2 + 135.**2)) # Expected std of the bead + expected std of psf sig = [0.95*sigE, 1.05*sigE] filters['sig'] = sig if 'fitError_z0' in pipeline.keys(): filters['fitError_z0'] = [0,30] recipe = pipeline.recipe filt_fiducials = FilterTable(recipe, inputName='Fiducials', outputName='filtered_fiducials', filters=filters) filt_fiducials.configure_traits(kind='modal') #print('Adding fiducial filter module') #recipe.add_module(filt_fiducials) dbs = DBSCANClustering(recipe,inputName='filtered_fiducials', outputName='clumped_fiducials', columns=['x', 'y'], searchRadius=500, minClumpSize=10, clumpColumnName='fiducialID') recipe.add_modules_and_execute([filt_fiducials, dbs]) fids = recipe.namespace['clumped_fiducials'] fid_ids = [int(id) for id in set(fids['fiducialID']) if id > 0] id_filter = FilterTableByIDs(recipe, inputName='clumped_fiducials', outputName='selected_fiducials', idColumnName='fiducialID', ids = fid_ids) #recipe.add_module(id_filter) fc = FiducialCorrection(recipe, inputLocalizations=pipeline.selectedDataSourceKey, inputFiducials='selected_fiducials', outputName='corrected_localizations', outputFiducials='corrected_fiducials') recipe.add_modules_and_execute([id_filter, fc]) pipeline.selectDataSource('corrected_localizations')
def OnFold(self, event=None): """ See multiview.foldX. At this point the origin of x should be the corner of the concatenated frame. Note that a probe key will be mapped into the data source to designate colour channel, but the colour channel will not be selectable in the GUI until after mergeClumps is called (due to the possibility of using this module with ratio-metric data). Parameters ---------- None, but requires metadata. Notes ----- """ from PYME.recipes.multiview import Fold from PYME.recipes.tablefilters import FilterTable recipe = self.pipeline.recipe #TODO - move me to building the pipeline recipe.add_module( FilterTable(recipe, inputName=self.pipeline.selectedDataSourceKey, outputName='filtered_input', filters={ 'error_x': [0, 30.], 'error_y': [0, 30.] })) recipe.add_module( Fold(recipe, input_name='filtered_input', output_name='folded')) recipe.execute() self.pipeline.selectDataSource('folded')
def OnSelROIFT(self, event): try: #old glcanvas x0, y0 = self.visFr.glCanvas.selectionStart x1, y1 = self.visFr.glCanvas.selectionFinish except AttributeError: #new glcanvas x0, y0 = self.visFr.glCanvas.selectionSettings.start x1, y1 = self.visFr.glCanvas.selectionSettings.finish filters = {} filters['x'] = [float(min(x0, x1)), float(max(x0, x1))] # must ensure all values are eventually scalars to avoid issue with recipe yaml output filters['y'] = [float(min(y0, y1)), float(max(y0, y1))] # ditto recipe = self.visFr.pipeline.recipe ftable = FilterTable(recipe, inputName=self.visFr.pipeline.selectedDataSourceKey, outputName='selectedROI', filters=filters) if not ftable.configure_traits(kind='modal'): return recipe.add_module(ftable) recipe.execute()
def OnTrackMolecules(self, event=None): import PYME.Analysis.points.DeClump.deClumpGUI as deClumpGUI #import PYME.Analysis.points.DeClump.deClump as deClump import PYME.Analysis.Tracking.trackUtils as trackUtils from PYME.LMVis.layers.tracks import TrackRenderLayer from PYME.recipes import tracking from PYME.recipes.tablefilters import FilterTable recipe = self.visFr.pipeline.recipe visFr = self.visFr pipeline = visFr.pipeline # type: PYME.LMVis.pipeline.Pipeline if hasattr(self, '_mol_tracking_module') and (self._mol_tracking_module in recipe.modules): # We have already tracked, edit existing tracking module instead wx.MessageBox('This dataset has already been tracked, edit parameters of existing tracking rather than starting again', 'Error', wx.OK|wx.ICON_ERROR, visFr) self._mol_tracking_module.configure_traits(kind='modal') return output_name = 'with_tracks' if output_name in recipe.namespace: # this should take care of, e.g. having tracked with feature based tracking or something in the recipe output_name = pipeline.new_ds_name('with_tracks') wx.MessageBox("Another module has already created a 'with_tracks' output, using the nonstandard name '%s' instead" % output_name, 'Warning', wx.OK|wx.ICON_WARNING, visFr) tracking_module = tracking.FindClumps(recipe, inputName=pipeline.selectedDataSourceKey, outputName=output_name, timeWindow=5, clumpRadiusVariable='1.0', clumpRadiusScale=250., minClumpSize=50) if tracking_module.configure_traits(kind='modal'): self._mol_tracking_module = tracking_module recipe.add_modules_and_execute([tracking_module, # Add dynamic filtering on track length, etc. FilterTable(recipe, inputName=tracking_module.outputName, outputName='filtered_{}'.format(tracking_module.outputName), filters={'clumpSize':[tracking_module.minClumpSize, 1e6]})]) self.visFr.pipeline.selectDataSource('filtered_{}'.format(tracking_module.outputName)) #self.visFr.CreateFoldPanel() #TODO: can we capture this some other way? layer = TrackRenderLayer(pipeline, dsname='filtered_{}'.format(tracking_module.outputName), method='tracks') visFr.add_layer(layer)
def OpenFile(self, filename='', ds=None, clobber_recipe=True, **kwargs): """Open a file - accepts optional keyword arguments for use with files saved as .txt and .mat. These are: FieldNames: a list of names for the fields in the text file or matlab variable. VarName: the name of the variable in the .mat file which contains the data. SkipRows: Number of header rows to skip for txt file data PixelSize: Pixel size if not in nm """ #close any files we had open previously while len(self.filesToClose) > 0: self.filesToClose.pop().close() # clear our state # nb - equivalent to clearing recipe namespace self.dataSources.clear() if clobber_recipe: # clear any processing modules from the pipeline # call with clobber_recipe = False in a 'Open a new file with the processing pipeline I've set up' use case # TODO: Add an "File-->Open [preserving recipe]" menu option or similar self.recipe.modules = [] if 'zm' in dir(self): del self.zm self.filter = None self.mapping = None self.colourFilter = None self.events = None self.mdh = MetaDataHandler.NestedClassMDHandler() self.filename = filename if ds is None: from PYME.IO import unifiedIO # TODO - what is the launch time penalty here for importing clusterUI and finding a nameserver? # load from file(/cluster, downloading a copy of the file if needed) with unifiedIO.local_or_temp_filename(filename) as fn: # TODO - check that loading isn't lazy (i.e. we need to make a copy of data in memory whilst in the # context manager in order to be safe with unifiedIO and cluster data). From a quick look, it would seem # that _ds_from_file() copies the data, but potentially keeps the file open which could be problematic. # This won't effect local file loading even if loading is lazy (i.e. shouldn't cause a regression) ds = self._ds_from_file(fn, **kwargs) self.events = getattr(ds, 'events', None) self.mdh.copyEntriesFrom(ds.mdh) # skip the MappingFilter wrapping, etc. in self.addDataSource and add this datasource as-is self.dataSources['FitResults'] = ds # Fit module specific filter settings # TODO - put all the defaults here and use a local variable rather than in __init__ (self.filterKeys is largely an artifact of pre-recipe based pipeline) if 'Analysis.FitModule' in self.mdh.getEntryNames(): fitModule = self.mdh['Analysis.FitModule'] if 'Interp' in fitModule: self.filterKeys['A'] = (5, 100000) if fitModule == 'SplitterShiftEstFR': self.filterKeys['fitError_dx'] = (0, 10) self.filterKeys['fitError_dy'] = (0, 10) if clobber_recipe: from PYME.recipes.localisations import ProcessColour, Pipelineify from PYME.recipes.tablefilters import FilterTable add_pipeline_variables = Pipelineify( self.recipe, inputFitResults='FitResults', pixelSizeNM=kwargs.get('PixelSize', 1.), outputLocalizations='Localizations') self.recipe.add_module(add_pipeline_variables) #self._get_dye_ratios_from_metadata() colour_mapper = ProcessColour(self.recipe, input='Localizations', output='colour_mapped') self.recipe.add_module(colour_mapper) self.recipe.add_module( FilterTable(self.recipe, inputName='colour_mapped', outputName='filtered_localizations', filters={ k: list(v) for k, v in self.filterKeys.items() if k in ds.keys() })) else: logger.warn( 'Opening file without clobbering recipe, filter and ratiometric colour settings might not be handled properly' ) # FIXME - should we update filter keys and/or make the filter more robust # FIXME - do we need to do anything about colour settings? self.recipe.execute() self.filterKeys = {} if 'filtered_localizations' in self.dataSources.keys(): self.selectDataSource( 'filtered_localizations') #NB - this rebuilds the pipeline else: # TODO - replace / remove this fallback with something better. This is currently required # when we use/abuse the pipeline in dh5view, but that should ideally be replaced with # something cleaner. This (and case above) should probably also be conditional on `clobber_recipe` # as if opening with an existing recipe we would likely want to keep selectedDataSource constant as well. self.selectDataSource('FitResults') # FIXME - we do this already in pipelinify, maybe we can avoid doubling up? self.ev_mappings, self.eventCharts = _processEvents( ds, self.events, self.mdh) # extract information from any events # Retrieve or estimate image bounds if False: # 'imgBounds' in kwargs.keys(): # TODO - why is this disabled? Current usage would appear to be when opening from LMAnalysis # during real-time localization, to force image bounds to match raw data, but also potentially useful # for other scenarios where metadata is not fully present. self.imageBounds = kwargs['imgBounds'] elif ('scanx' not in self.selectedDataSource.keys() or 'scany' not in self.selectedDataSource.keys() ) and 'Camera.ROIWidth' in self.mdh.getEntryNames(): self.imageBounds = ImageBounds.extractFromMetadata(self.mdh) else: self.imageBounds = ImageBounds.estimateFromSource( self.selectedDataSource)
def dye_kinetics(self, event): import sys from time import time t0 = time() self.pipeline = self.visFr.pipeline recipe = self.visFr.pipeline.recipe script_init = kin_rcps.DyeKineticsInit( recipe, inputName=self.pipeline.selectedDataSourceKey, outputName='localisations', clusteringRadius=40.0, blinkRadius=20.0, clusterColumnName='clusterID', fitHistograms=True, blinkGapTolerance=0, onTimesColumnName='on_times', offTimesColumnName='off_times', minimumClusterSize=2, minimumKeptBlinkSize=2, minimumOffTimeInSecondsFit=0, maximumOffTimeInSecondsToFit=10000.0, offTimeFitBinWidthInSeconds=1) if script_init.configure_traits(kind='modal'): recipe.trait_set(execute_on_invalidation=False) recipe.add_module(script_init) recipe.add_module( kin_rcps.FindClumps_DK( recipe, inputName='localisations', outputName='with_clumps', timeWindow=script_init.blinkGapTolerance, clumpRadiusVariable=u'1.0', clumpRadiusScale=script_init.blinkRadius, minClumpSize=script_init.minimumKeptBlinkSize)) minblink = script_init.minimumKeptBlinkSize minblink -= 1 recipe.add_module( FilterTable(recipe, inputName='with_clumps', outputName='good_clumps', filters={'clumpSize': [minblink, sys.maxsize]})) recipe.add_module( kin_rcps.MergeClumpsDyeKinetics(recipe, inputName='good_clumps', outputName='coalesced', lebelKey='clumpIndex')) recipe.add_module( FilterTable(recipe, inputName='coalesced', outputName='filtered_coalesced', filters={'clumpSize': [minblink, sys.maxsize]})) try: import hdbscan except: print('HDBScan not installed, defaulting to regular DBScan') clusterer = dbscan( recipe, inputName='filtered_coalesced', searchRadius=script_init.clusteringRadius, minClumpSize=script_init.minimumClusterSize, clumpColumnName=script_init.clusterColumnName, outputName='clusters') else: print('HDBScan installed, clustering with that') clusterer = kin_rcps.HDBSCANClustering( recipe, input_name='filtered_coalesced', search_radius=script_init.clusteringRadius, min_clump_size=script_init.minimumClusterSize, clump_column_name=script_init.clusterColumnName, output_name='clusters') recipe.add_module(clusterer) recipe.add_module( FilterTable(recipe, inputName='clusters', outputName='filt_clusters', filters={ script_init.clusterColumnName: [0.0, sys.maxsize] })) recipe.add_module( kin_rcps.FindBlinkStateDurations( recipe, inputName='filt_clusters', outputName='unfiltered_kinetics', labelKey=script_init.clusterColumnName, onTimesColName=script_init.onTimesColumnName, offTimesColName=script_init.offTimesColumnName)) recipe.add_module( FilterTable(recipe, inputName='unfiltered_kinetics', outputName='filtered_kinetics', filters={ script_init.offTimesColumnName: [0.0, sys.maxsize], script_init.onTimesColumnName: [minblink, sys.maxsize] })) recipe.add_module( kin_rcps.FitSwitchingRates( recipe, inputName='filtered_kinetics', outputName='kinetics_out', onTimesColName=script_init.onTimesColumnName, offTimesColName=script_init.offTimesColumnName, minOffTimeSecondsToFit=script_init. minimumOffTimeInSecondsToFit, maxOffTimeSecondsToFit=script_init. maximumOffTimeInSecondsToFit, offTimesFitBinSizeInSeconds=script_init. offTimeFitBinWidthInSeconds)) recipe.execute() self.visFr.pipeline.selectDataSource('filtered_kinetics') print 'RUNTIME OF ' + str(time() - t0)
def OpenFile(self, filename= '', ds = None, **kwargs): """Open a file - accepts optional keyword arguments for use with files saved as .txt and .mat. These are: FieldNames: a list of names for the fields in the text file or matlab variable. VarName: the name of the variable in the .mat file which contains the data. SkipRows: Number of header rows to skip for txt file data PixelSize: Pixel size if not in nm """ #close any files we had open previously while len(self.filesToClose) > 0: self.filesToClose.pop().close() #clear our state self.dataSources.clear() if 'zm' in dir(self): del self.zm self.filter = None self.mapping = None self.colourFilter = None self.events = None self.mdh = MetaDataHandler.NestedClassMDHandler() self.filename = filename if ds is None: #load from file ds = self._ds_from_file(filename, **kwargs) #wrap the data source with a mapping so we can fiddle with things #e.g. combining z position and focus mapped_ds = tabular.MappingFilter(ds) if 'PixelSize' in kwargs.keys(): mapped_ds.addVariable('pixelSize', kwargs['PixelSize']) mapped_ds.setMapping('x', 'x*pixelSize') mapped_ds.setMapping('y', 'y*pixelSize') #extract information from any events self.ev_mappings, self.eventCharts = _processEvents(mapped_ds, self.events, self.mdh) #Fit module specific filter settings if 'Analysis.FitModule' in self.mdh.getEntryNames(): fitModule = self.mdh['Analysis.FitModule'] #print 'fitModule = %s' % fitModule if 'Interp' in fitModule: self.filterKeys['A'] = (5, 100000) if 'LatGaussFitFR' in fitModule: mapped_ds.addColumn('nPhotons', getPhotonNums(mapped_ds, self.mdh)) if 'SplitterFitFNR' in fitModule: mapped_ds.addColumn('nPhotonsg', getPhotonNums({'A': mapped_ds['fitResults_Ag'], 'sig': mapped_ds['fitResults_sigma']}, self.mdh)) mapped_ds.addColumn('nPhotonsr', getPhotonNums({'A': mapped_ds['fitResults_Ar'], 'sig': mapped_ds['fitResults_sigma']}, self.mdh)) mapped_ds.setMapping('nPhotons', 'nPhotonsg+nPhotonsr') if fitModule == 'SplitterShiftEstFR': self.filterKeys['fitError_dx'] = (0,10) self.filterKeys['fitError_dy'] = (0,10) #self._get_dye_ratios_from_metadata() self.addDataSource('Localizations', mapped_ds) # Retrieve or estimate image bounds if False: # 'imgBounds' in kwargs.keys(): self.imageBounds = kwargs['imgBounds'] elif (not ( 'scanx' in mapped_ds.keys() or 'scany' in mapped_ds.keys())) and 'Camera.ROIWidth' in self.mdh.getEntryNames(): self.imageBounds = ImageBounds.extractFromMetadata(self.mdh) else: self.imageBounds = ImageBounds.estimateFromSource(mapped_ds) from PYME.recipes.localisations import ProcessColour from PYME.recipes.tablefilters import FilterTable self.colour_mapper = ProcessColour(self.recipe, input='Localizations', output='colour_mapped') #we keep a copy of this so that the colour panel can find it. self.recipe.add_module(self.colour_mapper) self.recipe.add_module(FilterTable(self.recipe, inputName='colour_mapped', outputName='filtered_localizations', filters={k:list(v) for k, v in self.filterKeys.items() if k in mapped_ds.keys()})) self.recipe.execute() self.filterKeys = {} self.selectDataSource('filtered_localizations') #NB - this rebuilds the pipeline