def execute(self, namespace): from PYME.Analysis.points import spherical_harmonics from PYME.IO import MetaDataHandler inp = namespace[self.input_localizations] points = tabular.MappingFilter(inp) shell = namespace[self.input_shell] if isinstance(shell, tabular.TabularBase): shell = spherical_harmonics.ScaledShell.from_tabular(shell) # map points to scaled spherical coordinates azimuth, zenith, r = shell.shell_coordinates( (points['x'], points['y'], points['z'])) # lookup shell radius at those angles r_shell = spherical_harmonics.reconstruct_shell( shell.modes, shell.coefficients, azimuth, zenith) points.addColumn(self.name_scaled_azimuth, azimuth) points.addColumn(self.name_scaled_zenith, zenith) points.addColumn(self.name_scaled_radius, r) points.addColumn(self.name_normalized_radius, r / r_shell) try: points.mdh = MetaDataHandler.DictMDHandler(inp.mdh) except AttributeError: pass namespace[self.output_mapped] = points
def execute(self, namespace): data = namespace[self.input] n_rows = len(data) if n_rows < self.num_to_select: if self.strict: raise IndexError( 'Trying to select %d from data with only %d rows. To allow truncation, use strict=False' % (self.num_to_select, n_rows)) else: logger.info( 'RandomSubset: Truncating from %d to %d rows as data only has %d rows. To make this an error, use strict=True' % (self.num_to_select, n_rows, n_rows)) if self.strict and (self.num_to_select > 0.5 * n_rows): logger.warning( 'RandomSubset: Selecting %d from %d rows will not be very random' % (self.num_to_select, n_rows)) out = tabular.RandomSelectionFilter(data, num_Samples=min( n_rows, self.num_to_select)) try: out.mdh = MetaDataHandler.DictMDHandler(data.mdh) except AttributeError: pass namespace[self.output] = out
def __init__(self, spooler, seriesName, analysisMetadata, resultsFilename=None, startAt=0, serverfilter=clusterIO.local_serverfilter, **kwargs): # TODO - reduce duplication of `LocalisationRule.__init__()` and `LocalisationRule._setup()` from PYME.IO import MetaDataHandler from PYME.Analysis import MetaData from PYME.IO.FileUtils.nameUtils import genClusterResultFileName from PYME.IO import unifiedIO self.spooler = spooler if resultsFilename is None: resultsFilename = genClusterResultFileName(seriesName) resultsFilename = verify_cluster_results_filename(resultsFilename) logger.info('Results file: ' + resultsFilename) resultsMdh = MetaDataHandler.DictMDHandler() # NB - anything passed in analysis MDH will wipe out corresponding entries in the series metadata resultsMdh.update(self.spooler.md) resultsMdh.update(analysisMetadata) resultsMdh['EstimatedLaserOnFrameNo'] = resultsMdh.getOrDefault( 'EstimatedLaserOnFrameNo', resultsMdh.getOrDefault('Analysis.StartAt', 0)) MetaData.fixEMGain(resultsMdh) self._setup(seriesName, resultsMdh, resultsFilename, startAt, serverfilter) Rule.__init__(self, **kwargs)
def __init__(self, size_x, size_y, n_frames, dtype='uint16', dim_order='XYCZT', shape=[-1, -1,1,1,1]): self.data = np.empty([size_x, size_y, n_frames], dtype=dtype) self.mdh = MetaDataHandler.DictMDHandler() # once we have proper xyztc support in the image viewer ds = XYZTCWrapper(ArrayDataSource(self.data), dim_order, shape[2], shape[3], shape[4]) #self.image = image.ImageStack(data=ds, mdh=self.mdh) # in the meantime - note that this will flatten the ctz dimensions self.image = image.ImageStack(data=ds, mdh=self.mdh)
def execute(self, namespace): from sklearn.mixture import GaussianMixture, BayesianGaussianMixture from PYME.IO import MetaDataHandler points = namespace[self.input_points] X = np.stack([points['x'], points['y'], points['z']], axis=1) if self.mode == 'n': gmm = GaussianMixture(n_components=self.n, covariance_type=self.covariance) predictions = gmm.fit_predict(X) elif self.mode == 'bic': n_components = range(1, self.n + 1) bic = np.zeros(len(n_components)) for ind in range(len(n_components)): gmm = GaussianMixture(n_components=n_components[ind], covariance_type=self.covariance) gmm.fit(X) bic[ind] = gmm.bic(X) logger.debug('%d BIC: %f' % (n_components[ind], bic[ind])) best = n_components[np.argmin(bic)] if best == self.n or (self.n > 10 and best > 0.9 * self.n): logger.warning( 'BIC optimization selected n components near n max') gmm = GaussianMixture(n_components=best, covariance_type=self.covariance) predictions = gmm.fit_predict(X) elif self.mode == 'bayesian': bgm = BayesianGaussianMixture(n_components=self.n, covariance_type=self.covariance) predictions = bgm.fit_predict(X) out = tabular.MappingFilter(points) try: out.mdh = MetaDataHandler.DictMDHandler(points.mdh) except AttributeError: pass out.addColumn(self.label_key, predictions) namespace[self.output_labeled] = out
def execute(self, namespace): from sklearn.mixture import GaussianMixture, BayesianGaussianMixture from PYME.IO import MetaDataHandler points = namespace[self.input_points] X = np.stack([points['x'], points['y'], points['z']], axis=1) if self.mode == 'n': gmm = GaussianMixture(n_components=self.n, covariance_type=self.covariance, max_iter=self.max_iter, init_params=self.init_params) predictions = gmm.fit_predict(X) + 1 # PYME labeling scheme log_prob = gmm.score_samples(X) if not gmm.converged_: logger.error('GMM fitting did not converge') predictions = np.zeros(len(points), int) log_prob = -np.inf * np.ones(len(points)) elif self.mode == 'bic': n_components = range(1, self.n + 1) bic = np.zeros(len(n_components)) for ind in range(len(n_components)): gmm = GaussianMixture(n_components=n_components[ind], covariance_type=self.covariance, max_iter=self.max_iter, init_params=self.init_params) gmm.fit(X) bic[ind] = gmm.bic(X) logger.debug('%d BIC: %f' % (n_components[ind], bic[ind])) best = n_components[np.argmin(bic)] if best == self.n or (self.n > 10 and best > 0.9 * self.n): logger.warning( 'BIC optimization selected n components near n max') gmm = GaussianMixture(n_components=best, covariance_type=self.covariance, max_iter=self.max_iter, init_params=self.init_params) predictions = gmm.fit_predict(X) + 1 # PYME labeling scheme log_prob = gmm.score_samples(X) if not gmm.converged_: logger.error('GMM fitting did not converge') predictions = np.zeros(len(points), int) log_prob = -np.inf * np.ones(len(points)) elif self.mode == 'bayesian': bgm = BayesianGaussianMixture(n_components=self.n, covariance_type=self.covariance, max_iter=self.max_iter, init_params=self.init_params) predictions = bgm.fit_predict(X) + 1 # PYME labeling scheme log_prob = bgm.score_samples(X) if not bgm.converged_: logger.error('GMM fitting did not converge') predictions = np.zeros(len(points), int) log_prob = -np.inf * np.ones(len(points)) out = tabular.MappingFilter(points) try: out.mdh = MetaDataHandler.DictMDHandler(points.mdh) except AttributeError: pass out.addColumn(self.label_key, predictions) out.addColumn(self.label_key + '_log_prob', log_prob) avg_log_prob = np.empty_like(log_prob) for label in np.unique(predictions): mask = label == predictions avg_log_prob[mask] = np.mean(log_prob[mask]) out.addColumn(self.label_key + '_avg_log_prob', avg_log_prob) namespace[self.output_labeled] = out
def start_spooling(self, fn=None, settings={}, preflight_mode='interactive'): """ Parameters ---------- fn : str, optional fn can be hardcoded here, otherwise differs to the seriesName property which will create one if need-be. settings : dict keys should be `SpoolController` attributes or properties with setters. Not all keys must be present, and example keys include: method : str One of 'File', 'Cluster', or 'Queue'(py2 only) hdf_compression_level: int zlib compression level that pytables should use (spool to file and queue) z_stepped : bool toggle z-stepping during acquisition z_dwell : int number of frames to acquire at each z level (predicated on `SpoolController.z_stepped` being True) cluster_h5 : bool Toggle spooling to single h5 file on cluster rather than pzf file per frame. Only applicable to 'Cluster' `method` and preferred for PYMEClusterOfOne. pzf_compression_settings : dict Compression settings relevant for 'Cluster' `method` if `cluster_h5` is False. See HTTPSpooler.defaultCompSettings. protocol_name : str Note that passing the protocol name will force a (re)load of the protocol file (even if it is already selected). max_frames : int, optional point at which to end the series automatically, by default sys.maxsize subdirectory : str, optional Directory within current set directory to spool this series. The directory will be created if it doesn't already exist. extra_metadata : dict, optional metadata to supplement this series for entries known prior to acquisition which do not have handlers to hook start metadata preflight_mode : str (default='interactive') What to do when the preflight check fails. Options are 'interactive', 'warn', 'abort' and 'skip' which will display a dialog and prompt the user, log a warning and continue, and log an error and abort, or skip completely. The former is suitable for interactive acquisition, whereas one of the latter modes is likely better for automated spooling via the action manager. """ # these settings were managed by the GUI, but are now managed by the # controller, still allow them to be passed in, but default to internals fn = self.seriesName if fn in ['', None] else fn stack = settings.get('z_stepped', self.z_stepped) compLevel = settings.get('hdf_compression_level', self.hdf_compression_level) pzf_compression_settings = settings.get('pzf_compression_settings', self.pzf_compression_settings) cluster_h5 = settings.get('cluster_h5', self.cluster_h5) maxFrames = settings.get('max_frames', sys.maxsize) stack_settings = settings.get('stack_settings', None) # try stack settings for z_dwell, then aq settings. # precedence is settings > stack_settings > self.z_dwell # The reasoning for allowing the dwell time to be set in either the spooling or stack settings is to allow # API users to choose which is most coherent for their use case (it would seem logical to put dwell time with # the other stack settings, but this becomes problematic when sharing stack settings across modalities - e.g. # PALM/STORM and widefield stacks which are likely to share most of the stack settings but have greatly different # z dwell times). PYMEAcquire specifies it in the spooling/series settings by default to allow shared usage # between modalities. if stack_settings: if isinstance(stack_settings, dict): z_dwell = stack_settings.get('DwellFrames', self.z_dwell) else: # have a StackSettings object # TODO - fix this to be a bit more sane and not use private attributes etc ... z_dwell = stack_settings._dwell_frames # z_dwell defaults to -1 (with a meaning of ignore) in StackSettings objects if not value is not # explicitly provided. In this case, use our internal value instead. The reason for the 'ignore' # special value is to allow the same StackSettings object to be used for widefield stacks and # localization series (where sharing everything except dwell time makes sense). if z_dwell < 1: z_dwell = self.z_dwell else: z_dwell = self.z_dwell z_dwell = settings.get('z_dwell', z_dwell) if (stack_settings is not None) and (not isinstance(stack_settings, stackSettings.StackSettings)): # let us pass stack settings as a dict, constructing a StackSettings instance as needed stack_settings = stackSettings.StackSettings(**dict(stack_settings)) protocol_name = settings.get('protocol_name') if protocol_name is None: protocol, protocol_z = self.protocol, self.protocolZ else: pmod = prot.get_protocol(protocol_name) protocol, protocol_z = pmod.PROTOCOL, pmod.PROTOCOL_STACK subdirectory = settings.get('subdirectory', None) # make directories as needed, makedirs(dir, exist_ok=True) once py2 support is dropped if (self.spoolType != 'Cluster') and (not os.path.exists(self.get_dirname(subdirectory))): os.makedirs(self.get_dirname(subdirectory)) if self._checkOutputExists(fn): #check to see if data with the same name exists self.seriesCounter +=1 self.seriesName = self._GenSeriesName() raise IOError('A series with the same name already exists') if stack: protocol = protocol_z protocol.dwellTime = z_dwell #print(protocol) else: protocol = protocol if (preflight_mode != 'skip') and not preflight.ShowPreflightResults(protocol.PreflightCheck(), preflight_mode): return #bail if we failed the pre flight check, and the user didn't choose to continue #fix timing when using fake camera if self.scope.cam.__class__.__name__ == 'FakeCamera': fakeCycleTime = self.scope.cam.GetIntegTime() else: fakeCycleTime = None frameShape = (self.scope.cam.GetPicWidth(), self.scope.cam.GetPicHeight()) if self.spoolType == 'Queue': from PYME.Acquire import QueueSpooler self.queueName = getRelFilename(self._get_queue_name(fn, subdirectory=subdirectory)) self.spooler = QueueSpooler.Spooler(self.queueName, self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, stack_settings=stack_settings) elif self.spoolType == 'Cluster': from PYME.Acquire import HTTPSpooler self.queueName = self._get_queue_name(fn, pcs=(not cluster_h5), subdirectory=subdirectory) self.spooler = HTTPSpooler.Spooler(self.queueName, self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, compressionSettings=pzf_compression_settings, aggregate_h5=cluster_h5, stack_settings=stack_settings) else: from PYME.Acquire import HDFSpooler self.spooler = HDFSpooler.Spooler(self._get_queue_name(fn, subdirectory=subdirectory), self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, stack_settings=stack_settings) #TODO - sample info is probably better handled with a metadata hook #if sampInf: # try: # sampleInformation.getSampleData(self, self.spooler.md) # except: # #the connection to the database will timeout if not present # #FIXME: catch the right exception (or delegate handling to sampleInformation module) # pass extra_metadata = settings.get('extra_metadata') if extra_metadata is not None: self.spooler.md.mergeEntriesFrom(MetaDataHandler.DictMDHandler(extra_metadata)) # stop the frameWrangler before we start spooling # this serves to ensure that a) we don't accidentally spool frames which were in the camera buffer when we hit start # and b) we get a nice clean timestamp for when the actual frames start (after any protocol init tasks) # it might also slightly improve performance. self.scope.frameWrangler.stop() try: self.spooler.onSpoolStop.connect(self.SpoolStopped) self.spooler.StartSpool() except: self.spooler.abort() raise # restart frame wrangler self.scope.frameWrangler.Prepare() self.scope.frameWrangler.start() self.onSpoolStart.send(self) #return a function which can be called to indicate if we are done return lambda : self.spooler.spool_complete
def execute(self, namespace): from PYME.Analysis.points import spherical_harmonics from PYME.IO import MetaDataHandler shell = namespace[self.input_shell] if isinstance(shell, tabular.TabularBase): shell = spherical_harmonics.ScaledShell.from_tabular(shell) bin_edges = np.arange(0, 1.0 + self.r_bin_spacing, self.r_bin_spacing) bin_centers = 0.5 * (bin_edges[1:] + bin_edges[:-1]) out_hist = np.zeros(len(bin_centers), float) # get shell bounds, make grid within shell_bounds = shell.approximate_image_bounds() xv = np.arange(shell_bounds.x0, shell_bounds.x1 + self.sampling_nm[0], self.sampling_nm[0]) yv = np.arange(shell_bounds.y0, shell_bounds.y1 + self.sampling_nm[1], self.sampling_nm[1]) zv = np.arange(shell_bounds.z0, shell_bounds.z1 + self.sampling_nm[2], self.sampling_nm[2]) x, y, z = np.meshgrid(xv, yv, zv, indexing='ij') v_estimates = [] sdev_estimates = [] n_choose = 10000 for _ in range(self.jitter_iterations): xr, yr, zr = np.random.rand(len(xv), len(yv), len(zv)), np.random.rand( len(xv), len(yv), len(zv)), np.random.rand( len(xv), len(yv), len(zv)) xr = (xr - 0.5) * self.sampling_nm[0] + x yr = (yr - 0.5) * self.sampling_nm[1] + y zr = (zr - 0.5) * self.sampling_nm[2] + z azi, zen, r = shell.shell_coordinates((xr, yr, zr)) r_shell = spherical_harmonics.reconstruct_shell( shell.modes, shell.coefficients, azi, zen) inside = r < r_shell N = np.sum(inside) r_norm = r[inside] / r_shell[inside] # sum-normalize this iteration and add to output out_hist += np.histogram(r_norm, bins=bin_edges)[0] / N # record volume estimate v_estimates.append(N) # estimate spread along principle axes of the shell X = np.vstack([xr[inside], yr[inside], zr[inside]]) if N > n_choose: # downsample to avoid memory error X = X[:, np.random.choice(N, n_choose, replace=False)] # TODO - do we need to be mean-centered? X = X - X.mean(axis=1)[:, None] _, s, _ = np.linalg.svd(X.T) # svd cov is not normalized, handle that sdev_estimates.append( s / np.sqrt(X.shape[1] - 1)) # with bessel's correction # finish the average out_hist = out_hist / self.jitter_iterations # finish the volume calculation, convert from nm^3 to um^3 volume = np.mean(v_estimates) * (np.prod(self.sampling_nm) / (1e9)) # average the standard deviation estimates standard_deviations = np.mean(np.stack(sdev_estimates), axis=0) # similar to Basser, P. J., et al. doi.org/10.1006/jmrb.1996.0086 # note that singular values are square roots of the eigenvalues. Use # the sample standard deviation rather than pop. anisotropy = np.sqrt(np.var(standard_deviations**2, ddof=1)) / ( np.sqrt(3) * np.mean(standard_deviations**2)) res = tabular.DictSource({ 'bin_centers': bin_centers, 'counts': out_hist }) try: res.mdh = MetaDataHandler.DictMDHandler(shell.mdh) except AttributeError: res.mdh = MetaDataHandler.DictMDHandler() res.mdh['SHShellRadiusDensityEstimate.Volume'] = float(volume) res.mdh[ 'SHShellRadiusDensityEstimate.StdDeviations'] = standard_deviations.tolist( ) res.mdh['SHShellRadiusDensityEstimate.Anisotropy'] = float(anisotropy) namespace[self.output] = res
def __init__(self, parent=None): wx.Frame.__init__(self, parent, wx.ID_ANY, 'The PYME Bakery') logger.debug('BatchFrame.__init__ start') self.rm = RecipeManager() #self.inputFiles = [] #self.inputFiles2 = [] self._default_md = MetaDataHandler.DictMDHandler(MetaData.ConfocDefault) self._file_lists = [] vsizer1=wx.BoxSizer(wx.VERTICAL) hsizer = wx.StaticBoxSizer(wx.StaticBox(self, -1, "Recipe:"), wx.HORIZONTAL) self.recipeView = RecipeView(self, self.rm) hsizer.Add(self.recipeView, 1, wx.ALL|wx.EXPAND, 2) vsizer1.Add(hsizer, 1, wx.ALL|wx.EXPAND, 2) hsizer1 = wx.BoxSizer(wx.HORIZONTAL) sbsizer = wx.StaticBoxSizer(wx.StaticBox(self, -1, 'Input files:'), wx.VERTICAL) self._file_lists.append(FileListPanel(self, -1)) sbsizer.Add(self._file_lists[-1], 1, wx.EXPAND, 0) hsizer1.Add(sbsizer, 1, wx.EXPAND, 10) sbsizer = wx.StaticBoxSizer(wx.StaticBox(self, -1, 'Input files (input2) [optional]:'), wx.VERTICAL) self._file_lists.append(FileListPanel(self, -1)) sbsizer.Add(self._file_lists[-1], 1, wx.EXPAND, 0) hsizer1.Add(sbsizer, 1, wx.EXPAND, 10) self._sb_metadata = wx.StaticBox(self, -1, 'Metadata defaults') sbsizer = wx.StaticBoxSizer(self._sb_metadata, wx.VERTICAL) sbsizer.Add(wx.StaticText(self, -1, 'If metadata is not found in input images,\nthe following defaults will be used:'), 0, wx.EXPAND,0) self._mdpan = MetadataTree.MetadataPanel(self, self._default_md, refreshable=False) sbsizer.Add(self._mdpan, 1, wx.EXPAND, 0) hsizer1.Add(sbsizer, 0, wx.EXPAND, 10) vsizer1.Add(hsizer1, 0, wx.EXPAND|wx.TOP, 10) hsizer2 = wx.StaticBoxSizer(wx.StaticBox(self, -1, 'Output Directory:'), wx.HORIZONTAL) self.dcOutput = wx.DirPickerCtrl(self, -1, style=wx.DIRP_USE_TEXTCTRL) hsizer2.Add(self.dcOutput, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) vsizer1.Add(hsizer2, 0, wx.EXPAND|wx.TOP, 10) hsizer = wx.BoxSizer(wx.HORIZONTAL) hsizer.AddStretchSpacer() self.cbSpawnWorkerProcs = wx.CheckBox(self, -1, 'spawn worker processes for each core') self.cbSpawnWorkerProcs.SetValue(True) hsizer.Add(self.cbSpawnWorkerProcs, 0, wx.ALL, 5) self.bBake = wx.Button(self, -1, 'Bake') hsizer.Add(self.bBake, 0, wx.ALL, 5) self.bBake.Bind(wx.EVT_BUTTON, self.OnBake) vsizer1.Add(hsizer, 0, wx.EXPAND|wx.TOP, 10) self.SetSizerAndFit(vsizer1) logger.debug('BatchFrame.__init__ done')
from PYME.IO.DataSources.RandomDataSource import DataSource from PYME.IO.buffers import dataBuffer from warpdrive.buffers import Buffer import numpy as np try: from PYME.localization.remFitBuf import CameraInfoManager from PYME.IO import MetaDataHandler mdh = MetaDataHandler.DictMDHandler() mdh['Camera.ReadNoise'] = 1.0 mdh['Camera.NoiseFactor'] = 1.0 mdh['Camera.ElectronsPerCount'] = 1.0 mdh['Camera.TrueEMGain'] = 1.0 mdh['voxelsize.x'] = 0.7 mdh['voxelsize.y'] = 0.7 mdh['Analysis.DetectionFilterSize'] = 3 mdh['Analysis.ROISize'] = 4.5 mdh['Analysis.GPUPCTBackground'] = False camera_info_manager = CameraInfoManager() except ImportError: print("PYME not installed") import pytest pytest.skip('python-microscopy environment (PYME) not installed') def gen_image(p=.95, disp=False): try: from PYME.simulation import wormlike2 except ImportError: # legacy PYME, pre March 2021 from PYME.Acquire.Hardware.Simulator import wormlike2