def execute(self, namespace): from sklearn.cluster import dbscan inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) # Note that sklearn gives unclustered points label of -1, and first value starts at 0. try: core_samp, dbLabels = dbscan(np.vstack( [inp[k] for k in self.columns]).T, self.searchRadius, self.minClumpSize, n_jobs=self.numberOfJobs) multiproc = True except: core_samp, dbLabels = dbscan( np.vstack([inp[k] for k in self.columns]).T, self.searchRadius, self.minClumpSize) multiproc = False if multiproc: logger.info('using dbscan multiproc version') else: logger.info('falling back to dbscan single-threaded version') # shift dbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped mapped.addColumn('dbscanClumpID', dbLabels + 1) # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): # print('testing showpoints again') # print(namespace['showplots']) inp = namespace[self.input_name] mapped = tabular.mappingFilter(inp) # vert_data = namespace[self.input_vert] import hdbscan clusterer = hdbscan.HDBSCAN(min_cluster_size=self.min_clump_size) clusterer.fit(np.vstack([inp[k] for k in self.columns]).T) # Note that hdbscan gives unclustered points label of -1, and first value starts at 0. # shift hdbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped mapped.addColumn(str(self.clump_column_name), clusterer.labels_ + 1) mapped.addColumn(str(self.clump_prob_column_name), clusterer.probabilities_) if not self.search_radius is None and self.search_radius > 0: #Extract dbscan clustering from hdbscan clusterer dbscan = clusterer.single_linkage_tree_.get_clusters( self.search_radius, self.min_clump_size) # shift dbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped mapped.addColumn(str(self.clump_dbscan_column_name), dbscan + 1) # propogate metadata, if present try: mapped.mdh = inp.mdh print('testing for mdh') except AttributeError: pass namespace[self.output_name] = mapped print('finished clustering')
def execute(self, namespace): self._namespace = namespace import multiprocessing # from PYME.util import mProfile # mProfile.profileOn(["frc.py"]) if self.multiprocessing: proccess_count = np.clip(2, 1, multiprocessing.cpu_count()-1) self._pool = multiprocessing.Pool(processes=proccess_count) pipeline = namespace[self.inputName] mapped_pipeline = tabular.mappingFilter(pipeline) self._pixel_size_in_nm = self.pixel_size_in_nm * np.ones(3, dtype=np.float) image_pair = self.generate_image_pair(mapped_pipeline) image_pair = self.preprocess_images(image_pair) # Should use DensityMapping recipe eventually when it is ready. mdh = MetaDataHandler.NestedClassMDHandler() mdh['Rendering.Method'] = "np.histogramdd" if 'imageID' in pipeline.mdh.getEntryNames(): mdh['Rendering.SourceImageID'] = pipeline.mdh['imageID'] try: mdh['Rendering.SourceFilename'] = pipeline.resultsSource.h5f.filename except: pass mdh.Source = MetaDataHandler.NestedClassMDHandler(pipeline.mdh) mdh['Rendering.NEventsRendered'] = [image_pair[0].sum(), image_pair[1].sum()] mdh['voxelsize.units'] = 'um' mdh['voxelsize.x'] = self.pixel_size_in_nm * 1E-3 mdh['voxelsize.y'] = self.pixel_size_in_nm * 1E-3 ims = ImageStack(data=np.stack(image_pair, axis=-1), mdh=mdh) namespace[self.output_images] = ims # if self.plot_graphs: # from PYME.DSView.dsviewer import ViewIm3D # ViewIm3D(ims) frc_res, rawdata = self.calculate_FRC_from_images(image_pair, pipeline.mdh) # smoothed_frc = self.SmoothFRC(frc_freq, frc_corr) # # self.CalculateThreshold(frc_freq, frc_corr, smoothed_frc) namespace[self.output_frc_dict] = frc_res namespace[self.output_frc_raw] = rawdata if self.multiprocessing: self._pool.close() self._pool.join() # mProfile.profileOff() # mProfile.report() self.save_to_file(namespace)
def execute(self, namespace): from PYMEcs.Analysis.objectVolumes import objectVolumes inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) volumes = objectVolumes( np.vstack([inp[k] for k in ('x', 'y')]).T, inp['objectID']) mapped.addColumn('volumes', volumes) namespace[self.outputName] = mapped
def mergeClumpsDyeKinetics(datasource, labelKey='clumpIndex'): from PYME.IO.tabular import mappingFilter, resultsFilter from PYME.Analysis.points.multiview import coalesce_dict_sorted import sys """ adds some steps to the default version for finding dye switching behavior """ datasource = mappingFilter(datasource) # TODO this is needed for faster off time calculation, but may not be the best place for adding this datasource.addColumn('t2', datasource['t']) datasource.addColumn('min_error_x', datasource['error_x']) datasource.addColumn('min_error_y', datasource['error_y']) ds_keys = datasource.keys() keys_to_aggregate = [ k for k in ds_keys if not (k.startswith('error') or k.startswith('slicesUsed') or k.startswith('fitError')) ] all_keys = list( keys_to_aggregate ) # this should be a copy otherwise we end up adding the weights to our list of stuff to aggregate # pair fit results and errors for weighting aggregation_weights = { k: 'error_' + k for k in keys_to_aggregate if 'error_' + k in datasource.keys() } all_keys += aggregation_weights.values() aggregation_weights['t'] = 'min' aggregation_weights['t2'] = 'max' aggregation_weights['clumpSize'] = 'max' aggregation_weights['nPhotons'] = 'sum' aggregation_weights['A'] = 'mean' I = np.argsort(datasource[labelKey]) sorted_src = {k: datasource[k][I] for k in all_keys} aggregation_weights['min_error_x'] = 'min' aggregation_weights['min_error_y'] = 'min' #TODO: make sure we've added all the correct aggregation keys for the datasource grouped = coalesce_dict_sorted(sorted_src, sorted_src[labelKey], keys_to_aggregate, aggregation_weights) ds = resultsFilter(grouped, x=[0, sys.maxsize]) return ds
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) qkey = self.qIndexkey scaled = inp[qkey] qigood = inp[qkey] > 0 scaled[ qigood] = inp[qkey][qigood] * self.NEquivalent / self.qindexValue self.newKey = '%sCal' % qkey mapped.addColumn(self.newKey, scaled) namespace[self.outputName] = mapped
def execute(self, namespace): # t_shift, shifts = namespace[self.input_drift] out = tabular.mappingFilter(namespace[self.input_localizations]) out.mdh = namespace[self.input_localizations].mdh t_out = out['t'] dx = namespace[self.input_drift_interpolator][0](t_out) dy = namespace[self.input_drift_interpolator][1](t_out) dz = namespace[self.input_drift_interpolator][2](t_out) if 'dx' in out.keys(): # getting around oddity with mappingFilter # addColumn adds a new column but also keeps the old column # __getitem__ returns the new column # but mappings usues the old column # Wrap with another level of mappingFilter so the new column becomes the 'old column' out.addColumn('dx', dx) out.addColumn('dy', dy) out.addColumn('dz', dz) out = tabular.mappingFilter(out) out.mdh = namespace[self.input_localizations].mdh out.setMapping('x', 'x + dx') out.setMapping('y', 'y + dy') out.setMapping('z', 'z + dz') else: out.addColumn('dx', dx) out.addColumn('dy', dy) out.addColumn('dz', dz) out.setMapping('x', 'x + dx') out.setMapping('y', 'y + dy') out.setMapping('z', 'z + dz') try: out.mdh = self.input_localizations.mdh except AttributeError: pass namespace[self.output_name] = out
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) qkey1 = self.qIndexDenom qkey2 = self.qIndexNumer v2 = inp[qkey2] ratio = np.zeros_like(v2, dtype='float64') qigood = v2 > 0 ratio[qigood] = inp[qkey1][qigood] / v2[qigood] mapped.addColumn(self.qIndexRatio, ratio) namespace[self.outputName] = mapped
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) fdialog = wx.FileDialog( None, 'Please select PSF to use ...', #defaultDir=os.path.split(self.image.filename)[0], wildcard='PSF Files|*.psf|TIFF files|*.tif', style=wx.FD_OPEN) succ = fdialog.ShowModal() if (succ == wx.ID_OK): psfn = filename = fdialog.GetPath() mdh = inp.mdh if mdh.getEntry('Analysis.FitModule') not in [ 'SplitterFitInterpBNR' ]: Warn('Plugin works only for Biplane analysis') return fitMod = __import__( 'PYME.localization.FitFactories.' + mdh.getEntry('Analysis.FitModule'), fromlist=['PYME', 'localization', 'FitFactories']) fr = populate_fresults(fitMod, inp) progress = wx.ProgressDialog("calculating photon numbers", "calculating...", maximum=100, parent=None, style=wx.PD_SMOOTH | wx.PD_AUTO_HIDE) nph = nPhotons(fitMod, fr, mdh, psfname=psfn, nmax=1e6, progressBar=progress, updateStep=100) progress.Destroy() mapped.addColumn('nPhotons', nph) mapped.addColumn('fitResults_background', inp['fitResults_bg'] + inp['fitResults_br']) #mapped.addColumn('sig',float(137.0)+np.zeros_like(inp['x'])) # this one is a straight kludge for mortensenError # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): inp = namespace[self.inputData] fiducial = namespace[self.inputFiducials] mapped = tabular.mappingFilter(inp) out_f = tabular.mappingFilter(fiducial) for dim in ['x', 'y', 'z']: fiducial_dim = finterpDS(inp, fiducial, 'fiducial_%s' % dim) mapped.addColumn('fiducial_%s' % dim, fiducial_dim) mapped.addColumn(dim, inp[dim] - fiducial_dim) out_f.setMapping(dim, '{0} - fiducial_{0}'.format(dim)) # propogate metadata, if present try: mapped.mdh = inp.mdh out_f.mdh = fiducial.mdh except AttributeError: pass namespace[self.outputName] = mapped namespace[self.outputFiducials] = out_f
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) timedSpecies = self.populateTimedSpecies() mapped.addColumn('ColourNorm', np.ones_like(mapped['t'], 'float')) for species in timedSpecies: mapped.addColumn('p_%s' % species['name'], (mapped['t'] >= species['t_start']) * (mapped['t'] < species['t_end'])) if 'mdh' in dir(inp): mapped.mdh = inp.mdh mapped.mdh['TimedSpecies'] = timedSpecies namespace[self.outputName] = mapped
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) for dim in ['x', 'y', 'z']: try: mapped.addColumn(dim, inp[dim] - inp['fiducial_%s' % dim]) except: logger.warn('Could not set dim %s' % dim) # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): import PYMEcs.Analysis.trackFiducials as tfs inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) if self.singleFiducial: # if all data is from a single fiducial we do not need to align # we then avoid problems with incomplete tracks giving rise to offsets between # fiducial track fragments align = False else: align = True t, x, y, z, isFiducial = tfs.extractTrajectoriesClump( inp, clumpRadiusVar='error_x', clumpRadiusMultiplier=self.radiusMultiplier, timeWindow=self.timeWindow, clumpMinSize=self.clumpMinSize, align=align) rawtracks = (t, x, y, z) tracks = tfs.AverageTrack(inp, rawtracks, filter=self.filterMethod, filterScale=self.filterScale, align=align) # add tracks for all calculated dims to output for dim in tracks.keys(): mapped.addColumn('fiducial_%s' % dim, tracks[dim]) mapped.addColumn('isFiducial', isFiducial) # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): from scipy.stats import binned_statistic inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) ids = inp[self.IDkey] t = inp['t'] maxid = int(ids.max()) edges = -0.5 + np.arange(maxid + 2) resmin = binned_statistic(ids, t, statistic='min', bins=edges) resmax = binned_statistic(ids, t, statistic='max', bins=edges) trange = resmax[0][ids] - resmin[0][ids] + 1 mapped.addColumn('trange', trange) # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) if 'mdh' not in dir(inp): raise RuntimeError('SnrCalculation needs metadata') else: mdh = inp.mdh nph = inp['nPhotons'] bgraw = inp['fitResults_background'] bgph = np.clip((bgraw) * mdh['Camera.ElectronsPerCount'] / mdh.getEntry('Camera.TrueEMGain'), 1, None) npixroi = (2 * mdh.getOrDefault('Analysis.ROISize', 5) + 1)**2 snr = 1.0 / npixroi * np.clip(nph, 0, None) / np.sqrt(bgph) mapped.addColumn('SNR', snr) mapped.addColumn('backgroundPhotons', bgph) mapped.mdh = inp.mdh namespace[self.outputName] = mapped
def execute(self, namespace): inp = namespace[self.inputName] valid = namespace[self.inputValid] mapped = tabular.mappingFilter(inp) # note: in coalesced data the clumpIndices are float! # this creates issues in comparisons unless these are converted to int before comparisons are made!! # that is the reason for the rint and astype conversions below ids = np.rint(inp[self.IDkey]).astype('i') validIDs = np.in1d(ids, np.unique(np.rint(valid[self.IDkey]).astype('i'))) mapped.addColumn('validID', validIDs.astype('f')) # should be float or int? # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def execute(self, namespace): from scipy.stats import binned_statistic inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) ids = inp[self.IDkey] # I imagine this needs to be an int type key prop = inp[self.StatKey] maxid = int(ids.max()) edges = -0.5 + np.arange(maxid + 2) resstat = binned_statistic(ids, prop, statistic=self.StatMethod, bins=edges) mapped.addColumn(self.StatKey + "_" + self.StatMethod, resstat[0][ids]) # propogate metadata, if present try: mapped.mdh = inp.mdh except AttributeError: pass namespace[self.outputName] = mapped
def _execute(self, namespace): # from PYME.util import mProfile # self._start_time = time.time() self.trait_setq(**{"_start_time": time.time()}) print("Starting drift correction module.") if self.multiprocessing: proccess_count = np.clip(multiprocessing.cpu_count()-1, 1, None) self.trait_setq(**{"_pool": multiprocessing.Pool(processes=proccess_count)}) locs = namespace[self.input_for_correction] # mProfile.profileOn(['localisations.py', 'processing.py']) drift_res = self.calc_corr_drift_from_locs(locs['x'], locs['y'], locs['z'] * (0 if self.flatten_z else 1), locs['t']) t_shift, shifts = self.rcc(self.shift_max, *drift_res) # mProfile.profileOff() # mProfile.report() if self.multiprocessing: self._pool.close() self._pool.join() # convert frame-to-frame drift to drift from origin shifts = np.cumsum(shifts, 0) out = tabular.mappingFilter(namespace[self.input_for_mapping]) t_out = out['t'] # cubic interpolate with no smoothing dx = interpolate.CubicSpline(t_shift, shifts[:, 0])(t_out) dy = interpolate.CubicSpline(t_shift, shifts[:, 1])(t_out) dz = interpolate.CubicSpline(t_shift, shifts[:, 2])(t_out) if 'dx' in out.keys(): # getting around oddity with mappingFilter # addColumn adds a new column but also keeps the old column # __getitem__ returns the new column # but mappings usues the old column # Wrap with another level of mappingFilter so the new column becomes the 'old column' out.addColumn('dx', dx) out.addColumn('dy', dy) out.addColumn('dz', dz) out = tabular.mappingFilter(out) # out.mdh = namespace[self.input_localizations].mdh out.setMapping('x', 'x + dx') out.setMapping('y', 'y + dy') out.setMapping('z', 'z + dz') else: out.addColumn('dx', dx) out.addColumn('dy', dy) out.addColumn('dz', dz) out.setMapping('x', 'x + dx') out.setMapping('y', 'y + dy') out.setMapping('z', 'z + dz') # propagate metadata, if present try: out.mdh = locs.mdh except AttributeError: pass namespace[self.outputName] = out namespace[self.output_drift] = t_shift, shifts # non essential, only for plotting out drift data namespace[self.output_drift_plot] = Plot(partial(generate_drift_plot, t_shift, shifts)) namespace[self.output_cross_cor] = self._cc_image
def execute(self, namespace): inp = namespace[self.inputName] mapped = tabular.mappingFilter(inp) namespace[self.outputName] = mapped