def execute(self, namespace):
from sklearn.cluster import dbscan
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
# Note that sklearn gives unclustered points label of -1, and first value starts at 0.
try:
core_samp, dbLabels = dbscan(np.vstack(
[inp[k] for k in self.columns]).T,
self.searchRadius,
self.minClumpSize,
n_jobs=self.numberOfJobs)
multiproc = True
except:
core_samp, dbLabels = dbscan(
np.vstack([inp[k] for k in self.columns]).T, self.searchRadius,
self.minClumpSize)
multiproc = False
if multiproc:
logger.info('using dbscan multiproc version')
else:
logger.info('falling back to dbscan single-threaded version')
# shift dbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped
mapped.addColumn('dbscanClumpID', dbLabels + 1)
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
# print('testing showpoints again')
# print(namespace['showplots'])
inp = namespace[self.input_name]
mapped = tabular.mappingFilter(inp)
# vert_data = namespace[self.input_vert]
import hdbscan
clusterer = hdbscan.HDBSCAN(min_cluster_size=self.min_clump_size)
clusterer.fit(np.vstack([inp[k] for k in self.columns]).T)
# Note that hdbscan gives unclustered points label of -1, and first value starts at 0.
# shift hdbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped
mapped.addColumn(str(self.clump_column_name), clusterer.labels_ + 1)
mapped.addColumn(str(self.clump_prob_column_name),
clusterer.probabilities_)
if not self.search_radius is None and self.search_radius > 0:
#Extract dbscan clustering from hdbscan clusterer
dbscan = clusterer.single_linkage_tree_.get_clusters(
self.search_radius, self.min_clump_size)
# shift dbscan labels up by one to match existing convention that a clumpID of 0 corresponds to unclumped
mapped.addColumn(str(self.clump_dbscan_column_name), dbscan + 1)
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
print('testing for mdh')
except AttributeError:
pass
namespace[self.output_name] = mapped
print('finished clustering')
def execute(self, namespace):
self._namespace = namespace
import multiprocessing
# from PYME.util import mProfile
# mProfile.profileOn(["frc.py"])
if self.multiprocessing:
proccess_count = np.clip(2, 1, multiprocessing.cpu_count()-1)
self._pool = multiprocessing.Pool(processes=proccess_count)
pipeline = namespace[self.inputName]
mapped_pipeline = tabular.mappingFilter(pipeline)
self._pixel_size_in_nm = self.pixel_size_in_nm * np.ones(3, dtype=np.float)
image_pair = self.generate_image_pair(mapped_pipeline)
image_pair = self.preprocess_images(image_pair)
# Should use DensityMapping recipe eventually when it is ready.
mdh = MetaDataHandler.NestedClassMDHandler()
mdh['Rendering.Method'] = "np.histogramdd"
if 'imageID' in pipeline.mdh.getEntryNames():
mdh['Rendering.SourceImageID'] = pipeline.mdh['imageID']
try:
mdh['Rendering.SourceFilename'] = pipeline.resultsSource.h5f.filename
except:
pass
mdh.Source = MetaDataHandler.NestedClassMDHandler(pipeline.mdh)
mdh['Rendering.NEventsRendered'] = [image_pair[0].sum(), image_pair[1].sum()]
mdh['voxelsize.units'] = 'um'
mdh['voxelsize.x'] = self.pixel_size_in_nm * 1E-3
mdh['voxelsize.y'] = self.pixel_size_in_nm * 1E-3
ims = ImageStack(data=np.stack(image_pair, axis=-1), mdh=mdh)
namespace[self.output_images] = ims
# if self.plot_graphs:
# from PYME.DSView.dsviewer import ViewIm3D
# ViewIm3D(ims)
frc_res, rawdata = self.calculate_FRC_from_images(image_pair, pipeline.mdh)
# smoothed_frc = self.SmoothFRC(frc_freq, frc_corr)
#
# self.CalculateThreshold(frc_freq, frc_corr, smoothed_frc)
namespace[self.output_frc_dict] = frc_res
namespace[self.output_frc_raw] = rawdata
if self.multiprocessing:
self._pool.close()
self._pool.join()
# mProfile.profileOff()
# mProfile.report()
self.save_to_file(namespace)
def execute(self, namespace):
from PYMEcs.Analysis.objectVolumes import objectVolumes
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
volumes = objectVolumes(
np.vstack([inp[k] for k in ('x', 'y')]).T, inp['objectID'])
mapped.addColumn('volumes', volumes)
namespace[self.outputName] = mapped
def mergeClumpsDyeKinetics(datasource, labelKey='clumpIndex'):
from PYME.IO.tabular import mappingFilter, resultsFilter
from PYME.Analysis.points.multiview import coalesce_dict_sorted
import sys
"""
adds some steps to the default version for finding dye switching behavior
"""
datasource = mappingFilter(datasource)
# TODO this is needed for faster off time calculation, but may not be the best place for adding this
datasource.addColumn('t2', datasource['t'])
datasource.addColumn('min_error_x', datasource['error_x'])
datasource.addColumn('min_error_y', datasource['error_y'])
ds_keys = datasource.keys()
keys_to_aggregate = [
k for k in ds_keys
if not (k.startswith('error') or k.startswith('slicesUsed')
or k.startswith('fitError'))
]
all_keys = list(
keys_to_aggregate
) # this should be a copy otherwise we end up adding the weights to our list of stuff to aggregate
# pair fit results and errors for weighting
aggregation_weights = {
k: 'error_' + k
for k in keys_to_aggregate if 'error_' + k in datasource.keys()
}
all_keys += aggregation_weights.values()
aggregation_weights['t'] = 'min'
aggregation_weights['t2'] = 'max'
aggregation_weights['clumpSize'] = 'max'
aggregation_weights['nPhotons'] = 'sum'
aggregation_weights['A'] = 'mean'
I = np.argsort(datasource[labelKey])
sorted_src = {k: datasource[k][I] for k in all_keys}
aggregation_weights['min_error_x'] = 'min'
aggregation_weights['min_error_y'] = 'min'
#TODO: make sure we've added all the correct aggregation keys for the datasource
grouped = coalesce_dict_sorted(sorted_src, sorted_src[labelKey],
keys_to_aggregate, aggregation_weights)
ds = resultsFilter(grouped, x=[0, sys.maxsize])
return ds
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
qkey = self.qIndexkey
scaled = inp[qkey]
qigood = inp[qkey] > 0
scaled[
qigood] = inp[qkey][qigood] * self.NEquivalent / self.qindexValue
self.newKey = '%sCal' % qkey
mapped.addColumn(self.newKey, scaled)
namespace[self.outputName] = mapped
def execute(self, namespace):
# t_shift, shifts = namespace[self.input_drift]
out = tabular.mappingFilter(namespace[self.input_localizations])
out.mdh = namespace[self.input_localizations].mdh
t_out = out['t']
dx = namespace[self.input_drift_interpolator][0](t_out)
dy = namespace[self.input_drift_interpolator][1](t_out)
dz = namespace[self.input_drift_interpolator][2](t_out)
if 'dx' in out.keys():
# getting around oddity with mappingFilter
# addColumn adds a new column but also keeps the old column
# __getitem__ returns the new column
# but mappings usues the old column
# Wrap with another level of mappingFilter so the new column becomes the 'old column'
out.addColumn('dx', dx)
out.addColumn('dy', dy)
out.addColumn('dz', dz)
out = tabular.mappingFilter(out)
out.mdh = namespace[self.input_localizations].mdh
out.setMapping('x', 'x + dx')
out.setMapping('y', 'y + dy')
out.setMapping('z', 'z + dz')
else:
out.addColumn('dx', dx)
out.addColumn('dy', dy)
out.addColumn('dz', dz)
out.setMapping('x', 'x + dx')
out.setMapping('y', 'y + dy')
out.setMapping('z', 'z + dz')
try:
out.mdh = self.input_localizations.mdh
except AttributeError:
pass
namespace[self.output_name] = out
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
qkey1 = self.qIndexDenom
qkey2 = self.qIndexNumer
v2 = inp[qkey2]
ratio = np.zeros_like(v2, dtype='float64')
qigood = v2 > 0
ratio[qigood] = inp[qkey1][qigood] / v2[qigood]
mapped.addColumn(self.qIndexRatio, ratio)
namespace[self.outputName] = mapped
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
fdialog = wx.FileDialog(
None,
'Please select PSF to use ...',
#defaultDir=os.path.split(self.image.filename)[0],
wildcard='PSF Files|*.psf|TIFF files|*.tif',
style=wx.FD_OPEN)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
psfn = filename = fdialog.GetPath()
mdh = inp.mdh
if mdh.getEntry('Analysis.FitModule') not in [
'SplitterFitInterpBNR'
]:
Warn('Plugin works only for Biplane analysis')
return
fitMod = __import__(
'PYME.localization.FitFactories.' +
mdh.getEntry('Analysis.FitModule'),
fromlist=['PYME', 'localization', 'FitFactories'])
fr = populate_fresults(fitMod, inp)
progress = wx.ProgressDialog("calculating photon numbers",
"calculating...",
maximum=100,
parent=None,
style=wx.PD_SMOOTH | wx.PD_AUTO_HIDE)
nph = nPhotons(fitMod,
fr,
mdh,
psfname=psfn,
nmax=1e6,
progressBar=progress,
updateStep=100)
progress.Destroy()
mapped.addColumn('nPhotons', nph)
mapped.addColumn('fitResults_background',
inp['fitResults_bg'] + inp['fitResults_br'])
#mapped.addColumn('sig',float(137.0)+np.zeros_like(inp['x'])) # this one is a straight kludge for mortensenError
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
inp = namespace[self.inputData]
fiducial = namespace[self.inputFiducials]
mapped = tabular.mappingFilter(inp)
out_f = tabular.mappingFilter(fiducial)
for dim in ['x', 'y', 'z']:
fiducial_dim = finterpDS(inp, fiducial, 'fiducial_%s' % dim)
mapped.addColumn('fiducial_%s' % dim, fiducial_dim)
mapped.addColumn(dim, inp[dim] - fiducial_dim)
out_f.setMapping(dim, '{0} - fiducial_{0}'.format(dim))
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
out_f.mdh = fiducial.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
namespace[self.outputFiducials] = out_f
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
timedSpecies = self.populateTimedSpecies()
mapped.addColumn('ColourNorm', np.ones_like(mapped['t'], 'float'))
for species in timedSpecies:
mapped.addColumn('p_%s' % species['name'],
(mapped['t'] >= species['t_start']) *
(mapped['t'] < species['t_end']))
if 'mdh' in dir(inp):
mapped.mdh = inp.mdh
mapped.mdh['TimedSpecies'] = timedSpecies
namespace[self.outputName] = mapped
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
for dim in ['x', 'y', 'z']:
try:
mapped.addColumn(dim, inp[dim] - inp['fiducial_%s' % dim])
except:
logger.warn('Could not set dim %s' % dim)
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
import PYMEcs.Analysis.trackFiducials as tfs
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
if self.singleFiducial:
# if all data is from a single fiducial we do not need to align
# we then avoid problems with incomplete tracks giving rise to offsets between
# fiducial track fragments
align = False
else:
align = True
t, x, y, z, isFiducial = tfs.extractTrajectoriesClump(
inp,
clumpRadiusVar='error_x',
clumpRadiusMultiplier=self.radiusMultiplier,
timeWindow=self.timeWindow,
clumpMinSize=self.clumpMinSize,
align=align)
rawtracks = (t, x, y, z)
tracks = tfs.AverageTrack(inp,
rawtracks,
filter=self.filterMethod,
filterScale=self.filterScale,
align=align)
# add tracks for all calculated dims to output
for dim in tracks.keys():
mapped.addColumn('fiducial_%s' % dim, tracks[dim])
mapped.addColumn('isFiducial', isFiducial)
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
from scipy.stats import binned_statistic
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
ids = inp[self.IDkey]
t = inp['t']
maxid = int(ids.max())
edges = -0.5 + np.arange(maxid + 2)
resmin = binned_statistic(ids, t, statistic='min', bins=edges)
resmax = binned_statistic(ids, t, statistic='max', bins=edges)
trange = resmax[0][ids] - resmin[0][ids] + 1
mapped.addColumn('trange', trange)
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
if 'mdh' not in dir(inp):
raise RuntimeError('SnrCalculation needs metadata')
else:
mdh = inp.mdh
nph = inp['nPhotons']
bgraw = inp['fitResults_background']
bgph = np.clip((bgraw) * mdh['Camera.ElectronsPerCount'] /
mdh.getEntry('Camera.TrueEMGain'), 1, None)
npixroi = (2 * mdh.getOrDefault('Analysis.ROISize', 5) + 1)**2
snr = 1.0 / npixroi * np.clip(nph, 0, None) / np.sqrt(bgph)
mapped.addColumn('SNR', snr)
mapped.addColumn('backgroundPhotons', bgph)
mapped.mdh = inp.mdh
namespace[self.outputName] = mapped
def execute(self, namespace):
inp = namespace[self.inputName]
valid = namespace[self.inputValid]
mapped = tabular.mappingFilter(inp)
# note: in coalesced data the clumpIndices are float!
# this creates issues in comparisons unless these are converted to int before comparisons are made!!
# that is the reason for the rint and astype conversions below
ids = np.rint(inp[self.IDkey]).astype('i')
validIDs = np.in1d(ids,
np.unique(np.rint(valid[self.IDkey]).astype('i')))
mapped.addColumn('validID',
validIDs.astype('f')) # should be float or int?
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def execute(self, namespace):
from scipy.stats import binned_statistic
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
ids = inp[self.IDkey] # I imagine this needs to be an int type key
prop = inp[self.StatKey]
maxid = int(ids.max())
edges = -0.5 + np.arange(maxid + 2)
resstat = binned_statistic(ids,
prop,
statistic=self.StatMethod,
bins=edges)
mapped.addColumn(self.StatKey + "_" + self.StatMethod, resstat[0][ids])
# propogate metadata, if present
try:
mapped.mdh = inp.mdh
except AttributeError:
pass
namespace[self.outputName] = mapped
def _execute(self, namespace):
# from PYME.util import mProfile
# self._start_time = time.time()
self.trait_setq(**{"_start_time": time.time()})
print("Starting drift correction module.")
if self.multiprocessing:
proccess_count = np.clip(multiprocessing.cpu_count()-1, 1, None)
self.trait_setq(**{"_pool": multiprocessing.Pool(processes=proccess_count)})
locs = namespace[self.input_for_correction]
# mProfile.profileOn(['localisations.py', 'processing.py'])
drift_res = self.calc_corr_drift_from_locs(locs['x'], locs['y'], locs['z'] * (0 if self.flatten_z else 1), locs['t'])
t_shift, shifts = self.rcc(self.shift_max, *drift_res)
# mProfile.profileOff()
# mProfile.report()
if self.multiprocessing:
self._pool.close()
self._pool.join()
# convert frame-to-frame drift to drift from origin
shifts = np.cumsum(shifts, 0)
out = tabular.mappingFilter(namespace[self.input_for_mapping])
t_out = out['t']
# cubic interpolate with no smoothing
dx = interpolate.CubicSpline(t_shift, shifts[:, 0])(t_out)
dy = interpolate.CubicSpline(t_shift, shifts[:, 1])(t_out)
dz = interpolate.CubicSpline(t_shift, shifts[:, 2])(t_out)
if 'dx' in out.keys():
# getting around oddity with mappingFilter
# addColumn adds a new column but also keeps the old column
# __getitem__ returns the new column
# but mappings usues the old column
# Wrap with another level of mappingFilter so the new column becomes the 'old column'
out.addColumn('dx', dx)
out.addColumn('dy', dy)
out.addColumn('dz', dz)
out = tabular.mappingFilter(out)
# out.mdh = namespace[self.input_localizations].mdh
out.setMapping('x', 'x + dx')
out.setMapping('y', 'y + dy')
out.setMapping('z', 'z + dz')
else:
out.addColumn('dx', dx)
out.addColumn('dy', dy)
out.addColumn('dz', dz)
out.setMapping('x', 'x + dx')
out.setMapping('y', 'y + dy')
out.setMapping('z', 'z + dz')
# propagate metadata, if present
try:
out.mdh = locs.mdh
except AttributeError:
pass
namespace[self.outputName] = out
namespace[self.output_drift] = t_shift, shifts
# non essential, only for plotting out drift data
namespace[self.output_drift_plot] = Plot(partial(generate_drift_plot, t_shift, shifts))
namespace[self.output_cross_cor] = self._cc_image
def execute(self, namespace):
inp = namespace[self.inputName]
mapped = tabular.mappingFilter(inp)
namespace[self.outputName] = mapped
