def locate(self, diameter=7, plot_progress=False, **configs):
configs['diameter'] = diameter
# self.locate_configs has the highest priority
configs.update(self.locate_configs)
# Calculate locations
tic = time.time()
# Locate
tp.quiet()
def after_locate(frame_no, features):
# Plot progress if required
if plot_progress:
self.show_text(
self.axes,
'Calculating {}/{} ...'.format(frame_no, self.n_frames))
self.canvas.draw()
console.print_progress(frame_no, self.n_frames)
return features
self.locations = tp.batch(self.objects,
processes=0,
after_locate=after_locate,
**configs)
console.show_status('Locating completed. Configurations:')
console.supplement(configs)
# Clear status
if plot_progress: self.axes.cla()
self.effective_locate_config = configs
def setUpClass(cls):
# Suppress logging messages
tp.quiet()
# Catch attempts to set values on an inadvertent copy of a Pandas object.
make_pandas_strict()
# Make numpy strict
np.seterr('raise')
def setUpClass(cls):
# Suppress logging messages
tp.quiet()
# Catch attempts to set values on an inadvertent copy of a Pandas object.
make_pandas_strict()
# Make numpy strict
np.seterr('raise')
def set_trajectories(self,
link=True,
search_range=2.,
verbose=True,
**kwargs):
df = self.trajectories
if df.empty:
for frame in self.frames:
df = df.append(frame.to_df())
if link:
if not verbose:
tp.quiet(suppress=True)
# display(df)
print(df)
df = df.rename(columns={
'x_p': 'x',
'y_p': 'y',
'framenumber': 'frame'
})
# display(df)
print(df)
df = tp.link_df(df, search_range, **kwargs)
df = df.rename(columns={
'x': 'x_p',
'y': 'y_p',
'frame': 'framenumber'
})
self._trajectories = df
def workerFunc(locData: Optional[pd.DataFrame], searchRange: float,
memory: int) -> Union[pd.DataFrame, None]:
"""Perform tracking
Parameters
----------
locData
Localization data for tracking
searchRange
`search_range` parameter to :py:func:`trackpy.link`
memory
`memory` parameter to :py:func:`trackpy.link`
Returns
-------
Tracked data
"""
if locData is None:
return None
if not locData.size:
ret = locData.copy()
ret["particle"] = []
return ret
trackpy.quiet()
return trackpy.link(locData, search_range=searchRange, memory=memory)
def test_convenience_funcs(self):
trackpy.quiet(True)
self.assertEqual(trackpy.logger.level, logging.WARN)
trackpy.quiet(False)
self.assertEqual(trackpy.logger.level, logging.INFO)
trackpy.ignore_logging()
self.assertEqual(len(trackpy.logger.handlers), 0)
self.assertEqual(trackpy.logger.level, logging.NOTSET)
self.assertTrue(trackpy.logger.propagate)
trackpy.handle_logging()
self.assertEqual(len(trackpy.logger.handlers), 1)
self.assertEqual(trackpy.logger.level, logging.INFO)
self.assertEqual(trackpy.logger.propagate, 1)
def test_convenience_funcs(self):
trackpy.quiet(True)
self.assertEqual(trackpy.logger.level, logging.WARN)
trackpy.quiet(False)
self.assertEqual(trackpy.logger.level, logging.INFO)
trackpy.ignore_logging()
self.assertEqual(len(trackpy.logger.handlers), 0)
self.assertEqual(trackpy.logger.level, logging.NOTSET)
self.assertTrue(trackpy.logger.propagate)
trackpy.handle_logging()
self.assertEqual(len(trackpy.logger.handlers), 1)
self.assertEqual(trackpy.logger.level, logging.INFO)
self.assertEqual(trackpy.logger.propagate, 1)
def link_df(obj, ParameterJsonFile, SearchFixedParticles = False, max_displacement = None, dark_time = None):
""" wrapper for the trackpy routine tp.link, which forms trajectories
out of particle positions, out of the json file
important parameters:
SearchFixedParticles = defines whether fixed or moving particles are under current investigation
dark_time = settings["Link"]["Dark time"] ... maximum number of frames a particle can disappear
max_displacement = ["Link"]["Max displacement"] ...maximum displacement between two frames
"""
settings = nd.handle_data.ReadJson(ParameterJsonFile)
dark_time = settings["Link"]["Dark time"]
if SearchFixedParticles == False:
max_displacement = settings["Link"]["Max displacement"]
else:
max_displacement = settings["Link"]["Max displacement fix"]
# here comes the linking
nd.logger.info("Linking particles to trajectories: starting...")
if nd.logger.getEffectiveLevel() >= 20:
# Switch the logging of for the moment
tp.quiet(suppress=True)
t1_orig = tp.link_df(obj, max_displacement, memory=dark_time)
nd.logger.info("Linking particles to trajectories: ...finished")
if nd.logger.getEffectiveLevel() >= 20:
# Switch the logging off for the moment
tp.quiet(suppress=False)
nd.handle_data.WriteJson(ParameterJsonFile, settings)
return t1_orig
import unittest
import numpy as np
import pandas as pd
from numpy.testing import assert_allclose
from trackpy import quiet
from trackpy.utils import validate_tuple
from trackpy.refine import refine_com, refine_leastsq
from trackpy.artificial import (feat_gauss, rot_2d, rot_3d, draw_feature,
draw_cluster, SimulatedImage)
from trackpy.refine.least_squares import (dimer, trimer, tetramer, dimer_global,
FitFunctions, vect_from_params)
from trackpy.tests.common import assert_coordinates_close
from scipy.optimize.slsqp import approx_jacobian
from nose import SkipTest
quiet()
EPSILON = 1E-7
ATOL = 0.001
RTOL = 0.01
SIGNAL = 160
NOISE_IMPERFECT = 16 # S/N of 10
NOISE_NOISY = 48 # S/N of 3
DISC_SIZE = 0.5
RING_THICKNESS = 0.2
SIZE_2D = 4.
SIZE_3D = 4.
SIZE_2D_ANISOTROPIC = (5., 3.)
SIZE_3D_ANISOTROPIC = (3., 5., 5.)
def __init__(self,
excitation_seq: str = "da",
registrator: Optional[multicolor.Registrator] = None,
link_radius: float = 5,
link_mem: int = 1,
min_length: int = 1,
feat_radius: int = 4,
bg_frame: int = 2,
bg_estimator: Union[str, Callable[[np.ndarray],
float]] = "mean",
neighbor_radius: Union[float, str] = "auto",
interpolate: bool = True,
coloc_dist: float = 2.0,
acceptor_channel: int = 2,
link_quiet: bool = True,
link_options: Dict = {},
columns: Dict = {}):
"""Parameters
----------
excitation_seq
Set the :py:attr:`excitation_seq` attribute.
registrator
Registrator used to overlay channels. If `None`, use the identity
transform.
link_radius
Maximum movement of features between frames. See `search_range`
option of :py:func:`trackpy.link_df`.
link_mem
Maximum number of frames for which a feature may not be detected.
See `memory` option of :py:func:`trackpy.link_df`.
min_length
Minimum length of tracks.
feat_radius
Radius of circle that is a little larger than features. See
`radius` option of :py:func:`brightness.from_raw_image`.
bg_frame
Size of frame around features for background determination. See
`bg_frame` option of :py:func:`brightness.from_raw_image`.
bg_estimator
Statistic to estimate background. See `bg_estimator` option of
:py:func:`brightness.from_raw_image`.
neighbor_radius
How far two features may be apart while still being considered
close enough so that one influences the brightness measurement of
the other. This is related to the `radius` option of
:py:func:`brightness.from_raw_image`. If "auto", use the smallest
value that avoids overlaps.
interpolate
Whether to interpolate coordinates of features that have been
missed by the localization algorithm.
coloc_dist
After overlaying donor and acceptor channel features, this gives
the maximum distance up to which donor and acceptor signal are
considered to come from the same molecule.
acceptor_channel
Whether the acceptor channel is number 1 or 2 in `registrator`.
link_options
Specify additional options to :py:func:`trackpy.link_df`.
"search_range" and "memory" will be overwritten by the
`link_radius` and `link_mem` parameters.
link_quiet
If `True`, call :py:func:`trackpy.quiet`.
Other parameters
----------------
columns
Override default column names as defined in
:py:attr:`config.columns`. Relevant names are `coords`, `time`,
`mass`, `signal`, `bg`, `bg_dev`. This means, if your DataFrame has
coordinate columns "x" and "z" and the time column "alt_frame", set
``columns={"coords": ["x", "z"], "time": "alt_frame"}``. This
parameters sets the :py:attr:`columns` attribute.
"""
self.frame_selector = multicolor.FrameSelector(excitation_seq)
self.registrator = (registrator if registrator is not None else
multicolor.Registrator())
self.link_options = link_options.copy()
self.link_options["search_range"] = link_radius
self.link_options["memory"] = link_mem
self.min_length = min_length
self.brightness_options = dict(radius=feat_radius,
bg_frame=bg_frame,
bg_estimator=bg_estimator,
mask="circle")
self.interpolate = interpolate
self.coloc_dist = coloc_dist
self.acceptor_channel = acceptor_channel
self.columns = columns
if isinstance(neighbor_radius, str):
# auto radius
neighbor_radius = 2 * feat_radius + 1
self.neighbor_radius = neighbor_radius
if link_quiet and trackpy_available:
trackpy.quiet()
import json
import trackpy as tp
tp.quiet(suppress=True)
import matplotlib.pyplot as plt
import numpy as np
import glob
import pandas as pd
import os
# Import data from JSON files into DataFrame
def loadJSON(directory, height):
array = np.empty([0, 3])
files = sorted(glob.glob(os.path.join(directory, '*.json')))
for j in range(0, len(files)):
with open(files[j], 'r') as read_file:
data = json.load(read_file)
for i in range(0, len(data)):
x = (data[i]['bottomright']['x'] + data[i]['topleft']['x']) / 2
y = (data[i]['bottomright']['y'] + data[i]['topleft']['y']) / 2
#y = height-(data[i]['bottomright']['y'] + data[i]['topleft']['y'])/2
array = np.vstack([array, [j, x, y]])
array = array.astype(int)
dataset = pd.DataFrame({
'frame': array[:, 0],
'x': array[:, 1],
'y': array[:, 2]
})
return dataset
Created on Wed Feb 27 10:51:28 2019
@author: Aswin Muralidharan
"""
# For compatibility with Python 2 and 3
from __future__ import division, unicode_literals, print_function
import pims
import trackpy as tp
import os
Filepath = '/Volumes/Samsung_T5/Experimental Data/Hans'
bp = 'MCF10A500bp' # Base pair to process
directory = Filepath + '/D_ROI_tiff/' + str(
bp) # The input directory after background image corrections
tp.quiet() # Set the trackpy steps off so that the code runs faster
for filename in os.listdir(directory):
"""Import data series """
print(filename)
frames = pims.ImageSequence(os.path.join(directory, filename) + '/*.tif',
as_grey=True)
f = tp.batch(frames[0:350],
11,
minmass=200,
maxsize=4,
noise_size=1,
smoothing_size=15)
"""Link features into particle trajectories"""
t = tp.link_df(f, 2, memory=3)
""" max displacement 2 pixels
# memory missed particle is 3"""
t.to_csv('./' + CaseName + 'Left.csv')
listParticle = list(t['particle'])
listParticle = list(set(listParticle))
print('There are %d trajectories' % len(listParticle))
# %% ---------------------------------------- Right Camera
f1 = tp.locate(frames[87], 21, 7000)
plt.figure()
tp.annotate(f1, frames[87]);
# %% run trajectory finding for Right
estimateFeatureSizeRight = 29
CameraName = 'Right'
tp.quiet()
f, frames = Detect(estimateFeatureSizeRight, CameraName, dynamicMinMass = True)
# %%
f = f[['y', 'x', 'frame']]
searchRange = 70
memory = 40
minFrames = 40
t = Link(searchRange, memory, minFrames)
# %% filter trajectory by minimum moving distance
minFrames = 10
minDistance = 1000
t = filterTrajectory(t, minDistance, minFrames)
plt.figure()
tp.plot_traj(t)
t.to_csv('./' + CaseName + 'Right.csv')
listParticle = list(t['particle'])
def link(path, FPS, MPP, SEARCH_RANGE_MICRONS, MEMORY, STUBS, MIN_VELOCITY, MIN_AREA, MAX_AREA):
"""
Given the path of a csv file output from ImageJ particle analysis, link, filter, and label the trajectories.
Outputs a dataframe for the csv file analyzed.
"""
df = pd.read_csv(path)
# Pull info from filename
filename = path.split('\\')[-1].split('.')[0]
print(f'Processing {path}')
# Clean up output data from fiji. It seems to change the label column randomly, so I've accounted for both here.
if "slice" in df['Label'].values[0]:
# Clean up results df for 1:slice:23 format
df['Label'] = df['Label'].astype('str')
df['Label'] = df['Label'].str.split(':').str[2]
df['frame'] = df['Label'].astype('int')
elif type(df['Label'].values[0]) != str:
# The column is already in int format
df['frame'] = df['Label']
elif df['Label'].values[0].split(' ')[-1] == 's': # if the exported csv has time data instead, convert back to frames...
# Other format
df['imagejtime'] = df['Label'].str.split(' ').str[0].str.split(':').str[-1].astype(float)
df['approx_frame'] = df['imagejtime'] * FPS
df['frame'] = df['approx_frame'].round().astype(int)
else:
# Other format
df['frame'] = df['Label'].str.split('_').str[2].str.lstrip('0').str.rstrip('.tif')
df['frame'] = pd.to_numeric(df['frame'])
df.drop(labels=['Label', ' '], inplace=True, axis=1)
df.rename({'X': 'x', 'Y': 'y'}, axis=1, inplace=True)
# Actual Linking
search_range = round(SEARCH_RANGE_MICRONS / MPP / FPS) # number of pixels away to look for the blob in each frame
print(f'Search range --> {search_range} pixels.')
tp.quiet() # supress output, makes linking quicker
t = tp.link_df(df, search_range=search_range, memory=MEMORY)
t1 = tp.filter_stubs(t, STUBS)
print('Before filtering:', t['particle'].nunique())
print('After filtering:', t1['particle'].nunique())
# Calculate velocities.
df_vel = calc_velocity(t1)
df_vel.index.name = None
t1.index.name = None
t2 = t1.merge(df_vel, on=['particle', 'frame', 'x', 'y'], how='left')
# Calculate useful quantities
t2['time'] = t2['frame'] / FPS
t2['dv'] = np.sqrt(t2['dx'] ** 2 + t2['dy'] ** 2) # total velocity in pix/frame
t2['dv_m'] = t2['dv'] * FPS * MPP # total velocity in microns/s
t2['Area_m'] = t2['Area'] * MPP ** 2 # area in microns^2
t2['dx_m'] = t2['dx'] * FPS * MPP # x velocity in microns/s, defined as the left/right movement.
# Keep wheels larger than MIN_AREA
bp = t2.groupby('particle').mean()['Area_m'] > MIN_AREA # bp = big_particles
bp = bp[bp].index.values
t2 = t2[t2['particle'].isin(bp)]
# Eliminate wheels smaller than MAX_AREA
hp = t2.groupby('particle').max()['Area_m'] > MAX_AREA # hp = huge particles
hp = hp[hp].index.values
t2 = t2[~t2['particle'].isin(hp)] # The unary (~) operator negates the conditional, i.e. takes everything except the huge particles
# Eliminate wheels whose means are slower than MIN_VELOCITY
# fp = t2.groupby('particle').mean()['dx_m'] > MIN_VELOCITY # fp = fast_particles
# fp = fp[fp].index.values
# t2 = t2[t2['particle'].isin(fp)]
t2['filename'] = filename
# Make a unique particle column, to prevent interference between videos.
t2['particle_u'] = t2['filename'] + '-' + t2['particle'].astype(str)
return t2
def OptimizeMinmassInTrackpy(img1,
diameter,
separation,
num_particles_zncc,
pos_particles,
minmass_start=1,
DoPreProcessing=True,
percentile=64,
DoLog=True):
"""
the particles are found accurately by zncc, which is accurate but time consuming
trackpy is faster but needs proper threshold to find particles - minmass
start with a low threshold and increase it till the found particles by zncc are lost
DoLog is required, because this function is sometimes called in parallel, where the logging fails
"""
#start value
minmass = minmass_start
# First Particle that NCC has but not trackpy
First_wrong_assignment = True
# First iteration
first_iteration = True
# loop exiting variable
stop_optimizing = False
# maximum distance between position of zncc and trackpy
if type(diameter) == int:
max_distance = diameter / 2
else:
max_distance = diameter[
0] / 2 # assume that diameters for x and y direction are equal
# save the optimal minmass
minmass_optimum = 0
# save the history
right_found_save = []
wrong_found_save = []
# wrong_to_right_save = []
minmass_save = []
if DoLog: nd.logger.info("Separation: %s", separation)
percentile = 0
if DoLog: nd.logger.info("percentile: %s", percentile)
# switch logger ouf for this optimization
if nd.logger.getEffectiveLevel() >= 20:
# Switch the logging of for the moment
tp.quiet(suppress=True)
count_loops = 0
# run the following till the optimization is aborted
while stop_optimizing == False:
# here comes trackpy.
# Trackpy is not running in parallel mode, since it loads quite long and we have only one frame here
if count_loops % 10 == 0:
#plot every ten iterations
if DoLog:
nd.logger.info("Iteration: %s with minmass: %s", count_loops,
minmass)
else:
if DoLog:
nd.logger.debug("Iteration: %s with minmass: %s", count_loops,
minmass)
count_loops = count_loops + 1
output = tp.batch(img1,
diameter,
minmass=minmass,
separation=separation,
max_iterations=10,
preprocess=DoPreProcessing,
percentile=percentile)
# num of found particles by trackpy
num_particles_trackpy = len(output)
if first_iteration == True:
# sanity check
if num_particles_trackpy < num_particles_zncc:
stop_optimizing = True
if DoLog:
nd.logger.error(
"Trackpy finds too few particles. Possible reasons: \n - start value of minmass is too low (unlikely) \n - specimen is too dense/too highly concentrated \n - Percentile filter gives you problems in tp.batch or tp.locate."
)
raise ValueError("")
# reset counters
# right_found: particle is found in ncc and trackpy. Location mismatch is smaller than diameter
right_found = 0
# wrong_found: particle only in zncc or trackpy found. This is not good
wrong_found = 0
# if there are more particles found by trackpy than in zncc. The difference adds to the number of wrong_found particles. If zncc finds more than trackpy, they are counted later, when a localized particle in zncc does not have a corresponding point in trackpy.
if num_particles_trackpy > num_particles_zncc:
num_particle_only_trackpy_finds = num_particles_trackpy - num_particles_zncc
else:
num_particle_only_trackpy_finds = 0
if DoLog:
nd.logger.debug("Found particles (trackpy): %s",
num_particles_trackpy)
if DoLog:
nd.logger.debug("Found particles (zncc): %s", num_particles_zncc)
if num_particles_trackpy > (5 * num_particles_zncc):
# if far too many particles are found the threshold must be increased significantly
if DoLog:
nd.logger.debug(
"5 times more feactures than expected. Enhance threshold!")
# + 1 is required to ensure that minmass is increasing, although the value might be small
minmass = np.int(minmass * 1.5) + 1
elif num_particles_trackpy > (2 * num_particles_zncc):
if DoLog:
nd.logger.debug(
"2 times more feactures than expected. Enhance threshold!")
minmass = np.int(minmass * 1.2) + 1
else:
# trackpy and znnc have similar results. so make some find tuning in small steps
# check for every particle found by zncc if trackpy finds a particle too, withing the diameter
for id_part, pos in enumerate(pos_particles):
pos_y, pos_x, frame = pos
# choose correct frame
output_frame = output[output.frame == frame]
# get distance to each particle found by trackpy
dist_each_det_particle = np.hypot(output_frame.y - pos_y,
output_frame.x - pos_x)
# get the nearest
closest_agreement = np.min(dist_each_det_particle)
# check if closer than the maximum allowed distance
if closest_agreement > max_distance:
# particle is to far away. That is not good. So a particle is wrong assigned
wrong_found = wrong_found + 1
# show position of first wrong particle. If more are plotted the console is just overfull
if First_wrong_assignment == True:
First_wrong_assignment = False
if DoLog:
nd.logger.debug(
"Particle found in ZNCC but not with trackpy")
if DoLog:
nd.logger.debug("Problem with particle: %s",
id_part)
if DoLog: nd.logger.debug("Position: %s", pos)
if DoLog:
nd.logger.debug("Closest point: %s",
closest_agreement)
else:
# This is what you want. Particle found by zncc and trackpy within a neighborhood.
# right found + 1
right_found = right_found + 1
# add number of particles trackpy finds to much
wrong_found = wrong_found + num_particle_only_trackpy_finds
# get the ratio of wrong to right assignments. This should be as small as possible
if right_found > 0:
wrong_to_right = wrong_found / right_found
else:
wrong_to_right = np.inf
#save numbers for plotting later on
right_found_save = np.append(right_found_save, right_found)
wrong_found_save = np.append(wrong_found_save, wrong_found)
# wrong_to_right_save = np.append(wrong_to_right_save, wrong_to_right)
minmass_save = np.append(minmass_save, minmass)
if DoLog: nd.logger.debug("right_found: %s", right_found)
if DoLog: nd.logger.debug("wrong_found: %s", wrong_found)
if DoLog:
nd.logger.debug("Wrong to right assignment: %s",
wrong_to_right)
if DoLog: nd.logger.debug("Still optimizing...")
# check how value is changing
# if Wrong_to_right > Wrong_to_right_optimum:
if (wrong_found
== 0) | (num_particles_trackpy < 0.8 * num_particles_zncc):
if DoLog:
nd.logger.debug(
"TrackPy finds much less particles than the zncc")
#value increasing so abort loop
stop_optimizing = True
plt.figure()
plt.plot(minmass_save,
right_found_save,
'.-',
label='right found')
plt.plot(minmass_save,
wrong_found_save,
'.-',
label='wrong found')
plt.xlabel("Minmass")
plt.ylabel("Number particles")
plt.legend()
plt.figure()
wrong_to_right_save = wrong_found_save / right_found_save
plt.plot(minmass_save, wrong_to_right_save, '.-')
plt.xlabel("Minmass")
plt.ylabel("Wrong-to-right")
pos_minmass_optimum = np.where(
wrong_to_right_save == np.min(wrong_to_right_save))[0][0]
minmass_optimum = minmass_save[pos_minmass_optimum]
# enhance minmass for next iteration
minmass = np.int(minmass * 1.02) + 10
first_iteration = False
#leave a bit of space to not work at the threshold
minmass_optimum = np.int(minmass_optimum * 0.90)
if DoLog:
nd.logger.info("Optimized Minmass threshold is: %s", minmass_optimum)
obj_all = tp.batch(img1,
diameter,
minmass=minmass_optimum,
separation=diameter,
max_iterations=10,
preprocess=DoPreProcessing)
if nd.logger.getEffectiveLevel() >= 20:
# Switch the logging back on
tp.quiet(suppress=False)
# num of found particles by trackpy
num_particles_trackpy = len(obj_all)
return minmass_optimum, num_particles_trackpy, obj_all
import pandas as pd
from pandas import DataFrame, Series
import unittest
import nose
from numpy.testing import assert_almost_equal, assert_allclose
from numpy.testing.decorators import slow
from pandas.util.testing import (assert_series_equal, assert_frame_equal,
assert_almost_equal, assert_produces_warning)
import trackpy as tp
from trackpy.try_numba import NUMBA_AVAILABLE
from trackpy.linking import PointND, Hash_table
from trackpy.utils import is_pandas_since_016, pandas_sort, validate_tuple
# suppress logging messages
tp.quiet()
# Catch attempts to set values on an inadvertent copy of a Pandas object.
tp.utils.make_pandas_strict()
path, _ = os.path.split(os.path.abspath(__file__))
path = os.path.join(path, 'data')
# Call lambda function for a fresh copy each time.
unit_steps = lambda: [[PointND(t, (x, 0))] for t, x in enumerate(range(5))]
np.random.seed(0)
random_x = np.random.randn(5).cumsum()
random_x -= random_x.min() # All x > 0
max_disp = np.diff(random_x).max()
