def test_storage(self):
STORE_NAME = 'temp_for_testing_{0}.h5'.format(_random_hash())
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v, output=s, engine='python', meta=False,
**self.PARAMS)
self.assertEqual(len(s), 2)
self.assertEqual(s.max_frame, 1)
count_total_dumped = s.dump()['frame'].nunique()
count_one_dumped = s.dump(1)['frame'].nunique()
self.assertEqual(count_total_dumped, 2)
self.assertEqual(count_one_dumped, 1)
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
# Putting an empty df should warn
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('ignore')
warnings.simplefilter('always', UserWarning)
s.put(pandas.DataFrame())
assert len(w) == 1
s.close()
os.remove(STORE_NAME)
def test_storage(self):
STORE_NAME = 'temp_for_testing_{0}.h5'.format(_random_hash())
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v[[0, 1]],
*self.PARAMS,
output=s,
engine='python',
meta=False)
self.assertEqual(len(s), 2)
self.assertEqual(s.max_frame, 1)
count_total_dumped = s.dump()['frame'].nunique()
count_one_dumped = s.dump(1)['frame'].nunique()
self.assertEqual(count_total_dumped, 2)
self.assertEqual(count_one_dumped, 1)
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
s.close()
os.remove(STORE_NAME)
def test_storage(self):
STORE_NAME = 'temp_for_testing_{0}.h5'.format(_random_hash())
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v,
output=s,
engine='python',
meta=False,
**self.PARAMS)
self.assertEqual(len(s), 2)
self.assertEqual(s.max_frame, 1)
count_total_dumped = s.dump()['frame'].nunique()
count_one_dumped = s.dump(1)['frame'].nunique()
self.assertEqual(count_total_dumped, 2)
self.assertEqual(count_one_dumped, 1)
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
# Putting an empty df should warn
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('ignore')
warnings.simplefilter('always', UserWarning)
s.put(pandas.DataFrame())
assert len(w) == 1
s.close()
os.remove(STORE_NAME)
def run_particle_detection(self, overwrite=False):
print('Run particle detection')
invert = self.gb_detection.check_invert.isChecked()
diameter = self.gb_detection.spn_diameter.value()
minmass = self.gb_detection.spn_minmass.value()
gv.h5f.attrs[gv.KEY_ATTR_DETECT_INV] = invert
gv.h5f.attrs[gv.KEY_ATTR_DETECT_DIA] = diameter
gv.h5f.attrs[gv.KEY_ATTR_DETECT_MINMASS] = minmass
print(f'Invert {invert}')
print(f'Diameter {diameter}')
print(f'Minmass {minmass}')
trackpy_filepath = file_handling.get_trackpy_path()
mode = 'a'
# If detection file exists, ask if should be removed
if os.path.exists(trackpy_filepath):
if not (overwrite):
confirm_dialog = QtWidgets.QMessageBox.question(
gv.w, 'Overwrite?',
'Previous particle detection exists. Overwrite?',
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No
| QtWidgets.QMessageBox.Cancel, QtWidgets.QMessageBox.No)
if confirm_dialog \
in [QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.Cancel]:
return
print('Remove previous detection file')
if gv.tpf is not None:
gv.tpf.close()
gv.tpf = None
mode = 'w'
# Disable window
gv.statusbar.showMessage('Run particle detection')
gv.w.setEnabled(False)
gv.app.processEvents()
# Run detection
with tp.PandasHDFStoreBig(trackpy_filepath, mode=mode) as s:
tp.batch(gv.h5f[gv.KEY_PROCESSED],
diameter,
invert=invert,
minmass=minmass,
output=s,
processes='auto')
# Enable window
gv.w.setEnabled(True)
gv.statusbar.set_ready()
# Open detection file
gv.tpf = tp.PandasHDFStoreBig(trackpy_filepath)
def test_PandasHDFStore(self):
STORE_NAME = 'temp_for_testing.h5'
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = tp.PandasHDFStore(STORE_NAME)
except:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v[[0, 1]], *self.PARAMS,
output=s, engine='python', meta=False)
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
os.remove(STORE_NAME)
def locate(self, diameter=7, plot_progress=False, **configs):
configs['diameter'] = diameter
# self.locate_configs has the highest priority
configs.update(self.locate_configs)
# Calculate locations
tic = time.time()
# Locate
tp.quiet()
def after_locate(frame_no, features):
# Plot progress if required
if plot_progress:
self.show_text(
self.axes,
'Calculating {}/{} ...'.format(frame_no, self.n_frames))
self.canvas.draw()
console.print_progress(frame_no, self.n_frames)
return features
self.locations = tp.batch(self.objects,
processes=0,
after_locate=after_locate,
**configs)
console.show_status('Locating completed. Configurations:')
console.supplement(configs)
# Clear status
if plot_progress: self.axes.cla()
self.effective_locate_config = configs
def example_with_trackpy_and_twv(filename):
"""
Example usecase from input file to particle and trap positions in .dat file.
"""
frames = pims.open(filename)
# Open file with pims. Works with many file extensions.
# This example assumes .twv file.
# metadata = frames.get_all_metadata()
# Optional access to additional metadata.
times, laser_powers, traps = frames.get_all_tweezer_positions()
# Obtain frame times, laser power at each frame time and
# traps powers and positions at each frame.
features = tp.batch(frames, 25, minmass=1000, invert=False)
# Obtain features (particle positions) using trackpy's batch function.
# It is verbose.
# The 25 in arguments is diameter. It be odd number.
# It is recommended to obtain parameters using GUI.
tracks = tp.link_df(features, 15, memory=10)
# Joins particles positions to tracks (connects them in time).
# See trackpy documentation for parameters.
save_tracked_data_pandas(filename[:-4] + '_out.dat', frames, tracks, times,
laser_powers, traps)
def generate_trajectories(self, subject='particles', plot=True, export=True, invert=True, memory=50):
"""
:param subject:
:param plot:
:param export:
:param invert:
:param memory:
:return:
"""
fv = tp.batch(self.image_sequence, self.particlesize, minmass=self.particleminmass, invert=invert)
t = tp.link_df(fv, 5, memory=memory)
if subject == 'particles':
self.particle_trajectory_list = TrajectorySequence(t)
elif subject == 'algae':
self.algae_trajectory_list = TrajectorySequence(t)
else:
raise Exception('The argument subject of the current method '
'(generate_trajectories) must be either particles or algae')
if plot:
tp.plot_traj(t, label=True)
plt.show()
if export:
t.to_csv(self.path + '\\t_' + subject + '.csv', index=None, header=True)
if subject == 'particles':
return self.particle_trajectory_list
if subject == 'algae':
return self.algae_trajectory_list
def extract_tracks_from_frames(frames):
"""Extract tracks as dataframe from movie frames.
Parameters
----------
frames: np.array
les différentes images chargées depuis le fichier .tif
en tant que numpy array
Returns
-------
tracks: pd.DataFrame
les déplacements des nanoparticules dans le film sous la forme
d'un dataframe pandas. C'est trackpy qui fait la plupart du travail
"""
Nbframe = frames.shape[0]
print('[extract.py] Running extraction from tracks')
print('Second Step: Localizaton of the nanoparticles in each frame')
f = tp.batch(frames[:Nbframe], 7, minmass=150, preprocess=False)
print('Third Step: Computation of the trajectories')
tracks = tp.link_df(f, search_range=5, memory=7)
# PB avec le choix des paramètres de memory. Certaines trajectoires sont mal interprétées
# exemple du film 411 : avec memory = 7, on a une traj qui commence sur un stop bordélique
# avec memory = 2, la trajectoire passe au dessus du stop en question : c'est ce qui se passe
# en réalité si on regarde le film. Le pb (je crois) est lié au fait que link_df ne prédit pas
# la prochaine position en fonction de la vitesse. Ca doit être possible mais comment ?
return tracks
def track_features(images, diameter=5, **kwargs):
"""Locate features - call them blobs, detections - in the given images.
This is a wrapper for the `trackpy.batch` function.
See its documentation for parameters and more information.
Parameters:
----------
diameter: odd int
the approximate diameter, in pixels, of the features to detect.
any keyword argument to be passed to the trackpy ``batch`` function
Returns:
--------
features: pd.DataFrame
the list of detected features
"""
df = tp.batch(images, diameter=diameter, **kwargs)
kwargs['diameter'] = diameter
features = cats.particles.Particles(df)
# Set up element attributes to be added once the particles are tracked
features._element_attributes = {
'source': images,
'tracking_parameters': kwargs
}
return features
def batch(self, array, filter=True, store=True):
# Check odd numbers
self.diameter = _nbr2odd(self.diameter)
# Run TrackPy
dataframe = tp.batch(
array,
self.diameter,
minmass=self.minmass,
maxsize=self.maxsize,
separation=self.separation,
noise_size=self.noise_size,
smoothing_size=self.smoothing_size,
threshold=self.threshold,
invert=self.invert,
percentile=self.percentile,
topn=self.topn,
preprocess=self.preprocess,
max_iterations=self.max_iterations,
characterize=self.characterize,
engine=self.engine,
)
# Store in the instance
if store:
self.spots = deepcopy(dataframe)
self.tracks = deepcopy(dataframe)
# Filter the trajectory
if filter:
dataframe = self.filter(dataframe, store=store)
return dataframe
def Detect(estimateFeatureSize, CameraName, minMass = None, dynamicMinMass = False):
ImagePath = 'E:/BubbleRisingUltimate/4mmGasBubbleRising/4mmGasBubbleRising'
Path = os.path.join(ImagePath, CameraName)
frames = pims.open(Path)
print('Valid frames length is %d' %len(frames))
# find five brightest
if not minMass:
f = tp.locate(frames[testFrame], estimateFeatureSize)
mass = list(f['mass']); mass.sort()
minMass = int(mass[-3]*0.9 + mass[-1]*0.1)
print(minMass)
#TopTen = np.argsort(f['mass'])[-5:]
#TopTenArray = f['mass'][TopTen]
# show mass histogram
# show subpixel accuracy of the detection
#minMass = list(TopTenArray)[0]
f = tp.locate(frames[testFrame], estimateFeatureSize, minmass= minMass)
plt.figure()
tp.annotate(f, frames[testFrame]);
# run batch processing for all frames
if dynamicMinMass:
f = f[0:0]
for i in range(len(frames)):
f_ele = tp.locate(frames[i], estimateFeatureSize)
mass = list(f_ele['mass']); mass.sort()
minMass = int(mass[-2]*0.9 + mass[-1]*0.1)
f_ele = tp.locate(frames[i], estimateFeatureSize, minmass = minMass)
f = f.append(f_ele)
else:
f = tp.batch(frames, estimateFeatureSize, minmass = minMass)
return f, frames
def _locate_batch_wrapper(diameter: int, minmass: int, maxmass: int,
maxsize: float, img_seq: np.ndarray) -> pd.DataFrame:
df = trackpy.batch(img_seq,
diameter=diameter,
minmass=minmass,
maxsize=maxsize)
df = df[df['mass'] < maxmass]
return df
def prepare(self):
directory = os.path.join(path, 'video', 'image_sequence')
self.v = tp.ImageSequence(directory)
self.PARAMS = (11, 3000)
self.expected = tp.batch(self.v[[0, 1]],
*self.PARAMS,
engine='python',
meta=False)
def setUpClass(cls):
super(TestReproducibility, cls).setUpClass()
# generate a new file
video = pims.ImageSequence(
os.path.join(path, 'video', 'image_sequence'))
actual = tp.batch(invert_image(video), diameter=9, minmass=240)
actual = tp.link_df(actual, search_range=5, memory=2)
actual.to_csv(reproduce_fn)
def prepare(self):
directory = os.path.join(path, 'video', 'image_sequence')
self.v = ImageSequence(os.path.join(directory, '*.png'))
# mass depends on pixel dtype, which differs per reader
minmass = self.v[0].max() * 2
self.PARAMS = {'diameter': 11, 'minmass': minmass, 'invert': True}
self.expected = tp.batch(self.v[[0, 1]], engine='python', meta=False,
**self.PARAMS)
def prepare(self):
directory = os.path.join(path, 'video', 'image_sequence')
self.v = tp.invert_image(ImageSequence(os.path.join(directory, '*.png')))
# mass depends on pixel dtype, which differs per reader
minmass = self.v[0].max() * 2
self.PARAMS = {'diameter': 11, 'minmass': minmass}
self.expected = tp.batch(self.v[[0, 1]], engine='python', meta=False,
**self.PARAMS)
def detect(self,
images): ## and has same output format as YOLO localizer
circles = []
for n in range(np.shape(images)[0]):
norm = images[n]
circ = ct.circletransform(norm, theory='orientTrans')
circles.append(circ / np.amax(circ))
return tp.batch(circles, 51, minmass=50)
def prepare(self):
directory = os.path.join(path, 'video', 'image_sequence')
v = TrackpyImageSequence(os.path.join(directory, '*.png'))
self.v = [tp.invert_image(v[i]) for i in range(2)]
# mass depends on pixel dtype, which differs per reader
minmass = self.v[0].max() * 2
self.PARAMS = {'diameter': 11, 'minmass': minmass}
self.expected = tp.batch(self.v, engine='python', meta=False,
**self.PARAMS)
def test_storage(self):
STORE_NAME = 'temp_for_testing_{0}.h5'.format(_random_hash())
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v[[0, 1]], *self.PARAMS,
output=s, engine='python', meta=False)
if len(s) != 2:
raise
assert len(s) == 2
assert s.max_frame == 1
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
s.close()
os.remove(STORE_NAME)
def nps_finder_tp_cluster(image_file, z_extent, filter_kernel, min_sep):
files, metadata = preprocessor.data_org(tifdir)
columns = ('x', 'y', 'z', 'a_total', 'intensity', 'particle', 'com_dist',
'trial')
all_parts = pd.DataFrame([], columns=columns)
for i in range(0, len(files)):
print(i)
print(files[i])
frames = pims.TiffStack('./tifs/' + folders[m] + '/tifs/' + files[i])
# nps = np.array(frames[::2])
# nuclei = np.array(frames[1::2])
features = tp.batch(nps[:], diameter=filter_kernel, separation=min_sep)
# Option to add a mass cut
# features = features[features['raw_mass'] > 0]
if len(features) == 0:
features['particle'] = []
if len(features) == 1:
features['particle'] = 1
elif len(features) > 1:
# Clustering to eliminate maxima that are too close together
positions = features[['x', 'y', 'frame']].values
# Distance matrix is n-particles x n-particles in size - reckon it gives the interparticle separation
# This gives the upper triangle of the distance matrix
dist_mat = dist.pdist(positions)
link_mat = hier.linkage(dist_mat)
# fcluster assigns each of the particles in positions a cluster to which it belongs
cluster_idx = hier.fcluster(link_mat, 5, criterion='distance')
features['particle'] = cluster_idx
n_parts = np.unique(features['particle'].values)
values_clustered = []
for j in n_parts:
current = features[features['particle'] == j]
if len(current) > z_extent:
norm = np.sum(current['raw_mass'])
x_av = np.mean(current['x'])
y_av = np.mean(current['y'])
# z-value is the intensity-weighted value
z_av = np.sum(current['raw_mass'] * current['frame']) / norm
a_total = np.sum(current['size'])
# Geometric cut-off to particle size
if a_total > 1:
values_clustered.append(
[x_av, y_av, z_av, a_total, norm, j])
columns_clustered = ('x', 'y', 'z', 'a_total', 'intensity', 'particle')
particles_clustered = pd.DataFrame(values_clustered,
columns=columns_clustered)
def prepare(self, batch_params=None):
directory = os.path.join(path, 'video', 'image_sequence')
v = TrackpyImageSequence(os.path.join(directory, '*.png'))
self.v = [tp.invert_image(v[i]) for i in range(2)]
# mass depends on pixel dtype, which differs per reader
minmass = self.v[0].max() * 2
self.PARAMS = {'diameter': 11, 'minmass': minmass}
if batch_params is not None:
self.PARAMS.update(batch_params)
self.expected = tp.batch(self.v,
engine='python',
meta=False,
**self.PARAMS)
def test_storage(self):
STORE_NAME = 'temp_for_testing_{0}.h5'.format(_random_hash())
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v[[0, 1]], output=s, engine='python', meta=False,
**self.PARAMS)
self.assertEqual(len(s), 2)
self.assertEqual(s.max_frame, 1)
count_total_dumped = s.dump()['frame'].nunique()
count_one_dumped = s.dump(1)['frame'].nunique()
self.assertEqual(count_total_dumped, 2)
self.assertEqual(count_one_dumped, 1)
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
s.close()
os.remove(STORE_NAME)
def batch_locate_particles(self, frames, diameter, minmass, filename='data.h5',
progress_listener=lambda *args: None):
self.log.info('Starting particle tracking to file: ' + filename)
self.log.info('Diameter: ' + str(diameter) + "; Minmass: " + str(minmass))
def batches_gen(iterator, size):
batch = []
i = 0
for item in iterator:
batch.append(item)
i += 1
if i == size:
yield batch
batch = []
i = 0
with trackpy.PandasHDFStore(filename, t_column='frame') as s:
for batch in batches_gen(frames, 5):
trackpy.batch(batch, diameter, minmass=minmass, invert=True,
output=s, engine='numba')
self.log.info('Particles located')
return self
def test_storage(self):
STORE_NAME = 'temp_for_testing.h5'
if os.path.isfile(STORE_NAME):
os.remove(STORE_NAME)
try:
s = self.storage_class(STORE_NAME)
except IOError:
nose.SkipTest('Cannot make an HDF5 file. Skipping')
else:
tp.batch(self.v[[0, 1]],
*self.PARAMS,
output=s,
engine='python',
meta=False)
print s.store.keys()
print dir(s.store.root)
assert len(s) == 2
assert s.max_frame == 1
assert_frame_equal(s.dump().reset_index(drop=True),
self.expected.reset_index(drop=True))
assert_frame_equal(s[0], s.get(0))
s.close()
os.remove(STORE_NAME)
def test_everything(self):
"""
End to end test.
Test the whole tracking pipeline from input file to particle and trap
positions in the output file.
"""
filename = "../../examples/data/test_example.twv"
frames = pims.open(filename)
times, laser_powers, traps = frames.get_all_tweezer_positions()
features = tp.batch(frames, 25, minmass=1000, invert=False)
tracks = tp.link_df(features, 15, memory=10)
save_tracked_data_pandas(filename[:-4] + '_out.dat', frames, tracks, times, laser_powers, traps)
with open(filename[:-4] + '_out.dat', 'r') as calculated_file:
with open(filename[:-4] + '_expected.dat', 'r') as expected_file:
for calculated, expected in zip(calculated_file, expected_file):
self.assertEqual(calculated, expected)
def compute_traj(filename):
vid = pims.Video('../test_video/' + filename)
frames = as_grey(vid)
midpoint = len(frames) / 2
start = int(midpoint - 60)
stop = int(midpoint + 60)
f = tp.batch(frames[start:stop],
11,
invert=False,
minmass=160,
maxsize=3.0,
engine="numba")
t = tp.link_df(f, 5, memory=3)
t1 = tp.filter_stubs(t, 60)
# Compare the number of particles in the unfiltered and filtered data.
print('Before:', t['particle'].nunique())
print('After:', t1['particle'].nunique())
data = []
for item in set(t1.particle):
sub = t1[t1.particle == item]
dvx = np.diff(sub.x)
dvy = np.diff(sub.y)
for x, y, dx, dy, frame, mass, size, ecc, signal, raw_mass, ep in \
zip(sub.x[:-1], sub.y[:-1], dvx, dvy, sub.frame[:-1], sub.mass[:-1], sub['size'][:-1], sub.ecc[:-1], sub.signal[:-1], sub.raw_mass[:-1], sub.ep[:-1]):
data.append({
'dx': dx,
'dy': dy,
'x': x,
'y': y,
'frame': frame,
'particle': item,
'size': size,
'ecc': ecc,
'signal': signal,
'mass': mass,
'raw_mass': raw_mass,
'ep': ep
})
df = pd.DataFrame(data)
df.to_csv('../csvs/extract.csv')
def Detect(estimateFeatureSize,
CameraName,
minMass=None,
dynamicMinMass=False,
Crop=False):
ImagePath = os.path.join('data',
CaseName.split('-')[0],
CaseName.split('-')[1])
Path = os.path.join(ImagePath, CameraName + '*.tif')
frames = pims.open(Path)
if Crop:
frames = pims.process.crop(frames, ((400, 700), (0, 0)))
print('Valid frames length is %d' % len(frames))
# check start frame and end frame with total frames number
if len(frames) != (endFrame - startFrame + 1):
print('Invalid frames length')
return
# find five brightest
if not minMass:
f = tp.locate(frames[testFrame], estimateFeatureSize)
mass = list(f['mass'])
mass.sort()
minMass = int(mass[-2] * 0.9 + mass[-1] * 0.1)
print(minMass)
#TopTen = np.argsort(f['mass'])[-5:]
#TopTenArray = f['mass'][TopTen]
# show mass histogram
# show subpixel accuracy of the detection
#minMass = list(TopTenArray)[0]
f = tp.locate(frames[testFrame], estimateFeatureSize, minmass=minMass)
plt.figure()
tp.annotate(f, frames[testFrame])
# run batch processing for all frames
if dynamicMinMass:
f = f[0:0]
for i in range(len(frames)):
f_ele = tp.locate(frames[i], estimateFeatureSize)
mass = list(f_ele['mass'])
mass.sort()
minMass = int(mass[-2] * 0.9 + mass[-1] * 0.1)
f_ele = tp.locate(frames[i], estimateFeatureSize, minmass=minMass)
f = f.append(f_ele)
else:
f = tp.batch(frames, estimateFeatureSize, minmass=minMass)
return f, frames
def cell_detect(file, var, opt):
print('Detecting cells')
# http://soft-matter.github.io/trackpy/v0.3.2/tutorial/walkthrough.html
# load, run object detection and track (then clean up)
raw_frames = pims.TiffStack(file, as_grey=True) # load
# only analyse n frames
if var['frames_keep'] is not 0:
raw_frames = raw_frames[0:var['frames_keep']]
# object detect
cellsdf = tp.batch(raw_frames,
var['diameter'],
minmass=var['minFluroMass'],
separation=var['separation'],
engine='numba',
max_iterations=1,
characterize=False,
noise_size=var['noise_smooth'])
# remove brightest objects
cellsdf = cellsdf.drop(cellsdf[cellsdf.mass > var['maxFluroMass']].index)
if opt['plot_inter_static']:
annotate_args = {"vmin": 0, "vmax": 200}
# Tweak styles
plt.ion()
plt.show()
fig_dims = np.multiply(0.01, raw_frames[0].shape)
mpl.rc('figure',
figsize=(fig_dims[1].astype(int), fig_dims[0].astype(int)))
mpl.rc('image', cmap='gray')
# plot final particles chosen
plt.figure()
tp.annotate(cellsdf[cellsdf.frame == var['frame_plot']],
raw_frames[var['frame_plot']],
imshow_style=annotate_args)
plt.title('Particles included in analysis'
'(at t=' + str(var['frame_plot']) + ')')
plt.draw()
plt.pause(0.001)
return cellsdf, raw_frames
def save_data(frame_dir, save_to_dir):
"""
Takes a dirpath containing video frames and extracts the data.
:param frame_dir: the directory containing the frames.
:param save_to_dir: the directory path where we save our data, without the last '/'.
:return:
"""
print("Getting frame data from " + frame_dir)
frames = pims.ImageSequence(frame_dir + '/' + FRAME_NAME + '*.jpg',
as_grey=True)
try:
data = tp.batch(frames[:-2],
PARTICLE_SIZE,
invert=True,
minmass=MIN_MASS,
percentile=PERCENTILE)
except OSError:
pass
frame_dirname = os.path.basename(frame_dir)
out_filepath = save_to_dir + '/' + frame_dirname + '.csv'
print("Writing frame data to " + out_filepath)
data.to_csv(out_filepath)
def evaluate_features(video_name, particle_size, particle_minmass, start_frame,
length, noise):
"""
The function that runs Trackpy's feature detection algorithm on the arrays.
Parameters:
video_name (String): The name of the video to be evaluated stored in the Recordings folder.
particle_size (int): The odd-number size of the feature to be detected by Trackpy.
particle_minmass (double): The minimum feature brightness to filter using Trackpy's filtering functions.
start_frame (int): The frame in the video from which to begin evaluation.
length (int): The number of frames to evaluate.
Returns:
frame (DataFrame): The DataFrame of the features found in the video.
"""
video_frames = process_video(video_name)
f = tp.batch(video_frames[start_frame:start_frame + length],
particle_size,
minmass=particle_minmass,
noise_size=noise)
return f
def get_data(outdir):
""" Loads the output of the preprocessing steps for feature extraction
Returns the formatted data
"""
frames = pims.ImageSequence("../"+outdir+"/*tif")
print(frames)
# particle diameter
diam = 11
features = tp.batch(frames[:frames._count], diameter=diam, minmass=1, invert=True)
# Link features in time: sigma_(max)
search_range = diam-2
# r, g, b images are loaded
lframes = int(np.floor(frames._count/3))
# default max 15% frame count
imax = int(np.floor(15*lframes/100))
t = tp.link_df(features, search_range, memory=imax)
# default neighbour strategy: KDTree
# Filter spurious trajectories
# default min 10% frame count
imin = int(np.floor(10*lframes/100))
# if seen in imin
t1 = tp.filter_stubs(t, imin)
# Compare the number of particles in the unfiltered and filtered data.
print("Unique number of particles (Before filtering):", t["particle"].nunique())
print("(After):", t1["particle"].nunique())
# export pandas data frame with filename being current date and time
timestr = time.strftime("%Y%m%d-%H%M%S")
data = pd.DataFrame({"x": t1.x, "y": t1.y, "z": t1.frame, "mass": t1.mass, "size": t1.size, "ecc": t1.ecc, "signal": t1.signal, "ep": t1.ep, "particle": t1.particle})
file_name = "../features_" + timestr + ".csv"
print("Exporting %s" % (file_name))
data.to_csv(file_name, sep="\t", encoding="utf-8")
return data
def FindSpots_tp(frames_np, diameter, minmass, separation, max_iterations, DoPreProcessing, percentile, DoParallel = True):
# run trackpy with the given parameters seriell or parallel (faster after "long" loading time)
num_frames = frames_np.shape[0]
if (DoParallel == False) or (num_frames < 100):
nd.logger.info("Find the particles - seriell: starting....")
output = tp.batch(frames_np, diameter, minmass = minmass, separation = separation, max_iterations = max_iterations, preprocess = DoPreProcessing, engine = 'auto', percentile = percentile)
else:
num_cores = multiprocessing.cpu_count()
nd.logger.info("Find the particles - parallel (Number of cores: %s): starting....", num_cores)
inputs = range(num_frames)
num_verbose = nd.handle_data.GetNumberVerbose()
output_list = Parallel(n_jobs=num_cores, verbose=num_verbose)(delayed(tp.batch)(frames_np[loop_frame:loop_frame+1,:,:].copy(), diameter, minmass = minmass, separation = separation, max_iterations = max_iterations, preprocess = DoPreProcessing, engine = 'auto', percentile = percentile) for loop_frame in inputs)
empty_frame = []
#parallel looses frame number, so we add it again
for frame_id,_ in enumerate(output_list):
output_list[frame_id].frame = frame_id
if len(output_list[frame_id]) == 0:
empty_frame.append(frame_id)
# go through empty frames (start from the back deleting otherwise indexing fails)
for frame_id in (np.flip(empty_frame)):
del output_list[frame_id]
# make list of pandas to one big pandas
output = pd.concat(output_list)
# reset the index which starts at every frame again
output = output.reset_index(drop=True)
nd.logger.info("Find the particles - finished")
return output
def nd2msd(nd_fh):
# print nd_fh
frames=pims.ND2_Reader(nd_fh)
logging.info('number of frames = %d' % len(np.shape(frames)))
if len(np.shape(frames))==4:
frames = average_z(frames)
threshold=np.percentile(frames,75)
f_batch = tp.batch(frames,diameter=11,threshold=threshold)
t = tp.link_df(f_batch, search_range=11, memory=3)
t_flt = tp.filter_stubs(t, 3*int(len(frames)/4))
try:
d = tp.compute_drift(t_flt)
t_cor = tp.subtract_drift(t_flt, d)
except:
t_cor=t_flt
logging.info("drift correction excepted")
# plt.figure()
# tp.plot_traj(t_flt)
# plt.figure()
# d.plot()
imsd=tp.imsd(t_cor,0.1,0.2, max_lagtime=100, statistic='msd')
emsd=tp.emsd(t_cor,0.1,0.2, max_lagtime=100)
return imsd,emsd
def tracking(video):
# print "running tracking"
pimsframes = pims.Video(video, as_grey = True)
fgbg = cv2.BackgroundSubtractorMOG()
framesmask = []
framecount = 0
blurredframes = []
# HACK to flip if has space in name. #TODO get all videos correctly alligned...
if " " in video:
pimsframes = [p[:, ::-1] for p in pimsframes]
pimsframes = [frame[:,400:] for frame in pimsframes]
for frame in pimsframes:
# frame = cv2.GaussianBlur(frame,(9,9),0)
# frame = cv2.medianBlur(frame, 7)
# if align remove
frame = cv2.GaussianBlur(frame,(11,11),7)
frame = cv2.medianBlur(frame, 3)
fgmask = fgbg.apply(frame, learningRate=1.0/history)
framesmask.append(fgmask)
framecount += 1
blurredframes.append(frame)
background_sub = [m * frame for m,frame in zip(framesmask, pimsframes)]
if False:
for i in range(100):
cv2.imshow("asdf", background_sub[i])
cv2.imshow("mask", framesmask[i])
cv2.imshow("orig", pimsframes[i])
cv2.imshow("blur", blurredframes[i])
cv2.waitKey(0)
# for i, f in enumerate(framesmask):
# half_show("asdf", f)
# cv2.waitKey(0)
# print i
cells = []
track = []
to_track = background_sub
minmass = 3000
f = tp.batch(to_track[:], 11, minmass=minmass, invert=False, noise_size=3)
# for j in range(20,100):
# f = tp.locate(to_track[j], 11, invert=False, minmass = minmass, noise_size=3)
# plt.figure(1)
# tp.annotate(f, to_track[j])
# plt.show()
# ipdb.set_trace()
print "linking"
try:
# t = tp.link_df(f, 100, memory=3)
t = tp.link_df(f, 100, memory=1)
except Exception:
print "FAILED on", video
return None
print "done"
# plt.figure(2)
# tp.plot_traj(t)
# plt.show()
return t
v = pims.ImageSequence(impath)
# take reader that provides uint8!
assert np.issubdtype(v.dtype, np.uint8)
v0 = tp.invert_image(v[0])
v0_bp = tp.bandpass(v0, lshort=1, llong=9)
expected_find = tp.grey_dilation(v0, separation=9)
expected_find_bandpass = tp.grey_dilation(v0_bp, separation=9)
expected_refine = tp.refine_com(v0, v0_bp, radius=4,
coords=expected_find_bandpass)
expected_refine = expected_refine[expected_refine['mass'] >= 140]
expected_refine_coords = expected_refine[pos_columns].values
expected_locate = tp.locate(v0, diameter=9, minmass=140)
expected_locate_coords = expected_locate[pos_columns].values
df = tp.locate(v0, diameter=9)
df = df[(df['x'] < 64) & (df['y'] < 64)]
expected_characterize = df[pos_columns + char_columns].values
f = tp.batch(tp.invert_image(v), 9, minmass=140)
f_crop = f[(f['x'] < 320) & (f['x'] > 280) & (f['y'] < 280) & (f['x'] > 240)]
f_linked = tp.link(f_crop, search_range=5, memory=0)
f_linked_memory = tp.link(f_crop, search_range=5, memory=2)
link_coords = f_linked[pos_columns + ['frame']].values
expected_linked = f_linked['particle'].values
expected_linked_memory = f_linked_memory['particle'].values
np.savez_compressed(npzpath, expected_find, expected_find_bandpass,
expected_refine_coords, expected_locate_coords,
link_coords, expected_linked, expected_linked_memory,
expected_characterize)
import trackpy as tp
from os import path
from pimsviewer import Viewer, AnnotatePlugin
from pims import pipeline
@pipeline
def as_grey(frame):
return 0.2125 * frame[:, :, 0] + 0.7154 * frame[:, :, 1] + 0.0721 * frame[:, :, 2]
filename = path.join(path.dirname(path.realpath(__file__)), '../screenshot.png')
viewer = Viewer(filename)
f = tp.batch(as_grey(viewer.reader), diameter=15)
plugin = AnnotatePlugin(viewer, f)
viewer.run()
def prepare(self):
directory = os.path.join(path, 'video', 'image_sequence')
self.v = tp.ImageSequence(directory)
self.PARAMS = (11, 3000)
self.expected = tp.batch(self.v[[0, 1]], *self.PARAMS,
engine='python', meta=False)
def tracking(video):
# print "running tracking"
pimsframes = pims.Video(video, as_grey=True)
fgbg = cv2.BackgroundSubtractorMOG()
framesmask = []
framecount = 0
blurredframes = []
# HACK to flip if has space in name. #TODO get all videos correctly alligned...
if " " in video:
pimsframes = [p[:, ::-1] for p in pimsframes]
pimsframes = [frame[:, 400:] for frame in pimsframes]
for frame in pimsframes:
# frame = cv2.GaussianBlur(frame,(9,9),0)
# frame = cv2.medianBlur(frame, 7)
# if align remove
frame = cv2.GaussianBlur(frame, (11, 11), 7)
frame = cv2.medianBlur(frame, 3)
fgmask = fgbg.apply(frame, learningRate=1.0 / history)
framesmask.append(fgmask)
framecount += 1
blurredframes.append(frame)
background_sub = [m * frame for m, frame in zip(framesmask, pimsframes)]
if False:
for i in range(100):
cv2.imshow("asdf", background_sub[i])
cv2.imshow("mask", framesmask[i])
cv2.imshow("orig", pimsframes[i])
cv2.imshow("blur", blurredframes[i])
cv2.waitKey(0)
# for i, f in enumerate(framesmask):
# half_show("asdf", f)
# cv2.waitKey(0)
# print i
cells = []
track = []
to_track = background_sub
minmass = 3000
f = tp.batch(to_track[:], 11, minmass=minmass, invert=False, noise_size=3)
# for j in range(20,100):
# f = tp.locate(to_track[j], 11, invert=False, minmass = minmass, noise_size=3)
# plt.figure(1)
# tp.annotate(f, to_track[j])
# plt.show()
# ipdb.set_trace()
print "linking"
try:
# t = tp.link_df(f, 100, memory=3)
t = tp.link_df(f, 100, memory=1)
except Exception:
print "FAILED on", video
return None
print "done"
# plt.figure(2)
# tp.plot_traj(t)
# plt.show()
return t
def get_data(outdir, red, green, blue, diam=11):
''' Loads the output of the preprocessing steps for particle extraction
Returns the formatted data
'''
frames = pims.ImageSequence("../" + outdir + "/*tif")
print frames
# particle diameter
features = tp.batch(frames[:frames._count], diameter=diam, \
minmass=1, invert=True)
# Link features in time
search_range = diam - 2 # sigma_(max)
lframes = int(np.floor(frames._count / 3)) # r, g, b images are loaded
imax = int(np.floor(15 * lframes / 100)) # default max 15% frame count
t = tp.link_df(features, search_range, memory=imax)
# default neighbour strategy: KDTree
# Filter spurious trajectories
imin = int(np.floor(10 * lframes / 100)) # default min 10% frame count
t1 = tp.filter_stubs(t, imin) # if seen in imin
# Compare the number of particles in the unfiltered and filtered data
print 'Unique number of particles (Before filtering):', t[
'particle'].nunique()
print '(After):', t1['particle'].nunique()
# export pandas data frame with filename being current date and time
timestr = time.strftime("%Y%m%d-%H%M%S")
data = pd.DataFrame({ 'x': t1.x, 'y': t1.y,'z':t1.frame,\
'mass':t1.mass, 'size':t1.size, 'ecc':t1.ecc,\
'signal':t1.signal, 'ep':t1.ep, 'particle':t1.particle\
})
# format the dataframe / original indexing
data["n"] = np.arange(len(data))
print("Sorting dataframe by time...")
data = data.sort(columns='z', ascending=True)
print("Extracting pixel values of particles...")
r, g, b = get_val(red, 2, data), get_val(green, 1,
data), get_val(blue, 0, data)
print("Normalising rgb values to relative quantities...")
r1, g1, b1 = np.array(r), np.array(g), np.array(b)
r = (r1 - np.min(r1)) * (65535 / np.max(r1))
g = (g1 - np.min(g1)) * (65535 / np.max(g1))
b = (b1 - np.min(b1)) * (65535 / np.max(b1))
print("Adding (r,g,b) values as columns to dataframe...")
strname, px_val = ["r", "g", "b"], [r, g, b]
add_arrays_df(strname, px_val, data)
# sort back to original state
data = data.sort(columns='n', ascending=True)
# remove the previously created column
data.drop('n', axis=1, inplace=True)
# format df with rgb values to uint8
data = format_df(data)
print "Dataframe summary:\n", data.describe()
file_name = "../particles_" + timestr + ".csv"
print "Exporting %s" % (file_name)
data.to_csv(file_name, sep='\t', encoding='utf-8')
return data
def frames2coords(frames,out_fh,
params_locate,params_msd,params_link_df={'search_range':20,},
mass_cutoff=0.5,size_cutoff=0.5,ecc_cutoff=0.5,
filter_stubs=True,flt_mass_size=True,flt_incomplete_trjs=True,
force=False,test=False):
dns=['f_batch','t','t1','t2']
dn2dp={dn:f'{out_fh}.{dn}.tsv' for dn in dns}
dn2df={}
if not exists(dn2dp['t2']) or force:
if not exists(dn2dp['t']) or force:
dn2df['f_batch']=tp.batch(frames,engine='numba',**params_locate)
dn2df['t']=tp.link_df(dn2df['f_batch'], **params_link_df)
print(params_link_df)
dn2df['f_batch'].to_csv(dn2dp['f_batch'],sep='\t')
dn2df['t'].to_csv(dn2dp['t'],sep='\t')
else:
dn2df['t']=pd.read_csv(dn2dp['t'])
max_lagtime_stubs=params_msd["max_lagtime"]*params_msd["fps"]
if filter_stubs:
dn2df['t1'] = tp.filter_stubs(dn2df['t'], max_lagtime_stubs*1.25)
logging.info('filter_stubs: particle counts: %s to %s' % (dn2df['t']['particle'].nunique(),dn2df['t1']['particle'].nunique()))
if t1['particle'].nunique()==0:
logging.error('filter_stubs: particle counts =0; using less stringent conditions')
dn2df['t1'] = tp.filter_stubs(dn2df['t'], max_lagtime_stubs*1)
else:
dn2df['t1'] = dn2df['t'].copy()
if test:
fig=plt.figure()
ax=plt.subplot(111)
tp.mass_size(dn2df['t1'].groupby('particle').mean(),ax=ax);
plt.tight_layout()
plt.savefig('%s.mass_size.svg' % out_fh,format='svg')
if flt_mass_size:
dn2df['t2'] = dn2df['t1'][((dn2df['t1']['mass'] > dn2df['t1']['mass'].quantile(mass_cutoff)) & (dn2df['t1']['size'] < dn2df['t1']['size'].quantile(size_cutoff)) &
(dn2df['t1']['ecc'] < ecc_cutoff))]
logging.info('filter_mass_size: particle counts: %s to %s' % (dn2df['t1']['particle'].nunique(),dn2df['t2']['particle'].nunique()))
if len(t2)==0:
dn2df['t2'] = dn2df['t1'].copy()
logging.warning('filter_mass_size produced 0 particles; using t2=t1.copy()')
else:
dn2df['t2'] = dn2df['t1'].copy()
if test:
fig=plt.figure()
ax=plt.subplot(111)
tp.mass_size(dn2df['t2'].groupby('particle').mean(),ax=ax);
plt.tight_layout()
plt.savefig('%s.mass_size_post_filtering.svg' % out_fh,format='svg')
if flt_incomplete_trjs:
dn2df['t2']=dn2df['t2'].reset_index()
vals=pd.DataFrame(dn2df['t2']['particle'].value_counts())
partis=[i for i in vals.index if vals.loc[i,'particle']>=int(vals.max())*0.95 ]
dn2df['t2']=dn2df['t2'].loc[[i for i in dn2df['t2'].index if (dn2df['t2'].loc[i,'particle'] in partis)],:]
dn2df['t2'].to_csv(dn2dp['t2'],sep='\t')
else:
dn2df['t2']=pd.read_csv(dn2dp['t2'],sep='\t')
if test:
for traj in ['t','t1','t2']:
ax=plot_traj(frames[-1],traj=dn2df[traj])
logging.info('getting plots hist')
cols=['mass','size','ecc','signal','raw_mass','ep']
fig=plt.figure()
ax=plt.subplot(111)
_=dn2df['t2'].loc[:,cols].hist(ax=ax)
return dn2df['t2']
