def test_run_tracking():
tf = tempfile.NamedTemporaryFile(suffix='.csv')
ij.track('http://fiji.sc/samples/FakeTracks.tif', tf.name)
assert op.exists(tf.name)
df = csv_to_pd(tf.name)
assert df.shape == (84, 8)
def download_and_track(filename):
import diff_classifier.imagej as ij
import diff_classifier.utils as ut
import diff_classifier.aws as aws
import os.path as op
import pandas as pd
aws.download_s3(filename, op.split(filename)[1])
outfile = 'Traj_' + op.split(filename)[1].split('.')[0] + '.csv'
local_im = op.split(filename)[1]
if not op.isfile(outfile):
ij.track(local_im, outfile, template=None, fiji_bin=None, radius=4.5, threshold=0.,
do_median_filtering=True, quality=4.5, median_intensity=300.0, snr=0.0,
linking_max_distance=8.0, gap_closing_max_distance=10.0, max_frame_gap=2,
track_displacement=10.0)
aws.upload_s3(outfile, op.split(filename)[0]+'/'+outfile)
print("Done with tracking. Should output file of name {}".format(op.split(filename)[0]+'/'+outfile))
def sensitivity_it(counter):
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
import diff_classifier.heatmaps as hm
from scipy.spatial import Voronoi
import scipy.stats as stats
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon
import matplotlib.cm as cm
import os
import os.path as op
import numpy as np
import numpy.ma as ma
import pandas as pd
import boto3
import itertools
#Sweep parameters
#----------------------------------
radius = [4.5, 6.0, 7.0]
do_median_filtering = [True, False]
quality = [1.5, 4.5, 8.5]
linking_max_distance = [6.0, 10.0, 15.0]
gap_closing_max_distance = [6.0, 10.0, 15.0]
max_frame_gap = [1, 2, 5]
track_displacement = [0.0, 10.0, 20.0]
sweep = [
radius, do_median_filtering, quality, linking_max_distance,
gap_closing_max_distance, max_frame_gap, track_displacement
]
all_params = list(itertools.product(*sweep))
#Variable prep
#----------------------------------
s3 = boto3.client('s3')
folder = '01_18_Experiment'
s_folder = '{}/sensitivity'.format(folder)
local_folder = '.'
prefix = "P1_S1_R_0001_2_2"
name = "{}.tif".format(prefix)
local_im = op.join(local_folder, name)
aws.download_s3(
'{}/{}/{}.tif'.format(folder,
prefix.split('_')[0], prefix),
'{}.tif'.format(prefix))
outputs = np.zeros((len(all_params), len(all_params[0]) + 2))
#Tracking and calculations
#------------------------------------
params = all_params[counter]
outfile = 'Traj_{}_{}.csv'.format(name.split('.')[0], counter)
msd_file = 'msd_{}_{}.csv'.format(name.split('.')[0], counter)
geo_file = 'geomean_{}_{}.csv'.format(name.split('.')[0], counter)
geoS_file = 'geoSEM_{}_{}.csv'.format(name.split('.')[0], counter)
msd_image = 'msds_{}_{}.png'.format(name.split('.')[0], counter)
iter_name = "{}_{}".format(prefix, counter)
ij.track(local_im,
outfile,
template=None,
fiji_bin=None,
radius=params[0],
threshold=0.,
do_median_filtering=params[1],
quality=params[2],
x=511,
y=511,
ylo=1,
median_intensity=300.0,
snr=0.0,
linking_max_distance=params[3],
gap_closing_max_distance=params[4],
max_frame_gap=params[5],
track_displacement=params[6])
traj = ut.csv_to_pd(outfile)
msds = msd.all_msds2(traj, frames=651)
msds.to_csv(msd_file)
gmean1, gSEM1 = hm.plot_individual_msds(iter_name, alpha=0.05)
np.savetxt(geo_file, gmean1, delimiter=",")
np.savetxt(geoS_file, gSEM1, delimiter=",")
aws.upload_s3(outfile, '{}/{}'.format(s_folder, outfile))
aws.upload_s3(msd_file, '{}/{}'.format(s_folder, msd_file))
aws.upload_s3(geo_file, '{}/{}'.format(s_folder, geo_file))
aws.upload_s3(geoS_file, '{}/{}'.format(s_folder, geoS_file))
aws.upload_s3(msd_image, '{}/{}'.format(s_folder, msd_image))
print('Successful parameter calculations for {}'.format(iter_name))
def tracking(subprefix, remote_folder, bucket, tparams,
regress_f='regress.obj', rows=4, cols=4, ires=(512, 512)):
'''Tracks particles in input image using Trackmate.
A function based on imagej.track that downloads the image from S3, tracks
particles using Trackmate, and uploads the resulting trajectory file to S3.
Designed to work with Cloudknot for parallelizable workflows. Typically,
this function is used in conjunction with kn.split and kn.assemble_msds for
a complete analysis.
Parameters
----------
subprefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
regress_f : string
Name of regress object used to predict quality parameter.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
tparams : dict
Dictionary containing tracking parameters to Trackmate analysis.
'''
import os
import os.path as op
import boto3
from sklearn.externals import joblib
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
local_folder = os.getcwd()
filename = '{}.tif'.format(subprefix)
remote_name = remote_folder+'/'+filename
local_name = local_folder+'/'+filename
outfile = 'Traj_' + subprefix + '.csv'
local_im = op.join(local_folder, '{}.tif'.format(subprefix))
row = int(subprefix.split('_')[-2])
col = int(subprefix.split('_')[-1])
aws.download_s3(remote_folder+'/'+regress_f, regress_f, bucket_name=bucket)
with open(regress_f, 'rb') as fp:
regress = joblib.load(fp)
s3 = boto3.client('s3')
aws.download_s3('{}/{}'.format(remote_folder,
'{}.tif'.format(subprefix)),
local_im, bucket_name=bucket)
tparams['quality'] = ij.regress_tracking_params(regress, subprefix,
regmethod='PassiveAggressiveRegressor')
if row == rows-1:
tparams['ydims'] = (tparams['ydims'][0], ires[1] - 27)
ij.track(local_im, outfile, template=None, fiji_bin=None,
tparams=tparams)
aws.upload_s3(outfile, remote_folder+'/'+outfile, bucket_name=bucket)
print("Done with tracking. Should output file of name {}".format(
remote_folder+'/'+outfile))
def download_split_track_msds(prefix):
"""
1. Checks to see if features file exists.
2. If not, checks to see if image partitioning has occured.
3. If yes, checks to see if tracking has occured.
4. Regardless, tracks, calculates MSDs and features.
"""
import matplotlib as mpl
mpl.use('Agg')
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
import diff_classifier.heatmaps as hm
from scipy.spatial import Voronoi
import scipy.stats as stats
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon
import matplotlib.cm as cm
import os
import os.path as op
import numpy as np
import numpy.ma as ma
import pandas as pd
import boto3
#Splitting section
###############################################################################################
remote_folder = "01_18_Experiment/{}".format(prefix.split('_')[0])
local_folder = os.getcwd()
ires = 512
frames = 651
filename = '{}.tif'.format(prefix)
remote_name = remote_folder+'/'+filename
local_name = local_folder+'/'+filename
msd_file = 'msd_{}.csv'.format(prefix)
ft_file = 'features_{}.csv'.format(prefix)
s3 = boto3.client('s3')
names = []
for i in range(0, 4):
for j in range(0, 4):
names.append('{}_{}_{}.tif'.format(prefix, i, j))
try:
obj = s3.head_object(Bucket='ccurtis7.pup', Key=remote_folder+'/'+ft_file)
except:
try:
for name in names:
aws.download_s3(remote_folder+'/'+name, name)
except:
aws.download_s3(remote_name, local_name)
names = ij.partition_im(local_name)
for name in names:
aws.upload_s3(name, remote_folder+'/'+name)
print("Done with splitting. Should output file of name {}".format(remote_folder+'/'+name))
#Tracking section
################################################################################################
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
local_im = op.join(local_folder, name)
row = int(name.split('.')[0].split('_')[4])
col = int(name.split('.')[0].split('_')[5])
try:
aws.download_s3(remote_folder+'/'+outfile, outfile)
except:
test_intensity = ij.mean_intensity(local_im)
if test_intensity > 500:
quality = 245
else:
quality = 4.5
if row==3:
y = 485
else:
y = 511
ij.track(local_im, outfile, template=None, fiji_bin=None, radius=4.5, threshold=0.,
do_median_filtering=True, quality=quality, x=511, y=y, ylo=1, median_intensity=300.0, snr=0.0,
linking_max_distance=8.0, gap_closing_max_distance=10.0, max_frame_gap=2,
track_displacement=10.0)
aws.upload_s3(outfile, remote_folder+'/'+outfile)
print("Done with tracking. Should output file of name {}".format(remote_folder+'/'+outfile))
#MSD and features section
#################################################################################################
files_to_big = False
size_limit = 10
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
local_im = name
file_size_MB = op.getsize(local_im)/1000000
if file_size_MB > size_limit:
file_to_big = True
if files_to_big:
print('One or more of the {} trajectory files exceeds {}MB in size. Will not continue with MSD calculations.'.format(
prefix, size_limit))
else:
counter = 0
for name in names:
row = int(name.split('.')[0].split('_')[4])
col = int(name.split('.')[0].split('_')[5])
filename = "Traj_{}_{}_{}.csv".format(prefix, row, col)
local_name = local_folder+'/'+filename
if counter == 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
merged = msd.all_msds2(to_add, frames=frames)
else:
if merged.shape[0] > 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
to_add['Track_ID'] = to_add['Track_ID'] + max(merged['Track_ID']) + 1
else:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
to_add['Track_ID'] = to_add['Track_ID']
merged = merged.append(msd.all_msds2(to_add, frames=frames))
print('Done calculating MSDs for row {} and col {}'.format(row, col))
counter = counter + 1
merged.to_csv(msd_file)
aws.upload_s3(msd_file, remote_folder+'/'+msd_file)
merged_ft = ft.calculate_features(merged)
merged_ft.to_csv(ft_file)
aws.upload_s3(ft_file, remote_folder+'/'+ft_file)
#Plots
features = ('AR', 'D_fit', 'alpha', 'MSD_ratio', 'Track_ID', 'X', 'Y', 'asymmetry1', 'asymmetry2', 'asymmetry3',
'boundedness', 'efficiency', 'elongation', 'fractal_dim', 'frames', 'kurtosis', 'straightness', 'trappedness')
vmin = (1.36, 0.015, 0.72, -0.09, 0, 0, 0, 0.5, 0.049, 0.089, 0.0069, 0.65, 0.26, 1.28, 0, 1.66, 0.087, -0.225)
vmax = (3.98, 2.6, 2.3, 0.015, max(merged_ft['Track_ID']), 2048, 2048, 0.99, 0.415, 0.53,
0.062, 3.44, 0.75, 1.79, 650, 3.33, 0.52, -0.208)
die = {'features': features,
'vmin': vmin,
'vmax': vmax}
di = pd.DataFrame(data=die)
for i in range(0, di.shape[0]):
hm.plot_heatmap(prefix, feature=di['features'][i], vmin=di['vmin'][i], vmax=di['vmax'][i])
hm.plot_scatterplot(prefix, feature=di['features'][i], vmin=di['vmin'][i], vmax=di['vmax'][i])
hm.plot_trajectories(prefix)
try:
hm.plot_histogram(prefix)
except ValueError:
print("Couldn't plot histogram.")
hm.plot_particles_in_frame(prefix)
gmean1, gSEM1 = hm.plot_individual_msds(prefix, alpha=0.05)
def tracking(
subprefix,
remote_folder,
bucket='nancelab.publicfiles',
regress_f='regress.obj',
rows=4,
cols=4,
ires=(512, 512),
tparams={
'frames': 651,
'radius': 3.0,
'threshold': 0.0,
'do_median_filtering': False,
'quality': 15.0,
'xdims': (0, 511),
'ydims': (1, 511),
'median_intensity': 300.0,
'snr': 0.0,
'linking_max_distance': 6.0,
'gap_closing_max_distance': 10.0,
'max_frame_gap': 3,
'track_duration': 20.0
}):
'''Tracks particles in input image using Trackmate.
A function based on imagej.track that downloads the image from S3, tracks
particles using Trackmate, and uploads the resulting trajectory file to S3.
Parameters
----------
subprefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
regress_f : string
Name of regress object used to predict quality parameter.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
tparams : dict
Dictionary containing tracking parameters to Trackmate analysis.
'''
import os
import os.path as op
import boto3
from sklearn.externals import joblib
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
local_folder = os.getcwd()
filename = '{}.tif'.format(subprefix)
remote_name = remote_folder + '/' + filename
local_name = local_folder + '/' + filename
outfile = 'Traj_' + subprefix + '.csv'
local_im = op.join(local_folder, '{}.tif'.format(subprefix))
row = int(subprefix.split('_')[-2])
col = int(subprefix.split('_')[-1])
aws.download_s3(remote_folder + '/' + regress_f,
regress_f,
bucket_name=bucket)
with open(regress_f, 'rb') as fp:
regress = joblib.load(fp)
s3 = boto3.client('s3')
aws.download_s3('{}/{}'.format(remote_folder, '{}.tif'.format(subprefix)),
local_im,
bucket_name=bucket)
tparams['quality'] = ij.regress_tracking_params(
regress, subprefix, regmethod='PassiveAggressiveRegressor')
if row == rows - 1:
tparams['ydims'] = (tparams['ydims'][0], ires[1] - 27)
ij.track(local_im, outfile, template=None, fiji_bin=None, tparams=tparams)
aws.upload_s3(outfile, remote_folder + '/' + outfile, bucket_name=bucket)
print("Done with tracking. Should output file of name {}".format(
remote_folder + '/' + outfile))