def plot_particles_in_frame(prefix, x_range=600, y_range=2000, upload=True,
remote_folder = "01_18_Experiment", bucket='ccurtis.data'):
"""
Plot number of particles per frame as a function of time.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
x_range: float64 or int
Desire x range of graph.
y_range: float64 or int
Desire y range of graph.
upload: boolean
True if you want to upload to s3.
"""
merged = pd.read_csv('msd_{}.csv'.format(prefix))
frames = int(max(merged['Frame']))
framespace = np.linspace(0, frames, frames)
particles = np.zeros((framespace.shape[0]))
for i in range(0, frames):
particles[i] = merged.loc[merged.Frame == i, 'MSDs'].dropna().shape[0]
fig = plt.figure(figsize=(5, 5))
plt.plot(framespace, particles, linewidth=4)
plt.xlim(0, x_range)
plt.ylim(0, y_range)
plt.xlabel('Frames', fontsize=20)
plt.ylabel('Particles', fontsize=20)
outfile = 'in_frame_{}.png'.format(prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile, remote_folder+'/'+outfile, bucket_name=bucket)
def plot_scatterplot(prefix, feature='asymmetry1', vmin=0, vmax=1, resolution=512, rows=4, cols=4,
dotsize=10, figsize=(12, 10), upload=True, remote_folder = "01_18_Experiment",
bucket='ccurtis.data'):
"""
Plot scatterplot of trajectories in video with colors corresponding to features.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
feature: string
Feature to be plotted. See features_analysis.py
vmin: float64
Lower intensity bound for heatmap.
vmax: float64
Upper intensity bound for heatmap.
resolution: int
Resolution of base image. Only needed to calculate bounds of image.
rows: int
Rows of base images used to build tiled image.
cols: int
Columns of base images used to build tiled images.
upload: boolean
True if you want to upload to s3.
"""
# Inputs
# ----------
merged_ft = pd.read_csv('features_{}.csv'.format(prefix))
string = feature
leveler = merged_ft[string]
t_min = vmin
t_max = vmax
ires = resolution
norm = mpl.colors.Normalize(t_min, t_max, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.viridis)
zs = ma.masked_invalid(merged_ft[string])
zs = ma.masked_where(zs <= t_min, zs)
zs = ma.masked_where(zs >= t_max, zs)
to_mask = ma.getmask(zs)
zs = ma.compressed(zs)
xs = ma.compressed(ma.masked_where(to_mask, merged_ft['X'].astype(int)))
ys = ma.compressed(ma.masked_where(to_mask, merged_ft['Y'].astype(int)))
fig = plt.figure(figsize=figsize)
plt.scatter(xs, ys, c=zs, s=dotsize)
mapper.set_array(10)
plt.colorbar(mapper)
plt.xlim(0, ires*cols)
plt.ylim(0, ires*rows)
plt.axis('off')
print('Plotted {} scatterplot successfully.'.format(prefix))
outfile = 'scatter_{}_{}.png'.format(feature, prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile, remote_folder+'/'+outfile, bucket_name=bucket)
def plot_trajectories(prefix,
resolution=512,
rows=4,
cols=4,
upload=True,
remote_folder="01_18_Experiment",
bucket='ccurtis.data',
figsize=(12, 12)):
"""
Plot trajectories in video.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
resolution: int
Resolution of base image. Only needed to calculate bounds of image.
rows: int
Rows of base images used to build tiled image.
cols: int
Columns of base images used to build tiled images.
upload: boolean
True if you want to upload to s3.
"""
merged = pd.read_csv('msd_{}.csv'.format(prefix))
particles = int(max(merged['Track_ID']))
ires = resolution
fig = plt.figure(figsize=figsize)
for part in range(0, particles):
x = merged[merged['Track_ID'] == part]['X']
y = merged[merged['Track_ID'] == part]['Y']
plt.plot(x, y, color='k', alpha=0.7)
plt.xlim(0, ires * cols)
plt.ylim(0, ires * rows)
plt.axis('off')
print('Plotted {} trajectories successfully.'.format(prefix))
outfile = 'traj_{}.png'.format(prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile,
remote_folder + '/' + outfile,
bucket_name=bucket)
def geomean_msd(prefix, umppx=0.16, fps=100.02, upload=True,
remote_folder="01_18_Experiment", bucket='ccurtis.data',
backup_frames=651):
import pandas as pd
import numpy as np
import numpy.ma as ma
import diff_classifier.aws as aws
import scipy.stats as stats
aws.download_s3('{}/msd_{}.csv'.format(remote_folder, prefix),
'msd_{}.csv'.format(prefix), bucket_name=bucket)
merged = pd.read_csv('msd_{}.csv'.format(prefix))
try:
particles = int(max(merged['Track_ID']))
frames = int(max(merged['Frame']))
ypos = np.zeros((particles+1, frames+1))
for i in range(0, particles+1):
ypos[i, :] = merged.loc[merged.Track_ID == i, 'MSDs']*umppx*umppx
xpos = merged.loc[merged.Track_ID == i, 'Frame']/fps
geo_mean = np.nanmean(ma.log(ypos), axis=0)
geo_stder = ma.masked_equal(stats.sem(ma.log(ypos), axis=0,
nan_policy='omit'), 0.0)
except ValueError:
geo_mean = np.nan*np.ones(backup_frames)
geo_stder = np.nan*np.ones(backup_frames)
np.savetxt('geomean_{}.csv'.format(prefix), geo_mean, delimiter=",")
np.savetxt('geoSEM_{}.csv'.format(prefix), geo_stder, delimiter=",")
if upload:
aws.upload_s3('geomean_{}.csv'.format(prefix),
remote_folder+'/'+'geomean_{}.csv'.format(prefix),
bucket_name=bucket)
aws.upload_s3('geoSEM_{}.csv'.format(prefix),
remote_folder+'/'+'geoSEM_{}.csv'.format(prefix),
bucket_name=bucket)
return geo_mean, geo_stder
def download_and_track(filename):
import diff_classifier.imagej as ij
import diff_classifier.utils as ut
import diff_classifier.aws as aws
import os.path as op
import pandas as pd
aws.download_s3(filename, op.split(filename)[1])
outfile = 'Traj_' + op.split(filename)[1].split('.')[0] + '.csv'
local_im = op.split(filename)[1]
if not op.isfile(outfile):
ij.track(local_im, outfile, template=None, fiji_bin=None, radius=4.5, threshold=0.,
do_median_filtering=True, quality=4.5, median_intensity=300.0, snr=0.0,
linking_max_distance=8.0, gap_closing_max_distance=10.0, max_frame_gap=2,
track_displacement=10.0)
aws.upload_s3(outfile, op.split(filename)[0]+'/'+outfile)
print("Done with tracking. Should output file of name {}".format(op.split(filename)[0]+'/'+outfile))
def download_and_split(filename):
import diff_classifier.imagej as ij
import diff_classifier.aws as aws
import os.path as op
local_name = op.split(filename)[1]
DIR = op.split(filename)[0]
try1 = filename.split('.')[0] + '_0_0.tif'
try2 = filename.split('.')[0] + '_3_3.tif'
s3 = boto3.client('s3')
try:
obj = s3.head_object(Bucket='ccurtis7.pup', Key=try1)
except:
try:
obj = s3.head_object(Bucket='ccurtis7.pup', Key=try2)
except:
aws.download_s3(filename, local_name)
names = ij.partition_im(local_name)
for name in names:
aws.upload_s3(name, op.split(filename)[0]+'/'+name)
print("Done with splitting. Should output file of name {}".format(op.split(filename)[0]+'/'+name))
def split(prefix, remote_folder, bucket,
rows=4, cols=4, ores=(2048, 2048), ires=(512, 512)):
'''Splits input image file into smaller images.
A function based on imagej.partition_im that download images from an S3
bucket, splits it into smaller images, and uploads these to S3. Designed to
work with Cloudknot for parallelizable workflows. Typically, this function
is used in conjunction with kn.tracking and kn.assemble_msds for a complete
analysis.
Parameters
----------
prefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
ores : tuple of int
Original resolution of input image.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
'''
import os
import boto3
import diff_classifier.aws as aws
import diff_classifier.imagej as ij
local_folder = os.getcwd()
filename = '{}.tif'.format(prefix)
remote_name = remote_folder+'/'+filename
local_name = local_folder+'/'+filename
msd_file = 'msd_{}.csv'.format(prefix)
ft_file = 'features_{}.csv'.format(prefix)
aws.download_s3(remote_name, local_name, bucket_name=bucket)
s3 = boto3.client('s3')
# Splitting section
names = ij.partition_im(local_name, irows=rows, icols=cols,
ores=ores, ires=ires)
# Names of subfiles
# names = []
# for i in range(0, 4):
# for j in range(0, 4):
# names.append('{}_{}_{}.tif'.format(prefix, i, j))
for name in names:
aws.upload_s3(name, remote_folder+'/'+name, bucket_name=bucket)
os.remove(name)
print("Done with splitting. Should output file of name {}".format(
remote_folder+'/'+name))
os.remove(filename)
def assemble_msds(prefix, remote_folder, bucket,
ires=(512, 512), frames=651):
'''Calculates MSDs and features from input trajectory files
A function based on msd.all_msds2 and features.calculate_features, creates
msd and feature csv files from input trajectory files and uploads to S3.
Designed to work with Cloudknot for parallelizable workflows. Typically,
this function is used in conjunction with kn.split and kn.tracking for an
entire workflow.
prefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
frames : int
Number of frames in input videos.
'''
import os
import boto3
import diff_classifier.aws as aws
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.utils as ut
filename = '{}.tif'.format(prefix)
remote_name = remote_folder+'/'+filename
msd_file = 'msd_{}.csv'.format(prefix)
ft_file = 'features_{}.csv'.format(prefix)
s3 = boto3.client('s3')
# names = []
# for i in range(0, 4):
# for j in range(0, 4):
# names.append('{}_{}_{}.tif'.format(prefix, i, j))
all_objects = s3.list_objects(Bucket=bucket,
Prefix='{}/{}_'.format(remote_folder,
prefix))
names = []
rows = 0
cols = 0
for entry in all_objects['Contents']:
name = entry['Key'].split('/')[-1]
names.append(name)
row = int(name.split(prefix)[1].split('.')[0].split('_')[-2])
col = int(name.split(prefix)[1].split('.')[0].split('_')[-1])
if row > rows:
rows = row
if col > cols:
cols = col
rows = rows + 1
cols = cols + 1
counter = 0
for name in names:
row = int(name.split(prefix)[1].split('.')[0].split('_')[-2])
col = int(name.split(prefix)[1].split('.')[0].split('_')[-1])
filename = "Traj_{}_{}_{}.csv".format(prefix, row, col)
aws.download_s3(remote_folder+'/'+filename, filename,
bucket_name=bucket)
local_name = filename
if counter == 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0]*col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1]*(rows-1-row)
merged = msd.all_msds2(to_add, frames=frames)
else:
if merged.shape[0] > 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0]*col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1]*(rows-1-row)
to_add['Track_ID'] = to_add['Track_ID'
] + max(merged['Track_ID']) + 1
else:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0]*col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1]*(rows-1-row)
to_add['Track_ID'] = to_add['Track_ID']
merged = merged.append(msd.all_msds2(to_add, frames=frames))
print('Done calculating MSDs for row {} and col {}'.format(row,
col))
counter = counter + 1
merged.to_csv(msd_file)
aws.upload_s3(msd_file, remote_folder+'/'+msd_file, bucket_name=bucket)
merged_ft = ft.calculate_features(merged)
merged_ft.to_csv(ft_file)
aws.upload_s3(ft_file, remote_folder+'/'+ft_file, bucket_name=bucket)
os.remove(ft_file)
os.remove(msd_file)
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
os.remove(outfile)
def plot_all_experiments(experiments, bucket='ccurtis.data', folder='test',
yrange=(10**-1, 10**1), fps=100.02,
xrange=(10**-2, 10**0), upload=True,
outfile='test.png', exponential=True):
"""Plots precision-weighted averages of MSD datasets.
Plots pre-calculated precision-weighted averages of MSD datasets calculated
from precision_averaging and stored in an AWS S3 bucket.
Parameters
----------
group : list of str
List of experiment names to plot. Each experiment must have an MSD and
SEM file associated with it in s3.
bucket : str
S3 bucket from which to download data.
folder : str
Folder in s3 bucket from which to download data.
yrange : list of float
Y range of plot
xrange: list of float
X range of plot
upload : bool
True to upload to S3
outfile : str
Filename of output image
"""
n = len(experiments)
color = iter(cm.viridis(np.linspace(0, 0.9, n)))
fig = plt.figure(figsize=(8.5, 8.5))
plt.xlim(xrange[0], xrange[1])
plt.ylim(yrange[0], yrange[1])
plt.xlabel('Tau (s)', fontsize=25)
plt.ylabel(r'Mean Squared Displacement ($\mu$m$^2$)', fontsize=25)
geo = {}
gstder = {}
counter = 0
for experiment in experiments:
aws.download_s3('{}/geomean_{}.csv'.format(folder, experiment),
'geomean_{}.csv'.format(experiment), bucket_name=bucket)
aws.download_s3('{}/geoSEM_{}.csv'.format(folder, experiment),
'geoSEM_{}.csv'.format(experiment), bucket_name=bucket)
geo[counter] = np.genfromtxt('geomean_{}.csv'.format(experiment))
gstder[counter] = np.genfromtxt('geoSEM_{}.csv'.format(experiment))
geo[counter] = ma.masked_equal(geo[counter], 0.0)
gstder[counter] = ma.masked_equal(gstder[counter], 0.0)
frames = np.shape(gstder[counter])[0]
xpos = np.linspace(0, frames-1, frames)/fps
c = next(color)
if exponential:
plt.loglog(xpos, np.exp(geo[counter]), c=c, linewidth=6,
label=experiment)
plt.loglog(xpos, np.exp(geo[counter] - 1.96*gstder[counter]),
c=c, dashes=[6, 2], linewidth=4)
plt.loglog(xpos, np.exp(geo[counter] + 1.96*gstder[counter]),
c=c, dashes=[6, 2], linewidth=4)
else:
plt.loglog(xpos, geo[counter], c=c, linewidth=6,
label=experiment)
plt.loglog(xpos, geo[counter] - 1.96*gstder[counter], c=c,
dashes=[6, 2], linewidth=4)
plt.loglog(xpos, geo[counter] + 1.96*gstder[counter], c=c,
dashes=[6, 2], linewidth=4)
counter = counter + 1
plt.legend(frameon=False, prop={'size': 16})
if upload:
fig.savefig(outfile, bbox_inches='tight')
aws.upload_s3(outfile, folder+'/'+outfile, bucket_name=bucket)
'range': fparams.yrange,
'rsd': fparams.rsd,
'amplitude': fparams.amp,
'pawcount': fparams.pawcount,
'pawdensity': fparams.pawdens,
'period': fparams.period,
'cross': fparams.cross,
'crossdensity': fparams.crossdens,
'stride': fparams.stride,
'stridestd': fparams.stridestd,
'run': prefix,
'LH range': fparams.yrangefoot['LH'],
'LH std': fparams.ystdfoot['LH'],
'LH rsd': fparams.yrsdfoot['LH'],
'RH range': fparams.yrangefoot['RH'],
'RH std': fparams.ystdfoot['RH'],
'LH rsd': fparams.yrsdfoot['RH'],
'LF range': fparams.yrangefoot['LF'],
'LF std': fparams.ystdfoot['LF'],
'LH rsd': fparams.yrsdfoot['LF'],
'RF range': fparams.yrangefoot['RF'],
'RF std': fparams.ystdfoot['RF'],
'LH rsd': fparams.yrsdfoot['RF']
},
ignore_index=True)
df.to_csv('ferret_stats.csv')
stats = 'ferret_stats.csv'
aws.upload_s3(stats, '{}/{}'.format(folder, stats), bucket_name='ccurtis.data')
def plot_individual_msds(prefix,
x_range=100,
y_range=20,
umppx=0.16,
fps=100.02,
alpha=0.01,
folder='.',
upload=True,
remote_folder="01_18_Experiment",
bucket='ccurtis.data',
figsize=(10, 10)):
"""
Plot MSDs of trajectories and the geometric average.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
x_range: float64 or int
Desire x range of graph.
y_range: float64 or int
Desire y range of graph.
fps: float64
Frames per second of video.
umppx: float64
Resolution of video in microns per pixel.
alpha: float64
Transparency factor. Between 0 and 1.
upload: boolean
True if you want to upload to s3.
Returns
-------
geo_mean: numpy array
Geometric mean of trajectory MSDs at all time points.
geo_SEM: numpy array
Geometric standard errot of trajectory MSDs at all time points.
"""
merged = pd.read_csv('{}/msd_{}.csv'.format(folder, prefix))
fig = plt.figure(figsize=figsize)
particles = int(max(merged['Track_ID']))
frames = int(max(merged['Frame']))
y = np.zeros((particles + 1, frames + 1))
for i in range(0, particles + 1):
y[i, :] = merged.loc[merged.Track_ID == i, 'MSDs'] * umppx * umppx
x = merged.loc[merged.Track_ID == i, 'Frame'] / fps
plt.plot(x, y[i, :], 'k', alpha=alpha)
geo_mean = np.nanmean(ma.log(y), axis=0)
geo_SEM = stats.sem(ma.log(y), axis=0, nan_policy='omit')
plt.plot(x, np.exp(geo_mean), 'k', linewidth=4)
plt.plot(x, np.exp(geo_mean - geo_SEM), 'k--', linewidth=2)
plt.plot(x, np.exp(geo_mean + geo_SEM), 'k--', linewidth=2)
plt.xlim(0, x_range)
plt.ylim(0, y_range)
plt.xlabel('Tau (s)', fontsize=25)
plt.ylabel(r'Mean Squared Displacement ($\mu$m$^2$)', fontsize=25)
outfile = '{}/msds_{}.png'.format(folder, prefix)
outfile2 = '{}/geomean_{}.csv'.format(folder, prefix)
outfile3 = '{}/geoSEM_{}.csv'.format(folder, prefix)
fig.savefig(outfile, bbox_inches='tight')
np.savetxt(outfile2, geo_mean, delimiter=",")
np.savetxt(outfile3, geo_SEM, delimiter=",")
if upload == True:
aws.upload_s3(outfile,
remote_folder + '/' + outfile,
bucket_name=bucket)
aws.upload_s3(outfile2,
remote_folder + '/' + outfile2,
bucket_name=bucket)
aws.upload_s3(outfile3,
remote_folder + '/' + outfile3,
bucket_name=bucket)
return geo_mean, geo_SEM
def tracking(
subprefix,
remote_folder,
bucket='nancelab.publicfiles',
regress_f='regress.obj',
rows=4,
cols=4,
ires=(512, 512),
tparams={
'frames': 651,
'radius': 3.0,
'threshold': 0.0,
'do_median_filtering': False,
'quality': 15.0,
'xdims': (0, 511),
'ydims': (1, 511),
'median_intensity': 300.0,
'snr': 0.0,
'linking_max_distance': 6.0,
'gap_closing_max_distance': 10.0,
'max_frame_gap': 3,
'track_duration': 20.0
}):
'''Tracks particles in input image using Trackmate.
A function based on imagej.track that downloads the image from S3, tracks
particles using Trackmate, and uploads the resulting trajectory file to S3.
Parameters
----------
subprefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
regress_f : string
Name of regress object used to predict quality parameter.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
tparams : dict
Dictionary containing tracking parameters to Trackmate analysis.
'''
import os
import os.path as op
import boto3
from sklearn.externals import joblib
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
local_folder = os.getcwd()
filename = '{}.tif'.format(subprefix)
remote_name = remote_folder + '/' + filename
local_name = local_folder + '/' + filename
outfile = 'Traj_' + subprefix + '.csv'
local_im = op.join(local_folder, '{}.tif'.format(subprefix))
row = int(subprefix.split('_')[-2])
col = int(subprefix.split('_')[-1])
aws.download_s3(remote_folder + '/' + regress_f,
regress_f,
bucket_name=bucket)
with open(regress_f, 'rb') as fp:
regress = joblib.load(fp)
s3 = boto3.client('s3')
aws.download_s3('{}/{}'.format(remote_folder, '{}.tif'.format(subprefix)),
local_im,
bucket_name=bucket)
tparams['quality'] = ij.regress_tracking_params(
regress, subprefix, regmethod='PassiveAggressiveRegressor')
if row == rows - 1:
tparams['ydims'] = (tparams['ydims'][0], ires[1] - 27)
ij.track(local_im, outfile, template=None, fiji_bin=None, tparams=tparams)
aws.upload_s3(outfile, remote_folder + '/' + outfile, bucket_name=bucket)
print("Done with tracking. Should output file of name {}".format(
remote_folder + '/' + outfile))
def assemble_msds(prefix,
remote_folder,
bucket='nancelab.publicfiles',
ires=(512, 512),
frames=651,
rows=4,
cols=4):
'''Calculates MSDs and features from input trajectory files
A function based on msd.all_msds2 and features.calculate_features, creates
msd and feature csv files from input trajectory files and uploads to S3.
prefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
frames : int
Number of frames in input videos.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
'''
import os
import boto3
import diff_classifier.aws as aws
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.utils as ut
filename = '{}.tif'.format(prefix)
remote_name = remote_folder + '/' + filename
msd_file = 'msd_{}.csv'.format(prefix)
ft_file = 'features_{}.csv'.format(prefix)
s3 = boto3.client('s3')
names = []
for i in range(0, 4):
for j in range(0, 4):
names.append('{}_{}_{}.tif'.format(prefix, i, j))
counter = 0
for name in names:
row = int(name.split(prefix)[1].split('.')[0].split('_')[1])
col = int(name.split(prefix)[1].split('.')[0].split('_')[2])
filename = "Traj_{}_{}_{}.csv".format(prefix, row, col)
aws.download_s3(remote_folder + '/' + filename,
filename,
bucket_name=bucket)
local_name = filename
if counter == 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0] * col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1] * (rows - 1 - row)
merged = msd.all_msds2(to_add, frames=frames)
else:
if merged.shape[0] > 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0] * col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1] * (rows - 1 -
row)
to_add['Track_ID'] = to_add['Track_ID'] + max(
merged['Track_ID']) + 1
else:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires[0] * col
to_add['Y'] = ires[1] - to_add['Y'] + ires[1] * (rows - 1 -
row)
to_add['Track_ID'] = to_add['Track_ID']
merged = merged.append(msd.all_msds2(to_add, frames=frames))
print('Done calculating MSDs for row {} and col {}'.format(
row, col))
counter = counter + 1
merged.to_csv(msd_file)
aws.upload_s3(msd_file, remote_folder + '/' + msd_file, bucket_name=bucket)
merged_ft = ft.calculate_features(merged)
merged_ft.to_csv(ft_file)
aws.upload_s3(ft_file, remote_folder + '/' + ft_file, bucket_name=bucket)
os.remove(ft_file)
os.remove(msd_file)
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
os.remove(outfile)
def tracking(subprefix, remote_folder, bucket, tparams,
regress_f='regress.obj', rows=4, cols=4, ires=(512, 512)):
'''Tracks particles in input image using Trackmate.
A function based on imagej.track that downloads the image from S3, tracks
particles using Trackmate, and uploads the resulting trajectory file to S3.
Designed to work with Cloudknot for parallelizable workflows. Typically,
this function is used in conjunction with kn.split and kn.assemble_msds for
a complete analysis.
Parameters
----------
subprefix : string
Prefix (everything except file extension and folder name) of image file
to be tracked. Must be available on S3.
remote_folder : string
Folder name where file is contained on S3 in the bucket specified by
'bucket'.
bucket : string
S3 bucket where file is contained.
regress_f : string
Name of regress object used to predict quality parameter.
rows : int
Number of rows to split image into.
cols : int
Number of columns to split image into.
ires : tuple of int
Resolution of split images. Really just a sanity check to make sure you
correctly splitting.
tparams : dict
Dictionary containing tracking parameters to Trackmate analysis.
'''
import os
import os.path as op
import boto3
from sklearn.externals import joblib
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
local_folder = os.getcwd()
filename = '{}.tif'.format(subprefix)
remote_name = remote_folder+'/'+filename
local_name = local_folder+'/'+filename
outfile = 'Traj_' + subprefix + '.csv'
local_im = op.join(local_folder, '{}.tif'.format(subprefix))
row = int(subprefix.split('_')[-2])
col = int(subprefix.split('_')[-1])
aws.download_s3(remote_folder+'/'+regress_f, regress_f, bucket_name=bucket)
with open(regress_f, 'rb') as fp:
regress = joblib.load(fp)
s3 = boto3.client('s3')
aws.download_s3('{}/{}'.format(remote_folder,
'{}.tif'.format(subprefix)),
local_im, bucket_name=bucket)
tparams['quality'] = ij.regress_tracking_params(regress, subprefix,
regmethod='PassiveAggressiveRegressor')
if row == rows-1:
tparams['ydims'] = (tparams['ydims'][0], ires[1] - 27)
ij.track(local_im, outfile, template=None, fiji_bin=None,
tparams=tparams)
aws.upload_s3(outfile, remote_folder+'/'+outfile, bucket_name=bucket)
print("Done with tracking. Should output file of name {}".format(
remote_folder+'/'+outfile))
def download_split_track_msds(prefix):
"""
1. Checks to see if features file exists.
2. If not, checks to see if image partitioning has occured.
3. If yes, checks to see if tracking has occured.
4. Regardless, tracks, calculates MSDs and features.
"""
import matplotlib as mpl
mpl.use('Agg')
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
import diff_classifier.heatmaps as hm
from scipy.spatial import Voronoi
import scipy.stats as stats
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon
import matplotlib.cm as cm
import os
import os.path as op
import numpy as np
import numpy.ma as ma
import pandas as pd
import boto3
#Splitting section
###############################################################################################
remote_folder = "01_18_Experiment/{}".format(prefix.split('_')[0])
local_folder = os.getcwd()
ires = 512
frames = 651
filename = '{}.tif'.format(prefix)
remote_name = remote_folder+'/'+filename
local_name = local_folder+'/'+filename
msd_file = 'msd_{}.csv'.format(prefix)
ft_file = 'features_{}.csv'.format(prefix)
s3 = boto3.client('s3')
names = []
for i in range(0, 4):
for j in range(0, 4):
names.append('{}_{}_{}.tif'.format(prefix, i, j))
try:
obj = s3.head_object(Bucket='ccurtis7.pup', Key=remote_folder+'/'+ft_file)
except:
try:
for name in names:
aws.download_s3(remote_folder+'/'+name, name)
except:
aws.download_s3(remote_name, local_name)
names = ij.partition_im(local_name)
for name in names:
aws.upload_s3(name, remote_folder+'/'+name)
print("Done with splitting. Should output file of name {}".format(remote_folder+'/'+name))
#Tracking section
################################################################################################
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
local_im = op.join(local_folder, name)
row = int(name.split('.')[0].split('_')[4])
col = int(name.split('.')[0].split('_')[5])
try:
aws.download_s3(remote_folder+'/'+outfile, outfile)
except:
test_intensity = ij.mean_intensity(local_im)
if test_intensity > 500:
quality = 245
else:
quality = 4.5
if row==3:
y = 485
else:
y = 511
ij.track(local_im, outfile, template=None, fiji_bin=None, radius=4.5, threshold=0.,
do_median_filtering=True, quality=quality, x=511, y=y, ylo=1, median_intensity=300.0, snr=0.0,
linking_max_distance=8.0, gap_closing_max_distance=10.0, max_frame_gap=2,
track_displacement=10.0)
aws.upload_s3(outfile, remote_folder+'/'+outfile)
print("Done with tracking. Should output file of name {}".format(remote_folder+'/'+outfile))
#MSD and features section
#################################################################################################
files_to_big = False
size_limit = 10
for name in names:
outfile = 'Traj_' + name.split('.')[0] + '.csv'
local_im = name
file_size_MB = op.getsize(local_im)/1000000
if file_size_MB > size_limit:
file_to_big = True
if files_to_big:
print('One or more of the {} trajectory files exceeds {}MB in size. Will not continue with MSD calculations.'.format(
prefix, size_limit))
else:
counter = 0
for name in names:
row = int(name.split('.')[0].split('_')[4])
col = int(name.split('.')[0].split('_')[5])
filename = "Traj_{}_{}_{}.csv".format(prefix, row, col)
local_name = local_folder+'/'+filename
if counter == 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
merged = msd.all_msds2(to_add, frames=frames)
else:
if merged.shape[0] > 0:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
to_add['Track_ID'] = to_add['Track_ID'] + max(merged['Track_ID']) + 1
else:
to_add = ut.csv_to_pd(local_name)
to_add['X'] = to_add['X'] + ires*col
to_add['Y'] = ires - to_add['Y'] + ires*(3-row)
to_add['Track_ID'] = to_add['Track_ID']
merged = merged.append(msd.all_msds2(to_add, frames=frames))
print('Done calculating MSDs for row {} and col {}'.format(row, col))
counter = counter + 1
merged.to_csv(msd_file)
aws.upload_s3(msd_file, remote_folder+'/'+msd_file)
merged_ft = ft.calculate_features(merged)
merged_ft.to_csv(ft_file)
aws.upload_s3(ft_file, remote_folder+'/'+ft_file)
#Plots
features = ('AR', 'D_fit', 'alpha', 'MSD_ratio', 'Track_ID', 'X', 'Y', 'asymmetry1', 'asymmetry2', 'asymmetry3',
'boundedness', 'efficiency', 'elongation', 'fractal_dim', 'frames', 'kurtosis', 'straightness', 'trappedness')
vmin = (1.36, 0.015, 0.72, -0.09, 0, 0, 0, 0.5, 0.049, 0.089, 0.0069, 0.65, 0.26, 1.28, 0, 1.66, 0.087, -0.225)
vmax = (3.98, 2.6, 2.3, 0.015, max(merged_ft['Track_ID']), 2048, 2048, 0.99, 0.415, 0.53,
0.062, 3.44, 0.75, 1.79, 650, 3.33, 0.52, -0.208)
die = {'features': features,
'vmin': vmin,
'vmax': vmax}
di = pd.DataFrame(data=die)
for i in range(0, di.shape[0]):
hm.plot_heatmap(prefix, feature=di['features'][i], vmin=di['vmin'][i], vmax=di['vmax'][i])
hm.plot_scatterplot(prefix, feature=di['features'][i], vmin=di['vmin'][i], vmax=di['vmax'][i])
hm.plot_trajectories(prefix)
try:
hm.plot_histogram(prefix)
except ValueError:
print("Couldn't plot histogram.")
hm.plot_particles_in_frame(prefix)
gmean1, gSEM1 = hm.plot_individual_msds(prefix, alpha=0.05)
def plot_heatmap(prefix,
feature='asymmetry1',
vmin=0,
vmax=1,
resolution=512,
rows=4,
cols=4,
upload=True,
dpi=None,
figsize=(12, 10),
remote_folder="01_18_Experiment",
bucket='ccurtis.data'):
"""
Plot heatmap of trajectories in video with colors corresponding to features.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
feature: string
Feature to be plotted. See features_analysis.py
vmin: float64
Lower intensity bound for heatmap.
vmax: float64
Upper intensity bound for heatmap.
resolution: int
Resolution of base image. Only needed to calculate bounds of image.
rows: int
Rows of base images used to build tiled image.
cols: int
Columns of base images used to build tiled images.
upload: boolean
True if you want to upload to s3.
dpi: int
Desired dpi of output image.
figsize: list
Desired dimensions of output image.
Returns
-------
"""
# Inputs
# ----------
merged_ft = pd.read_csv('features_{}.csv'.format(prefix))
string = feature
leveler = merged_ft[string]
t_min = vmin
t_max = vmax
ires = resolution
# Building points and color schemes
# ----------
zs = ma.masked_invalid(merged_ft[string])
zs = ma.masked_where(zs <= t_min, zs)
zs = ma.masked_where(zs >= t_max, zs)
to_mask = ma.getmask(zs)
zs = ma.compressed(zs)
xs = ma.compressed(ma.masked_where(to_mask, merged_ft['X'].astype(int)))
ys = ma.compressed(ma.masked_where(to_mask, merged_ft['Y'].astype(int)))
points = np.zeros((xs.shape[0], 2))
points[:, 0] = xs
points[:, 1] = ys
vor = Voronoi(points)
# Plot
# ----------
fig = plt.figure(figsize=figsize, dpi=dpi)
regions, vertices = voronoi_finite_polygons_2d(vor)
my_map = cm.get_cmap('viridis')
norm = mpl.colors.Normalize(t_min, t_max, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.viridis)
test = 0
p2 = 0
counter = 0
for i in range(0, points.shape[0] - 1):
try:
polygon = vertices[regions[p2]]
point1 = Point(points[test, :])
poly1 = Polygon(polygon)
check = poly1.contains(point1)
if check:
plt.fill(*zip(*polygon),
color=my_map(norm(zs[test])),
alpha=0.7)
p2 = p2 + 1
test = test + 1
else:
p2 = p2
test = test + 1
except IndexError:
print('Index mismatch possible.')
mapper.set_array(10)
plt.colorbar(mapper)
plt.xlim(0, ires * cols)
plt.ylim(0, ires * rows)
plt.axis('off')
print('Plotted {} heatmap successfully.'.format(prefix))
outfile = 'hm_{}_{}.png'.format(feature, prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile,
remote_folder + '/' + outfile,
bucket_name=bucket)
def geomean_msdisp(prefix, umppx=0.16, fps=100.02, upload=True,
remote_folder="01_18_Experiment", bucket='ccurtis.data',
backup_frames=651):
"""Comptes geometric averages of mean squared displacement datasets
Calculates geometric averages and stadard errors for MSD datasets. Might
error out if not formatted as output from all_msds2.
Parameters
----------
prefix : string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
umppx : float
Microns per pixel of original images.
fps : float
Frames per second of video.
upload : bool
True if you want to upload to s3.
remote_folder : string
Folder in S3 bucket to upload to.
bucket : string
Name of S3 bucket to upload to.
Returns
-------
geo_mean : numpy.ndarray
Geometric mean of trajectory MSDs at all time points.
geo_stder : numpy.ndarray
Geometric standard errot of trajectory MSDs at all time points.
"""
merged = pd.read_csv('msd_{}.csv'.format(prefix))
try:
particles = int(max(merged['Track_ID']))
frames = int(max(merged['Frame']))
ypos = np.zeros((particles+1, frames+1))
for i in range(0, particles+1):
ypos[i, :] = merged.loc[merged.Track_ID == i, 'MSDs']*umppx*umppx
xpos = merged.loc[merged.Track_ID == i, 'Frame']/fps
geo_mean = np.nanmean(ma.log(ypos), axis=0)
geo_stder = ma.masked_equal(stats.sem(ma.log(ypos), axis=0,
nan_policy='omit'), 0.0)
except ValueError:
geo_mean = np.nan*np.ones(backup_frames)
geo_stder = np.nan*np.ones(backup_frames)
np.savetxt('geomean_{}.csv'.format(prefix), geo_mean, delimiter=",")
np.savetxt('geoSEM_{}.csv'.format(prefix), geo_stder, delimiter=",")
if upload:
aws.upload_s3('geomean_{}.csv'.format(prefix),
remote_folder+'/'+'geomean_{}.csv'.format(prefix),
bucket_name=bucket)
aws.upload_s3('geoSEM_{}.csv'.format(prefix),
remote_folder+'/'+'geoSEM_{}.csv'.format(prefix),
bucket_name=bucket)
return geo_mean, geo_stder
def plot_histogram(prefix,
xlabel='Log Diffusion Coefficient Dist',
ylabel='Trajectory Count',
fps=100.02,
umppx=0.16,
frames=651,
y_range=100,
frame_interval=20,
frame_range=100,
analysis='log',
theta='D',
upload=True,
remote_folder="01_18_Experiment",
bucket='ccurtis.data'):
"""
Plot heatmap of trajectories in video with colors corresponding to features.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
xlabel: string
X axis label.
ylabel: string
Y axis label.
fps: float64
Frames per second of video.
umppx: float64
Resolution of video in microns per pixel.
frames: int
Number of frames in video.
y_range: float64 or int
Desire y range of graph.
frame_interval: int
Desired spacing between MSDs/Deffs to be plotted.
analysis: string
Desired output format. If log, will plot log(MSDs/Deffs)
theta: string
Desired output. D for diffusion coefficients. Anything else, MSDs.
upload: boolean
True if you want to upload to s3.
"""
merged = pd.read_csv('msd_{}.csv'.format(prefix))
data = merged
frame_range = range(frame_interval, frame_range + frame_interval,
frame_interval)
# load data
# generate keys for legend
bar = {}
keys = []
entries = []
for i in range(0, len(list(frame_range))):
keys.append(i)
entries.append(str(10 * frame_interval * (i + 1)) + 'ms')
set_x_limit = False
set_y_limit = True
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
fig = plt.figure(figsize=(16, 6))
counter = 0
for i in frame_range:
toi = i / fps
if theta == "MSD":
factor = 1
else:
factor = 4 * toi
if analysis == 'log':
dist = np.log(umppx * umppx *
merged.loc[merged.Frame == i, 'MSDs'].dropna() /
factor)
test_bins = np.linspace(-5, 5, 76)
else:
dist = umppx * umppx * merged.loc[merged.Frame == i,
'MSDs'].dropna() / factor
test_bins = np.linspace(0, 20, 76)
histogram, test_bins = np.histogram(dist, bins=test_bins)
# Plot_general_histogram_code
avg = np.mean(dist)
plt.rc('axes', linewidth=2)
plot = histogram
bins = test_bins
width = 0.7 * (bins[1] - bins[0])
center = (bins[:-1] + bins[1:]) / 2
bar[keys[counter]] = plt.bar(center,
plot,
align='center',
width=width,
color=colors[counter],
label=entries[counter])
plt.axvline(avg, color=colors[counter])
plt.xlabel(xlabel, fontsize=30)
plt.ylabel(ylabel, fontsize=30)
plt.tick_params(axis='both', which='major', labelsize=20)
counter = counter + 1
if set_y_limit:
plt.gca().set_ylim([0, y_range])
if set_x_limit:
plt.gca().set_xlim([0, x_range])
plt.legend(fontsize=20, frameon=False)
outfile = 'hist_{}.png'.format(prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile,
remote_folder + '/' + outfile,
bucket_name=bucket)
def precision_averaging(group, geomean, geo_stder, weights, save=True,
bucket='ccurtis.data', folder='test',
experiment='test'):
"""Calculates precision-weighted averages of MSD datasets.
Parameters
----------
group : list of str
List of experiment names to average. Each element corresponds to a key
in geo_stder and geomean.
geomean : dict of numpy.ndarray
Each entry in dictionary corresponds to an MSD profiles, they key
corresponding to an experiment name.
geo_stder : dict of numpy.ndarray
Each entry in dictionary corresponds to the standard errors of an MSD
profile, the key corresponding to an experiment name.
weights : numpy.ndarray
Precision weights to be used in precision averaging.
Returns
-------
geo : numpy.ndarray
Precision-weighted averaged MSDs from experiments specified in group
geo_stder : numpy.ndarray
Precision-weighted averaged SEMs from experiments specified in group
"""
frames = np.shape(geo_stder[group[0]])[0]
slices = len(group)
video_counter = 0
geo_holder = np.zeros((slices, frames))
gstder_holder = np.zeros((slices, frames))
w_holder = np.zeros((slices, frames))
for sample in group:
w_holder[video_counter, :] = (1/(geo_stder[sample]*geo_stder[sample])
)/weights
geo_holder[video_counter, :] = w_holder[video_counter, :
] * geomean[sample]
gstder_holder[video_counter, :] = 1/(geo_stder[sample]*geo_stder[sample]
)
video_counter = video_counter + 1
w_holder = ma.masked_equal(w_holder, 0.0)
w_holder = ma.masked_equal(w_holder, 1.0)
geo_holder = ma.masked_equal(geo_holder, 0.0)
geo_holder = ma.masked_equal(geo_holder, 1.0)
gstder_holder = ma.masked_equal(gstder_holder, 0.0)
gstder_holder = ma.masked_equal(gstder_holder, 1.0)
geo = ma.sum(geo_holder, axis=0)
geo_stder = ma.sqrt((1/ma.sum(gstder_holder, axis=0)))
if save:
geo_f = 'geomean_{}.csv'.format(experiment)
gstder_f = 'geoSEM_{}.csv'.format(experiment)
np.savetxt(geo_f, geo, delimiter=',')
np.savetxt(gstder_f, geo_stder, delimiter=',')
aws.upload_s3(geo_f, '{}/{}'.format(folder, geo_f), bucket_name=bucket)
aws.upload_s3(gstder_f, '{}/{}'.format(folder, gstder_f),
bucket_name=bucket)
geodata = Bunch(geomean=geo, geostd=geo_stder, weighthold=w_holder,
geostdhold=gstder_holder)
return geodata
def sensitivity_it(counter):
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import diff_classifier.aws as aws
import diff_classifier.utils as ut
import diff_classifier.msd as msd
import diff_classifier.features as ft
import diff_classifier.imagej as ij
import diff_classifier.heatmaps as hm
from scipy.spatial import Voronoi
import scipy.stats as stats
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon
import matplotlib.cm as cm
import os
import os.path as op
import numpy as np
import numpy.ma as ma
import pandas as pd
import boto3
import itertools
#Sweep parameters
#----------------------------------
radius = [4.5, 6.0, 7.0]
do_median_filtering = [True, False]
quality = [1.5, 4.5, 8.5]
linking_max_distance = [6.0, 10.0, 15.0]
gap_closing_max_distance = [6.0, 10.0, 15.0]
max_frame_gap = [1, 2, 5]
track_displacement = [0.0, 10.0, 20.0]
sweep = [
radius, do_median_filtering, quality, linking_max_distance,
gap_closing_max_distance, max_frame_gap, track_displacement
]
all_params = list(itertools.product(*sweep))
#Variable prep
#----------------------------------
s3 = boto3.client('s3')
folder = '01_18_Experiment'
s_folder = '{}/sensitivity'.format(folder)
local_folder = '.'
prefix = "P1_S1_R_0001_2_2"
name = "{}.tif".format(prefix)
local_im = op.join(local_folder, name)
aws.download_s3(
'{}/{}/{}.tif'.format(folder,
prefix.split('_')[0], prefix),
'{}.tif'.format(prefix))
outputs = np.zeros((len(all_params), len(all_params[0]) + 2))
#Tracking and calculations
#------------------------------------
params = all_params[counter]
outfile = 'Traj_{}_{}.csv'.format(name.split('.')[0], counter)
msd_file = 'msd_{}_{}.csv'.format(name.split('.')[0], counter)
geo_file = 'geomean_{}_{}.csv'.format(name.split('.')[0], counter)
geoS_file = 'geoSEM_{}_{}.csv'.format(name.split('.')[0], counter)
msd_image = 'msds_{}_{}.png'.format(name.split('.')[0], counter)
iter_name = "{}_{}".format(prefix, counter)
ij.track(local_im,
outfile,
template=None,
fiji_bin=None,
radius=params[0],
threshold=0.,
do_median_filtering=params[1],
quality=params[2],
x=511,
y=511,
ylo=1,
median_intensity=300.0,
snr=0.0,
linking_max_distance=params[3],
gap_closing_max_distance=params[4],
max_frame_gap=params[5],
track_displacement=params[6])
traj = ut.csv_to_pd(outfile)
msds = msd.all_msds2(traj, frames=651)
msds.to_csv(msd_file)
gmean1, gSEM1 = hm.plot_individual_msds(iter_name, alpha=0.05)
np.savetxt(geo_file, gmean1, delimiter=",")
np.savetxt(geoS_file, gSEM1, delimiter=",")
aws.upload_s3(outfile, '{}/{}'.format(s_folder, outfile))
aws.upload_s3(msd_file, '{}/{}'.format(s_folder, msd_file))
aws.upload_s3(geo_file, '{}/{}'.format(s_folder, geo_file))
aws.upload_s3(geoS_file, '{}/{}'.format(s_folder, geoS_file))
aws.upload_s3(msd_image, '{}/{}'.format(s_folder, msd_image))
print('Successful parameter calculations for {}'.format(iter_name))
def BF_cell_features(prefix, folder, bucket='ccurtis.data'):
ffilename = 'features_{}.csv'.format(prefix)
mfilename = 'msd_{}.csv'.format(prefix)
bffilename = 'BF_cells_{}.tif'.format(prefix)
biim = 'bi_BF_cells_{}.tif'.format(prefix)
bimages = 'biproc_BF_cells_{}.png'.format(prefix)
aws.download_s3('{}/{}'.format(folder, ffilename),
ffilename,
bucket_name=bucket)
aws.download_s3('{}/{}'.format(folder, mfilename),
mfilename,
bucket_name=bucket)
aws.download_s3('{}/{}'.format(folder, bffilename),
bffilename,
bucket_name=bucket)
print('Successfully downloaded files')
fstats = pd.read_csv(ffilename, encoding="ISO-8859-1")
msds = pd.read_csv(mfilename, encoding="ISO-8859-1")
bfimage = plt.imread(bffilename)
tophimage = binary_BF(bfimage,
opense=disk(12),
bi_thresh=1.2,
tophatse=disk(20))
plt.savefig(bimages)
euimage = EuclideanTransform(tophimage) + EuclideanTransform(~tophimage)
print('Successfully performed image processing')
xa = -np.reshape(np.clip(
(fstats.Y.values - 1).astype(int), a_min=0, a_max=2043),
newshape=(fstats.Y.shape[0], 1))
ya = np.reshape(np.clip((fstats.X.values - 1).astype(int),
a_min=0,
a_max=2043),
newshape=(fstats.X.shape[0], 1))
xya = [tuple(l) for l in np.concatenate((xa, ya), axis=1).tolist()]
fstats['Cell Status'] = itemgetter(*xya)(tophimage)
fstats['Cell Distance'] = itemgetter(*xya)(euimage)
print('Successfully calculated Cell Status Params')
frames = 651
xb = -np.reshape(np.clip(
(msds.Y.values - 1).astype(int), a_min=0, a_max=2043),
newshape=(int(msds.Y.shape[0]), 1))
yb = np.reshape(np.clip((msds.X.values - 1).astype(int),
a_min=0,
a_max=2043),
newshape=(int(msds.X.shape[0]), 1))
xyb = [tuple(l) for l in np.concatenate((xb, yb), axis=1).tolist()]
msds['Cell Status'] = itemgetter(*xyb)(tophimage)
msds['Cell Distance'] = itemgetter(*xyb)(euimage)
msds_cell_status = np.reshape(msds['Cell Status'].values,
newshape=(int(msds.X.shape[0] / frames),
frames))
msds_cell_distance = np.reshape(msds['Cell Distance'].values,
newshape=(int(msds.X.shape[0] / frames),
frames))
fstats['Membrane Xing'] = np.sum(np.diff(msds_cell_status, axis=1) == True,
axis=1)
fstats['Distance Towards Cell'] = np.sum(np.diff(msds_cell_distance,
axis=1),
axis=1)
fstats['Percent Towards Cell'] = np.mean(
np.diff(msds_cell_distance, axis=1) > 0, axis=1)
print('Successfully calculated Membrane Xing Params')
fstats.to_csv(ffilename, sep=',', encoding="ISO-8859-1")
msds.to_csv(mfilename, sep=',', encoding="ISO-8859-1")
plt.imsave(biim, tophimage, cmap='gray')
aws.upload_s3(ffilename,
'{}/{}'.format(folder, ffilename),
bucket_name=bucket)
aws.upload_s3(mfilename,
'{}/{}'.format(folder, mfilename),
bucket_name=bucket)
aws.upload_s3(biim, '{}/{}'.format(folder, biim), bucket_name=bucket)
aws.upload_s3(bimages, '{}/{}'.format(folder, bimages, bucket_name=bucket))
print('Successfully uploaded files')
return fstats
