def calculate_food_cnt(mask_file,
method='NN',
_is_debug=False,
solidity_th=0.98):
assert method in ['NN', 'MORPH']
if method == 'NN':
food_cnt, food_prob, cnt_solidity = get_food_contour_nn(
mask_file, _is_debug=_is_debug)
if cnt_solidity < solidity_th:
food_cnt = np.zeros(0)
elif method == 'MORPH':
food_cnt = get_food_contour_morph(mask_file, _is_debug=_is_debug)
else:
raise ValueError('Invalid method argument.')
#transform contour from pixels to microns
microns_per_pixel = read_microns_per_pixel(mask_file)
food_cnt *= microns_per_pixel
#smooth contours
food_cnt = smooth_cnt(food_cnt, _is_debug=_is_debug)
return food_cnt
def _h_add_stage_position_pix(mask_file, skeletons_file):
# if the stage was aligned correctly add the information into the mask file
microns_per_pixel = read_microns_per_pixel(mask_file)
with tables.File(mask_file, 'r+') as fid:
timestamp_c = fid.get_node('/timestamp/raw')[:]
timestamp = np.arange(np.min(timestamp_c), np.max(timestamp_c) + 1)
stage_vec_inv, ind_ff = _h_get_stage_inv(skeletons_file, timestamp)
stage_vec_pix = stage_vec_inv[ind_ff] / microns_per_pixel
if '/stage_position_pix' in fid:
fid.remove_node('/', 'stage_position_pix')
fid.create_array('/', 'stage_position_pix', obj=stage_vec_pix)
def obtain_food_cnt(mask_video):
#%%
microns_per_pixel = read_microns_per_pixel(mask_video)
circy, circx = get_food_contour(mask_video, is_debug=False)
#%%
food_cnt = np.vstack((circx, circy)).T*microns_per_pixel
#polar coordinates from the centroid
food_centroid = np.mean(food_cnt, axis=0)
food_r = np.linalg.norm(food_cnt-food_centroid, axis=1)
return food_cnt, food_r, food_centroid
def read_food_contour(skeletons_file):
try:
with tables.File(skeletons_file, 'r') as fid:
food_cnt_pix = fid.get_node('/food_cnt_coord')[:]
#smooth contours
microns_per_pixel = read_microns_per_pixel(skeletons_file)
food_cnt = microns_per_pixel * food_cnt_pix
food_cnt = _h_smooth_cnt(food_cnt)
except tables.exceptions.NoSuchNodeError:
food_cnt = None
return food_cnt
def __init__(self,
file_name,
worm_index,
use_skel_filter=True,
worm_index_type='worm_index_joined',
smooth_window=-1,
POL_DEGREE_DFLT=3):
# Populates an empty normalized worm.
#if it does not exists return 1 as a default, like that we can still calculate the features in pixels and frames, instead of micrometers and seconds.
self.microns_per_pixel = read_microns_per_pixel(file_name, dflt=1)
self.fps = read_fps(file_name, dflt=1)
# savitzky-golay filter polynomial order default
self.POL_DEGREE_DFLT = POL_DEGREE_DFLT
# save the input parameters
self.file_name = file_name
self.worm_index = worm_index
self.use_skel_filter = use_skel_filter
self.worm_index_type = worm_index_type
# set to less than POL_DEGREE_DFLT to eliminate smoothing
self.smooth_window = smooth_window
# smooth window must be an odd number larger than the polynomial degree
# (savitzky-golay filter requirement)
if self.smooth_window >= self.POL_DEGREE_DFLT and self.smooth_window % 2 == 0:
self.smooth_window += 1
self.ventral_side = 'unknown'
self._h_read_data()
# smooth data if required
if self.smooth_window > self.POL_DEGREE_DFLT:
# print('Smoothing...')
self.skeleton = _h_smooth_curve_all(self.skeleton,
window=self.smooth_window)
self.widths = _h_smooth_curve_all(self.widths,
window=self.smooth_window)
# assert the dimenssions of the read data are correct
self._h_assert_data_dim()
def _r_fill_blob_features(skeletons_file, trajectories_data_f, is_WT2):
'''
Read previously calculated blob features, convert features with units
`pixels` into microns and if it is_WT2 add the stage movement to the centroid
coordinates.
'''
microns_per_pixel = read_microns_per_pixel(skeletons_file)
with pd.HDFStore(skeletons_file, 'r') as fid:
if not 'blob_features' in fid:
return
blob_features = fid['/blob_features']
blob_features['area'] *= microns_per_pixel**2
for feat in ['coord_x', 'coord_y', 'perimeter', 'box_length', 'box_width']:
blob_features[feat] *= microns_per_pixel
blob_features = blob_features.merge(
trajectories_data_f[['old_trajectory_data_index']],
left_index=True,
right_on='old_trajectory_data_index',
how = 'right'
)
del blob_features['old_trajectory_data_index']
if is_WT2:
stage_vec_inv, _ = _h_get_stage_inv(skeletons_file,
trajectories_data_f['timestamp_raw'].values)
is_stage_move = np.isnan(stage_vec_inv[:, 0])
blob_features[is_stage_move] = np.nan
blob_features['coord_x'] += stage_vec_inv[:, 0]
blob_features['coord_y'] += stage_vec_inv[:, 1]
blob_features = blob_features.interpolate()
return blob_features
def _test_read():
from tierpsy.helper.params import read_microns_per_pixel
microns_per_pixel = read_microns_per_pixel(feat_file)
with pd.HDFStore(feat_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
roi_generator = generateMoviesROI(mask_file, trajectories_data, roi_size=128)
frame_data = next(roi_generator)
for irow, (img_roi, roi_corner) in frame_data.items():
img_roi_N = (img_roi.astype(np.float32)-90)/255
row = trajectories_data.loc[irow]
plt.figure()
plt.imshow(img_roi_N, interpolation=None, cmap='gray')
skel_id = int(row['skeleton_id'])
if skel_id > 0:
with tables.File(feat_file, 'r') as fid:
skel = fid.get_node('/coordinates/skeletons')[skel_id]
skel /= microns_per_pixel
skel -= roi_corner[ None, :]
plt.plot(skel[..., 0], skel[..., 1])
plt.plot(skel[0, 0], skel[0, 1], 'o')
def test_plots(mask_file, skeletons_file, first_ind_plot=0, max_n_plots=25):
with pd.HDFStore(skeletons_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
microns_per_pixel = read_microns_per_pixel(skeletons_file)
with tables.File(skeletons_file, 'r+') as fid:
food_cnt_pix = fid.get_node('/', 'food_cnt_coord')[:]
food_cnt = food_cnt_pix * microns_per_pixel
g_data = trajectories_data.groupby('worm_index_joined')
with tables.File(skeletons_file, 'r') as fid:
for ii, (worm_index, worm_data) in enumerate(g_data):
if worm_index < first_ind_plot:
continue
print(ii)
skeletons = fid.get_node('/skeleton')[
worm_data['skeleton_id'].values, :, :]
skeletons *= microns_per_pixel
get_cnt_feats(skeletons, food_cnt, _is_debug=True)
plt.suptitle(worm_index)
if ii > max_n_plots:
break
import torch.optim as optim
from torch.autograd import Variable
import torch.nn.functional as F
import matplotlib.pylab as plt
from tierpsy.helper.params import read_microns_per_pixel
from tierpsy.analysis.ske_create.helperIterROI import getROIfromInd
from scipy.ndimage.filters import laplace, median_filter, gaussian_filter
mask_file = '/Users/ajaver/OneDrive - Imperial College London/aggregation/N2_1_Ch1_29062017_182108_comp3.hdf5'
feat_file = mask_file.replace('.hdf5', '_featuresN.hdf5')
microns_per_pixel = read_microns_per_pixel(feat_file)
with pd.HDFStore(feat_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
#%%
skel_data = trajectories_data[(trajectories_data['skeleton_id'] >= 0)]
skel_g = skel_data.groupby('worm_index_joined')
with tables.File(feat_file, 'r') as fid:
skel_h = fid.get_node('/coordinates/skeletons')[:, 0, 0]
#%%
traj_with_gaps = skel_data.loc[np.isnan(skel_h), 'worm_index_joined'].unique()
w_ind = 264
def _match_units(filter_params, fps, fname):
"""
author: EM
The filtering thresholds must match the timeseries units. If the right
conversion is not possible, then check_ok is False, and the feature
summaries will not be calculated for this file.
"""
from copy import deepcopy
if filter_params is None:
return filter_params, True
all_units = filter_params['units'] + [filter_params['time_units']]
cfilter_params = deepcopy(filter_params)
if fps == -1:
# In this case, all time-related timeseries will be in frames.
# If thresholds have been defined in seconds there is no way to convert.
if 'seconds' in all_units:
no_attr_flush('fps', fname)
return cfilter_params, False
else:
# In this case, all time-related timeseries will be in seconds.
# We always want the time_units for traj_length in frames
if filter_params['time_units']=='seconds' and \
filter_params['min_traj_length'] is not None:
cfilter_params['min_traj_length'] = \
filter_params['min_traj_length']*fps
# If the timeseries therholds are defined in seconds, no conversion is
# necessary
# If the timeseries thresholds are defined in frames, we need to convert
# to seconds
if 'frame_numbers' in filter_params['units']:
ids = [
i for i, x in enumerate(filter_params['units'])
if x == 'frame_numbers'
]
for i in ids:
if filter_params['min_thresholds'][i] is not None:
cfilter_params['min_thresholds'][i]= \
filter_params['min_thresholds'][i]/fps
if filter_params['max_thresholds'][i] is not None:
cfilter_params['max_thresholds'][i]= \
filter_params['max_thresholds'][i]/fps
mpp = read_microns_per_pixel(fname)
if mpp == -1:
# In this case, all distance-related timeseries will be in pixels.
# If thresholds have been defined in microns there is no way to convert.
if 'microns' in all_units:
no_attr_flush('mpp', fname)
return cfilter_params, False
else:
# In this case, all distance-related timeseries will be in microns.
# If the timeseries threholds are defined in micorns, no conversion is
# necessary
# If the timeseries thresholds are defined in pixels, we need to convert
# to microns
if filter_params['distance_units']=='pixels' and \
filter_params['min_distance_traveled'] is not None:
cfilter_params['min_distance_traveled'] = \
filter_params['min_distance_traveled']*mpp
if 'pixels' in filter_params['units']:
ids = [
i for i, x in enumerate(filter_params['units'])
if x == 'pixels'
]
for i in ids:
if filter_params['min_thresholds'][i] is not None:
cfilter_params['min_thresholds'][i]= \
filter_params['min_thresholds'][i]*mpp
if filter_params['max_thresholds'][i] is not None:
cfilter_params['max_thresholds'][i]= \
filter_params['max_thresholds'][i]*mpp
return cfilter_params, True
def updateSkelFile(self, skel_file, dflt_skel_size=10):
super().updateSkelFile(skel_file)
self.ui.spinBox_skelBlock.setMaximum(max(len(self.skel_block) - 1, 0))
self.ui.spinBox_skelBlock.setMinimum(0)
if self.skel_block_n != 0:
self.skel_block_n = 0
self.ui.spinBox_skelBlock.setValue(0)
else:
self.changeSkelBlock(0)
self.skel_block = []
self.is_stage_move = []
self.stage_position_pix = None
self.is_feat_file = False
VALID_ERRORS = (IOError, KeyError, tables.exceptions.HDF5ExtError,
tables.exceptions.NoSuchNodeError)
#try to read the information from the features file if possible
if not self.trajectories_data is None:
try:
with tables.File(self.skeletons_file, 'r') as fid:
self.stage_position_pix = fid.get_node(
'/stage_movement/stage_vec')[:]
#only used for skeletons, and to test the head/tail orientation. I leave it but probably should be removed for in the future
prov_str = fid.get_node(
'/provenance_tracking/INT_SKE_ORIENT').read()
func_arg_str = json.loads(
prov_str.decode("utf-8"))['func_arguments']
gap_size = json.loads(func_arg_str)['gap_size']
good = (self.trajectories_data['int_map_id'] > 0).values
has_skel_group = createBlocks(good, min_block_size=0)
if len(has_skel_group) > 0:
self.skel_block = _fuseOverlapingGroups(
has_skel_group, gap_size=gap_size)
except VALID_ERRORS:
pass
else:
try:
#load skeletons from _features.hdf5
if '/stage_position_pix' in self.fid:
self.stage_position_pix = self.fid.get_node(
'/stage_position_pix')[:]
else:
n_frames = self.fid.get_node('/mask').shape[0]
self.stage_position_pix = np.full((n_frames, 2), np.nan)
timestamp = self.fid.get_node('/timestamp/raw')[:]
self.microns_per_pixel = read_microns_per_pixel(
self.skeletons_file)
with pd.HDFStore(self.skeletons_file, 'r') as ske_file_id:
#this could be better so I do not have to load everything into memory, but this is faster
self.trajectories_data = ske_file_id[
'/features_timeseries']
if self.trajectories_data['worm_index'].unique().size != 1:
QMessageBox.critical(
self, '',
"There is more than one worm index. This file does not seem to have been analyzed with the WT2 option.",
QMessageBox.Ok)
raise KeyError()
good = self.trajectories_data['timestamp'].isin(timestamp)
self.trajectories_data = self.trajectories_data[good]
self.trajectories_data.sort_values(by='timestamp',
inplace=True)
if np.any(
self.trajectories_data['timestamp'] < 0) or np.any(
self.trajectories_data['timestamp'].isnull()):
QMessageBox.critical(
self, '',
'There are invalid values in the timestamp. I cannot get the stage movement information.',
QMessageBox.Ok)
raise KeyError()
first_frame = np.where(
timestamp ==
self.trajectories_data['timestamp'].min())[0][0]
last_frame = np.where(
timestamp ==
self.trajectories_data['timestamp'].max())[0][0]
self.trajectories_data['frame_number'] = np.arange(
first_frame, last_frame + 1, dtype=np.int)
self.trajectories_data[
'skeleton_id'] = self.trajectories_data.index
self.traj_time_grouped = self.trajectories_data.groupby(
'frame_number')
self.is_feat_file = True
except VALID_ERRORS:
self.trajectories_data = None
self.traj_time_grouped = None
self.is_feat_file = False
if self.stage_position_pix is not None:
self.is_stage_move = np.isnan(self.stage_position_pix[:, 0])
self.updateImage()
def getFoodFeatures(mask_file,
skeletons_file,
features_file=None,
cnt_method='NN',
solidity_th=0.98,
batch_size=100000,
_is_debug=False):
if features_file is None:
features_file = remove_ext(skeletons_file) + '_featuresN.hdf5'
base_name = get_base_name(mask_file)
progress_timer = TimeCounter('')
print_flush("{} Calculating food features {}".format(
base_name, progress_timer.get_time_str()))
food_cnt = calculate_food_cnt(mask_file,
method=cnt_method,
solidity_th=solidity_th,
_is_debug=_is_debug)
microns_per_pixel = read_microns_per_pixel(skeletons_file)
#store contour coordinates in pixels into the skeletons file for visualization purposes
food_cnt_pix = food_cnt / microns_per_pixel
with tables.File(skeletons_file, 'r+') as fid:
if '/food_cnt_coord' in fid:
fid.remove_node('/food_cnt_coord')
if _is_valid_cnt(food_cnt):
tab = fid.create_array('/', 'food_cnt_coord', obj=food_cnt_pix)
tab._v_attrs['method'] = cnt_method
print_flush("{} Calculating food features {}".format(
base_name, progress_timer.get_time_str()))
feats_names = [
'orient_to_food_cnt', 'dist_from_food_cnt', 'closest_cnt_ind'
]
feats_dtypes = [(x, np.float32) for x in feats_names]
with tables.File(skeletons_file, 'r') as fid:
tot_rows = fid.get_node('/skeleton').shape[0]
features_df = np.full(tot_rows, np.nan, dtype=feats_dtypes)
if food_cnt.size > 0:
for ii in range(0, tot_rows, batch_size):
skeletons = fid.get_node('/skeleton')[ii:ii + batch_size]
skeletons *= microns_per_pixel
outputs = get_cnt_feats(skeletons,
food_cnt,
_is_debug=_is_debug)
for irow, row in enumerate(zip(*outputs)):
features_df[irow + ii] = row
with tables.File(features_file, 'a') as fid:
if '/food' in fid:
fid.remove_node('/food', recursive=True)
fid.create_group('/', 'food')
if _is_valid_cnt(food_cnt):
fid.create_carray('/food',
'cnt_coordinates',
obj=food_cnt,
filters=TABLE_FILTERS)
fid.create_table('/food',
'features',
obj=features_df,
filters=TABLE_FILTERS)
#%%
print_flush("{} Calculating food features {}".format(
base_name, progress_timer.get_time_str()))
