def correct_schafer_worm(self):
if hasattr(self, 'stage_vec_inv'):
print(
'The worm has been previously corrected. The attribute "stage_vec_inv" exists. '
)
return
self.ventral_side = read_ventral_side(self.file_name)
assert isGoodStageAligment(self.file_name)
self.stage_vec_inv, _ = _h_get_stage_inv(self.file_name,
self.timestamp)
#remove data where the stage is moving (the blurred image can induce artifacts)
self.is_stage_move = np.isnan(self.stage_vec_inv[:, 0])
self.widths[self.is_stage_move, :] = np.nan
for field in ['skeleton', 'ventral_contour', 'dorsal_contour']:
if hasattr(self, field):
tmp_dat = getattr(self, field)
# rotate the skeletons
# for ii in range(tot_skel):
#tmp_dat[ii] = np.dot(rotation_matrix, tmp_dat[ii].T).T
tmp_dat = tmp_dat + self.stage_vec_inv[:, np.newaxis, :]
setattr(self, field, tmp_dat)
def _get_ventral_side(features_file):
ventral_side = read_ventral_side(features_file)
if not ventral_side or ventral_side == 'unknown':
ventral_type = '?'
else:
#we will merge the ventral and dorsal contours so the ventral contour is clockwise
ventral_type = 'CW'
return ventral_type
def save_timeseries_feats_table(features_file,
derivate_delta_time,
fovsplitter_param={}):
timeseries_features = []
fps = read_fps(features_file)
# initialise class for splitting fov
if len(fovsplitter_param) > 0:
is_fov_tosplit = True
assert all(key in fovsplitter_param
for key in ['total_n_wells', 'whichsideup', 'well_shape'])
assert fovsplitter_param['total_n_wells'] > 0
else:
is_fov_tosplit = False
print('is fov to split?', is_fov_tosplit)
if is_fov_tosplit:
# split fov in wells
masked_image_file = features_file.replace('Results', 'MaskedVideos')
masked_image_file = masked_image_file.replace('_featuresN.hdf5',
'.hdf5')
# fovsplitter = FOVMultiWellsSplitter(masked_image_file=masked_image_file,
# total_n_wells=fovsplitter_param['total_n_wells'],
# whichsideup=fovsplitter_param['whichsideup'],
# well_shape=fovsplitter_param['well_shape'])
fovsplitter = FOVMultiWellsSplitter(masked_image_file,
**fovsplitter_param)
# store wells data in the features file
fovsplitter.write_fov_wells_to_file(features_file)
with pd.HDFStore(features_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
trajectories_data_g = trajectories_data.groupby('worm_index_joined')
progress_timer = TimeCounter('')
base_name = get_base_name(features_file)
tot_worms = len(trajectories_data_g)
def _display_progress(n):
# display progress
dd = " Calculating tierpsy features. Worm %i of %i done." % (n + 1,
tot_worms)
print_flush(base_name + dd + ' Total time:' +
progress_timer.get_time_str())
_display_progress(0)
with tables.File(features_file, 'r+') as fid:
for gg in [
'/timeseries_data', '/event_durations', '/timeseries_features'
]:
if gg in fid:
fid.remove_node(gg)
feat_dtypes = [(x, np.float32) for x in timeseries_all_columns]
feat_dtypes = [('worm_index', np.int32), ('timestamp', np.int32),
('well_name', 'S3')] + feat_dtypes
timeseries_features = fid.create_table('/',
'timeseries_data',
obj=np.recarray(0, feat_dtypes),
filters=TABLE_FILTERS)
if '/food_cnt_coord' in fid:
food_cnt = fid.get_node('/food_cnt_coord')[:]
else:
food_cnt = None
#If i find the ventral side in the multiworm case this has to change
ventral_side = read_ventral_side(features_file)
for ind_n, (worm_index, worm_data) in enumerate(trajectories_data_g):
skel_id = worm_data['skeleton_id'].values
#deal with any nan in the skeletons
good_id = skel_id >= 0
skel_id_val = skel_id[good_id]
traj_size = skel_id.size
args = []
for p in ('skeletons', 'widths', 'dorsal_contours',
'ventral_contours'):
node_str = '/coordinates/' + p
if node_str in fid:
node = fid.get_node(node_str)
dat = np.full((traj_size, *node.shape[1:]), np.nan)
if skel_id_val.size > 0:
if len(node.shape) == 3:
dd = node[skel_id_val, :, :]
else:
dd = node[skel_id_val, :]
dat[good_id] = dd
else:
dat = None
args.append(dat)
timestamp = worm_data['timestamp_raw'].values.astype(np.int32)
feats = get_timeseries_features(
*args,
timestamp=timestamp,
food_cnt=food_cnt,
fps=fps,
ventral_side=ventral_side,
derivate_delta_time=derivate_delta_time)
#save timeseries features data
feats = feats.astype(np.float32)
feats['worm_index'] = worm_index
if is_fov_tosplit:
feats[
'well_name'] = fovsplitter.find_well_from_trajectories_data(
worm_data)
else:
feats['well_name'] = 'n/a'
# cast well_name to the correct type
# (before shuffling columns, so it remains the last entry)
# needed because for some reason this does not work:
# feats['well_name'] = feats['well_name'].astype('S3')
feats['_well_name'] = feats['well_name'].astype('S3')
feats.drop(columns='well_name', inplace=True)
feats.rename(columns={'_well_name': 'well_name'}, inplace=True)
#move the last fields to the first columns
cols = feats.columns.tolist()
cols = cols[-2:] + cols[:-2]
cols[1], cols[2] = cols[2], cols[1]
feats = feats[cols]
feats['worm_index'] = feats['worm_index'].astype(np.int32)
feats['timestamp'] = feats['timestamp'].astype(np.int32)
feats = feats.to_records(index=False)
timeseries_features.append(feats)
_display_progress(ind_n)
def save_timeseries_feats_table(features_file, derivate_delta_time):
timeseries_features = []
fps = read_fps(features_file)
with pd.HDFStore(features_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
#only use data that was skeletonized
#trajectories_data = trajectories_data[trajectories_data['skeleton_id']>=0]
trajectories_data_g = trajectories_data.groupby('worm_index_joined')
progress_timer = TimeCounter('')
base_name = get_base_name(features_file)
tot_worms = len(trajectories_data_g)
def _display_progress(n):
# display progress
dd = " Calculating tierpsy features. Worm %i of %i done." % (n + 1,
tot_worms)
print_flush(base_name + dd + ' Total time:' +
progress_timer.get_time_str())
_display_progress(0)
with tables.File(features_file, 'r+') as fid:
for gg in [
'/timeseries_data', '/event_durations', '/timeseries_features'
]:
if gg in fid:
fid.remove_node(gg)
feat_dtypes = [(x, np.float32) for x in timeseries_all_columns]
feat_dtypes = [('worm_index', np.int32),
('timestamp', np.int32)] + feat_dtypes
timeseries_features = fid.create_table('/',
'timeseries_data',
obj=np.recarray(0, feat_dtypes),
filters=TABLE_FILTERS)
if '/food_cnt_coord' in fid:
food_cnt = fid.get_node('/food_cnt_coord')[:]
else:
food_cnt = None
#If i find the ventral side in the multiworm case this has to change
ventral_side = read_ventral_side(features_file)
for ind_n, (worm_index, worm_data) in enumerate(trajectories_data_g):
with tables.File(features_file, 'r') as fid:
skel_id = worm_data['skeleton_id'].values
#deal with any nan in the skeletons
good_id = skel_id >= 0
skel_id_val = skel_id[good_id]
traj_size = skel_id.size
args = []
for p in ('skeletons', 'widths', 'dorsal_contours',
'ventral_contours'):
node_str = '/coordinates/' + p
if node_str in fid:
node = fid.get_node(node_str)
dat = np.full((traj_size, *node.shape[1:]), np.nan)
if skel_id_val.size > 0:
if len(node.shape) == 3:
dd = node[skel_id_val, :, :]
else:
dd = node[skel_id_val, :]
dat[good_id] = dd
else:
dat = None
args.append(dat)
timestamp = worm_data['timestamp_raw'].values.astype(np.int32)
feats = get_timeseries_features(
*args,
timestamp=timestamp,
food_cnt=food_cnt,
fps=fps,
ventral_side=ventral_side,
derivate_delta_time=derivate_delta_time)
#save timeseries features data
feats = feats.astype(np.float32)
feats['worm_index'] = worm_index
#move the last fields to the first columns
cols = feats.columns.tolist()
cols = cols[-1:] + cols[:-1]
feats = feats[cols]
feats['worm_index'] = feats['worm_index'].astype(np.int32)
feats['timestamp'] = feats['timestamp'].astype(np.int32)
feats = feats.to_records(index=False)
timeseries_features.append(feats)
_display_progress(ind_n)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 16 14:52:11 2017
@author: ajaver
"""
import glob
import os
import tables
from tierpsy.helper.params import read_ventral_side
main_dir = '/Volumes/behavgenom_archive$/single_worm/finished'
fnames = glob.glob(os.path.join(main_dir, '**', '*_featuresN.hdf5'),
recursive=True)
for ii, fname in enumerate(fnames):
print(ii + 1, len(fnames))
skeletons_file = fname.replace('_featuresN', '_skeletons')
try:
with tables.File(fname, 'r+') as fid:
fid.get_node('/trajectories_data')._v_attrs[
'ventral_side'] = read_ventral_side(skeletons_file)
except tables.exceptions.NoSuchNodeError:
pass
def smooth_skeletons_table(skeletons_file,
features_file,
is_WT2=False,
skel_smooth_window=5,
coords_smooth_window_s=0.25,
gap_to_interp_s=0.25):
#%%
#%%
fps = read_fps(skeletons_file)
coords_smooth_window = int(np.round(fps * coords_smooth_window_s))
gap_to_interp = int(np.round(fps * gap_to_interp_s))
if coords_smooth_window <= 3: #do not interpolate
coords_smooth_window = None
trajectories_data = _r_fill_trajectories_data(skeletons_file)
#%%
trajectories_data_g = trajectories_data.groupby('worm_index_joined')
progress_timer = TimeCounter('')
base_name = get_base_name(skeletons_file)
tot_worms = len(trajectories_data_g)
def _display_progress(n):
# display progress
dd = " Smoothing skeletons. Worm %i of %i done." % (n, tot_worms)
print_flush(base_name + dd + ' Total time:' +
progress_timer.get_time_str())
_display_progress(0)
#%%
#initialize arrays
food_cnt = read_food_contour(skeletons_file)
with tables.File(skeletons_file, 'r') as fid:
n_segments = fid.get_node('/skeleton').shape[1]
with tables.File(features_file, 'w') as fid_features:
if food_cnt is not None:
fid_features.create_array('/',
'food_cnt_coord',
obj=food_cnt.astype(np.float32))
worm_coords_array = {}
w_node = fid_features.create_group('/', 'coordinates')
for array_name in [
'skeletons', 'dorsal_contours', 'ventral_contours', 'widths'
]:
if array_name != 'widths':
a_shape = (0, n_segments, 2)
else:
a_shape = (0, n_segments)
worm_coords_array[array_name] = fid_features.create_earray(
w_node,
array_name,
shape=a_shape,
atom=tables.Float32Atom(shape=()),
filters=TABLE_FILTERS)
tot_skeletons = 0
for ind_n, (worm_index, worm_data) in enumerate(trajectories_data_g):
if worm_data['was_skeletonized'].sum() < 2:
continue
worm = WormFromTable(skeletons_file,
worm_index,
worm_index_type='worm_index_joined')
if is_WT2:
worm.correct_schafer_worm()
if np.sum(~np.isnan(worm.skeleton[:, 0, 0])) <= 2:
warnings.warn('Not enough data to smooth. Empty file?')
wormN = worm
else:
wormN = SmoothedWorm(worm.skeleton,
worm.widths,
worm.ventral_contour,
worm.dorsal_contour,
skel_smooth_window=skel_smooth_window,
coords_smooth_window=coords_smooth_window,
gap_to_interp=gap_to_interp)
dat_index = pd.Series(False,
index=worm_data['timestamp_raw'].values)
try:
dat_index[worm.timestamp] = True
except ValueError:
import pdb
pdb.set_trace()
#%%
skeleton_id = np.arange(wormN.skeleton.shape[0]) + tot_skeletons
tot_skeletons = skeleton_id[-1] + 1
row_ind = worm_data.index[dat_index.values]
trajectories_data.loc[row_ind, 'skeleton_id'] = skeleton_id
#%%
#add data
worm_coords_array['skeletons'].append(getattr(wormN, 'skeleton'))
worm_coords_array['dorsal_contours'].append(
getattr(wormN, 'dorsal_contour'))
worm_coords_array['ventral_contours'].append(
getattr(wormN, 'ventral_contour'))
worm_coords_array['widths'].append(getattr(wormN, 'widths'))
#display progress
_display_progress(ind_n + 1)
#save trajectories data
newT = fid_features.create_table(
'/',
'trajectories_data',
obj=trajectories_data.to_records(index=False),
filters=TABLE_FILTERS)
copy_unit_conversions(newT, skeletons_file)
newT._v_attrs['is_WT2'] = is_WT2
newT._v_attrs['ventral_side'] = read_ventral_side(skeletons_file)
#save blob features interpolating in dropped frames and stage movement (WT2)
blob_features = _r_fill_blob_features(skeletons_file,
trajectories_data, is_WT2)
if blob_features is not None:
fid_features.create_table(
'/',
'blob_features',
obj=blob_features.to_records(index=False),
filters=TABLE_FILTERS)
def _get_timeseries_feats(features_file, delta_time=1 / 3):
'''
Get the all the time series features from the skeletons
'''
timeseries_features = []
fps = read_fps(features_file)
with pd.HDFStore(features_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
#only use data that was skeletonized
#trajectories_data = trajectories_data[trajectories_data['skeleton_id']>=0]
trajectories_data_g = trajectories_data.groupby('worm_index_joined')
progress_timer = TimeCounter('')
base_name = get_base_name(features_file)
tot_worms = len(trajectories_data_g)
def _display_progress(n):
# display progress
dd = " Calculating tierpsy features. Worm %i of %i done." % (n + 1,
tot_worms)
print_flush(base_name + dd + ' Total time:' +
progress_timer.get_time_str())
_display_progress(0)
with tables.File(features_file, 'r') as fid:
if '/food_cnt_coord' in fid:
food_cnt = fid.get_node('/food_cnt_coord')[:]
else:
food_cnt = None
#If i find the ventral side in the multiworm case this has to change
ventral_side = read_ventral_side(features_file)
timeseries_features = []
for ind_n, (worm_index, worm_data) in enumerate(trajectories_data_g):
with tables.File(features_file, 'r') as fid:
skel_id = worm_data['skeleton_id'].values
#deal with any nan in the skeletons
good_id = skel_id >= 0
skel_id_val = skel_id[good_id]
traj_size = skel_id.size
args = []
for p in ('skeletons', 'widths', 'dorsal_contours',
'ventral_contours'):
node = fid.get_node('/coordinates/' + p)
dat = np.full((traj_size, *node.shape[1:]), np.nan)
if skel_id_val.size > 0:
if len(node.shape) == 3:
dd = node[skel_id_val, :, :]
else:
dd = node[skel_id_val, :]
dat[good_id] = dd
args.append(dat)
timestamp = worm_data['timestamp_raw'].values.astype(np.int32)
feats = get_timeseries_features(*args,
timestamp=timestamp,
food_cnt=food_cnt,
fps=fps,
ventral_side=ventral_side)
#save timeseries features data
feats = feats.astype(np.float32)
feats['worm_index'] = worm_index
#move the last fields to the first columns
cols = feats.columns.tolist()
cols = cols[-1:] + cols[:-1]
feats = feats[cols]
feats['worm_index'] = feats['worm_index'].astype(np.int32)
feats['timestamp'] = feats['timestamp'].astype(np.int32)
timeseries_features.append(feats)
_display_progress(ind_n)
timeseries_features = pd.concat(timeseries_features, ignore_index=True)
return timeseries_features
import os
import pandas as pd
from tierpsy.helper.params import read_fps, read_ventral_side
if __name__ == '__main__':
experiments_df = pd.read_csv('ageing_celine.csv')
bad_f = []
good_f = []
for irow, row in experiments_df.iterrows():
print(irow, len(experiments_df))
skel_file = os.path.join(row['directory'],
row['base_name'] + '_skeletons.hdf5')
print(skel_file)
ventral_side = read_ventral_side(skel_file)
if '_CW_' in skel_file and ventral_side != 'clockwise':
bad_f.append(skel_file)
print(skel_file)
elif '_CCW_' in skel_file and ventral_side != 'anticlockwise':
bad_f.append(skel_file)
print(skel_file)
else:
good_f.append((ventral_side, skel_file))
#%%
# import tables
#
# for ii, skel_file in enumerate(bad_f):
# print(ii, len(bad_f))
#
# bn = skel_file.replace('_skeletons.hdf5', '')
