def calculate_food_cnt(mask_file, method='NN', _is_debug=False, solidity_th=0.98): assert method in ['NN', 'MORPH'] if method == 'NN': food_cnt, food_prob, cnt_solidity = get_food_contour_nn( mask_file, _is_debug=_is_debug) if cnt_solidity < solidity_th: food_cnt = np.zeros(0) elif method == 'MORPH': food_cnt = get_food_contour_morph(mask_file, _is_debug=_is_debug) else: raise ValueError('Invalid method argument.') #transform contour from pixels to microns microns_per_pixel = read_microns_per_pixel(mask_file) food_cnt *= microns_per_pixel #smooth contours food_cnt = smooth_cnt(food_cnt, _is_debug=_is_debug) return food_cnt
def _h_add_stage_position_pix(mask_file, skeletons_file): # if the stage was aligned correctly add the information into the mask file microns_per_pixel = read_microns_per_pixel(mask_file) with tables.File(mask_file, 'r+') as fid: timestamp_c = fid.get_node('/timestamp/raw')[:] timestamp = np.arange(np.min(timestamp_c), np.max(timestamp_c) + 1) stage_vec_inv, ind_ff = _h_get_stage_inv(skeletons_file, timestamp) stage_vec_pix = stage_vec_inv[ind_ff] / microns_per_pixel if '/stage_position_pix' in fid: fid.remove_node('/', 'stage_position_pix') fid.create_array('/', 'stage_position_pix', obj=stage_vec_pix)
def obtain_food_cnt(mask_video): #%% microns_per_pixel = read_microns_per_pixel(mask_video) circy, circx = get_food_contour(mask_video, is_debug=False) #%% food_cnt = np.vstack((circx, circy)).T*microns_per_pixel #polar coordinates from the centroid food_centroid = np.mean(food_cnt, axis=0) food_r = np.linalg.norm(food_cnt-food_centroid, axis=1) return food_cnt, food_r, food_centroid
def read_food_contour(skeletons_file): try: with tables.File(skeletons_file, 'r') as fid: food_cnt_pix = fid.get_node('/food_cnt_coord')[:] #smooth contours microns_per_pixel = read_microns_per_pixel(skeletons_file) food_cnt = microns_per_pixel * food_cnt_pix food_cnt = _h_smooth_cnt(food_cnt) except tables.exceptions.NoSuchNodeError: food_cnt = None return food_cnt
def __init__(self, file_name, worm_index, use_skel_filter=True, worm_index_type='worm_index_joined', smooth_window=-1, POL_DEGREE_DFLT=3): # Populates an empty normalized worm. #if it does not exists return 1 as a default, like that we can still calculate the features in pixels and frames, instead of micrometers and seconds. self.microns_per_pixel = read_microns_per_pixel(file_name, dflt=1) self.fps = read_fps(file_name, dflt=1) # savitzky-golay filter polynomial order default self.POL_DEGREE_DFLT = POL_DEGREE_DFLT # save the input parameters self.file_name = file_name self.worm_index = worm_index self.use_skel_filter = use_skel_filter self.worm_index_type = worm_index_type # set to less than POL_DEGREE_DFLT to eliminate smoothing self.smooth_window = smooth_window # smooth window must be an odd number larger than the polynomial degree # (savitzky-golay filter requirement) if self.smooth_window >= self.POL_DEGREE_DFLT and self.smooth_window % 2 == 0: self.smooth_window += 1 self.ventral_side = 'unknown' self._h_read_data() # smooth data if required if self.smooth_window > self.POL_DEGREE_DFLT: # print('Smoothing...') self.skeleton = _h_smooth_curve_all(self.skeleton, window=self.smooth_window) self.widths = _h_smooth_curve_all(self.widths, window=self.smooth_window) # assert the dimenssions of the read data are correct self._h_assert_data_dim()
def _r_fill_blob_features(skeletons_file, trajectories_data_f, is_WT2): ''' Read previously calculated blob features, convert features with units `pixels` into microns and if it is_WT2 add the stage movement to the centroid coordinates. ''' microns_per_pixel = read_microns_per_pixel(skeletons_file) with pd.HDFStore(skeletons_file, 'r') as fid: if not 'blob_features' in fid: return blob_features = fid['/blob_features'] blob_features['area'] *= microns_per_pixel**2 for feat in ['coord_x', 'coord_y', 'perimeter', 'box_length', 'box_width']: blob_features[feat] *= microns_per_pixel blob_features = blob_features.merge( trajectories_data_f[['old_trajectory_data_index']], left_index=True, right_on='old_trajectory_data_index', how = 'right' ) del blob_features['old_trajectory_data_index'] if is_WT2: stage_vec_inv, _ = _h_get_stage_inv(skeletons_file, trajectories_data_f['timestamp_raw'].values) is_stage_move = np.isnan(stage_vec_inv[:, 0]) blob_features[is_stage_move] = np.nan blob_features['coord_x'] += stage_vec_inv[:, 0] blob_features['coord_y'] += stage_vec_inv[:, 1] blob_features = blob_features.interpolate() return blob_features
def _test_read(): from tierpsy.helper.params import read_microns_per_pixel microns_per_pixel = read_microns_per_pixel(feat_file) with pd.HDFStore(feat_file, 'r') as fid: trajectories_data = fid['/trajectories_data'] roi_generator = generateMoviesROI(mask_file, trajectories_data, roi_size=128) frame_data = next(roi_generator) for irow, (img_roi, roi_corner) in frame_data.items(): img_roi_N = (img_roi.astype(np.float32)-90)/255 row = trajectories_data.loc[irow] plt.figure() plt.imshow(img_roi_N, interpolation=None, cmap='gray') skel_id = int(row['skeleton_id']) if skel_id > 0: with tables.File(feat_file, 'r') as fid: skel = fid.get_node('/coordinates/skeletons')[skel_id] skel /= microns_per_pixel skel -= roi_corner[ None, :] plt.plot(skel[..., 0], skel[..., 1]) plt.plot(skel[0, 0], skel[0, 1], 'o')
def test_plots(mask_file, skeletons_file, first_ind_plot=0, max_n_plots=25): with pd.HDFStore(skeletons_file, 'r') as fid: trajectories_data = fid['/trajectories_data'] microns_per_pixel = read_microns_per_pixel(skeletons_file) with tables.File(skeletons_file, 'r+') as fid: food_cnt_pix = fid.get_node('/', 'food_cnt_coord')[:] food_cnt = food_cnt_pix * microns_per_pixel g_data = trajectories_data.groupby('worm_index_joined') with tables.File(skeletons_file, 'r') as fid: for ii, (worm_index, worm_data) in enumerate(g_data): if worm_index < first_ind_plot: continue print(ii) skeletons = fid.get_node('/skeleton')[ worm_data['skeleton_id'].values, :, :] skeletons *= microns_per_pixel get_cnt_feats(skeletons, food_cnt, _is_debug=True) plt.suptitle(worm_index) if ii > max_n_plots: break
import torch.optim as optim from torch.autograd import Variable import torch.nn.functional as F import matplotlib.pylab as plt from tierpsy.helper.params import read_microns_per_pixel from tierpsy.analysis.ske_create.helperIterROI import getROIfromInd from scipy.ndimage.filters import laplace, median_filter, gaussian_filter mask_file = '/Users/ajaver/OneDrive - Imperial College London/aggregation/N2_1_Ch1_29062017_182108_comp3.hdf5' feat_file = mask_file.replace('.hdf5', '_featuresN.hdf5') microns_per_pixel = read_microns_per_pixel(feat_file) with pd.HDFStore(feat_file, 'r') as fid: trajectories_data = fid['/trajectories_data'] #%% skel_data = trajectories_data[(trajectories_data['skeleton_id'] >= 0)] skel_g = skel_data.groupby('worm_index_joined') with tables.File(feat_file, 'r') as fid: skel_h = fid.get_node('/coordinates/skeletons')[:, 0, 0] #%% traj_with_gaps = skel_data.loc[np.isnan(skel_h), 'worm_index_joined'].unique() w_ind = 264
def _match_units(filter_params, fps, fname): """ author: EM The filtering thresholds must match the timeseries units. If the right conversion is not possible, then check_ok is False, and the feature summaries will not be calculated for this file. """ from copy import deepcopy if filter_params is None: return filter_params, True all_units = filter_params['units'] + [filter_params['time_units']] cfilter_params = deepcopy(filter_params) if fps == -1: # In this case, all time-related timeseries will be in frames. # If thresholds have been defined in seconds there is no way to convert. if 'seconds' in all_units: no_attr_flush('fps', fname) return cfilter_params, False else: # In this case, all time-related timeseries will be in seconds. # We always want the time_units for traj_length in frames if filter_params['time_units']=='seconds' and \ filter_params['min_traj_length'] is not None: cfilter_params['min_traj_length'] = \ filter_params['min_traj_length']*fps # If the timeseries therholds are defined in seconds, no conversion is # necessary # If the timeseries thresholds are defined in frames, we need to convert # to seconds if 'frame_numbers' in filter_params['units']: ids = [ i for i, x in enumerate(filter_params['units']) if x == 'frame_numbers' ] for i in ids: if filter_params['min_thresholds'][i] is not None: cfilter_params['min_thresholds'][i]= \ filter_params['min_thresholds'][i]/fps if filter_params['max_thresholds'][i] is not None: cfilter_params['max_thresholds'][i]= \ filter_params['max_thresholds'][i]/fps mpp = read_microns_per_pixel(fname) if mpp == -1: # In this case, all distance-related timeseries will be in pixels. # If thresholds have been defined in microns there is no way to convert. if 'microns' in all_units: no_attr_flush('mpp', fname) return cfilter_params, False else: # In this case, all distance-related timeseries will be in microns. # If the timeseries threholds are defined in micorns, no conversion is # necessary # If the timeseries thresholds are defined in pixels, we need to convert # to microns if filter_params['distance_units']=='pixels' and \ filter_params['min_distance_traveled'] is not None: cfilter_params['min_distance_traveled'] = \ filter_params['min_distance_traveled']*mpp if 'pixels' in filter_params['units']: ids = [ i for i, x in enumerate(filter_params['units']) if x == 'pixels' ] for i in ids: if filter_params['min_thresholds'][i] is not None: cfilter_params['min_thresholds'][i]= \ filter_params['min_thresholds'][i]*mpp if filter_params['max_thresholds'][i] is not None: cfilter_params['max_thresholds'][i]= \ filter_params['max_thresholds'][i]*mpp return cfilter_params, True
def updateSkelFile(self, skel_file, dflt_skel_size=10): super().updateSkelFile(skel_file) self.ui.spinBox_skelBlock.setMaximum(max(len(self.skel_block) - 1, 0)) self.ui.spinBox_skelBlock.setMinimum(0) if self.skel_block_n != 0: self.skel_block_n = 0 self.ui.spinBox_skelBlock.setValue(0) else: self.changeSkelBlock(0) self.skel_block = [] self.is_stage_move = [] self.stage_position_pix = None self.is_feat_file = False VALID_ERRORS = (IOError, KeyError, tables.exceptions.HDF5ExtError, tables.exceptions.NoSuchNodeError) #try to read the information from the features file if possible if not self.trajectories_data is None: try: with tables.File(self.skeletons_file, 'r') as fid: self.stage_position_pix = fid.get_node( '/stage_movement/stage_vec')[:] #only used for skeletons, and to test the head/tail orientation. I leave it but probably should be removed for in the future prov_str = fid.get_node( '/provenance_tracking/INT_SKE_ORIENT').read() func_arg_str = json.loads( prov_str.decode("utf-8"))['func_arguments'] gap_size = json.loads(func_arg_str)['gap_size'] good = (self.trajectories_data['int_map_id'] > 0).values has_skel_group = createBlocks(good, min_block_size=0) if len(has_skel_group) > 0: self.skel_block = _fuseOverlapingGroups( has_skel_group, gap_size=gap_size) except VALID_ERRORS: pass else: try: #load skeletons from _features.hdf5 if '/stage_position_pix' in self.fid: self.stage_position_pix = self.fid.get_node( '/stage_position_pix')[:] else: n_frames = self.fid.get_node('/mask').shape[0] self.stage_position_pix = np.full((n_frames, 2), np.nan) timestamp = self.fid.get_node('/timestamp/raw')[:] self.microns_per_pixel = read_microns_per_pixel( self.skeletons_file) with pd.HDFStore(self.skeletons_file, 'r') as ske_file_id: #this could be better so I do not have to load everything into memory, but this is faster self.trajectories_data = ske_file_id[ '/features_timeseries'] if self.trajectories_data['worm_index'].unique().size != 1: QMessageBox.critical( self, '', "There is more than one worm index. This file does not seem to have been analyzed with the WT2 option.", QMessageBox.Ok) raise KeyError() good = self.trajectories_data['timestamp'].isin(timestamp) self.trajectories_data = self.trajectories_data[good] self.trajectories_data.sort_values(by='timestamp', inplace=True) if np.any( self.trajectories_data['timestamp'] < 0) or np.any( self.trajectories_data['timestamp'].isnull()): QMessageBox.critical( self, '', 'There are invalid values in the timestamp. I cannot get the stage movement information.', QMessageBox.Ok) raise KeyError() first_frame = np.where( timestamp == self.trajectories_data['timestamp'].min())[0][0] last_frame = np.where( timestamp == self.trajectories_data['timestamp'].max())[0][0] self.trajectories_data['frame_number'] = np.arange( first_frame, last_frame + 1, dtype=np.int) self.trajectories_data[ 'skeleton_id'] = self.trajectories_data.index self.traj_time_grouped = self.trajectories_data.groupby( 'frame_number') self.is_feat_file = True except VALID_ERRORS: self.trajectories_data = None self.traj_time_grouped = None self.is_feat_file = False if self.stage_position_pix is not None: self.is_stage_move = np.isnan(self.stage_position_pix[:, 0]) self.updateImage()
def getFoodFeatures(mask_file, skeletons_file, features_file=None, cnt_method='NN', solidity_th=0.98, batch_size=100000, _is_debug=False): if features_file is None: features_file = remove_ext(skeletons_file) + '_featuresN.hdf5' base_name = get_base_name(mask_file) progress_timer = TimeCounter('') print_flush("{} Calculating food features {}".format( base_name, progress_timer.get_time_str())) food_cnt = calculate_food_cnt(mask_file, method=cnt_method, solidity_th=solidity_th, _is_debug=_is_debug) microns_per_pixel = read_microns_per_pixel(skeletons_file) #store contour coordinates in pixels into the skeletons file for visualization purposes food_cnt_pix = food_cnt / microns_per_pixel with tables.File(skeletons_file, 'r+') as fid: if '/food_cnt_coord' in fid: fid.remove_node('/food_cnt_coord') if _is_valid_cnt(food_cnt): tab = fid.create_array('/', 'food_cnt_coord', obj=food_cnt_pix) tab._v_attrs['method'] = cnt_method print_flush("{} Calculating food features {}".format( base_name, progress_timer.get_time_str())) feats_names = [ 'orient_to_food_cnt', 'dist_from_food_cnt', 'closest_cnt_ind' ] feats_dtypes = [(x, np.float32) for x in feats_names] with tables.File(skeletons_file, 'r') as fid: tot_rows = fid.get_node('/skeleton').shape[0] features_df = np.full(tot_rows, np.nan, dtype=feats_dtypes) if food_cnt.size > 0: for ii in range(0, tot_rows, batch_size): skeletons = fid.get_node('/skeleton')[ii:ii + batch_size] skeletons *= microns_per_pixel outputs = get_cnt_feats(skeletons, food_cnt, _is_debug=_is_debug) for irow, row in enumerate(zip(*outputs)): features_df[irow + ii] = row with tables.File(features_file, 'a') as fid: if '/food' in fid: fid.remove_node('/food', recursive=True) fid.create_group('/', 'food') if _is_valid_cnt(food_cnt): fid.create_carray('/food', 'cnt_coordinates', obj=food_cnt, filters=TABLE_FILTERS) fid.create_table('/food', 'features', obj=features_df, filters=TABLE_FILTERS) #%% print_flush("{} Calculating food features {}".format( base_name, progress_timer.get_time_str()))