def copyFilesFromFinal(self):
#check that the program didn't finished and that not forced start point was not specified
if ((self.final_start_point == checkpoint['END']) or self.final_start_point > self.end_point) and not self.force_start_point_str:
#If the program has finished there is nothing to do here.
print_flush('%s The files from completed results analysis were found in %s. Remove them if you want to recalculated them again.' % (self.base_name, self.results_dir))
sys.exit(0)
#find what files we need to copy from the final destination if the analysis is going to resume from a later point
files2copy = []
if self.analysis_start_point <= checkpoint['INT_PROFILE']:
#we do not need the mask to calculate the features
try:
#check if there is already a finished/readable temporary mask file in current directory otherwise copy the
with h5py.File(self.tmp_mask_file, "r") as mask_fid:
if mask_fid['/mask'].attrs['has_finished'] < 1:
#go to the exception if the mask has any other flag
raise
except:
with h5py.File(self.masked_image_file, "r") as mask_fid:
#check if the video to mask conversion did indeed finished correctly
assert mask_fid['/mask'].attrs['has_finished'] >= 1
files2copy += [(self.masked_image_file, self.tmp_mask_dir)]
if self.tmp_start_point < self.final_start_point:
#copy files from an incomplete analysis files.
if self.final_start_point > checkpoint['TRAJ_CREATE']: #and final_start_point <= checkpoint['SKE_CREATE']:
files2copy += [(self.trajectories_file, self.tmp_results_dir)]
if self.final_start_point > checkpoint['SKE_CREATE']:
files2copy += [(self.skeletons_file, self.tmp_results_dir)]
if self.final_start_point > checkpoint['INT_PROFILE']:
files2copy += [(self.intensities_file, self.tmp_results_dir)]
copyFilesLocal(files2copy)
def exec_all(self): try: self.start() self.main_code() self.clean() except: raise print_flush(base_name + ' Error')
def compressSingleWorker(video_file, mask_dir, param_file, is_single_worm,
cmd_original):
if mask_dir[-1] != os.sep: mask_dir += os.sep
#get function parameters
param = tracker_param(param_file)
base_name = video_file.rpartition('.')[0].rpartition(os.sep)[-1]
masked_image_file = os.path.join(mask_dir, base_name + '.hdf5')
print(masked_image_file)
try:
#try to get the has_finished flag, if this fails the file is likely to be corrupted so we can start over again
with h5py.File(masked_image_file, "r") as mask_fid:
has_finished = mask_fid['/mask'].attrs['has_finished']
if is_single_worm and not all(
dd in mask_fid for dd in ['stage_data', 'xml_info']):
#if a single_worm file does not have the stage_data or xml_info there is something really bad,
#let's recalculate everything again
has_finished = 0
except:
has_finished = 0
if has_finished:
print_flush(
'File alread exists: %s. If you want to calculate the mask again delete the existing file.'
% masked_image_file)
return
else:
initial_time = time.time()
#check if the video file exists
assert os.path.exists(video_file)
#create the mask path if it does not exist
if not os.path.exists(mask_dir): os.makedirs(mask_dir)
#This function will throw an exception if the additional files do not exists, before attempting to calculate anything
if is_single_worm: getAdditionalFiles(video_file)
#compress video the provenance tracking functions
compressVideoWProv(video_file, masked_image_file,
param.compress_vid_param, cmd_original)
#get the store the additional data for the single worm case. It needs that the
#masked_image_file has been created.
if is_single_worm: storeAdditionalDataSW(video_file, masked_image_file)
time_str = str(
datetime.timedelta(seconds=round(time.time() - initial_time)))
progress_str = 'Processing Done. Total time = %s' % time_str
print_flush(base_name + ' ' + progress_str)
return masked_image_file
def copyFilesLocal(files2copy):
#copy the necessary files (maybe we can create a daemon later)
for files in files2copy:
file_name, destination = files
#if not os.path.exists(file_name): continue
assert(os.path.exists(destination))
if os.path.abspath(os.path.dirname(file_name)) != os.path.abspath(destination):
print_flush('Copying %s to %s' % (file_name, destination))
shutil.copy(file_name, destination)
def copyFilesLocal(files2copy):
#copy the necessary files (maybe we can create a daemon later)
for files in files2copy:
file_name, destination = files
#if not os.path.exists(file_name): continue
assert (os.path.exists(destination))
if os.path.abspath(
os.path.dirname(file_name)) != os.path.abspath(destination):
print_flush('Copying %s to %s' % (file_name, destination))
shutil.copy(file_name, destination)
def compressSingleWorker(video_file, mask_dir, param_file, is_single_worm, cmd_original):
if mask_dir[-1] != os.sep: mask_dir += os.sep
#get function parameters
param = tracker_param(param_file)
base_name = video_file.rpartition('.')[0].rpartition(os.sep)[-1]
masked_image_file = os.path.join(mask_dir, base_name + '.hdf5')
print(masked_image_file)
try:
#try to get the has_finished flag, if this fails the file is likely to be corrupted so we can start over again
with h5py.File(masked_image_file, "r") as mask_fid:
has_finished = mask_fid['/mask'].attrs['has_finished']
if is_single_worm and not all(dd in mask_fid for dd in ['stage_data', 'xml_info']):
#if a single_worm file does not have the stage_data or xml_info there is something really bad,
#let's recalculate everything again
has_finished = 0
except:
has_finished = 0
if has_finished:
print_flush('File alread exists: %s. If you want to calculate the mask again delete the existing file.' % masked_image_file)
return
else:
initial_time = time.time()
#check if the video file exists
assert os.path.exists(video_file)
#create the mask path if it does not exist
if not os.path.exists(mask_dir): os.makedirs(mask_dir)
#This function will throw an exception if the additional files do not exists, before attempting to calculate anything
if is_single_worm: getAdditionalFiles(video_file)
#compress video the provenance tracking functions
compressVideoWProv(video_file, masked_image_file, param.compress_vid_param, cmd_original)
#get the store the additional data for the single worm case. It needs that the
#masked_image_file has been created.
if is_single_worm: storeAdditionalDataSW(video_file, masked_image_file)
time_str = str(datetime.timedelta(seconds=round(time.time()-initial_time)))
progress_str = 'Processing Done. Total time = %s' % time_str
print_flush(base_name + ' ' + progress_str)
return masked_image_file
def start(self):
self.start_time = time.time()
if not os.path.exists(self.tmp_mask_dir): os.makedirs(self.tmp_mask_dir)
self.get_file_names()
self.get_start_point()
#copy the video file locally if requiered. See getFileNames
if self.final_has_finished == 0 and self.tmp_video_file != self.video_file:
print_flush(self.base_name + ' Copying video file to local temporary directory.')
shutil.copy(self.video_file, self.tmp_video_file)
if self.is_single_worm:
try:
dd = self.video_file.rpartition('.')[0]
shutil.copy(dd + '.log.csv', self.tmp_mask_dir)
shutil.copy(dd + '.info.xml', self.tmp_mask_dir)
except FileNotFoundError:
try:
dd = self.video_file.rpartition('.')[0]
dd, base_name = os.path.split(dd)
dd = os.path.join(dd, '.data', base_name)
shutil.copy(dd + '.log.csv', self.tmp_mask_dir)
shutil.copy(dd + '.info.xml', self.tmp_mask_dir)
except FileNotFoundError:
pass
#this might be more logically group in main_code, but this operation can and should be do remotely if required
if self.final_has_finished == 1:
#The file finished to be processed but the additional data was not stored. We can do this remotely.
os.chflags(self.masked_image_file, not stat.UF_IMMUTABLE)
compressSingleWorker(self.video_file, self.mask_dir, self.json_file, self.is_single_worm, self.cmd_original)
os.chflags(self.masked_image_file, stat.UF_IMMUTABLE)
if self.final_has_finished == 2:
print_flush('File alread exists: %s. If you want to calculate the mask again delete the existing file.' % self.masked_image_file)
#parameters to calculate the mask, this will return a list with necessary commands in case I want to use it as a subprocess
if self.is_calculate_mask:
self.main_input_params = [self.tmp_video_file, self.tmp_mask_dir, self.json_file, self.is_single_worm, self.cmd_original]
else:
self.main_input_params = []
return self.create_script()
def clean(self):
if self.final_has_finished == 0:
print_flush(self.base_name + ' Moving files to final destination and removing temporary files.')
if os.path.abspath(self.tmp_video_file) != os.path.abspath(self.video_file):
#print_flush(self.base_name + ' Removing video file from local temporary directory.')
assert os.path.abspath(self.tmp_video_file) != os.path.abspath(self.video_file)
os.remove(self.tmp_video_file)
assert os.path.exists(self.video_file)
if self.is_single_worm:
dd = self.tmp_video_file.rpartition('.')[0]
os.remove(dd + '.log.csv')
os.remove(dd + '.info.xml')
#assert the local file did finished
with h5py.File(self.tmp_mask_file, "r") as mask_fid:
assert mask_fid['/mask'].attrs['has_finished'] > 0
if os.path.abspath(self.tmp_mask_file) != os.path.abspath(self.masked_image_file):
#it is very important to use os.path.abspath() otherwise there could be some
#confunsion in the same file name
#print_flush(self.base_name + " Copying temporal masked file into the final directory.")
shutil.copy(self.tmp_mask_file, self.masked_image_file)
#print_flush(self.base_name + " Removing temporary files.")
assert os.path.exists(self.masked_image_file)
os.remove(self.tmp_mask_file)
#Change the permissions so everybody can read/write.
#Otherwise only the owner would be able to change the ummutable flag.
os.chmod(self.masked_image_file, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH)
#Protect file from deletion.
os.chflags(self.masked_image_file, stat.UF_IMMUTABLE)
#print_flush(self.base_name + " Finished to create masked file")
time_str = str(datetime.timedelta(seconds=round(time.time()-self.start_time)))
print_flush('%s Finished. Total time = %s' % (self.base_name, time_str))
def copyFilesFromFinal(self):
#check that the program didn't finished and that not forced start point was not specified
if ((self.final_start_point == checkpoint['END'])
or self.final_start_point > self.end_point
) and not self.force_start_point_str:
#If the program has finished there is nothing to do here.
print_flush(
'%s The files from completed results analysis were found in %s. Remove them if you want to recalculated them again.'
% (self.base_name, self.results_dir))
sys.exit(0)
#find what files we need to copy from the final destination if the analysis is going to resume from a later point
files2copy = []
if self.analysis_start_point <= checkpoint['INT_PROFILE']:
#we do not need the mask to calculate the features
try:
#check if there is already a finished/readable temporary mask file in current directory otherwise copy the
with h5py.File(self.tmp_mask_file, "r") as mask_fid:
if mask_fid['/mask'].attrs['has_finished'] < 1:
#go to the exception if the mask has any other flag
raise
except:
with h5py.File(self.masked_image_file, "r") as mask_fid:
#check if the video to mask conversion did indeed finished correctly
assert mask_fid['/mask'].attrs['has_finished'] >= 1
files2copy += [(self.masked_image_file, self.tmp_mask_dir)]
if self.tmp_start_point < self.final_start_point:
#copy files from an incomplete analysis files.
if self.final_start_point > checkpoint[
'TRAJ_CREATE']: #and final_start_point <= checkpoint['SKE_CREATE']:
files2copy += [(self.trajectories_file, self.tmp_results_dir)]
if self.final_start_point > checkpoint['SKE_CREATE']:
files2copy += [(self.skeletons_file, self.tmp_results_dir)]
if self.final_start_point > checkpoint['INT_PROFILE']:
files2copy += [(self.intensities_file, self.tmp_results_dir)]
copyFilesLocal(files2copy)
def execAllPoints(self):
initial_time = time.time()
#if start_point is larger than end_point there is nothing else to do
if self.start_point > self.end_point:
print_flush(self.base_name + ' Finished in ' +
checkpoint_label[self.end_point])
return
print_flush(self.base_name + ' Starting checkpoint: ' +
checkpoint_label[self.start_point])
for ii in range(self.start_point, self.end_point + 1):
current_point = checkpoint_label[ii]
if self.is_single_worm and current_point == 'INT_PROFILE':
execThisPoint(current_point,
**self.points_parameters['STAGE_ALIGMENT'],
commit_hash=self.commit_hash,
cmd_original=self.cmd_original)
with tables.File(self.skeletons_file, 'r') as fid:
try:
good_aligment = fid.get_node(
'/stage_movement')._v_attrs['has_finished'][:]
except (KeyError, IndexError):
good_aligment = 0
if good_aligment != 1:
#break, bad video we do not need to calculate anything else.
last_check_point = 'STAGE_ALIGMENT'
break
if not self.use_manual_join and current_point == 'FEAT_MANUAL_CREATE':
continue
last_check_point = checkpoint_label[self.end_point]
if current_point == 'END': break
#print(current_point, self.points_parameters[current_point]['func'])
#print(self.points_parameters[current_point]['argkws'])
execThisPoint(current_point,
**self.points_parameters[current_point],
commit_hash=self.commit_hash,
cmd_original=self.cmd_original)
time_str = str(
datetime.timedelta(seconds=round(time.time() - initial_time)))
print_flush('%s Finished in %s. Total time %s' %
(self.base_name, last_check_point, time_str))
def alignStageMotion(masked_image_file,
skeletons_file,
tmp_dir=os.path.expanduser('~/Tmp')):
assert os.path.exists(masked_image_file)
assert os.path.exists(skeletons_file)
base_name = os.path.split(masked_image_file)[1].partition('.hdf5')[0]
#check if it was finished before
with h5py.File(skeletons_file, 'r+') as fid:
try:
has_finished = fid['/stage_movement'].attrs['has_finished'][:]
except (KeyError, IndexError):
has_finished = 0
if has_finished > 0:
print_flush('%s The stage motion was previously aligned.' % base_name)
return
#get the current to add as a matlab path
current_dir = os.path.dirname(os.path.abspath(__file__))
start_cmd = ('matlab -nojvm -nosplash -nodisplay -nodesktop <').split()
script_cmd = "addpath('{0}'); " \
"try, alignStageMotionSegwormFun('{1}', '{2}'); " \
"catch ME, disp(getReport(ME)); " \
"end; exit; "
script_cmd = script_cmd.format(current_dir, masked_image_file,
skeletons_file)
#create temporary file to read as matlab script, works better than passing a string in the command line.
tmp_fid, tmp_script_file = tempfile.mkstemp(suffix='.m',
dir=tmp_dir,
text=True)
with open(tmp_script_file, 'w') as fid:
fid.write(script_cmd)
matlab_cmd = start_cmd + [tmp_script_file]
#call matlab and align the stage motion
print_flush('%s Aligning Stage Motion.' % base_name)
sp.call(matlab_cmd)
print_flush('%s Alignment finished.' % base_name)
#delete temporary file.
os.close(tmp_fid)
os.remove(tmp_script_file)
def execAllPoints(self):
initial_time = time.time()
#if start_point is larger than end_point there is nothing else to do
if self.start_point > self.end_point:
print_flush(self.base_name + ' Finished in ' + checkpoint_label[self.end_point])
return
print_flush(self.base_name + ' Starting checkpoint: ' + checkpoint_label[self.start_point])
for ii in range(self.start_point, self.end_point+1):
current_point = checkpoint_label[ii]
if self.is_single_worm and current_point == 'INT_PROFILE':
execThisPoint(current_point, **self.points_parameters['STAGE_ALIGMENT'],
commit_hash=self.commit_hash, cmd_original=self.cmd_original)
with tables.File(self.skeletons_file, 'r') as fid:
try:
good_aligment = fid.get_node('/stage_movement')._v_attrs['has_finished'][:]
except (KeyError,IndexError):
good_aligment = 0;
if good_aligment != 1:
#break, bad video we do not need to calculate anything else.
last_check_point = 'STAGE_ALIGMENT'
break
if not self.use_manual_join and current_point == 'FEAT_MANUAL_CREATE': continue
last_check_point = checkpoint_label[self.end_point]
if current_point == 'END': break
#print(current_point, self.points_parameters[current_point]['func'])
#print(self.points_parameters[current_point]['argkws'])
execThisPoint(current_point, **self.points_parameters[current_point],
commit_hash=self.commit_hash, cmd_original=self.cmd_original)
time_str = str(datetime.timedelta(seconds=round(time.time()-initial_time)))
print_flush('%s Finished in %s. Total time %s' % (self.base_name, last_check_point, time_str))
def alignStageMotion(masked_image_file, skeletons_file, tmp_dir = os.path.expanduser('~/Tmp')):
assert os.path.exists(masked_image_file)
assert os.path.exists(skeletons_file)
base_name = os.path.split(masked_image_file)[1].partition('.hdf5')[0];
#check if it was finished before
with h5py.File(skeletons_file, 'r+') as fid:
try:
has_finished = fid['/stage_movement'].attrs['has_finished'][:]
except (KeyError,IndexError):
has_finished = 0;
if has_finished > 0:
print_flush('%s The stage motion was previously aligned.' % base_name)
return
#get the current to add as a matlab path
current_dir = os.path.dirname(os.path.abspath(__file__))
start_cmd = ('matlab -nojvm -nosplash -nodisplay -nodesktop <').split()
script_cmd = "addpath('{0}'); " \
"try, alignStageMotionSegwormFun('{1}', '{2}'); " \
"catch ME, disp(getReport(ME)); " \
"end; exit; "
script_cmd = script_cmd.format(current_dir, masked_image_file, skeletons_file)
#create temporary file to read as matlab script, works better than passing a string in the command line.
tmp_fid, tmp_script_file = tempfile.mkstemp(suffix='.m', dir=tmp_dir, text=True)
with open(tmp_script_file, 'w') as fid:
fid.write(script_cmd)
matlab_cmd = start_cmd + [tmp_script_file]
#call matlab and align the stage motion
print_flush('%s Aligning Stage Motion.' % base_name)
sp.call(matlab_cmd)
print_flush('%s Alignment finished.' % base_name)
#delete temporary file.
os.close(tmp_fid)
os.remove(tmp_script_file)
trajectories_file = ff[:-5] + '_trajectories.hdf5'
skeletons_file = ff[:-5] + '_skeletons.hdf5'
intensities_file = ff[:-5] + '_intensities.hdf5'
#check the file finished in the correct step
#with tables.File(skeletons_file, 'r') as fid:
# assert fid.get_node('/skeleton')._v_attrs['has_finished'] >= 4
with pd.HDFStore(skeletons_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
grouped_trajectories = trajectories_data.groupby('worm_index_joined')
tot_worms = len(grouped_trajectories)
#variables to report progress
base_name = skeletons_file.rpartition('.')[0].rpartition(
os.sep)[-1].rpartition('_')[0]
print_flush(base_name +
" Checking if the final Head-Tail orientation is correct")
for index_n, (worm_index,
trajectories_worm) in enumerate(grouped_trajectories):
p_tot, skel_group, int_group = checkFinalOrientation(
skeletons_file, intensities_file, trajectories_worm,
head_tail_param)
print(p_tot)
if p_tot < 0.5:
switchBlocks(skel_group, skeletons_file, int_group,
intensities_file)
def getIntensityProfile(masked_image_file,
skeletons_file,
intensities_file,
width_resampling=15,
length_resampling=131,
min_num_skel=100,
smooth_win=11,
pol_degree=3,
width_percentage=0.5,
save_int_maps=False):
assert smooth_win > pol_degree
assert min_num_skel > 0
assert 0 < width_percentage < 1
#we want to use symetrical distance centered in the skeleton
if length_resampling % 2 == 0: length_resampling += 1
if width_resampling % 2 == 0: width_resampling += 1
#get the limits to be averaged from the intensity map
if save_int_maps:
width_win_ind = getWidthWinLimits(width_resampling, width_percentage)
else:
width_win_ind = (0, width_resampling)
#filters for the tables structures
table_filters = tables.Filters(complevel=5,
complib='zlib',
shuffle=True,
fletcher32=True)
#Get a reduced version of the trajectories_data table with only the valid skeletons.
#The rows of this new table are going to be saved into skeletons_file
trajectories_data_valid = setIntMapIndexes(skeletons_file, min_num_skel)
#let's save this new table into the intensities file
with tables.File(intensities_file, 'w') as fid:
fid.create_table('/', 'trajectories_data_valid', \
obj = trajectories_data_valid.to_records(index=False), filters=table_filters)
tot_rows = len(trajectories_data_valid)
if tot_rows == 0:
with tables.File(intensities_file, "r+") as int_file_id:
#nothing to do here let's save empty data and go out
worm_int_avg_tab = int_file_id.create_array(
"/", "straighten_worm_intensity_median", obj=np.zeros(0))
worm_int_avg_tab._v_attrs['has_finished'] = 1
return
with tables.File(masked_image_file, 'r') as mask_fid, \
tables.File(skeletons_file, 'r') as ske_file_id, \
tables.File(intensities_file, "r+") as int_file_id:
#pointer to the compressed videos
mask_dataset = mask_fid.get_node("/mask")
#pointer to skeletons
skel_tab = ske_file_id.get_node('/skeleton')
skel_width_tab = ske_file_id.get_node('/width_midbody')
filters = tables.Filters(complevel=5, complib='zlib', shuffle=True)
#we are using Float16 to save space, I am assuing the intensities are between uint8
worm_int_avg_tab = int_file_id.create_carray("/", "straighten_worm_intensity_median", \
tables.Float16Atom(dflt=np.nan), \
(tot_rows, length_resampling), \
chunkshape = (1, length_resampling),\
filters = table_filters)
worm_int_avg_tab._v_attrs['has_finished'] = 0
worm_int_avg_tab.attrs['width_win_ind'] = width_win_ind
if save_int_maps:
worm_int_tab = int_file_id.create_carray("/", "straighten_worm_intensity", \
tables.Float16Atom(dflt=np.nan), \
(tot_rows, length_resampling,width_resampling), \
chunkshape = (1, length_resampling,width_resampling),\
filters = table_filters)
#variables used to report progress
base_name = skeletons_file.rpartition('.')[0].rpartition(
os.sep)[-1].rpartition('_')[0]
progressTime = timeCounterStr('Obtaining intensity maps.')
for frame, frame_data in trajectories_data_valid.groupby(
'frame_number'):
img = mask_dataset[frame, :, :]
for ii, row_data in frame_data.iterrows():
skeleton_id = int(row_data['skeleton_id'])
worm_index = int(row_data['worm_index_joined'])
int_map_id = int(row_data['int_map_id'])
#read ROI and skeleton, and put them in the same coordinates map
worm_img, roi_corner = getWormROI(img, row_data['coord_x'],
row_data['coord_y'],
row_data['roi_size'])
skeleton = skel_tab[skeleton_id, :, :] - roi_corner
half_width = skel_width_tab[skeleton_id] / 2
assert not np.isnan(skeleton[0, 0])
skel_smooth = smoothSkeletons(
skeleton,
length_resampling=length_resampling,
smooth_win=smooth_win,
pol_degree=pol_degree)
straighten_worm, grid_x, grid_y = getStraightenWormInt(
worm_img,
skel_smooth,
half_width=half_width,
width_resampling=width_resampling)
#if you use the mean it is better to do not use float16
int_avg = np.median(
straighten_worm[width_win_ind[0]:width_win_ind[1], :],
axis=0)
worm_int_avg_tab[int_map_id] = int_avg
#only save the full map if it is specified by the user
if save_int_maps:
worm_int_tab[int_map_id] = straighten_worm.T
if frame % 500 == 0:
progress_str = progressTime.getStr(frame)
print_flush(base_name + ' ' + progress_str)
worm_int_avg_tab._v_attrs['has_finished'] = 1
type=int,
default=-1,
help='Force the program to start at a specific point in the analysis.')
parser.add_argument('--end_point',
type=int,
default=checkpoint['END'],
help='End point of the analysis.')
parser.add_argument(
'--is_single_worm',
action='store_true',
help=
'This flag indicates if the video corresponds to the single worm case.'
)
parser.add_argument(
'--use_manual_join',
action='store_true',
help='Use this flag to calculate features on manually joined data.')
parser.add_argument(
'--use_skel_filter',
action='store_true',
help='Flag to filter valid skeletons using the movie robust averages.')
parser.add_argument('--cmd_original',
default='',
help='Internal. Used in provenance tracking.')
if len(sys.argv) > 1:
args = parser.parse_args()
getTrajectoriesWorkerL(**vars(args))
else:
print_flush('Bad', sys.argv)
def correctHeadTailIntensity(skeletons_file, intensities_file, smooth_W = 5,
gap_size = 0, min_block_size = 10, local_avg_win = 25, min_frac_in = 0.85, head_tail_param = {}):
#get the trajectories table
with pd.HDFStore(skeletons_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
#at this point the int_map_id with the intensity maps indexes must exist in the table
assert 'int_map_id' in trajectories_data
grouped_trajectories = trajectories_data.groupby('worm_index_joined')
tot_worms = len(grouped_trajectories)
#variables to report progress
base_name = skeletons_file.rpartition('.')[0].rpartition(os.sep)[-1].rpartition('_')[0]
progress_timer = timeCounterStr('');
bad_worms = [] #worms with not enough difference between the normal and inverted median intensity profile
switched_blocks = [] #data from the blocks that were switched
#ind2check = [765]
for index_n, (worm_index, trajectories_worm) in enumerate(grouped_trajectories):
#if not worm_index in ind2check: continue
if index_n % 10 == 0:
dd = " Correcting Head-Tail using intensity profiles. Worm %i of %i." % (index_n+1, tot_worms)
dd = base_name + dd + ' Total time:' + progress_timer.getTimeStr()
print_flush(dd)
#correct head tail using the intensity profiles
dd = correctHeadTailIntWorm(trajectories_worm, skeletons_file, intensities_file, \
smooth_W, gap_size, min_block_size, local_avg_win, min_frac_in)
switched_blocks += [(worm_index, t0, tf) for t0, tf in dd]
#check that the final orientation is correct, otherwise switch the whole trajectory
p_tot, skel_group, int_group = \
checkFinalOrientation(skeletons_file, intensities_file, trajectories_worm, min_block_size, head_tail_param)
if p_tot < 0.5:
switchBlocks(skel_group, skeletons_file, int_group, intensities_file)
#label the process as finished and store the indexes of the switched worms
with tables.File(skeletons_file, 'r+') as fid:
if not '/Intensity_Analysis' in fid:
fid.create_group('/', 'Intensity_Analysis')
if '/Intensity_Analysis/Bad_Worms' in fid:
fid.remove_node('/Intensity_Analysismin_block_size/Bad_Worms')
if '/Intensity_Analysis/Switched_Head_Tail' in fid:
fid.remove_node('/Intensity_Analysis/Switched_Head_Tail')
if bad_worms:
fid.create_array('/Intensity_Analysis', 'Bad_Worms', np.array(bad_worms))
if switched_blocks:
#to rec array
switched_blocks = np.array(switched_blocks, \
dtype = [('worm_index',np.int), ('ini_frame',np.int), ('last_frame',np.int)])
fid.create_table('/Intensity_Analysis', 'Switched_Head_Tail', switched_blocks)
fid.get_node('/skeleton')._v_attrs['has_finished'] = 4
print_flush(base_name + ' Head-Tail correction using intensity profiles finished: ' + progress_timer.getTimeStr())
if current_point == 'END': break
#print(current_point, self.points_parameters[current_point]['func'])
#print(self.points_parameters[current_point]['argkws'])
execThisPoint(current_point, **self.points_parameters[current_point],
commit_hash=self.commit_hash, cmd_original=self.cmd_original)
time_str = str(datetime.timedelta(seconds=round(time.time()-initial_time)))
print_flush('%s Finished in %s. Total time %s' % (self.base_name, last_check_point, time_str))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Track woms in an individual video hdf5 file.")
parser.add_argument('masked_image_file', help = 'hdf5 video file.')
parser.add_argument('results_dir', help = 'Directory where results are going to be saved.')
parser.add_argument('--json_file', default = '', help='File (.json) containing the tracking parameters.')
parser.add_argument('--start_point', type=int, default = -1, help = 'Force the program to start at a specific point in the analysis.')
parser.add_argument('--end_point', type=int, default = checkpoint['END'], help='End point of the analysis.')
parser.add_argument('--is_single_worm', action='store_true', help = 'This flag indicates if the video corresponds to the single worm case.')
parser.add_argument('--use_manual_join', action='store_true', help = 'Use this flag to calculate features on manually joined data.')
parser.add_argument('--use_skel_filter', action='store_true', help = 'Flag to filter valid skeletons using the movie robust averages.')
parser.add_argument('--cmd_original', default = '', help = 'Internal. Used in provenance tracking.')
if len(sys.argv)>1:
args = parser.parse_args()
getTrajectoriesWorkerL(**vars(args))
else:
print_flush('Bad', sys.argv)
def main_code(self): if self.is_calculate_mask: #start to calculate the mask from raw video print_flush(self.base_name + " Creating temporal masked file.") compressSingleWorker(*self.main_input_params)
#save event data as a subgroup per worm
worm_node = features_fid.create_group(group_feat_events, 'worm_%i' % worm_index )
worm_node._v_attrs['worm_index'] = worm_index
worm_node._v_attrs['frame_range'] = (worm.timestamp[0], worm.timestamp[-1])
for feat in events_data:
tmp_data = events_data[feat]
#consider the cases where the output is a single number, empty or None
if isinstance(tmp_data, (float, int)): tmp_data = np.array([tmp_data])
if tmp_data is None or tmp_data.size == 0: tmp_data = np.array([np.nan])
features_fid.create_carray(worm_node, feat, \
obj = tmp_data, filters = filters_tables)
#save the skeletons in the same group
features_fid.create_carray(worm_node, 'skeletons', \
obj = skeletons, filters = filters_tables)
#store the average for each worm feature
mean_features_df[ind_N] = worm_stats
#%%
#report progress
dd = " Extracting features. Worm %i of %i done." % (ind+1, tot_worms)
print_flush(base_name + dd + ' Total time:' + progress_timer.getTimeStr())
#create and save a table containing the averaged worm feature for each worm
feat_mean = features_fid.create_table('/', 'features_means', obj = mean_features_df, filters=filters_tables)
#flag and report a success finish
feat_mean._v_attrs['has_finished'] = 1
print_flush(base_name + ' Feature extraction finished: ' + progress_timer.getTimeStr())
def getIntensityProfile(masked_image_file, skeletons_file, intensities_file,
width_resampling = 15, length_resampling = 131, min_num_skel = 100,
smooth_win = 11, pol_degree = 3, width_percentage = 0.5, save_int_maps = False):
assert smooth_win > pol_degree
assert min_num_skel > 0
assert 0 < width_percentage < 1
#we want to use symetrical distance centered in the skeleton
if length_resampling % 2 == 0: length_resampling += 1
if width_resampling % 2 == 0: width_resampling += 1
#get the limits to be averaged from the intensity map
if save_int_maps:
width_win_ind = getWidthWinLimits(width_resampling, width_percentage)
else:
width_win_ind = (0, width_resampling)
#filters for the tables structures
table_filters = tables.Filters(complevel=5, complib='zlib',
shuffle=True, fletcher32=True)
#Get a reduced version of the trajectories_data table with only the valid skeletons.
#The rows of this new table are going to be saved into skeletons_file
trajectories_data_valid = setIntMapIndexes(skeletons_file, min_num_skel)
#let's save this new table into the intensities file
with tables.File(intensities_file, 'w') as fid:
fid.create_table('/', 'trajectories_data_valid', \
obj = trajectories_data_valid.to_records(index=False), filters=table_filters)
tot_rows = len(trajectories_data_valid);
if tot_rows == 0:
with tables.File(intensities_file, "r+") as int_file_id:
#nothing to do here let's save empty data and go out
worm_int_avg_tab = int_file_id.create_array("/", "straighten_worm_intensity_median", obj=np.zeros(0))
worm_int_avg_tab._v_attrs['has_finished'] = 1;
return
with tables.File(masked_image_file, 'r') as mask_fid, \
tables.File(skeletons_file, 'r') as ske_file_id, \
tables.File(intensities_file, "r+") as int_file_id:
#pointer to the compressed videos
mask_dataset = mask_fid.get_node("/mask")
#pointer to skeletons
skel_tab = ske_file_id.get_node('/skeleton')
skel_width_tab = ske_file_id.get_node('/width_midbody')
filters = tables.Filters(complevel=5, complib='zlib', shuffle=True)
#we are using Float16 to save space, I am assuing the intensities are between uint8
worm_int_avg_tab = int_file_id.create_carray("/", "straighten_worm_intensity_median", \
tables.Float16Atom(dflt=np.nan), \
(tot_rows, length_resampling), \
chunkshape = (1, length_resampling),\
filters = table_filters);
worm_int_avg_tab._v_attrs['has_finished'] = 0;
worm_int_avg_tab.attrs['width_win_ind'] = width_win_ind
if save_int_maps:
worm_int_tab = int_file_id.create_carray("/", "straighten_worm_intensity", \
tables.Float16Atom(dflt=np.nan), \
(tot_rows, length_resampling,width_resampling), \
chunkshape = (1, length_resampling,width_resampling),\
filters = table_filters);
#variables used to report progress
base_name = skeletons_file.rpartition('.')[0].rpartition(os.sep)[-1].rpartition('_')[0]
progressTime = timeCounterStr('Obtaining intensity maps.');
for frame, frame_data in trajectories_data_valid.groupby('frame_number'):
img = mask_dataset[frame,:,:]
for ii, row_data in frame_data.iterrows():
skeleton_id = int(row_data['skeleton_id'])
worm_index = int(row_data['worm_index_joined'])
int_map_id = int(row_data['int_map_id'])
#read ROI and skeleton, and put them in the same coordinates map
worm_img, roi_corner = getWormROI(img, row_data['coord_x'], row_data['coord_y'], row_data['roi_size'])
skeleton = skel_tab[skeleton_id,:,:]-roi_corner
half_width = skel_width_tab[skeleton_id]/2
assert not np.isnan(skeleton[0,0])
skel_smooth = smoothSkeletons(skeleton, length_resampling = length_resampling, smooth_win = smooth_win, pol_degree = pol_degree)
straighten_worm, grid_x, grid_y = getStraightenWormInt(worm_img, skel_smooth, half_width=half_width, width_resampling=width_resampling)
#if you use the mean it is better to do not use float16
int_avg = np.median(straighten_worm[width_win_ind[0]:width_win_ind[1],:], axis = 0)
worm_int_avg_tab[int_map_id] = int_avg
#only save the full map if it is specified by the user
if save_int_maps:
worm_int_tab[int_map_id] = straighten_worm.T
if frame % 500 == 0:
progress_str = progressTime.getStr(frame)
print_flush(base_name + ' ' + progress_str);
worm_int_avg_tab._v_attrs['has_finished'] = 1;
#variables to report progress
base_name = skeletons_file.rpartition('.')[0].rpartition(os.sep)[-1].rpartition('_')[0]
progress_timer = timeCounterStr('');
bad_worms = [] #worms with not enough difference between the normal and inverted median intensity profile
switched_blocks = [] #data from the blocks that were switched
ind2check = [52]
for index_n, (worm_index, trajectories_worm) in enumerate(grouped_trajectories):
if not worm_index in ind2check: continue
if index_n % 10 == 0:
dd = " Correcting Head-Tail using intensity profiles. Worm %i of %i." % (index_n+1, tot_worms)
dd = base_name + dd + ' Total time:' + progress_timer.getTimeStr()
print_flush(dd)
good = trajectories_worm['int_map_id']!=-1;
int_map_id = trajectories_worm.loc[good, 'int_map_id'].values
int_skeleton_id = trajectories_worm.loc[good, 'skeleton_id'].values
int_frame_number = trajectories_worm.loc[good, 'frame_number'].values
#only analyze data that contains at least min_block_size intensity profiles
if int_map_id.size < min_block_size:
continue
#%%
#read the worm intensity profiles
with tables.File(intensities_file, 'r') as fid:
worm_int_profile = fid.get_node('/straighten_worm_intensity_median')[int_map_id,:]
#%%
def getWormFeatures(skeletons_file, features_file, good_traj_index, expected_fps = 25, \
use_skel_filter = True, use_manual_join = False, is_single_worm = False):
if is_single_worm:
if not isValidSingleWorm(skeletons_file, good_traj_index):
#the stage was not aligned correctly. Return empty features file.
good_traj_index = np.array([])
fps, is_default_timestamp = getFPS(skeletons_file, expected_fps)
micronsPerPixel = getMicronsPerPixel(skeletons_file)
#function to calculate the progress time. Useful to display progress
base_name = skeletons_file.rpartition('.')[0].rpartition(
os.sep)[-1].rpartition('_')[0]
#filter used for each fo the tables
filters_tables = tables.Filters(complevel=5, complib='zlib', shuffle=True)
#Time series
progress_timer = timeCounterStr('')
#get total number of valid worms and break if it is zero
tot_worms = len(good_traj_index)
#initialize by getting the specs data subdivision
wStats = WormStatsClass()
#initialize rec array with the averaged features of each worm
mean_features_df = np.full(tot_worms, np.nan, dtype=wStats.feat_avg_dtype)
with tables.File(features_file, 'w') as features_fid:
#initialize groups for the timeseries and event features
header_timeseries = {
feat: tables.Float32Col(pos=ii)
for ii, (feat, _) in enumerate(wStats.feat_timeseries_dtype)
}
table_timeseries = features_fid.create_table('/',
'features_timeseries',
header_timeseries,
filters=filters_tables)
#save some data used in the calculation as attributes
table_timeseries._v_attrs['micronsPerPixel'] = micronsPerPixel
table_timeseries._v_attrs[
'is_default_timestamp'] = is_default_timestamp
table_timeseries._v_attrs['fps'] = fps
#start to calculate features for each worm trajectory
for ind_N, worm_index in enumerate(good_traj_index):
#initialize worm object, and extract data from skeletons file
worm = WormFromTable(skeletons_file,
worm_index,
use_skel_filter=use_skel_filter,
use_manual_join=use_manual_join,
micronsPerPixel=micronsPerPixel,
fps=fps,
smooth_window=5)
if is_single_worm:
assert worm_index == 1 and ind_N == 0
worm = correctSingleWorm(worm, skeletons_file)
#calculate features
timeseries_data, events_data, worm_stats, skeletons = \
getFeaturesOpenWorm(worm, wStats)
#%%
#save timeseries data
table_timeseries.append(timeseries_data)
table_timeseries.flush()
#save event data as a subgroup per worm
worm_node = features_fid.create_group('/', 'worm_%i' % worm_index)
worm_node._v_attrs['worm_index'] = worm_index
worm_node._v_attrs['frame_range'] = (worm.timestamp[0],
worm.timestamp[-1])
worm_feat_events = features_fid.create_group(
worm_node, 'features_events')
for feat in events_data:
tmp_data = events_data[feat]
#consider the cases where the output is a single number, empty or None
if isinstance(tmp_data, (float, int)):
tmp_data = np.array([tmp_data])
if tmp_data is None or tmp_data.size == 0:
tmp_data = np.array([np.nan])
features_fid.create_carray(worm_feat_events, feat, \
obj = tmp_data, filters = filters_tables)
#save the skeletons in the same group
features_fid.create_carray(worm_node, 'skeletons', \
obj = skeletons, filters = filters_tables)
#store the average for each worm feature
mean_features_df[ind_N] = worm_stats
#%%
#report progress
dd = " Extracting features. Worm %i of %i done." % (ind_N + 1,
tot_worms)
print_flush(base_name + dd + ' Total time:' +
progress_timer.getTimeStr())
#create and save a table containing the averaged worm feature for each worm
feat_mean = features_fid.create_table('/',
'features_means',
obj=mean_features_df,
filters=filters_tables)
#flag and report a success finish
feat_mean._v_attrs['has_finished'] = 1
print_flush(base_name + ' Feature extraction finished: ' +
progress_timer.getTimeStr())
def getWormFeatures(skeletons_file, features_file, good_traj_index, expected_fps = 25, \
use_skel_filter = True, use_manual_join = False, is_single_worm = False):
if is_single_worm:
if not isValidSingleWorm(skeletons_file, good_traj_index):
#the stage was not aligned correctly. Return empty features file.
good_traj_index = np.array([])
fps, is_default_timestamp = getFPS(skeletons_file, expected_fps)
micronsPerPixel = getMicronsPerPixel(skeletons_file)
#function to calculate the progress time. Useful to display progress
base_name = skeletons_file.rpartition('.')[0].rpartition(os.sep)[-1].rpartition('_')[0]
#filter used for each fo the tables
filters_tables = tables.Filters(complevel = 5, complib='zlib', shuffle=True)
#Time series
progress_timer = timeCounterStr('');
#get total number of valid worms and break if it is zero
tot_worms = len(good_traj_index)
#initialize by getting the specs data subdivision
wStats = WormStatsClass()
#initialize rec array with the averaged features of each worm
mean_features_df = np.full(tot_worms, np.nan, dtype=wStats.feat_avg_dtype)
with tables.File(features_file, 'w') as features_fid:
#initialize groups for the timeseries and event features
header_timeseries = {feat:tables.Float32Col(pos=ii) for ii, (feat,_) in enumerate(wStats.feat_timeseries_dtype)}
table_timeseries = features_fid.create_table('/', 'features_timeseries', header_timeseries, filters=filters_tables)
#save some data used in the calculation as attributes
table_timeseries._v_attrs['micronsPerPixel'] = micronsPerPixel
table_timeseries._v_attrs['is_default_timestamp'] = is_default_timestamp
table_timeseries._v_attrs['fps'] = fps
#start to calculate features for each worm trajectory
for ind_N, worm_index in enumerate(good_traj_index):
#initialize worm object, and extract data from skeletons file
worm = WormFromTable(skeletons_file, worm_index,
use_skel_filter = use_skel_filter, use_manual_join = use_manual_join,
micronsPerPixel = micronsPerPixel,fps=fps, smooth_window = 5)
if is_single_worm:
assert worm_index == 1 and ind_N == 0
worm = correctSingleWorm(worm, skeletons_file)
#calculate features
timeseries_data, events_data, worm_stats, skeletons = \
getFeaturesOpenWorm(worm, wStats)
#%%
#save timeseries data
table_timeseries.append(timeseries_data)
table_timeseries.flush()
#save event data as a subgroup per worm
worm_node = features_fid.create_group('/', 'worm_%i' % worm_index )
worm_node._v_attrs['worm_index'] = worm_index
worm_node._v_attrs['frame_range'] = (worm.timestamp[0], worm.timestamp[-1])
worm_feat_events = features_fid.create_group(worm_node, 'features_events')
for feat in events_data:
tmp_data = events_data[feat]
#consider the cases where the output is a single number, empty or None
if isinstance(tmp_data, (float, int)): tmp_data = np.array([tmp_data])
if tmp_data is None or tmp_data.size == 0: tmp_data = np.array([np.nan])
features_fid.create_carray(worm_feat_events, feat, \
obj = tmp_data, filters = filters_tables)
#save the skeletons in the same group
features_fid.create_carray(worm_node, 'skeletons', \
obj = skeletons, filters = filters_tables)
#store the average for each worm feature
mean_features_df[ind_N] = worm_stats
#%%
#report progress
dd = " Extracting features. Worm %i of %i done." % (ind_N + 1, tot_worms)
print_flush(base_name + dd + ' Total time:' + progress_timer.getTimeStr())
#create and save a table containing the averaged worm feature for each worm
feat_mean = features_fid.create_table('/', 'features_means', obj = mean_features_df, filters=filters_tables)
#flag and report a success finish
feat_mean._v_attrs['has_finished'] = 1
print_flush(base_name + ' Feature extraction finished: ' + progress_timer.getTimeStr())