def change_attrs(fname, field_name):
print(os.path.basename(fname))
read_unit_conversions(fname)
with tables.File(fname, 'r+') as fid:
group_to_save = fid.get_node(field_name)
set_unit_conversions(group_to_save,
expected_fps=expected_fps,
microns_per_pixel=microns_per_pixel)
read_unit_conversions(fname)
def change_attrs(fname, field_name):
print(os.path.basename(fname))
read_unit_conversions(fname)
with tables.File(fname, 'r+') as fid:
group_to_save = fid.get_node(field_name)
set_unit_conversions(group_to_save,
expected_fps=expected_fps,
microns_per_pixel=microns_per_pixel)
read_unit_conversions(fname)
#for fname in masked_files:
# change_attrs(fname, '/mask')
#for fname in skeletons_files:
# change_attrs(fname, '/trajectories_data')
def check_attrs(fname):
fps_out, microns_per_pixel_out, is_light_background = read_unit_conversions(
fname)
if fps_out != (25.0, 25.0, 'seconds') or \
microns_per_pixel_out != (10.0, 'micrometers'):
print('Fix %s' % os.path.basename(fname))
return
def get_bgnd_from_masked(masked_image_file, is_use_existing=False):
"""
- Opens the masked_image_file hdf5 file, reads the /full_data node and
creates a "background" by taking the maximum value of each pixel over time.
- if is_use_existing, read instead the /bgnd field
(and if /bgnd not there, fall back to method above)
- Parses the file name to find a camera serial number
- reads the pixel/um ratio from the masked_image_file
"""
import numpy as np
from tierpsy.helper.params import read_unit_conversions
# read attributes of masked_image_file
_, (microns_per_pixel, xy_units) , is_light_background = read_unit_conversions(masked_image_file)
# get "background" and px2um
with pd.HDFStore(masked_image_file, 'r') as fid:
assert is_light_background, \
'MultiWell recognition is only available for brightfield at the moment'
if is_use_existing and '/bgnd' in fid:
print('bgnd found :) ')
img = fid.get_node('/bgnd').read()
else:
img = np.max(fid.get_node('/full_data'), axis=0)
camera_serial = parse_camera_serial(masked_image_file)
return img, camera_serial, microns_per_pixel
def _getUnits(features_file, READ_FEATURES=False):
fps_out, microns_per_pixel_out, _ = read_unit_conversions(features_file)
xy_units = microns_per_pixel_out[1]
time_units = fps_out[2]
units = OrderedDict()
units["size"] = "mm" #size of the plate
units['t'] = time_units #frames or seconds
for field in ['x', 'y', 'px', 'py']:
units[field] = xy_units #(pixels or micrometers)
if READ_FEATURES:
#TODO how to change microns to pixels when required
ws = WormStats()
for field, unit in ws.features_info['units'].iteritems():
units['@OMG ' + field] = unit
return units
def calculate_bgnd_from_masked_fulldata(masked_image_file):
"""
- Opens the masked_image_file hdf5 file, reads the /full_data node and
creates a "background" by taking the maximum value of each pixel over time.
- Parses the file name to find a camera serial number
- reads the pixel/um ratio from the masked_image_file
"""
import numpy as np
from tierpsy.helper.params import read_unit_conversions
# read attributes of masked_image_file
_, (microns_per_pixel, xy_units) , is_light_background = read_unit_conversions(masked_image_file)
# get "background" and px2um
with pd.HDFStore(masked_image_file, 'r') as fid:
assert is_light_background, \
'MultiWell recognition is only available for brightfield at the moment'
img = np.max(fid.get_node('/full_data'), axis=0)
camera_serial = parse_camera_serial(masked_image_file)
return img, camera_serial, microns_per_pixel
def updateSkelFile(self, selected_file, dflt_skel_size=5):
self.trajectories_data = None
self.traj_time_grouped = None
self.traj_worm_index_grouped = None
self.skel_dat = {}
self.skeletons_file = selected_file
self.ui.lineEdit_skel.setText(self.skeletons_file)
try:
#find were to read the skeletons and pixel2microns transforming factor
self.stage_position_pix = None
#read units data
fps_out, microns_per_pixel_out, _ = read_unit_conversions(
self.skeletons_file)
self.fps, _, self.time_units = fps_out
self.microns_per_pixel, self.xy_units = microns_per_pixel_out
with tables.File(self.skeletons_file, 'r') as skel_file_id:
if '/coordinates' in skel_file_id:
self.coordinates_group = '/coordinates/'
self.coordinates_fields = {
'dorsal_contours': 'contour_side2',
'skeletons': 'skeleton',
'ventral_contours': 'contour_side1'
}
if (not self.isimgstore) and ('/stage_position_pix'
in self.fid):
self.stage_position_pix = self.fid.get_node(
'/stage_position_pix')[:]
else:
self.microns_per_pixel = 1.
self.coordinates_group = '/'
self.coordinates_fields = {
'contour_side1': 'contour_side1',
'skeleton': 'skeleton',
'contour_side2': 'contour_side2'
}
self.coordinates_fields = {
k: v
for k, v in self.coordinates_fields.items()
if (self.coordinates_group + k) in skel_file_id
}
#read trajectories data, and other useful factors
with pd.HDFStore(self.skeletons_file, 'r') as ske_file_id:
if 'trajectories_data' in ske_file_id:
self.trajectories_data = ske_file_id['/trajectories_data']
self.is_estimated_trajectories_data = False
else:
timestamp = [np.nan]
if '/timestamp/raw' in self.fid:
timestamp = self.fid.get_node('/timestamp/raw')[:]
if np.any(np.isnan(timestamp)):
tot = self.fid.get_node('/mask').shape[0]
timestamp = np.arange(tot)
self.trajectories_data = _estimate_trajectories_data(
self.skeletons_file, timestamp, self.microns_per_pixel,
self.stage_position_pix)
self.is_estimated_trajectories_data = True
#group data
self.traj_time_grouped = self.trajectories_data.groupby(
'frame_number')
self.traj_worm_index_grouped = self.trajectories_data.groupby(
'worm_index_joined')
# read the size of the structural element used in to calculate
# the mask
if '/provenance_tracking/int_ske_orient' in ske_file_id:
prov_str = ske_file_id.get_node(
'/provenance_tracking/ske_create').read()
func_arg_str = json.loads(
prov_str.decode("utf-8"))['func_arguments']
strel_size = json.loads(func_arg_str)['strel_size']
if isinstance(strel_size, (list, tuple)):
strel_size = strel_size[0]
self.strel_size = strel_size
else:
# use default
self.strel_size = dflt_skel_size
except (IOError, KeyError, tables.exceptions.HDF5ExtError,
tables.exceptions.NoSuchNodeError):
self.trajectories_data = None
self.traj_time_grouped = None
self.skel_dat = {}
#here i am updating the image
self.ui.spinBox_frame.setValue(0)
def getBlobsTable(masked_image_file,
trajectories_file,
buffer_size=None,
min_area=25,
min_box_width=5,
worm_bw_thresh_factor=1.,
strel_size=(5, 5),
analysis_type="WORM",
thresh_block_size=15,
n_cores_used=1,
bgnd_param={}):
#correct strel if it is not a tuple or list
if not isinstance(strel_size, (tuple, list)):
strel_size = (strel_size, strel_size)
assert len(strel_size) == 2
#read properties
fps_out, _, is_light_background = read_unit_conversions(masked_image_file)
expected_fps = fps_out[0]
#find if it is using background subtraction
if len(bgnd_param) > 0:
bgnd_param['is_light_background'] = int(is_light_background)
buffer_size = traj_create_defaults(masked_image_file, buffer_size)
def _ini_plate_worms(traj_fid, masked_image_file):
# intialize main table
int_dtypes = [('worm_index_blob', np.int),
('worm_index_joined', np.int), ('frame_number', np.int)]
dd = [
'coord_x', 'coord_y', 'box_length', 'box_width', 'angle', 'area',
'bounding_box_xmin', 'bounding_box_xmax', 'bounding_box_ymin',
'bounding_box_ymax', 'threshold'
]
float32_dtypes = [(x, np.float32) for x in dd]
plate_worms_dtype = np.dtype(int_dtypes + float32_dtypes)
plate_worms = traj_fid.create_table('/',
"plate_worms",
plate_worms_dtype,
"Worm feature List",
filters=TABLE_FILTERS)
#find if it is a mask from fluorescence and save it in the new group
plate_worms._v_attrs['is_light_background'] = is_light_background
plate_worms._v_attrs['expected_fps'] = expected_fps
#make sure it is in a "serializable" format
plate_worms._v_attrs['bgnd_param'] = bytes(json.dumps(bgnd_param),
'utf-8')
read_and_save_timestamp(masked_image_file, trajectories_file)
return plate_worms
progress_str = ' Calculating trajectories.'
if len(bgnd_param) == 0:
buff_generator = generateROIBuff(masked_image_file,
buffer_size,
progress_str=progress_str)
blob_params = (is_light_background, min_area, min_box_width,
worm_bw_thresh_factor, strel_size, analysis_type,
thresh_block_size)
f_blob_data = partial(getBlobsData, blob_params=blob_params)
else:
blob_params = (min_area, worm_bw_thresh_factor, strel_size)
buff_generator = generateImages(masked_image_file,
bgnd_param=bgnd_param,
progress_str=progress_str)
f_blob_data = partial(getBlobsSimple, blob_params=blob_params)
if n_cores_used > 1:
p = mp.Pool(n_cores_used)
blobs_generator = p.imap(f_blob_data, buff_generator)
else:
blobs_generator = map(f_blob_data, buff_generator)
with tables.open_file(trajectories_file, mode='w') as traj_fid:
plate_worms = _ini_plate_worms(traj_fid, masked_image_file)
for ibuf, blobs_data in enumerate(blobs_generator):
if blobs_data:
plate_worms.append(blobs_data)
def extract_rois(row, root_dir, roi_worm_size, roi_total_size):
mask_file = root_dir / row['mask_prefix']
feat_file = root_dir / row['feat_prefix']
with pd.HDFStore(mask_file, 'r') as fid:
full_data = fid.get_node('/full_data')[:]
save_interval = int(
fid.get_node('/full_data')._v_attrs['save_interval'])
#tot_frames = fid.get_node('/mask').shape[0]
#frames2read = np.arange(0, tot_frames + 1, save_interval)
frames2read = np.array(
[x * save_interval for x in range(full_data.shape[0])])
assert frames2read.size == full_data.shape[0]
masks = fid.get_node('/mask')[frames2read, :, :]
stage_position_pix = None
if '/stage_position_pix' in fid:
dat = fid.get_node('/stage_position_pix')
if dat.shape[0] >= frames2read.max():
stage_position_pix = dat[frames2read, :]
else:
stage_position_pix = np.full((len(frames2read), 2), np.nan)
_, (microns_per_pixel_f, _), _ = read_unit_conversions(str(feat_file), 'r')
all_rois = []
with pd.HDFStore(feat_file, 'r') as fid:
trajectories_data = fid['/trajectories_data']
trajectories_data = trajectories_data[
trajectories_data['frame_number'].isin(frames2read)]
skels_g = fid.get_node('/coordinates/skeletons')
widths_g = fid.get_node('/coordinates/widths')
skel_ids = trajectories_data['skeleton_id'].astype(np.int)
skel_ids = skel_ids[skel_ids >= 0]
if len(skel_ids) == 0:
return
skeletons = skels_g[skel_ids]
max_size = np.nanmax(
np.max(skeletons, axis=1) - np.min(skeletons, axis=1))
max_size = max_size / microns_per_pixel_f
if np.isnan(max_size):
return
_scale = roi_worm_size / max_size
new_img_shape = tuple(
[int(round(x * _scale)) for x in full_data.shape[1:]])
full_data = np.array([
cv2.resize(x, new_img_shape[::-1], interpolation=cv2.INTER_LINEAR)
for x in full_data
])
masks = np.array([
cv2.resize(x, new_img_shape[::-1], interpolation=cv2.INTER_LINEAR)
for x in masks
])
trajectories_data['coord_x'] *= _scale
trajectories_data['coord_y'] *= _scale
traj_group_by_frame = trajectories_data.groupby('frame_number')
w_mean = np.nanmean(widths_g, axis=0)
w_mean *= _scale / microns_per_pixel_f
for ii, current_frame in enumerate(frames2read):
try:
frame_data = traj_group_by_frame.get_group(current_frame)
except KeyError:
continue
full_img = full_data[ii]
mask_img = masks[ii]
#dictionary where keys are the table row and the values the worms ROIs
full_in_frame = getAllImgROI(full_img, frame_data, roi_total_size)
mask_in_frame = getAllImgROI(mask_img, frame_data, roi_total_size)
for irow in mask_in_frame.keys():
roi_mask, corner = mask_in_frame[irow]
if roi_mask.shape != (roi_total_size, roi_total_size):
#border roi, problematic, ignore it
continue
roi_full, _ = full_in_frame[irow]
row_data = frame_data.loc[irow]
skeleton_id = int(row_data['skeleton_id'])
if skeleton_id >= 0:
skel = skels_g[skeleton_id]
skel /= microns_per_pixel_f
if stage_position_pix is not None:
skel -= stage_position_pix[ii, None, :]
skel *= _scale
skel -= corner[None, :]
else:
skel = np.full((49, 2), np.nan)
is_skeletonized = int(~np.any(np.isnan(skel)))
dd = row['movie_id'], current_frame, int(
row_data['worm_index_joined']
), skeleton_id, is_skeletonized
all_rois.append((dd, roi_mask, roi_full, skel, w_mean))
if len(all_rois) == 0:
return
rois_data, roi_masks, roi_fulls, skels, ws_mean = map(
np.array, zip(*all_rois))
rois_data = pd.DataFrame(rois_data.astype(np.int32), columns=ROI_DATA_COLS)
return rois_data, roi_masks, roi_fulls, skels, ws_mean
def _process_row(row):
#%%
mask_file = root_dir / row['mask_prefix']
feat_file = root_dir / row['feat_prefix']
with pd.HDFStore(mask_file, 'r') as fid:
full_data = fid.get_node('/full_data')[:]
save_interval = int(
fid.get_node('/full_data')._v_attrs['save_interval'])
frames2read = np.array(
[x * save_interval for x in range(full_data.shape[0])])
assert frames2read.size == full_data.shape[0]
masks = fid.get_node('/mask')[frames2read, :, :]
stage_position_pix = None
if '/stage_position_pix' in fid:
dat = fid.get_node('/stage_position_pix')
if dat.shape[0] >= frames2read.max():
stage_position_pix = dat[frames2read, :]
else:
stage_position_pix = np.full((len(frames2read), 2), np.nan)
_, (microns_per_pixel_f, _), _ = read_unit_conversions(str(feat_file), 'r')
#%%
with pd.HDFStore(feat_file, 'r') as fid:
trajectories_data_ori = fid['/trajectories_data']
trajectories_data = trajectories_data_ori[
trajectories_data_ori['frame_number'].isin(frames2read)]
trajectories_data = trajectories_data[
trajectories_data['skeleton_id'] >= 0]
skels_g = fid.get_node('/coordinates/skeletons')
cnt1_g = fid.get_node('/coordinates/dorsal_contours')
cnt2_g = fid.get_node('/coordinates/ventral_contours')
widths_g = fid.get_node('/coordinates/widths')
skel_ids = trajectories_data['skeleton_id'].astype(np.int)
skel_ids = skel_ids[skel_ids >= 0]
if len(skel_ids) == 0:
return
w_means = {}
traj_group_by_worm_id = trajectories_data_ori.groupby(
'worm_index_joined')
for worm_ind in trajectories_data['worm_index_joined'].unique():
dat = traj_group_by_worm_id.get_group(worm_ind)
skel_ids = dat['skeleton_id']
skel_ids = skel_ids[skel_ids >= 0]
ws = np.nanmean(widths_g[skel_ids], axis=0)
w_means[worm_ind] = ws[None] / microns_per_pixel_f
traj_group_by_frame = trajectories_data.groupby('frame_number')
for ii, current_frame in enumerate(frames2read):
try:
frame_data = traj_group_by_frame.get_group(current_frame)
except KeyError:
continue
full_img = full_data[ii]
mask_img = masks[ii]
#dictionary where keys are the table row and the values the worms ROIs
full_in_frame = getAllImgROI(full_img, frame_data)
mask_in_frame = getAllImgROI(mask_img, frame_data)
for irow in mask_in_frame.keys():
roi_mask, corner = mask_in_frame[irow]
roi_full, _ = full_in_frame[irow]
row_data = frame_data.loc[irow]
skeleton_id = int(row_data['skeleton_id'])
assert skeleton_id >= 0
def _reformat(x_g):
x = x_g[skeleton_id]
x /= microns_per_pixel_f
if stage_position_pix is not None:
x -= stage_position_pix[ii, None, :]
x -= corner[None, :]
return x[None]
skel, cnt1, cnt2 = map(_reformat, [skels_g, cnt1_g, cnt2_g])
worm_ind = int(row_data['worm_index_joined'])
movie_id = int(row['movie_id'])
save_name = save_dir / f'T{movie_id}_frame-{current_frame}_worm{worm_ind}.hdf5'
save_labelled_roi(save_name, roi_mask, roi_full, skel,
w_means[worm_ind], cnt1, cnt2)
if _debug:
import matplotlib.pylab as plt
fig, axs = plt.subplots(1, 2, sharex=True, sharey=True)
axs[0].imshow(roi_mask, cmap='gray')
axs[1].imshow(roi_full, cmap='gray')
for ax in axs:
for ss, c1, c2 in zip(skel, cnt1, cnt2):
ax.plot(ss[:, 0], ss[:, 1])
ax.plot(c1[:, 0], c1[:, 1])
ax.plot(c2[:, 0], c2[:, 1])
#fnames = glob.glob('/Volumes/behavgenom_archive$/Avelino/screening/CeNDR/MaskedVideos/CeNDR_Exp_250417/*.hdf5')
#fnames = glob.glob('/Volumes/behavgenom_archive$/Avelino/screening/CeNDR/Results/**/*_features.hdf5')
dname = '/Volumes/behavgenom_archive$/Bertie/BertieRaw/MaskedVideos'
fnames = glob.glob(os.path.join(dname, '**', '*.hdf5'), recursive=True)
params = TrackerParams('filter_worms.json')
microns_per_pixel = params.p_dict['microns_per_pixel']
expected_fps = params.p_dict['expected_fps']
is_light_background = params.p_dict['is_light_background']
print(microns_per_pixel, expected_fps)
for ii, fname in enumerate(fnames):
dd = read_unit_conversions(fname);
if dd[0][-1] != 'seconds':
print(ii, fname)
with tables.File(fname, 'r+') as fid:
group_to_save = fid.get_node('/mask')
set_unit_conversions(group_to_save, expected_fps, microns_per_pixel, is_light_background)
dd = read_unit_conversions(fname);
#print(fid.get_node('/features_timeseries').attrs['is_light_background'])
# print(fid.get_node('/full_data').attrs['is_light_background'])
# fid.get_node('/full_data').attrs['is_light_background'] = 1
# print(fid.get_node('/full_data').attrs['is_light_background'])
'L1_Late' : 1/0.363,
'L2' : 1/0.363,
'L3' : 1/0.289,
'L4' : 1/0.224,
'New_Adult' : 1/0.178,
'Old_Adult' : 1/0.178,
}
root_dir = '/Volumes/behavgenom_archive$/Avelino/screening/Pratheeban_Development'
for stage, microns_per_pixel in microns_per_pixel_s.items():
print(stage)
attr_params = dict(
expected_fps = 25,
microns_per_pixel = microns_per_pixel,
is_light_background = 1
)
fnames = glob.glob(os.path.join(root_dir, stage, '**', '*.hdf5'))
#filter to get only the maskfiles
fnames = [x + '.hdf5' for x in set(remove_ext(x) for x in fnames)]
for fname in fnames:
read_unit_conversions(fname)
with tables.File(fname, 'r+') as fid:
mask_dataset = fid.get_node('/mask')
full_dataset = fid.get_node("/full_data")
set_unit_conversions(mask_dataset, **attr_params)
set_unit_conversions(full_dataset, **attr_params)