def input_folder(inputfiles, ext="[tT][iI][fF]"):
"""
Determines if the input is a file or a folder.
If it's a folder it returns a list of the tif files inside
Parameters:
------
inputfiles : List or str
A list of files or a string which is the path to a folder containing
the files
ext : str, optional
extension to match; NB using e.g. [tT][iI][fF] matches
lower and uppercase extensions
Returns
------
A list of files to analyse
"""
if isinstance(inputfiles, str):
inputfiles = [inputfiles, ]
# If input is a file-list, do nothing
if os.path.isfile(inputfiles[0]):
logger.debug("First input is a file; assuming file(s) given")
return inputfiles
logger.debug("First input not a file; assuming folder(s) given")
# Should have paths
filelist = []
for path in inputfiles:
if os.path.isdir(path):
filelist.extend(glob.glob(
os.path.join(path, "*.{}".format(ext))
))
else:
filelist.extend(glob.glob(path))
return sorted(filelist)
def detect_bacteria_in_all_wells(
wellimages,
maxsize=1500,
minsize=20,
absolwidth=1,
min_av_width=3,
sigma_list=(2.0, 6.0),
scale_factor=1,
debug="",
):
"""
Takes a dictionary of wells, detects any bacteria in each well and returns
a new dictionary with each value containing a labelled image of detected bacteria
Parameters
------
wellimages : Dictionary
Key is the well number and the value is a ndarray (2D) of the well
maxsize : float, optional
Maximum area (in pixels) of an object to be considered a bacteria (default : 1500)
minsize : float, optional
Maximum area (in pixels) of an object to be considered a bacteria (default : 20)
absolwidth : float, optional
Width (in pixels) at which something is definitely a bacteria (default : 1)
min_av_width : float, optional
Minimum average width (in pixels) of a bacteria
sigma_list : scalar arguments, optional
Sigma from ndi.gaussian_laplace, list of the standard deviations of the Gaussian filter
Scaled with the scale factor (default : (2, 6))
scale_factor : Float, optional
Used to scale other parameters depending on the image magnification (default : 1)
debug : Boolean, optional
Whether to add debugging outputs, save debug images with this basename (default : False)
returns
------
segs2 : Dictionary
The key is the well coordinates and the value is a labelled image of detected bacteria
"""
maxsize = scale_factor * maxsize
minsize = scale_factor * minsize
absolwidth = scale_factor * absolwidth
# divide by 2 first as its the half width (because of the skeleton)
min_av_width = scale_factor * (min_av_width / 2)
segs = {}
segs2 = {}
logger.debug("Detecting bacteria in wells")
#sigma_list = np.arange(0.5, 3.5, 0.1)
sigma_list = np.arange(*sigma_list) * math.sqrt(scale_factor)
for well_num, im1 in wellimages.items():
# using scale space
gl_images = [
-(ndi.gaussian_laplace(im1, s, mode="nearest")) * s**2
for s in sigma_list
]
newwell = np.max(gl_images, axis=0)
segs[well_num] = newwell
# can we "normalise" the numbers after the filter so it's between a set value?
# or just change filter
thresh = skfilt.threshold_li(
np.concatenate([s.flatten() for s in segs.values()]))
if debug:
import matplotlib.pyplot as plt
plt.figure()
imfiltdebug = np.hstack(segs.values())
plt.imshow(imfiltdebug, cmap='gray')
maskdebug = imfiltdebug > thresh
plt.imshow(np.ma.masked_where(~maskdebug, maskdebug), cmap='hsv')
plt.savefig(debug)
plt.close()
#outpath_debug_tif = "_FILTERED".join(os.path.splitext(debug))+".tif"
#skio.imsave(outpath_debug_tif, imfiltdebug)
logger.debug(" Calculated threshold for bacteria detection : %s" %
str(thresh))
smax = 0
allstats = {}
for well_number, img0 in segs.items():
initialbac = detect_bacteria_in_well(
img0,
thresh,
absolwidth,
well_number=well_number,
debug=debug,
)
filteredbac, stats = filter_bacteria(
initialbac,
min_av_width,
minsize,
maxsize,
)
for k, v in stats.items():
allstats.setdefault(k, 0)
allstats[k] += v
segs2[well_number] = filteredbac
logger.debug(
" Final number of detected bacteria following morphological property filtering : %d"
% (smax))
return segs2
def detect_bacteria_in_well(
img0,
thresh,
absolwidth,
well_number=-1,
debug="",
):
"""
Detects bacteria in a single well image `img0`.
This is usually the result of a filtering step (e.g. `LoG`).
Parameters
------
img0 : ndarray (2D)
Image of a single well
Thresh : float
Threshold to use (determined from all wells)
absolwidth : float
Width (in pixels) at which something is definitely a bacteria (default : 1)
min_av_width : float
Minimum average width (in pixels) of a bacteria
minsize : float
Maximum area (in pixels) of an object to be considered a bacteria (default : 20)
maxsize : float
Maximum area (in pixels) of an object to be considered a bacteria (default : 1500)
well_number : int
Label of current well for logging purposes
smax : int
Maximum label of full well image (for this frame)
debug : boolean (or equivalent), (optional)
Equated to True/False to determine if debug logging output should be generated
returns
------
newbac : ndarray (2D, ints)
Labelled array of bacteria
smax : int
Maximum label of full well image (updated)
"""
bw0 = img0 > thresh
bw1 = ndi.morphology.binary_erosion(bw0)
if debug:
nbac0 = ndi.label(bw1)[1]
logger.debug(" Well %d - bacteria in initial binary image: %d" %
(well_number, nbac0))
bw2 = ndi.morphology.binary_dilation(bw0)
bw3 = bw2 ^ bw1
# perform distance transform on filtered image
dist = ndi.distance_transform_edt(bw1)
# create markers for watershed
markers = np.zeros(img0.shape, dtype=bool)
markers[dist >= absolwidth] = True
markers = ndi.label(markers)[0]
markers = markers + 1
markers[bw3] = 0
# Perform watershed
segmentation = watershed(img0, markers)
segmentation = ndi.binary_fill_holes(segmentation != 1)
# label image
labeled_bacteria, nbac = ndi.label(segmentation)
logger.debug(
" Well %d - bacteria detected after initial filtering : %d" %
(well_number, nbac))
return labeled_bacteria
def run_analysis_pipeline(
# pylint: disable=too-many-statements,too-many-branches,too-many-locals
# pylint: disable=too-many-arguments
filenames,
output=None,
tmax=None,
invert=False,
show=False,
debug=False,
brightchannel=False,
loader='default',
channel=None,
tdim=0,
fluo=None,
fluoresc=False,
batchrun=False,
area_num=None,
scale_factor=1,
num_fluo=0,
exit_on_error=False,
# logger=logger,
):
"""Main analysis pipeline
Executes the full analysis pipeline, from input images to output images and
tables.
Parameters
----------
filenames : list of strings
List of files to load (can be single file)
output : String
Name of output file base (default : input file name)
tmax : Integer
Frame to run analysis until (default : all)
invert : Boolean
Whether the image needs to be inverted (default : False)
show : Boolean
Whether to "show" the plots on the screen (default : False)
debug : Boolean
Whether to add debugging outputs (default : False)
brightchannel : Boolean
Detect channel as a bright instead of dark line (default : False)
loader : string (options: tifffile, default)
Which image loader to use (default : default)
channel : Integer
Select channel dimension from multi-channel
image (default : None - single channel image)
tdim : Integer
Select time dimension (default : 0)
fluo : List of strings
List of fluorescence images to load (default : None)
fluoresc : Boolean
Image stack contains alternating fluorescence images (default : False)
batchrun : Boolean
Whether input is a folder containing multiple image areas to
run concurrently (default : False)
area_num : Integer
Area number for dealing with multiple areas (default : None)
scale_factor : float, optional
Scale factor for detection parameters (default : 1)
num_fluo : int, optional
The number of fluorescence channels for each brightfield image
exit_on_error : Boolean, optional
Exit if any errors are encountered (default : False)
"""
# See if the input is a folder or files
filenames = mio.input_folder(filenames)
dir_name, output, image_dir = mmeas.find_input_filename(filenames[0],
out=output,
batch=batchrun,
im_area=area_num,
debug=debug)
logger.debug("Starting run_analysis_pipeline")
logger.debug(" invert: %s", invert)
logger.debug(" show: %s", show)
logger.info("Loading data...")
if fluoresc and not num_fluo:
num_fluo = 1
fluo_data = None
if loader == "tifffile":
try:
import tifffile
except BaseException:
loader = "default"
if loader == "default":
data = mio.load_data(filenames)
if fluo is not None:
fluo_data = mio.load_data(fluo)
elif loader == "tifffile":
data = tifffile.imread(filenames).astype(float)
else:
logger.error("Invalid loader specified [%s]", loader)
return ''
logger.debug("Initial load: %s", data.shape)
if num_fluo >= 1:
data, fluo_data = mio.split_fluorescence(data, num_fluo)
if data.ndim == 2:
data = data[None]
if tdim:
data = np.rollaxis(data, tdim)
if tmax:
data = data[:tmax]
if channel is not None:
data = data[:, channel, ...]
if invert:
if isinstance(data, (list, tuple)):
data = [d.max() - d for d in data]
else:
data = data.max() - data
logger.info("Data loaded: %s of type %s", data.shape, data.dtype)
logger.info("Detecting channels, wells, and bacteria...")
allwellcoords = []
allwellimages = []
# allridges = []
allbacteria = []
# ------------------------------
# Run detection on every frame
# ------------------------------
for tpoint, frame in enumerate(data):
empty_image = np.zeros(frame.shape, dtype="int16")
try:
if debug:
dirpart, filenamepart = os.path.split(filenames[0])
filenamebase = "debug_%0.4d_%s" % (tpoint, filenamepart)
debugnow = os.path.join(dirpart, filenamebase)
logger.debug("Saving any debugging data to [%s]", dirpart)
logger.debug(" using the prefix [%s]", filenamebase)
else:
debugnow = ""
(labelled_wellimg, bacteria_image, wellcoords,
bacteria) = mdet.run_detection(frame,
brightchannel,
debug=debugnow,
scale_factor=scale_factor)
mout.output_detection_figures(frame, labelled_wellimg,
bacteria_image, tpoint, image_dir)
allwellcoords.append(wellcoords)
allbacteria.append(bacteria)
allwellimages.append(labelled_wellimg)
logger.info("Detection complete on frame %s", tpoint + 1)
except KeyboardInterrupt:
logger.info("Keyboard interrupt detected, exiting")
return
except Exception:
logger.error("Detection failed for area number: %s timepoint %s",
area_num, tpoint + 1)
logger.error("Exception: %s", traceback.format_exc())
if exit_on_error:
raise
# dictionary of well ids : coords of well pixels
allwellcoords.append({})
# should be dictionary well id : label array of bacteria
allbacteria.append({})
# should be 2d label array
allwellimages.append(empty_image)
# ------------------------------
# Run tracking on every frame (after the first)
# ------------------------------
logger.info("Tracking...")
(allwellimages, allwellcoords, allbacteria,
bacteria_lineage) = mtrack.run_tracking(data, allwellimages,
allwellcoords, allbacteria)
logger.info("Creating tracking figures...")
mout.output_tracking_figures(data, allwellimages, allwellcoords,
allbacteria, image_dir, bacteria_lineage)
# ------------------------------
# Run analysis and output
# ------------------------------
logger.info("Measuring...")
measurements = mmeas.get_measurements(data, fluo_data, allwellimages,
allwellcoords, allbacteria,
bacteria_lineage)[0]
mout.final_output(measurements, dir_name)
# for k, v in measurements.items():
# print("ID:",k)
# for t,vv in v.items():
# print(" T:",t,"Measurements:", vv)
# mout.final_output(dir_name, output, fluo, fluoresc, final_timepoint+1)
# Generate output images
return dir_name
def run_cli():
"""
Main CLI run function. This should be called to interface
with command-line arguments
"""
args, parser = get_args(return_parser=True)
if args.debug:
logger.setLevel(logging.DEBUG)
from IPython.core import ultratb
sys.excepthook = ultratb.FormattedTB(mode='Verbose',
color_scheme='Linux',
call_pdb=1)
else:
logger.setLevel(logging.INFO)
if args.limitmem:
# ------------------------------
# Memory managment
# ------------------------------
gb_in_bytes = 1024.**3
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
mem_gib = mem_bytes / gb_in_bytes # e.g. 3.74
lim = mem_bytes // 3
logger.debug("Managing memory usage...")
logger.debug("System memory: %0.2f GB", mem_gib)
logger.debug("Limiting program to %0.2f GB", lim)
rsrc = resource.RLIMIT_AS
_, hard = resource.getrlimit(rsrc)
resource.setrlimit(rsrc, (lim, hard)) # limit
# Run appropriate analysis
if args.gui:
run_gui(args.debug)
sys.exit()
if not args.filename:
print("At least one filename needed when not in gui mode")
parser.print_help()
sys.exit()
if args.ba:
batch_run(
args.filename,
output=args.output,
tmax=args.t,
invert=args.invert,
show=args.show,
debug=args.debug,
brightchannel=args.brightchannel,
loader=args.loader,
channel=args.channel,
tdim=args.tdim,
fluo=args.fluo,
fluoresc=args.f,
batchrun=args.ba,
scale_factor=args.sf,
num_fluo=args.nf,
exit_on_error=args.exit_on_error,
)
else:
run_analysis_pipeline(
args.filename,
output=args.output,
tmax=args.t,
invert=args.invert,
show=args.show,
debug=args.debug,
brightchannel=args.brightchannel,
loader=args.loader,
channel=args.channel,
tdim=args.tdim,
fluo=args.fluo,
fluoresc=args.f,
batchrun=args.ba,
scale_factor=args.sf,
num_fluo=args.nf,
exit_on_error=args.exit_on_error,
)
def bacteria_tracking(last_wells, current_wells, bacteria_lineage):
"""
Takes a dictionary of wells from the previous frame and a second
dictionary with the corresponding wells for the new frame and
determines the most likely way the bacteria within them have
moved, died, divided then relabels them accordingly
Parameters
------
last_wells : Dictionary
The previous timepoint. The key is the well coordinates and the value
is a labelled image of detected bacteria
current_wells : Dictionary
The current timepoint. The key is the well coordinates and the value
is a labelled image of detected bacteria
bacteria_lineage : dictionary
A dictionary that links the physical unique label of a bacteria
to one which shows information on its lineage
Returns
------
out_wells : Dictionary
The current timepoint. The key is the well coordinates and the value
is a labelled image of tracked bacteria
bacteria_lineage : dictionary
Updated dictionary that links the physical unique label of a bacteria
to one which shows information on its lineage
"""
out_wells = {}
for num, well in last_wells.items():
if num not in current_wells.keys():
continue
new_well = current_wells[num]
in_list = []
#option_list = []
for region in regionprops(well):
# list the bacteria labels from the current frame
in_list.append(region.label)
# create a list of all the possible combinations
logger.debug("Creating combination of %d items repeated %d times" %
(len(in_list), len(regionprops(new_well))))
options = itertools.combinations_with_replacement(
in_list, len(regionprops(new_well)))
if not in_list:
if len(regionprops(new_well)) > 0:
# if there is now a new bacteria we don't want to ignore as it may have just
# been missed in previous frame
smax = max(bacteria_lineage, key=int)
newwell = np.zeros(new_well.shape, dtype=new_well.dtype)
for new_bac in regionprops(new_well):
# so lets give each "new" bacteria a new label
smax += 1
newwell[new_well == new_bac.label] = smax
bacteria_lineage[smax] = str(smax)
elif not next(options, None):
# if the out well is also empty then there is nothing to track
# and simply return an empty well
newwell = np.zeros(new_well.shape, dtype=new_well.dtype)
else: # determine probabilities and label matching/new bacteria
#options_dict[n] = [in_list,option_list]
change_options = bactrack.find_changes(in_list, options, well,
new_well)
best_option = None
best_prob = 0
for option, probs in change_options:
probs_ = bactrack.find_probs(probs)
if probs_ > best_prob:
best_prob = probs_
best_option = option
newwell, bacteria_lineage = bactrack.label_most_likely(
best_option, new_well, bacteria_lineage)
out_wells[num] = newwell
return out_wells, bacteria_lineage
def run_tracking(data, allwellimages, allwellcoords, allbacteria, debug=False):
"""
Takes in two frames of data, wells, and bacteria, and performs
tracking
Parameters
------
data : list of ndarrays
List of initial image that detection was run on
allwellimages : list of ndarrays
List of labelled images showing the detected wells
allwellcoords : list of arrays
Each entry contains a further list where each entry contains well coordinates
allbacteria : list of arrays
List of labelled images showing the detected bacteria
debug : Boolean
Whether to add debugging outputs (default : False)
Returns
------
allwellimages : list of ndarrays
List of labelled images showing the tracked wells
allwellcoords : list of arrays
Each entry contains a further list where each entry contains well coordinates
allbacteria : list of arrays
List of labelled images showing the tracked bacteria
bacteria_lineage : dictionary
A dictionary that links the physical unique label of a bacteria
to one which shows information on its lineage
"""
previous_wellimage = allwellimages[0]
bacteria_lineage = {}
for key, bacteria_image in allbacteria[0].items():
bacteria_lineage.update(
{l.label: str(l.label)
for l in regionprops(bacteria_image)})
for tpoint in range(1, len(data)):
# Frame shift
frame_shift = frametracker(data[tpoint - 1], data[tpoint], debug=debug)
logger.debug("\tFrame tracking registered a transform of: %s",
str(frame_shift))
# Well tracking - find mappings from the previous image to the current
# image
if (len(np.unique(previous_wellimage)) > 1) and (len(
np.unique(allwellimages[tpoint])) > 1):
logger.debug("\tRunning well tracking...")
wellimage_tracked, well_map = welltracking(previous_wellimage,
allwellimages[tpoint],
frame_shift)
# Apply the mappings to the well dictionary and the bacteria
# dictionary
allwellcoords[tpoint] = {
idkey: allwellcoords[tpoint][listindex]
for listindex, idkey in well_map.items()
}
allbacteria[tpoint] = {
idkey: allbacteria[tpoint][listindex]
for listindex, idkey in well_map.items()
}
previous_wellimage = wellimage_tracked
logger.debug("\tRunning bacterial lineage tracking...")
# Do lineage tracking
allbacteria[tpoint], bacteria_lineage = bacteria_tracking(
allbacteria[tpoint - 1],
allbacteria[tpoint],
bacteria_lineage,
)
elif (len(np.unique(previous_wellimage))
== 1) and (len(np.unique(allwellimages[tpoint])) > 1):
previous_wellimage = allwellimages[tpoint]
if not bacteria_lineage:
maxid = 0
else:
maxid = max(bacteria_lineage.keys())
for key, bacteria_image in allbacteria[tpoint].items():
newim = bacteria_image.copy()
for i, bac in enumerate(regionprops(bacteria_image),
start=maxid + 1):
newim[bacteria_image == bac.label] = i
bacteria_lineage[i] = str(i)
maxid = i
allbacteria[tpoint][key] = newim
else:
previous_wellimage = allwellimages[tpoint]
logger.debug("\trun_tracking finished!")
return allwellimages, allwellcoords, allbacteria, bacteria_lineage
"""