def main(imgDir: Path,
imgPattern: str,
ffDir: Path,
brightPattern: str,
outDir: Path,
darkPattern: typing.Optional[str] = None,
photoPattern: typing.Optional[str] = None) -> None:
''' Start a process for each set of brightfield/darkfield/photobleach patterns '''
# Create the FilePattern objects to handle file access
ff_files = FilePattern(ffDir, brightPattern)
fp = FilePattern(imgDir, imgPattern)
if darkPattern != None and darkPattern != '':
dark_files = FilePattern(ffDir, darkPattern)
if photoPattern != None and photoPattern != '':
photo_files = FilePattern(
str(Path(ffDir).parents[0].joinpath('metadata').absolute()),
photoPattern)
group_by = [v for v in fp.variables if v not in ff_files.variables]
GROUPED = group_by + ['file']
ProcessManager.init_processes('main', 'unshade')
for files in fp(group_by=group_by):
flat_path = ff_files.get_matching(
**{k.upper(): v
for k, v in files[0].items() if k not in GROUPED})[0]['file']
if flat_path is None:
logger.warning("Could not find a flatfield image, skipping...")
continue
if darkPattern is not None and darkPattern != '':
dark_path = dark_files.get_matching(**{
k.upper(): v
for k, v in files[0].items() if k not in GROUPED
})[0]['file']
if dark_path is None:
logger.warning("Could not find a darkfield image, skipping...")
continue
if photoPattern is not None and photoPattern != '':
photo_path = photo_files.get_matching(**{
k.upper(): v
for k, v in files[0].items() if k not in GROUPED
})[0]['file']
if photo_path is None:
logger.warning(
"Could not find a photobleach file, skipping...")
continue
ProcessManager.submit_process(unshade_batch, files, outDir, flat_path,
dark_path, photo_path)
ProcessManager.join_processes()
def main(input_dir: pathlib.Path,
output_dir: pathlib.Path,
file_pattern: typing.Optional[str] = None,
group_by: typing.Optional[str] = None,
get_darkfield: typing.Optional[bool] = None,
get_photobleach: typing.Optional[bool] = None,
metadata_dir: pathlib.Path = None) -> None:
if group_by is None:
group_by = 'xyp'
if get_darkfield is None:
get_darkfield = False
if get_photobleach is None:
get_photobleach = False
if file_pattern is None:
filepattern = '.*'
fp = FilePattern(input_dir, file_pattern)
ProcessManager.init_processes("basic")
for files in fp(group_by=group_by):
ProcessManager.submit_process(basic.basic, files, output_dir,
metadata_dir, get_darkfield,
get_photobleach)
ProcessManager.join_processes()
def main(
inpDir: Path,
filePattern: str,
outDir: Path,
) -> None:
ProcessManager.init_processes("main", "zarr")
fp = FilePattern(inpDir, filePattern)
for files in fp():
for file in files:
ProcessManager.submit_process(image_to_zarr, file["file"], outDir)
ProcessManager.join_processes()
def bench(
input_dir: Path,
output_dir: Path,
file_pattern: str,
replicate: int,
):
components_dir = output_dir.joinpath('components')
components_dir.mkdir(exist_ok=True)
coefficients_dir = output_dir.joinpath('csvs')
coefficients_dir.mkdir(exist_ok=True)
corrected_dir = output_dir.joinpath('images')
corrected_dir.mkdir(exist_ok=True)
heatmaps_dir = output_dir.joinpath('plots')
heatmaps_dir.mkdir(exist_ok=True)
channel_ordering = [1, 0, 3, 2, 4, 5, 7, 6, 8, 9]
fp = FilePattern(input_dir, file_pattern)
for group in fp(['c']):
write_corrected_images(
group=group,
channel_ordering=channel_ordering,
components_dir=components_dir,
output_dir=corrected_dir,
)
plot_pearson_coefficients(
group=group,
pattern=file_pattern,
channel_ordering=channel_ordering,
corrected_dir=corrected_dir,
selector_name='HighMeanIntensity',
heatmaps_dir=heatmaps_dir,
replicate=replicate,
)
return
'%(asctime)s - %(name)s - Process [{0},{1},{2}] - %(levelname)s - %(message)s'
.format(R, T, C),
datefmt='%d-%b-%y %H:%M:%S')
logger = logging.getLogger("BaSiC")
logger.setLevel(logging.INFO)
# Start the javabridge
log_config = Path(__file__).parent.joinpath("log4j.properties")
jutil.start_vm(args=[
"-Dlog4j.configuration=file:{}".format(str(log_config.absolute()))
],
class_path=bioformats.JARS)
# Parse files into list to access by variable
regex, variables = get_regex(inp_regex)
test = FilePattern(fpath, inp_regex)
files = [i['file'] for i in test.get_matching(R=R, T=T, C=C)]
# Load files and sort
logger.info('Loading and sorting images...')
img_stk, X, Y = _get_resized_image_stack(files)
img_stk_sort = np.sort(img_stk)
# Initialize options
new_options = _initialize_options(img_stk_sort, get_darkfield, OPTIONS)
# Initialize flatfield/darkfield matrices
logger.info('Beginning flatfield estimation')
flatfield_old = np.ones((new_options['size'], new_options['size']),
dtype=np.float64)
darkfield_old = np.random.normal(size=(new_options['size'],
help='Filename pattern used to separate data', required=True)
parser.add_argument('--inpDir', dest='inpDir', type=str,
help='Input image collection to be processed by this plugin', required=True)
parser.add_argument('--outDir', dest='outDir', type=str,
help='Output collection', required=True)
# Parse the arguments
args = parser.parse_args()
filePattern = args.filePattern
if not filePattern.strip():
filePattern = '.*'
logger.info('filePattern = {}'.format(filePattern))
inpDir = args.inpDir
logger.info('inpDir = {}'.format(inpDir))
outDir = args.outDir
logger.info('outDir = {}'.format(outDir))
# Get all file names in inpDir image collection
fp = FilePattern(inpDir,pattern=filePattern)
# Loop through files in inpDir image collection and process
for files in fp():
if isinstance(files,list):
# files is a list of file dictionaries
for f in files:
input_path = Path(f['file'])
output_path = Path(outDir).joinpath(input_path.name)
logger.info('Copying file: {}'.format(input_path.name))
shutil.copy2(str(input_path.absolute()), str(output_path.absolute()))
logger.info('Finished copying all files!')
def main(pattern: str,
inpDir: pathlib.Path,
layout: typing.List[str],
outDir: pathlib.Path,
imageSpacing: typing.Optional[int] = None,
gridSpacing: typing.Optional[int] = None
) -> None:
global SPACING
global MULTIPLIER
# Set new image spacing and grid spacing arguments if present
if image_spacing != None:
SPACING = int(image_spacing)
if grid_spacing != None:
MULTIPLIER = int(grid_spacing)
# Set up the file pattern parser
logger.info('Parsing the file pattern...')
fp = FilePattern(inpDir,pattern)
# Parse the layout
logger.info('Parsing the layout...')
regex, variables = get_regex(pattern)
layout = layout.replace(' ','')
layout = layout.split(',')
for l in layout: # Error checking
for v in l:
if v not in VARIABLES:
logger.error("Variables must be one of {}".format(VARIABLES))
raise ValueError("Variables must be one of {}".format(VARIABLES))
if len(layout)>2 or len(layout)<1:
logger.error("Each layout subgrid must have one or two variables assigned to it.")
raise ValueError("Each layout subgrid must have one or two variables assigned to it.")
for v in reversed(variables): # Add supergrids if a variable is undefined in layout
is_defined = False
for l in layout:
if v in l:
is_defined = True
break
if not is_defined:
layout.append(v)
# Layout dimensions, used to calculate positions later on
layout_dimensions = {'grid_size':[[] for r in range(len(layout))], # number of tiles in each dimension in the subgrid
'size':[[] for r in range(len(layout))], # total size of subgrid in pixels
'tile_size':[[] for r in range(len(layout))]} # dimensions of each tile in the grid
# Get the size of each image
logger.info('Get the size of every image...')
grid_width = 0
grid_height = 0
for files in fp(group_by=layout[0]):
# Determine number of rows and columns in the smallest subgrid
grid_size = _get_xy_index(files,layout[0],layout)
layout_dimensions['grid_size'][len(layout)-1].append(grid_size)
# Get the height and width of each image
for f in files:
f['width'], f['height'] = BioReader.image_size(f['file'])
if grid_width < f['width']:
grid_width = f['width']
if grid_height < f['height']:
grid_height = f['height']
logger.info('Got the size of {} images...'.format(len(files)))
# Set the pixel and tile dimensions
layout_dimensions['tile_size'][len(layout)-1].append([grid_width,grid_height])
layout_dimensions['size'][len(layout)-1].append([grid_width*grid_size[0],grid_height*grid_size[1]])
# Find the largest subgrid size for the lowest subgrid
grid_size = [0,0]
for g in layout_dimensions['grid_size'][len(layout)-1]:
if g[0] > grid_size[0]:
grid_size[0] = g[0]
if g[1] > grid_size[1]:
grid_size[1] = g[1]
tile_size = [0,0]
for t in layout_dimensions['tile_size'][len(layout)-1]:
if t[0] > tile_size[0]:
tile_size[0] = t[0]
if t[1] > tile_size[1]:
tile_size[1] = t[1]
layout_dimensions['grid_size'][len(layout)-1] = grid_size
layout_dimensions['tile_size'][len(layout)-1] = [tile_size[0] + SPACING, tile_size[1] + SPACING]
layout_dimensions['size'][len(layout)-1] = [layout_dimensions['grid_size'][len(layout)-1][0] * layout_dimensions['tile_size'][len(layout)-1][0],
layout_dimensions['grid_size'][len(layout)-1][1] * layout_dimensions['tile_size'][len(layout)-1][1]]
logger.info('Grid size for layer ({}): {}'.format(layout[0],grid_size))
# Build the rest of the subgrid indexes
for i in range(1,len(layout)):
# Get the largest size subgrid image in pixels
index = len(layout) - 1 - i
layout_dimensions['tile_size'][index] = layout_dimensions['size'][index+1]
for files in fp(group_by=''.join(layout[:i+1])):
# determine number of rows and columns in the current subgrid
grid_size = _get_xy_index(files,layout[i],layout)
layout_dimensions['grid_size'][index].append(grid_size)
# Get the current subgrid size
grid_size = [0,0]
for g in layout_dimensions['grid_size'][index]:
if g[0] > grid_size[0]:
grid_size[0] = g[0]
if g[1] > grid_size[1]:
grid_size[1] = g[1]
layout_dimensions['grid_size'][index] = grid_size
layout_dimensions['tile_size'][index] = [layout_dimensions['tile_size'][index][0] + (MULTIPLIER**i) * SPACING, layout_dimensions['tile_size'][index][1] + (MULTIPLIER**i) * SPACING]
layout_dimensions['size'][len(layout)-1] = [layout_dimensions['grid_size'][index][0] * layout_dimensions['tile_size'][index][0],
layout_dimensions['grid_size'][index][1] * layout_dimensions['tile_size'][index][1]]
logger.info('Grid size for layer ({}): {}'.format(layout[i],grid_size))
# Build stitching vector
logger.info('Building the stitching vector....')
fpath = str(pathlib.Path(outDir).joinpath("img-global-positions-1.txt").absolute())
max_dim = len(layout_dimensions['grid_size'])-1
with open(fpath,'w') as fw:
correlation = 0
for file in fp():
f = file[0]
file_name = pathlib.Path(f['file']).name
# Calculate the image position
gridX = 0
gridY = 0
posX = 0
posY = 0
for i in reversed(range(max_dim + 1)):
posX += f[str(i) + '_gridX'] * layout_dimensions['tile_size'][i][0]
posY += f[str(i) + '_gridY'] * layout_dimensions['tile_size'][i][1]
if i == max_dim:
gridX += f[str(i) + '_gridX']
gridY += f[str(i) + '_gridY']
else:
gridX += f[str(i) + '_gridX'] * layout_dimensions['grid_size'][i+1][0]
gridY += f[str(i) + '_gridY'] * layout_dimensions['grid_size'][i+1][1]
# Write the position to the stitching vector
fw.write("file: {}; corr: {}; position: ({}, {}); grid: ({}, {});\n".format(file_name,
correlation,
posX,
posY,
gridX,
gridY))
logger.info('Done!')
def basic(files: typing.List[Path],
out_dir: Path,
metadata_dir: typing.Optional[Path] = None,
darkfield: bool = False,
photobleach: bool = False):
# Try to infer a filename
try:
pattern = infer_pattern([f['file'].name for f in files])
fp = FilePattern(files[0]['file'].parent,pattern)
base_output = fp.output_name()
# Fallback to the first filename
except:
base_output = files[0]['file'].name
extension = ''.join(files[0]['file'].suffixes)
with ProcessManager.process(base_output):
# Load files and sort
ProcessManager.log('Loading and sorting images...')
img_stk,X,Y = _get_resized_image_stack(files)
img_stk_sort = np.sort(img_stk)
# Initialize options
new_options = _initialize_options(img_stk_sort,darkfield,OPTIONS)
# Initialize flatfield/darkfield matrices
ProcessManager.log('Beginning flatfield estimation')
flatfield_old = np.ones((new_options['size'],new_options['size']),dtype=np.float64)
darkfield_old = np.random.normal(size=(new_options['size'],new_options['size'])).astype(np.float64)
# Optimize until the change in values is below tolerance or a maximum number of iterations is reached
for w in range(new_options['max_reweight_iterations']):
# Optimize using inexact augmented Legrangian multiplier method using L1 loss
A, E1, A_offset = _inexact_alm_l1(copy.deepcopy(img_stk_sort),new_options)
# Calculate the flatfield/darkfield images and update training weights
flatfield, darkfield, new_options = _get_flatfield_and_reweight(A,E1,A_offset,new_options)
# Calculate the change in flatfield and darkfield images between iterations
mad_flat = np.sum(np.abs(flatfield-flatfield_old))/np.sum(np.abs(flatfield_old))
temp_diff = np.sum(np.abs(darkfield - darkfield_old))
if temp_diff < 10**-7:
mad_dark =0
else:
mad_dark = temp_diff/np.max(np.sum(np.abs(darkfield_old)),initial=10**-6)
flatfield_old = flatfield
darkfield_old = darkfield
# Stop optimizing if the change in flatfield/darkfield is below threshold
ProcessManager.log('Iteration {} loss: {}'.format(w+1,mad_flat))
if np.max(mad_flat,initial=mad_dark) < new_options['reweight_tol']:
break
# Calculate photobleaching effects if specified
if photobleach:
pb = _get_photobleach(copy.deepcopy(img_stk),flatfield,darkfield)
# Resize images back to original image size
ProcessManager.log('Saving outputs...')
flatfield = cv2.resize(flatfield,(Y,X),interpolation=cv2.INTER_CUBIC).astype(np.float32)
if new_options['darkfield']:
darkfield = cv2.resize(darkfield,(Y,X),interpolation=cv2.INTER_CUBIC).astype(np.float32)
# Export the flatfield image as a tiled tiff
flatfield_out = base_output.replace(extension,'_flatfield' + extension)
with BioReader(files[0]['file'],max_workers=2) as br:
metadata = br.metadata
with BioWriter(out_dir.joinpath(flatfield_out),metadata=metadata,max_workers=2) as bw:
bw.dtype = np.float32
bw.x = X
bw.y = Y
bw[:] = np.reshape(flatfield,(Y,X,1,1,1))
# Export the darkfield image as a tiled tiff
if new_options['darkfield']:
darkfield_out = base_output.replace(extension,'_darkfield' + extension)
with BioWriter(out_dir.joinpath(darkfield_out),metadata=metadata,max_workers=2) as bw:
bw.dtype = np.float32
bw.x = X
bw.y = Y
bw[:] = np.reshape(darkfield,(Y,X,1,1,1))
# Export the photobleaching components as csv
if photobleach:
offsets_out = base_output.replace(extension,'_offsets.csv')
with open(metadata_dir.joinpath(offsets_out),'w') as fw:
fw.write('file,offset\n')
for f,o in zip(files,pb[0,:].tolist()):
fw.write("{},{}\n".format(f,o))
'Oh no! Something went wrong:',
*_error_messages,
'See the README for more details.',
))
logger.error(_message)
raise ValueError(_message)
logger.info(f'inpDir = {_input_dir}')
logger.info(f'filePattern = {_pattern}')
logger.info(f'groupBy = {_group_by}')
logger.info(f'selectionCriterion = {_selector_name}')
logger.info(f'model = {_model_name}')
logger.info(f'channelOverlap = {_channel_overlap}')
logger.info(f'kernelSize = {_kernel_size}x{_kernel_size}')
logger.info(f'channelOrdering = {_channel_ordering}')
logger.info(f'outDir = {_output_dir}')
logger.info(f'csvDir = {_csv_dir}')
_fp = FilePattern(_input_dir, _pattern)
_kwargs = dict(
pattern=_pattern,
selector_name=_selector_name,
model_name=_model_name,
channel_overlap=_channel_overlap,
kernel_size=_kernel_size,
channel_ordering=_channel_ordering,
output_dir=_output_dir,
metadata_dir=_csv_dir,
)
main(_fp, _group_by, _kwargs)
vectorInMetadata = args.vectorInMetadata == 'true'
logger.info('vectorInMetadata = {}'.format(vectorInMetadata))
if vectorInMetadata:
outVectors = Path(outImages).joinpath('metadata_files')
outVectors.mkdir()
outVectors = str(outVectors.absolute())
outImages = Path(outImages).joinpath('images')
outImages.mkdir()
outImages = str(outImages.absolute())
else:
outVectors = args.outVectors
logger.info('outImages = {}'.format(outImages))
logger.info('outVectors = {}'.format(outVectors))
# Set up the fileparser
fp = FilePattern(inpDir,'.*.ome.tif')
# Parse the stitching vector
logger.info('Parsing stitching vectors...')
widths, heights = _parse_stitch(vector,fp)
# Parse the features
logger.info('Parsing features...')
feature_list = _parse_features(features,fp,method)
# Determine the min, max, and unique values for each data set
logger.info('Setting feature scales...')
feature_mins = {}
feature_ranges = {}
for key,val in feature_list.items():
valid_vals = [v for v in val if v is not 'NaN']
outDir = args.outDir
logger.info('outDir = {}'.format(outDir))
image_spacing = args.imageSpacing
logger.info('image_spacing = {}'.format(image_spacing))
grid_spacing = args.gridSpacing
logger.info('grid_spacing = {}'.format(grid_spacing))
# Set new image spacing and grid spacing arguments if present
if image_spacing != None:
SPACING = int(image_spacing)
if grid_spacing != None:
MULTIPLIER = int(grid_spacing)
# Set up the file pattern parser
logger.info('Parsing the file pattern...')
fp = FilePattern(inpDir,pattern)
# Parse the layout
logger.info('Parsing the layout...')
regex, variables = get_regex(pattern)
layout = layout.replace(' ','')
layout = layout.split(',')
for l in layout: # Error checking
for v in l:
if v not in VARIABLES:
logger.error("Variables must be one of {}".format(VARIABLES))
ValueError("Variables must be one of {}".format(VARIABLES))
if len(layout)>2 or len(layout)<1:
logger.error("Each layout subgrid must have one or two variables assigned to it.")
ValueError("Each layout subgrid must have one or two variables assigned to it.")
v))
if darkPattern != None and darkPattern != '':
if (v in dark_variables) != (
v in img_variables): # v must be in both or neither (xor)
logger.error(
'Variable {} is not in both darkPattern and imgPattern.'.
format(v))
if photoPattern != None and photoPattern != '':
if (v in photo_variables) != (
v in img_variables): # v must be in both or neither (xor)
logger.error(
'Variable {} is not in both photoPattern and imgPattern.'.
format(v))
''' Start a process for each set of brightfield/darkfield/photobleach patterns '''
# Create the FilePattern objects to handle file access
ff_files = FilePattern(ffDir, brightPattern)
if darkPattern != None and darkPattern != '':
dark_files = FilePattern(ffDir, darkPattern)
if photoPattern != None and photoPattern != '':
photo_files = FilePattern(
str(Path(ffDir).parents[0].joinpath('metadata').absolute()),
photoPattern)
# Initialize variables for process management
processes = []
process_timer = []
pnum = 0
total_jobs = len([p for p in ff_files.iterate()])
# Loop through files in ffDir image collection and process
base_pstring = "python3 apply_flatfield.py --inpDir {} --outDir {} --filepattern {}".format(
darkfield = args.darkfield
photobleach = args.photobleach
R = int(args.R)
T = int(args.T)
C = int(args.C)
''' Initialize the logger '''
logging.basicConfig(
format=
'%(asctime)s - %(name)-8s - Process [{0},{1},{2}] - %(levelname)-8s - %(message)s'
.format(R, T, C),
datefmt='%d-%b-%y %H:%M:%S')
logger = logging.getLogger("do_flat")
logger.setLevel(logging.INFO)
# Set up the FilePattern object
images = FilePattern(inpDir, filepattern)
''' Start the javabridge '''
logger.info("Starting the javabridge...")
log_config = Path(__file__).parent.joinpath("log4j.properties")
jutil.start_vm(args=[
"-Dlog4j.configuration=file:{}".format(str(log_config.absolute()))
],
class_path=bioformats.JARS)
''' Load the flatfielding data '''
logger.info("Loading the flatfield data...")
flat_br = BioReader(brightfield)
flat_image = np.squeeze(flat_br.read_image())
del flat_br
# Normalize the brightfield image if it isn't done already
flat_image = flat_image.astype(np.float32)
dest='outDir',
type=str,
help='Output collection',
required=True)
# Parse the arguments
args = parser.parse_args()
filePattern = args.filePattern
logger.info('filePattern = {}'.format(filePattern))
inpDir = args.inpDir
logger.info('inpDir = {}'.format(inpDir))
outDir = args.outDir
logger.info('outDir = {}'.format(outDir))
# Get all file names in inpDir image collection
inpDir_files = FilePattern(inpDir, filePattern)
# Loop through files in inpDir image collection and process
for files in inpDir_files.iterate():
if isinstance(files, list):
# if a filename pattern is used, then files is a list of file dictionaries
for f in files:
input_path = Path(f['file'])
output_path = Path(outDir).joinpath(input_path.name)
logger.info('Copying file: {}'.format(input_path.name))
shutil.copy(str(input_path.absolute()),
str(output_path.absolute()))
else:
# if no filename pattern was used, only a single file dictionary is returned
input_path = Path(files['file'])
output_path = Path(outDir).joinpath(input_path.name)
def main():
""" Initialize argument parser """
logger.info("Parsing arguments...")
parser = argparse.ArgumentParser(
prog='main',
description='Calculate flatfield information from an image collection.'
)
""" Define the arguments """
parser.add_argument(
'--inpDir', # Name of the bucket
dest='inpDir',
type=str,
help='Path to input images.',
required=True)
parser.add_argument(
'--darkfield', # Path to the data within the bucket
dest='darkfield',
type=str,
help='If true, calculate darkfield contribution.',
required=False)
parser.add_argument(
'--photobleach', # Path to the data within the bucket
dest='photobleach',
type=str,
help='If true, calculates a photobleaching scalar.',
required=False)
parser.add_argument(
'--inpRegex', # Output directory
dest='inp_regex',
type=str,
help='Input file name pattern.',
required=False)
parser.add_argument(
'--outDir', # Output directory
dest='output_dir',
type=str,
help='The output directory for the flatfield images.',
required=True)
""" Get the input arguments """
args = parser.parse_args()
fpath = args.inpDir
"""Checking if there is images subdirectory"""
if (Path.is_dir(Path(args.inpDir).joinpath('images'))):
fpath = Path(args.inpDir).joinpath('images')
get_darkfield = str(args.darkfield).lower() == 'true'
output_dir = Path(args.output_dir).joinpath('images')
output_dir.mkdir(exist_ok=True)
metadata_dir = Path(args.output_dir).joinpath('metadata_files')
metadata_dir.mkdir(exist_ok=True)
inp_regex = args.inp_regex
get_photobleach = str(args.photobleach).lower() == 'true'
logger.info('input_dir = {}'.format(fpath))
logger.info('get_darkfield = {}'.format(get_darkfield))
logger.info('get_photobleach = {}'.format(get_photobleach))
logger.info('inp_regex = {}'.format(inp_regex))
logger.info('output_dir = {}'.format(output_dir))
# Set up lists for tracking processes
processes = []
process_timer = []
pnum = 0
# Iterator to group files with constant r,t and c values
file = FilePattern(fpath, inp_regex)
total_no = len(list(file.iterate(group_by='xyz')))
for i in file.iterate(group_by='xyz'):
if len(processes) >= multiprocessing.cpu_count() - 1:
free_process = -1
while free_process < 0:
for process in range(len(processes)):
if processes[process].poll() is not None:
free_process = process
break
# Wait between checks to free up some processing power
time.sleep(3)
pnum += 1
logger.info("Finished process {} of {} in {}s!".format(
pnum, total_no,
time.time() - process_timer[free_process]))
del processes[free_process]
del process_timer[free_process]
logger.info("Starting process [r,t,c]: [{},{},{}]".format(
i[0]['r'], i[0]['t'], i[0]['c']))
processes.append(
subprocess.Popen(
"python3 basic.py --inpDir {} --outDir {} --darkfield {} --photobleach {} --inpRegex {} --R {} --T {} --C {}"
.format(fpath, args.output_dir, get_darkfield, get_photobleach,
inp_regex, i[0]['r'], i[0]['t'], i[0]['c']),
shell=True))
process_timer.append(time.time())
while len(processes) > 1:
free_process = -1
while free_process < 0:
for process in range(len(processes)):
if processes[process].poll() is not None:
free_process = process
break
# Wait between checks to free up some processing power
time.sleep(3)
pnum += 1
logger.info("Finished process {} of {} in {}s!".format(
pnum, total_no,
time.time() - process_timer[free_process]))
del processes[free_process]
del process_timer[free_process]
processes[0].wait()
logger.info("Finished process {} of {} in {}s!".format(
total_no, total_no,
time.time() - process_timer[0]))
logger.info("Finished all processes!")
def parse_collection(directory_path, file_pattern, registration_variable,
similarity_variable, template_image):
"""
This function parses the input directory and returns a dictionary. Each key in the dictionary is a tuple
consisting of a template and a moving image. The value corresponding to each key is a list of images that have
similar transformation as the moving image.
Note: The code produces the expected output when len(registration_variable)==len(similarity_variable)==1.
The code will NOT spit out an error when the more than one variable is passed as registration or
similarity variable, but additional testing needs to be done to validate the script for this usecase.
inputs :
directory_path: path to the input collection
file_pattern: file name pattern of the input images
registration_variable : variable to help determine the set of moving and template images
similarity variable: variable to help determine the set of images having a similar transformation
corresponding to each set of moving and template images
template_image: name of a template image
outputs : result_dic
example of result_dic is shown below
result_dic = { (template_img1 : moving_img1) : [set1_img1,set1_img2, set1_img3....],
(template_img2 : moving_img2) : [set2_img1,set2_img2, set2_img3....],
.
. }
"""
# Predefined variables order
#var_order = 'rtczyx'
# get all variables in the file pattern
_, variables = get_regex(file_pattern)
# get variables except the registration and similarity variable
moving_variables = [
var for var in variables
if var not in registration_variable and var not in similarity_variable
]
# uvals is dictionary with all the possible variables as key
# corresponding to each key is a list of all values which that variable can take for the input collection
_, uvals = parse_directory(directory_path, file_pattern)
parser_object = FilePattern(directory_path, file_pattern)
image_set = []
# extract the index values from uvals for each variable in moving_variables
moving_variables_set = [uvals[var] for var in moving_variables]
# iterate over the similar transformation variables
# Code produced expected output when len(registration_variable)==len(similarity_variable)==1
# refer to function description
for char in similarity_variable:
# append the variable to the moving variable set
moving_variables.append(char)
# iterate over all possible index values of the similar transf. variable
for ind in uvals[char]:
registration_set = []
# append the fixed value of the index to the moving variables set
moving_variables_set.append([ind])
# get all the possible combinations of the index values in the moving variables set
registration_indices_combinations = list(
itertools.product(*moving_variables_set))
all_dicts = []
# iterate over all combinations and create a dictionary for each combination
# the dictionary is of the form {'C'=1, 'X'=2...etc} which can be used as an input
# to the get_matching() function
for index_comb in registration_indices_combinations:
inter_dict = {}
for i in range(len(moving_variables)):
inter_dict.update(
{moving_variables[i].upper(): index_comb[i]})
# store all dictionaries
all_dicts.append(inter_dict)
# iterate over all dictionaries
for reg_dict in all_dicts:
intermediate_set = []
# use get_matching function to get all filenames with defined variable values in the dictionary
files = parser_object.get_matching(**reg_dict)
# files is a list of dictionaries
for file_dict in files:
intermediate_set.append(file_dict['file'])
registration_set.append(intermediate_set)
# delete the fixed index value of the similar transf. variable to prepare for the next iteration
moving_variables_set.pop(-1)
image_set.append(registration_set)
# parse image set to form the result dictionary
result_dic = {}
old_set = np.array(image_set)
for j in range(old_set.shape[1]):
inter = old_set[:, j, :]
for k in range(inter.shape[1]):
ky = (inter[0, 0], inter[0, k])
items = inter[1:, k]
result_dic.update({ky: items})
return result_dic