def tspecjob(collection, render, tilespecs, estimate=True):
result = {}
result['tilespecs'] = tilespecs
matches = renderapi.pointmatch.get_matches_from_tile_to_tile(
collection,
tilespecs[0].layout.sectionId,
tilespecs[0].tileId,
tilespecs[1].layout.sectionId,
tilespecs[1].tileId,
render=render)
if len(matches) != 1:
estr = "\n expected 1 matching tile pair, found %d for" % (
len(matches))
estr += '\n %s\n group: %s\n %s\n group: %s' % (
tilespecs[0].tileId,
tilespecs[0].layout.sectionId,
tilespecs[1].tileId,
tilespecs[1].layout.sectionId)
raise RenderModuleException(estr)
match = matches[0]
if estimate:
tf = renderapi.transform.RigidModel()
src = np.array(match['matches']['p']).transpose()
dst = np.array(match['matches']['q']).transpose()
if match['pGroupId'] == tilespecs[0].layout.sectionId:
src = np.array(match['matches']['q']).transpose()
dst = np.array(match['matches']['p']).transpose()
ind = np.argwhere(
np.isclose(
np.array(match['matches']['w']),
1.0)).flatten()
tf.estimate(src[ind, :], dst[ind, :])
result['transform'] = tf
result['avg_residual'] = {
'z0': tilespecs[0].z,
'z1': tilespecs[1].z,
'avg_residual': avg_residual(match, tilespecs[0], tilespecs[1])
}
return result
def run(self):
if len(self.zValues) != len(self.args['new_zValues']):
raise RenderModuleException(
"zValues with length {} cannot be mapped to "
"zValues with length {}".format(len(self.zValues),
len(self.args['new_zValues'])))
for z, newz in zip(self.zValues, self.args['new_zValues']):
resolvedtiles = renderapi.resolvedtiles.get_resolved_tiles_from_z(
self.input_stack, z, render=self.render)
for ts in resolvedtiles.tilespecs:
ts.z = newz
if self.args.get('remap_sectionId'):
ts.layout.sectionId = self.sectionId_from_z(newz)
self.output_tilespecs_to_stack(
resolvedtiles.tilespecs,
sharedTransforms=resolvedtiles.transforms,
zValues=[z])
self.output({
"zValues": self.zValues,
"output_stack": self.output_stack
})
def run(self):
self.logger.debug('Montage scape stack generation module')
# get the list of z indices
zvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['montage_stack'])
zvalues1 = self.zValues
zvalues = list(set(zvalues1).intersection(set(zvalues)))
if not zvalues:
raise RenderModuleException(
'No sections found for stack {}'.format(
self.args['montage_stack']))
# generate tuple of old and new Zs
# setting a new z range does not check whether the range
# overlaps with existing sections/chunks in the output stack
if self.args['set_new_z']:
zvalues = list(np.sort(np.array(zvalues)))
diffarray = [x - zvalues[0] for x in zvalues]
newzvalues = [self.args['new_z_start'] + x for x in diffarray]
else:
newzvalues = zvalues
Z = [[int(oldz), int(newz)] for oldz, newz in zip(zvalues, newzvalues)]
out_stack_exists = self.args['output_stack'] in self.render.run(
renderapi.render.get_stacks_by_owner_project)
if out_stack_exists:
# check whether overwrite z is set to false. If so, then remove those z that is already in output stack
outzvalues = renderapi.stack.get_z_values_for_stack(
self.args['output_stack'], render=self.render)
if self.overwrite_zlayer:
# stack has to be in loading state
renderapi.stack.set_stack_state(self.args['output_stack'],
'LOADING',
render=self.render)
for oldz, newz in zip(zvalues, newzvalues):
# delete the section from output stack
renderapi.stack.delete_section(self.args['output_stack'],
newz,
render=self.render)
# generate the tag string to add to output tilespec json file name
tagstr = "%s_%s" % (min(zvalues), max(zvalues))
tilespecdir = os.path.join(self.args['image_directory'],
self.args['render']['project'],
self.args['montage_stack'],
'sections_at_%s' % str(self.args['scale']),
'tilespecs_%s' % tagstr)
try:
os.makedirs(tilespecdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
pass
render_materialize = renderapi.connect(**self.render.make_kwargs(
memGB=self.args['memGB_materialize']))
# process for each z
mypartial = partial(create_montage_scape_tile_specs,
render_materialize,
self.args['montage_stack'],
self.args['image_directory'],
self.args['scale'],
self.args['render']['project'],
tagstr,
self.args['imgformat'],
apply_scale=self.args['apply_scale'],
level=self.args['level'],
pool_size=1,
doFilter=self.args['doFilter'],
fillWithNoise=self.args['fillWithNoise'],
uuid_prefix=self.args["uuid_prefix"],
uuid_prefix_length=self.args["uuid_length"],
do_mp=False)
with renderapi.client.WithPool(
self.args['pool_size_materialize']) as pool:
pool.map(mypartial, Z)
# get all the output tilespec json files
tspath = os.path.join(self.args['image_directory'],
self.args['render']['project'],
self.args['montage_stack'],
'sections_at_%s' % str(self.args['scale']),
'tilespecs_%s' % tagstr)
jsonfiles = glob.glob("%s/*.json" % tspath)
if not jsonfiles:
raise RenderModuleException(
'No tilespecs json files were generated')
# create the stack if it doesn't exist
if self.output_stack not in self.render.run(
renderapi.render.get_stacks_by_owner_project):
# stack does not exist
# TODO configurable stack metadata
self.render.run(renderapi.stack.create_stack,
self.output_stack,
cycleNumber=5,
cycleStepNumber=1,
stackResolutionX=1,
stackResolutionY=1)
# import tilespecs to render
self.render.run(renderapi.client.import_jsonfiles_parallel,
self.output_stack, jsonfiles)
if self.close_stack:
# set stack state to complete
self.render.run(renderapi.stack.set_stack_state,
self.output_stack,
state='COMPLETE')
self.output({'output_stack': self.output_stack})
def run(self):
if "MCRROOT" not in os.environ:
raise ValidationError("MCRROOT not set")
if self.clone_section_stack:
tmp_stack = "{}_zs{}_ze{}_t{}".format(
self.args['source_collection']['stack'],
self.args['first_section'], self.args['last_section'],
time.strftime("%m%d%y_%H%M%S"))
zvalues = renderapi.stack.get_z_values_for_stack(
self.args['source_collection']['stack'], render=self.render)
zs = [z for z in zvalues if \
(z>=self.args['first_section']) \
and (z<=self.args['last_section'])]
renderapi.stack.clone_stack(
self.args['source_collection']['stack'],
tmp_stack,
zs=zs,
close_stack=True,
render=self.render)
self.args['source_collection']['stack'] = tmp_stack
# the clone stack option should also set the first and last section z appropriately in self.args for solver
self.args['first_section'] = min(zs)
self.args['last_section'] = max(zs)
# Check if the target stack exists and whether we need to overwrite the z
target_host = self.args['target_collection']['service_host']
num = target_host[-4:]
port = None
if num.isdigit():
target_host = target_host[:-5]
if target_host.find('http://') < 0:
target_host = 'http://%s' % (target_host)
port = int(num)
list_of_stacks = renderapi.render.get_stacks_by_owner_project(
owner=self.args['target_collection']['owner'],
project=self.args['target_collection']['project'],
host=target_host,
port=port,
render=self.render)
if self.args['target_collection']['stack'] in list_of_stacks:
for z in xrange(int(self.args['first_section']),
int(self.args['last_section']) + 1):
renderapi.stack.delete_section(
self.args['target_collection']['stack'],
z,
host=target_host,
port=port,
owner=self.args['target_collection']['owner'],
project=self.args['target_collection']['project'],
render=self.render)
# generate a temporary json to feed in to the solver
tempjson = tempfile.NamedTemporaryFile(suffix=".json",
mode='w',
delete=False)
tempjson.close()
with open(tempjson.name, 'w') as f:
json.dump(self.args, f, indent=4)
f.close()
#assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable
cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'],
tempjson.name)
ret = os.system(cmd)
# one successful completion remove the input json file
if ret == 0:
os.remove(tempjson.name)
else:
raise RenderModuleException("solve failed with input_json {}",
self.args)
#try to return the z's solved in the output json
try:
d = {'zs': zs}
self.output(d)
except:
raise RenderModuleException("unable to output json {}", d)
#if you made a tmp stack destroy it
if self.clone_section_stack:
renderapi.stack.delete_stack(tmp_stack, render=self.render)
def run(self):
allzvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['montage_stack'])
allzvalues = np.array(allzvalues)
Z = [[a, b] for a, b in zip(self.args['old_z'], self.args['new_z'])
if a in allzvalues]
mypartial = partial(
apply_rough_alignment,
self.render,
self.args['montage_stack'],
self.args['prealigned_stack'],
self.args['lowres_stack'],
self.args['output_stack'],
self.args['tilespec_directory'],
self.args['scale'],
self.args['mask_input_dir'],
self.args['update_lowres_with_masks'],
self.args['read_masks_from_lowres_stack'],
self.args['filter_montage_output_with_masks'],
self.args['mask_exts'],
apply_scale=self.args['apply_scale'],
consolidateTransforms=self.args['consolidate_transforms'],
remap_section_ids=self.args['remap_section_ids'])
# Create the output stack if it doesn't exist
if self.args['output_stack'] not in self.render.run(
renderapi.render.get_stacks_by_owner_project):
# stack does not exist
self.render.run(renderapi.stack.create_stack,
self.args['output_stack'])
else:
# stack exists and has to be set to LOADING state
self.render.run(renderapi.stack.set_stack_state,
self.args['output_stack'], 'LOADING')
lowres_state = self.render.run(renderapi.stack.get_full_stack_metadata,
self.args['lowres_stack'])['state']
with renderapi.client.WithPool(self.args['pool_size']) as pool:
results = pool.map(mypartial, Z)
# raise an exception if all the z values to apply alignment were not
if not all([r is None for r in results]):
failed_zs = [(result, z) for result, z in zip(results, Z)
if result is not None]
raise RenderModuleException(
"Failed to rough align z values {}".format(failed_zs))
if self.args["close_stack"]:
# set stack state to complete
self.render.run(renderapi.stack.set_stack_state,
self.args['output_stack'],
state='COMPLETE')
if self.args['update_lowres_with_masks']:
# return the lowres stack to original state
# if any imports were done
self.render.run(renderapi.stack.set_stack_state,
self.args['lowres_stack'],
state=lowres_state)
self.output({
'zs': np.array(Z),
'output_stack': self.args['output_stack']
})
def run(self, run_lc_bsh=None):
run_lc_bsh = self.default_bsh if run_lc_bsh is None else run_lc_bsh
outfn = self.args.get('outfile', os.path.abspath(os.path.join(
self.args['project_path'], 'lens_correction.json')))
delete_procdir = False
procdir = self.args.get('processing_directory',
self.args['project_path'])
if procdir is None:
delete_procdir = True
procdir = tempfile.mkdtemp()
# command line argument for beanshell script
bsh_call = [
"xvfb-run",
"-a",
self.args['fiji_path'],
"-Xms{}g".format(self.args['heap_size']),
"-Xmx{}g".format(self.args['heap_size']), "-Xincgc",
"-Dman=" + self.args['manifest_path'],
"-Ddir=" + self.args['project_path'],
"-Dgrid=" + str(self.args['grid_size']),
"-Disig=" + str(self.args['SIFT_params']['initialSigma']),
"-Dsteps=" + str(self.args['SIFT_params']['steps']),
"-Dminos=" + str(self.args['SIFT_params']['minOctaveSize']),
"-Dmaxos=" + str(self.args['SIFT_params']['maxOctaveSize']),
"-Dfdsize=" + str(self.args['SIFT_params']['fdSize']),
"-Dfdbins=" + str(self.args['SIFT_params']['fdBins']),
"-Drod=" + str(self.args['align_params']['rod']),
"-Dmaxe=" + str(self.args['align_params']['maxEpsilon']),
"-Dmir=" + str(self.args['align_params']['minInlierRatio']),
"-Dmni=" + str(self.args['align_params']['minNumInliers']),
"-Demi=" + str(self.args['align_params']['expectedModelIndex']),
"-Dmh=" + str(self.args['align_params']['multipleHypotheses']),
"-Dri=" + str(self.args['align_params']['rejectIdentity']),
"-Dit=" + str(self.args['align_params']['identityTolerance']),
"-Dtaip=" + str(self.args['align_params']['tilesAreInPlace']),
"-Ddmi=" + str(self.args['align_params']['desiredModelIndex']),
"-Dreg=" + str(self.args['align_params']['regularize']),
"-Dmio={}".format(
self.args['align_params']['maxIterationsOptimize']),
"-Dmpwo={}".format(
self.args['align_params']['maxPlateauWidthOptimize']),
"-Ddim=" + str(self.args['align_params']['dimension']),
"-Dlam=" + str(self.args['align_params']['lambdaVal']),
"-Dprocdir={}".format(procdir),
"-Doutfn={}".format(outfn),
"-Dctrans=" + str(self.args['align_params']['clearTransform']),
"-Dvis=" + str(self.args['align_params']['visualize']),
"-DthreadsSIFT={}".format(self.args['max_threads_SIFT']),
"--", "--no-splash",
run_lc_bsh]
self.logger.debug('bsh_cmd: {}'.format(bsh_call))
try:
subprocess.check_call(bsh_call)
except subprocess.CalledProcessError as e:
raise(RenderModuleException("{}".format(e)))
# self.logger.debug('ret_code: {}'.format(ret))
if delete_procdir:
shutil.rmtree(procdir)
try:
self.output({'output_json': outfn})
except AttributeError as e:
# output validation will need to wait for argschema PR
self.logger.error(e)
def run(self):
zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['output_downsampled_stack'])
zrange = range(self.args['minZ'], self.args['maxZ'] + 1)
zvalues = list(set(zvalues1).intersection(set(zrange)))
zvalues.sort()
if len(zvalues) == 0:
raise RenderModuleException(
('No valid zvalues found in stack '
'for given range {} - {}').format(self.args['minZ'],
self.args['maxZ']))
# get the boundary polygon for each section
mypartial1 = partial(get_poly, self.args['output_downsampled_stack'],
self.render)
with renderapi.client.WithPool(self.args['pool_size']) as pool:
boundary_polygons = pool.map(mypartial1, zvalues)
post_polys = {}
for poly, z in zip(boundary_polygons, zvalues):
post_polys[z] = poly
mypartial2 = partial(get_poly, self.args['input_downsampled_stack'],
self.render)
with renderapi.client.WithPool(self.args['pool_size']) as pool:
pre_boundary_polygons = pool.map(mypartial2, zvalues)
pre_polys = {}
for poly, z in zip(pre_boundary_polygons, zvalues):
pre_polys[z] = poly
if self.args['output_dir'] is None:
self.args['output_dir'] = tempfile.mkdtemp()
# compute ious
ious = compute_ious(post_polys, zvalues)
# compute distortion
distortion = []
dio = []
doi = []
for i, z in enumerate(zvalues):
di, do, dist = compute_distortion(pre_boundary_polygons[i],
boundary_polygons[i], z)
dio.append(di)
doi.append(do)
distortion.append(dist)
dist_plt_name = None
iou_plt_name = None
if self.args['out_file_format'] == 'pdf': # pdf plots
dist_plt_name, iou_plt_name = generate_pdf_plots(
ious, zrange, self.args['output_dir'], pre_boundary_polygons,
boundary_polygons, dio, doi, distortion, zvalues)
else:
plot_name = generate_bokeh_plots(ious, zrange,
self.args['output_dir'],
pre_boundary_polygons,
boundary_polygons, dio, doi,
distortion, zvalues)
dist_plt_name = plot_name
iou_plt_name = plot_name
self.output({
"iou_plot": iou_plt_name,
"distortion_plot": dist_plt_name
})
def run(self):
# inits
alltilespecs = []
numtiles = 0
firstts = []
outtilespecs = []
ind = 0
# get tilespecs for z
for z in range(self.args['minZ'], self.args['maxZ'] + 1):
tilespecs = self.render.run(
renderapi.tilespec.get_tile_specs_from_z,
self.args['input_stack'], z)
alltilespecs.extend(tilespecs)
# used for easy creation of tilespecs for output stack
firstts.append(tilespecs[0])
numtiles += len(tilespecs)
# subsample in the case where the number of tiles is too large
if self.args['num_images'] > 0:
alltilespecs = randomly_subsample_tilespecs(
alltilespecs, self.args['num_images'])
stackInfo = renderapi.stack.get_full_stack_metadata(
self.args['input_stack'], render=self.render)
channel_names = stackInfo['stats'].get('channelNames', [])
# Quick fix to have the empty channel logic work when channelNames=[]
if len(channel_names) == 0:
channel_names = [None]
# figure out which of our channels is the default channel
if channel_names[0] is not None:
try:
default_chan = next(
(ch.name for ch in firstts[0].channels
if ch.ip[0].imageUrl == firstts[0].ip[0].imageUrl))
except StopIteration:
raise RenderModuleException(
"default channel doesn't match any channel")
outdir = self.args['output_directory']
if not os.path.exists(outdir):
os.makedirs(outdir)
out_images = []
for chan_name in channel_names:
chan_str = chan_name if chan_name is not None else ""
outImage = outdir + \
"/%s_%s_%s_%d-%d.tif" % (self.args['file_prefix'],
self.args['output_stack'],
chan_str,
self.args['minZ'],
self.args['maxZ'])
make_median_image(alltilespecs,
numtiles,
outImage,
self.args['pool_size'],
chan=chan_name)
out_images.append(outImage)
for ind, z in enumerate(range(self.args['minZ'],
self.args['maxZ'] + 1)):
# want many variations on this tilespec so making a
# copy by serializing/deserializing
ts = renderapi.tilespec.TileSpec(json=firstts[ind].to_dict())
# want a different z for each one
ts.z = z
if channel_names[0] is None:
mml_o = ts.ip[0]
mml = renderapi.image_pyramid.MipMap(imageUrl=out_images[0],
maskUrl=mml_o.maskUrl)
ts.ip[0] = mml
else:
channels = []
for chan_name, out_image in zip(channel_names, out_images):
chan_orig = next(
(ch for ch in ts.channels if ch.name == chan_name))
mml = renderapi.image_pyramid.MipMap(imageUrl=out_image)
ip = renderapi.image_pyramid.ImagePyramid()
ip[0] = mml
chan_orig.ip = ip
channels.append(chan_orig)
# we want the default_chan median to be the
# default image pyramid
ts.ip = next((med_ch.ip for med_ch in channels
if med_ch.name == default_chan))
ts.channels = channels
outtilespecs.append(ts)
# upload to render
renderapi.stack.create_stack(self.args['output_stack'],
cycleNumber=2,
cycleStepNumber=1,
render=self.render)
renderapi.stack.set_stack_state(self.args['output_stack'],
"LOADING",
render=self.render)
renderapi.client.import_tilespecs_parallel(
self.args['output_stack'],
outtilespecs,
poolsize=self.args['pool_size'],
render=self.render,
close_stack=self.args['close_stack'])
def run(self):
zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['poststitched_stack'])
zrange = range(self.args['minZ'], self.args['maxZ'] + 1)
zvalues = list(set(zvalues1).intersection(set(zrange)))
if len(zvalues) == 0:
raise RenderModuleException(
('No valid zvalues found in stack '
'for given range {} - {}').format(
self.args['minZ'], self.args['maxZ']))
# check if pre or post stitched stack is in LOADING state.
# if so, clone them to new stacks
status1, new_prestitched = check_status_of_stack(
self.render,
self.args['prestitched_stack'],
zvalues)
status2, new_poststitched = check_status_of_stack(
self.render,
self.args['poststitched_stack'],
zvalues)
(disconnected_tiles, gap_tiles, seam_centroids,
distorted_zs, post_tspecs, matches, stats) = detect_stitching_mistakes(
self.render,
new_prestitched,
new_poststitched,
self.args['match_collection'],
self.args['match_collection_owner'],
self.args['threshold_cutoff'],
self.args['residual_threshold'],
self.args['neighbors_distance'],
self.args['min_cluster_size'],
zvalues,
pool_size=self.args['pool_size'])
# find the indices of sections having holes
hole_indices = [i for i, dt in enumerate(disconnected_tiles)
if len(dt) > 0]
gaps_indices = [i for i, gt in enumerate(gap_tiles) if len(gt) > 0]
seams_indices = [i for i, sm in enumerate(seam_centroids)
if len(sm) > 0]
holes = [zvalues[i] for i in hole_indices]
gaps = [zvalues[i] for i in gaps_indices]
seams = [zvalues[i] for i in seams_indices]
distorted_zs = [z[0] for z in distorted_zs if len(z) > 0]
combinedz = list(set(holes + gaps + seams + distorted_zs))
qc_passed_sections = set(zvalues) - set(combinedz)
centroids = [seam_centroids[i] for i in seams_indices]
print(distorted_zs)
self.args['output_html'] = []
if self.args['plot_sections']:
self.args['output_html'] = plot_section_maps(
self.render,
self.args['poststitched_stack'],
post_tspecs,
matches,
disconnected_tiles,
gap_tiles,
seam_centroids,
stats,
zvalues,
out_html_dir=self.args['out_html_dir'])
self.output({'output_html': self.args['output_html'],
'qc_passed_sections': qc_passed_sections,
'hole_sections': holes,
'gap_sections': gaps,
'seam_sections': seams,
'distorted_sections': distorted_zs,
'seam_centroids': np.array(centroids, dtype=object)})
print(self.output)
# delete the stacks that were cloned
if status1 == 'LOADING':
self.render.run(renderapi.stack.delete_stack, new_prestitched)
if status2 == 'LOADING':
self.render.run(renderapi.stack.delete_stack, new_poststitched)