def run(self):
self.logger.debug('Mipmap generation module')
"""
# get the list of z indices
zvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['input_stack'])
zvalues = list(set(self.zValues).intersection(set(zvalues)))
"""
zvalues = self.get_overlapping_inputstack_zvalues()
if not zvalues:
raise RenderModuleException(
"No sections found for stack {} for specified zs".format(
self.args['input_stack']))
self.logger.debug("Creating mipmaps...")
mipmap_args = make_tilespecs_and_cmds(
self.render, self.args['input_stack'], self.args['output_prefix'],
zvalues, self.args['levels'], self.args['imgformat'],
self.args['convert_to_8bit'], self.args['force_redo'],
self.args['pool_size'], self.args['method'])
self.output({
"levels": self.args["levels"],
"output_prefix": self.args["output_prefix"]
})
def run(self):
if self.args['transform_json']:
with open(self.args['transform_json'], 'r') as f:
j = json.load(f)
tfj = RegexDict({
k: renderapi.transform.AffineModel(json=v)
for (k, v) in j.items()
})
else:
tfj = RegexDict(
get_patch_transforms_from_xml(self.args['transform_xml']))
transform_zs = [int(k.split('_')[0]) for k in tfj]
z_overlap = self.get_overlapping_inputstack_zvalues(
zValues=transform_zs)
new_specs = []
for z in z_overlap:
tilespec = renderapi.tilespec.get_tile_specs_from_z(
self.args['input_stack'], z, render=self.render)
if len(tilespec) != 1:
raise RenderModuleException(
"expected 1 tilespec for z = %d in stack %s, "
"found %d" % (z, self.args['input_stack'], len(tilespec)))
tilespec[0].tforms = list(tfj.get_matching('%d_*' % z))
new_specs.append(tilespec[0])
self.output_tilespecs_to_stack(new_specs)
def run(self):
if "MCRROOT" not in os.environ:
raise ValidationError("MCRROOT not set")
if self.args['first_section'] >= self.args['last_section']:
raise RenderModuleException(
"First section z cannot be greater or equal to last section z")
zvalues = renderapi.stack.get_z_values_for_stack(
self.args['source_collection']['stack'], render=self.render)
self.args['first_section'] = min(
zvalues) if self.args['first_section'] > min(
zvalues) else self.args['first_section']
self.args['last_section'] = max(
zvalues) if self.args['last_section'] > max(
zvalues) else self.args['last_section']
# generate a temporary json to feed in to the solver
tempjson = tempfile.NamedTemporaryFile(suffix=".json",
mode="w",
delete=False)
tempjson.close()
with open(tempjson.name, 'w') as f:
json.dump(self.args, f, indent=4)
f.close()
#assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable
cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'],
tempjson.name)
ret = os.system(cmd)
# on successful completion remove the input json file
if ret == 0:
os.remove(tempjson.name)
else:
raise RenderModuleException("solve failed with input_json {}",
self.args)
d = {
'minz': self.args['first_section'],
'maxz': self.args['last_section']
}
self.output(d)
def run(self):
# get match collections
collecs = renderapi.pointmatch.get_matchcollections(
owner=self.args['match_owner'], render=self.render)
collections = [c['collectionId']['name'] for c in collecs]
if (self.args['source_collection']
not in collections) or (self.args['target_collection']
not in collections):
raise RenderModuleException(
"One of source or target collections does not exist")
# get all groupIds from source and target collections
source_ids = renderapi.pointmatch.get_match_groupIds(
self.args['source_collection'],
owner=self.args['match_owner'],
render=self.render)
target_ids = renderapi.pointmatch.get_match_groupIds(
self.args['target_collection'],
owner=self.args['match_owner'],
render=self.render)
# check existence of zvalues
ids = [
s for s in self.args['zValues']
if "{}.0".format(str(s)) in source_ids
and "{}.0".format(str(s)) in target_ids
]
# create a temp collection to copy the data
zvalues = []
mypartial = partial(swap_pt_matches,
self.render,
self.args['source_collection'],
self.args['target_collection'],
match_owner=self.args['match_owner'])
with renderapi.client.WithPool(self.args['pool_size']) as pool:
output_bool = pool.map(mypartial, ids)
zvalues = [z for z, n in zip(ids, output_bool) if n]
self.output({
"source_collection":
self.args['source_collection'],
"target_collection":
self.args['target_collection'],
"swapped_zs":
zvalues,
"nonswapped_zs":
set(self.args['zValues']).difference(zvalues)
})
def remove_mask_from_tilespec(self, z):
tspec = self.get_tilespec(z)
if not tspec:
return tspec
if tspec.ip[0].maskUrl is None:
raise RenderModuleException(
"Tilespec does not have a mask for z = %d" % (z))
tspec.ip[0].maskUrl = None
return tspec
def get_tilespec(self, z):
try:
tspecs = renderapi.tilespec.get_tile_specs_from_z(
self.args['stack'], z, render=self.render)
except RenderError:
return None
if len(tspecs) != 1:
raise RenderModuleException(
"Expected 1 tilespec for z=%d, found %d" % (z, len(tspecs)))
return tspecs[0]
def run(self):
# with open(self.args['metafile'], 'r') as f:
# meta = json.load(f)
meta = json.loads(uri_utils.uri_readbytes(self.args['metafile_uri']))
roidata = meta[0]['metadata']
imgdata = meta[1]['data']
img_coords = {
img['img_path']: self.image_coords_from_stage(
img['img_meta']['stage_pos'],
img['img_meta']['pixel_size_x_move'],
img['img_meta']['pixel_size_y_move'],
numpy.radians(img['img_meta']['angle']))
for img in imgdata
}
if not imgdata:
raise RenderModuleException(
"No relevant image metadata found for metadata at {}".format(
self.args['metafile_uri']))
minX, minY = numpy.min(numpy.array(list(img_coords.values())), axis=0)
# assume isotropic pixels
pixelsize = roidata['calibration']['highmag']['x_nm_per_pix']
imgdir = self.args.get('image_prefix',
uri_utils.uri_prefix(self.args['metafile_uri']))
tspecs = [
self.ts_from_imgdata(img,
imgdir,
img_coords[img['img_path']][0] - minX,
img_coords[img['img_path']][1] - minY,
minint=self.args['minimum_intensity'],
maxint=self.args['maximum_intensity'],
width=roidata['camera_info']['width'],
height=roidata['camera_info']['height'],
z=self.zValues[0],
sectionId=self.args.get('sectionId'),
scopeId=roidata['temca_id'],
cameraId=roidata['camera_info']['camera_id'],
pixelsize=pixelsize,
maskUrl=self.args['maskUrl_uri'])
for img in imgdata
]
self.output_tilespecs_to_stack(tspecs)
try:
self.output({'stack': self.output_stack})
except AttributeError as e:
self.logger.error(e)
def add_mask_to_tilespec(self, z, exts):
tspec = self.get_tilespec(z)
if not tspec:
return tspec
if tspec.ip[0].maskUrl is not None:
raise RenderModuleException(
"Tilespec already has a mask for z = %d" % (z))
fnames = []
for ext in exts:
fnames += glob.glob(
os.path.join(self.args['mask_dir'],
'%d_*' % z + os.extsep + ext))
if len(fnames) != 1:
raise RenderModuleException(
"Expected 1 mask file for z = %d, found %d in %s" %
(z, len(fnames), self.args['mask_dir']))
tspec.ip['0'].maskUrl = pathlib.Path(fnames[0]).as_uri()
return tspec
def dereference_tforms(tforms, ref_tforms):
deref_tforms = []
for tf in tforms:
if isinstance(tf, ReferenceTransform):
try:
mtf = next(mt for mt in ref_tforms
if mt.transformId == tf.refId)
deref_tforms.append(mtf)
except StopIteration as e:
raise RenderModuleException((
"reference transform: {} not found in provided refererence transforms {}"
.format(tf.refId, ref_tforms)))
else:
deref_tforms.append(tf)
return deref_tforms
def tspecjob(collection, render, tilespecs, estimate=True):
result = {}
result['tilespecs'] = tilespecs
matches = renderapi.pointmatch.get_matches_from_tile_to_tile(
collection,
tilespecs[0].layout.sectionId,
tilespecs[0].tileId,
tilespecs[1].layout.sectionId,
tilespecs[1].tileId,
render=render)
if len(matches) != 1:
estr = "\n expected 1 matching tile pair, found %d for" % (
len(matches))
estr += '\n %s\n group: %s\n %s\n group: %s' % (
tilespecs[0].tileId, tilespecs[0].layout.sectionId,
tilespecs[1].tileId, tilespecs[1].layout.sectionId)
raise RenderModuleException(estr)
match = matches[0]
if estimate:
tf = renderapi.transform.RigidModel()
src = np.array(match['matches']['p']).transpose()
dst = np.array(match['matches']['q']).transpose()
if match['pGroupId'] == tilespecs[0].layout.sectionId:
src = np.array(match['matches']['q']).transpose()
dst = np.array(match['matches']['p']).transpose()
ind = np.argwhere(np.isclose(np.array(match['matches']['w']),
1.0)).flatten()
tf.estimate(src[ind, :], dst[ind, :])
result['transform'] = tf
result['avg_residual'] = {
'z0': tilespecs[0].z,
'z1': tilespecs[1].z,
'avg_residual': avg_residual(match, tilespecs[0], tilespecs[1])
}
return result
def run(self):
if len(self.zValues) != len(self.args['new_zValues']):
raise RenderModuleException(
"zValues with length {} cannot be mapped to "
"zValues with length {}".format(len(self.zValues),
len(self.args['new_zValues'])))
for z, newz in zip(self.zValues, self.args['new_zValues']):
resolvedtiles = renderapi.resolvedtiles.get_resolved_tiles_from_z(
self.input_stack, z, render=self.render)
for ts in resolvedtiles.tilespecs:
ts.z = newz
if self.args.get('remap_sectionId'):
ts.layout.sectionId = self.sectionId_from_z(newz)
self.output_tilespecs_to_stack(
resolvedtiles.tilespecs,
sharedTransforms=resolvedtiles.transforms,
zValues=[z])
self.output({
"zValues": self.zValues,
"output_stack": self.output_stack
})
def run(self, run_lc_bsh=None):
run_lc_bsh = self.default_bsh if run_lc_bsh is None else run_lc_bsh
outfn = self.args.get(
'outfile',
os.path.abspath(
os.path.join(self.args['project_path'],
'lens_correction.json')))
delete_procdir = False
procdir = self.args.get('processing_directory',
self.args['project_path'])
if procdir is None:
delete_procdir = True
procdir = tempfile.mkdtemp()
# command line argument for beanshell script
bsh_call = [
"xvfb-run", "-a", self.args['fiji_path'], "-Xms{}g".format(
self.args['heap_size']), "-Xmx{}g".format(
self.args['heap_size']), "-Xincgc", "-Dman=" +
self.args['manifest_path'], "-Ddir=" + self.args['project_path'],
"-Dgrid=" + str(self.args['grid_size']),
"-Disig=" + str(self.args['SIFT_params']['initialSigma']),
"-Dsteps=" + str(self.args['SIFT_params']['steps']),
"-Dminos=" + str(self.args['SIFT_params']['minOctaveSize']),
"-Dmaxos=" + str(self.args['SIFT_params']['maxOctaveSize']),
"-Dfdsize=" + str(self.args['SIFT_params']['fdSize']),
"-Dfdbins=" + str(self.args['SIFT_params']['fdBins']),
"-Drod=" + str(self.args['align_params']['rod']),
"-Dmaxe=" + str(self.args['align_params']['maxEpsilon']),
"-Dmir=" + str(self.args['align_params']['minInlierRatio']),
"-Dmni=" + str(self.args['align_params']['minNumInliers']),
"-Demi=" + str(self.args['align_params']['expectedModelIndex']),
"-Dmh=" + str(self.args['align_params']['multipleHypotheses']),
"-Dri=" + str(self.args['align_params']['rejectIdentity']),
"-Dit=" + str(self.args['align_params']['identityTolerance']),
"-Dtaip=" + str(self.args['align_params']['tilesAreInPlace']),
"-Ddmi=" + str(self.args['align_params']['desiredModelIndex']),
"-Dreg=" + str(self.args['align_params']['regularize']),
"-Dmio={}".format(
self.args['align_params']['maxIterationsOptimize']),
"-Dmpwo={}".format(
self.args['align_params']['maxPlateauWidthOptimize']),
"-Ddim=" + str(self.args['align_params']['dimension']),
"-Dlam=" + str(self.args['align_params']['lambdaVal']),
"-Dprocdir={}".format(procdir), "-Doutfn={}".format(outfn),
"-Dctrans=" + str(self.args['align_params']['clearTransform']),
"-Dvis=" + str(self.args['align_params']['visualize']),
"-DthreadsSIFT={}".format(
self.args['max_threads_SIFT']), "--", "--no-splash", run_lc_bsh
]
self.logger.debug('bsh_cmd: {}'.format(bsh_call))
try:
subprocess.check_call(bsh_call)
except subprocess.CalledProcessError as e:
raise (RenderModuleException("{}".format(e)))
# self.logger.debug('ret_code: {}'.format(ret))
if delete_procdir:
shutil.rmtree(procdir)
try:
self.output({'output_json': outfn})
except AttributeError as e:
# output validation will need to wait for argschema PR
self.logger.error(e)
def run(self):
if "MCRROOT" not in os.environ:
raise ValidationError("MCRROOT not set")
if self.clone_section_stack:
tmp_stack = "{}_zs{}_ze{}_t{}".format(
self.args['source_collection']['stack'],
self.args['first_section'], self.args['last_section'],
time.strftime("%m%d%y_%H%M%S"))
zvalues = renderapi.stack.get_z_values_for_stack(
self.args['source_collection']['stack'], render=self.render)
zs = [z for z in zvalues if \
(z>=self.args['first_section']) \
and (z<=self.args['last_section'])]
renderapi.stack.clone_stack(
self.args['source_collection']['stack'],
tmp_stack,
zs=zs,
close_stack=True,
render=self.render)
self.args['source_collection']['stack'] = tmp_stack
# the clone stack option should also set the first and last section z appropriately in self.args for solver
self.args['first_section'] = min(zs)
self.args['last_section'] = max(zs)
# Check if the target stack exists and whether we need to overwrite the z
target_host = self.args['target_collection']['service_host']
num = target_host[-4:]
port = None
if num.isdigit():
target_host = target_host[:-5]
if target_host.find('http://') < 0:
target_host = 'http://%s' % (target_host)
port = int(num)
list_of_stacks = renderapi.render.get_stacks_by_owner_project(
owner=self.args['target_collection']['owner'],
project=self.args['target_collection']['project'],
host=target_host,
port=port,
render=self.render)
if self.args['target_collection']['stack'] in list_of_stacks:
for z in xrange(int(self.args['first_section']),
int(self.args['last_section']) + 1):
renderapi.stack.delete_section(
self.args['target_collection']['stack'],
z,
host=target_host,
port=port,
owner=self.args['target_collection']['owner'],
project=self.args['target_collection']['project'],
render=self.render)
# generate a temporary json to feed in to the solver
tempjson = tempfile.NamedTemporaryFile(suffix=".json",
mode='w',
delete=False)
tempjson.close()
with open(tempjson.name, 'w') as f:
json.dump(self.args, f, indent=4)
f.close()
#assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable
cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'],
tempjson.name)
ret = os.system(cmd)
# one successful completion remove the input json file
if ret == 0:
os.remove(tempjson.name)
else:
raise RenderModuleException("solve failed with input_json {}",
self.args)
#try to return the z's solved in the output json
try:
d = {'zs': zs}
self.output(d)
except:
raise RenderModuleException("unable to output json {}", d)
#if you made a tmp stack destroy it
if self.clone_section_stack:
renderapi.stack.delete_stack(tmp_stack, render=self.render)
def run(self):
if len(self.args['source_stack']) != len(self.args['target_stack']):
raise RenderModuleException(
"Count of source and target stacks do not match")
list_of_stacks = renderapi.render.get_stacks_by_owner_project(
owner=self.render.DEFAULT_OWNER,
project=self.render.DEFAULT_PROJECT,
host=self.render.DEFAULT_HOST,
port=self.render.DEFAULT_PORT,
render=self.render)
source_stacks = []
target_stacks = []
zvalues = []
for source_stack, target_stack, zVals in zip(self.args['source_stack'],
self.args['target_stack'],
self.args['zValues']):
if source_stack not in list_of_stacks or target_stack not in list_of_stacks: # pragma: no cover
continue
#if source_stack not in list_of_stacks:
# raise RenderModuleException("Source stack {} not in render".format(source_stack))
#if target_stack not in list_of_stacks:
# raise RenderModuleException("Target stack {} not in render".format(target_stack))
szvalues = renderapi.stack.get_z_values_for_stack(
source_stack, render=self.render)
tzvalues = renderapi.stack.get_z_values_for_stack(
target_stack, render=self.render)
avail_zs = set(szvalues).intersection(tzvalues)
final_zs = list(set(zVals).intersection(avail_zs))
#source_stacks.append(source_stack)
#target_stacks.append(target_stack)
#zvalues.append(final_zs)
print(final_zs)
#mypartial = partial(swap_section, self.render, source_stack, target_stack)
#with renderapi.client.WithPool(self.args['pool_size']) as pool:
# output_bool = pool.map(mypartial, final_zs)
output_bool = []
for z in final_zs:
outbool = swap_section(self.render, source_stack, target_stack,
z)
output_bool.append(outbool)
for z, n in zip(final_zs, output_bool):
if n:
zvalues.append(z)
source_stacks.append(source_stack)
target_stacks.append(target_stack)
#zs = [z for z, n in zip(final_zs, output_bool) if n]
#zvalues.append(zs)
if self.args['complete_source_stack']:
renderapi.stack.set_stack_state(source_stack,
state='COMPLETE',
render=self.render)
if self.args['complete_target_stack']:
renderapi.stack.set_stack_state(target_stack,
state='COMPLETE',
render=self.render)
# delete source stack if specified in input
if self.args['delete_source_stack']: # pragma: no cover
renderapi.stack.delete_stack(source_stack, render=self.render)
#self.swap_zs_from_stacks(source_stack, target_stack, final_zs)
self.output({
"source_stacks": source_stacks,
"target_stacks": target_stacks,
"swapped_zvalues": zvalues
})
def run(self):
zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['poststitched_stack'])
zrange = range(self.args['minZ'], self.args['maxZ']+1)
zvalues = list(set(zvalues1).intersection(set(zrange)))
if len(zvalues) == 0:
raise RenderModuleException('No valid zvalues found in stack for given range {} - {}'.format(self.args['minZ'], self.args['maxZ']))
# check if pre or post stitched stack is in LOADING state.
# if so, clone them to new stacks
status1, new_prestitched = check_status_of_stack(self.render,
self.args['prestitched_stack'],
zvalues)
status2, new_poststitched = check_status_of_stack(self.render,
self.args['poststitched_stack'],
zvalues)
disconnected_tiles, gap_tiles, seam_centroids, post_tspecs, matches, stats = detect_stitching_mistakes(
self.render,
new_prestitched,
new_poststitched,
self.args['match_collection'],
self.args['match_collection_owner'],
self.args['residual_threshold'],
self.args['neighbors_distance'],
self.args['min_cluster_size'],
zvalues,
pool_size=self.args['pool_size'])
# find the indices of sections having holes
hole_indices = [i for i, dt in enumerate(disconnected_tiles) if len(dt) > 0]
gaps_indices = [i for i, gt in enumerate(gap_tiles) if len(gt) > 0]
seams_indices = [i for i, sm in enumerate(seam_centroids) if len(sm) > 0]
holes = [zvalues[i] for i in hole_indices]
gaps = [zvalues[i] for i in gaps_indices]
seams = [zvalues[i] for i in seams_indices]
combinedz = list(set(holes + gaps + seams))
qc_passed_sections = set(zvalues) - set(combinedz)
centroids = [seam_centroids[i] for i in seams_indices]
self.args['output_html'] = []
if self.args['plot_sections']:
self.args['output_html'] = plot_section_maps(self.render,
self.args['poststitched_stack'],
post_tspecs,
matches,
disconnected_tiles,
gap_tiles,
seam_centroids,
stats,
zvalues,
out_html_dir=self.args['out_html_dir'])
self.output({'output_html':self.args['output_html'],
'qc_passed_sections': qc_passed_sections,
'hole_sections': holes,
'gap_sections':gaps,
'seam_sections':seams,
'seam_centroids':np.array(centroids)})
# delete the stacks that were cloned
if status1 == 'LOADING':
self.render.run(renderapi.stack.delete_stack, new_prestitched)
if status2 == 'LOADING':
self.render.run(renderapi.stack.delete_stack, new_poststitched)