def tspecjob(collection, render, tilespecs, estimate=True): result = {} result['tilespecs'] = tilespecs matches = renderapi.pointmatch.get_matches_from_tile_to_tile( collection, tilespecs[0].layout.sectionId, tilespecs[0].tileId, tilespecs[1].layout.sectionId, tilespecs[1].tileId, render=render) if len(matches) != 1: estr = "\n expected 1 matching tile pair, found %d for" % ( len(matches)) estr += '\n %s\n group: %s\n %s\n group: %s' % ( tilespecs[0].tileId, tilespecs[0].layout.sectionId, tilespecs[1].tileId, tilespecs[1].layout.sectionId) raise RenderModuleException(estr) match = matches[0] if estimate: tf = renderapi.transform.RigidModel() src = np.array(match['matches']['p']).transpose() dst = np.array(match['matches']['q']).transpose() if match['pGroupId'] == tilespecs[0].layout.sectionId: src = np.array(match['matches']['q']).transpose() dst = np.array(match['matches']['p']).transpose() ind = np.argwhere( np.isclose( np.array(match['matches']['w']), 1.0)).flatten() tf.estimate(src[ind, :], dst[ind, :]) result['transform'] = tf result['avg_residual'] = { 'z0': tilespecs[0].z, 'z1': tilespecs[1].z, 'avg_residual': avg_residual(match, tilespecs[0], tilespecs[1]) } return result
def run(self): if len(self.zValues) != len(self.args['new_zValues']): raise RenderModuleException( "zValues with length {} cannot be mapped to " "zValues with length {}".format(len(self.zValues), len(self.args['new_zValues']))) for z, newz in zip(self.zValues, self.args['new_zValues']): resolvedtiles = renderapi.resolvedtiles.get_resolved_tiles_from_z( self.input_stack, z, render=self.render) for ts in resolvedtiles.tilespecs: ts.z = newz if self.args.get('remap_sectionId'): ts.layout.sectionId = self.sectionId_from_z(newz) self.output_tilespecs_to_stack( resolvedtiles.tilespecs, sharedTransforms=resolvedtiles.transforms, zValues=[z]) self.output({ "zValues": self.zValues, "output_stack": self.output_stack })
def run(self): self.logger.debug('Montage scape stack generation module') # get the list of z indices zvalues = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['montage_stack']) zvalues1 = self.zValues zvalues = list(set(zvalues1).intersection(set(zvalues))) if not zvalues: raise RenderModuleException( 'No sections found for stack {}'.format( self.args['montage_stack'])) # generate tuple of old and new Zs # setting a new z range does not check whether the range # overlaps with existing sections/chunks in the output stack if self.args['set_new_z']: zvalues = list(np.sort(np.array(zvalues))) diffarray = [x - zvalues[0] for x in zvalues] newzvalues = [self.args['new_z_start'] + x for x in diffarray] else: newzvalues = zvalues Z = [[int(oldz), int(newz)] for oldz, newz in zip(zvalues, newzvalues)] out_stack_exists = self.args['output_stack'] in self.render.run( renderapi.render.get_stacks_by_owner_project) if out_stack_exists: # check whether overwrite z is set to false. If so, then remove those z that is already in output stack outzvalues = renderapi.stack.get_z_values_for_stack( self.args['output_stack'], render=self.render) if self.overwrite_zlayer: # stack has to be in loading state renderapi.stack.set_stack_state(self.args['output_stack'], 'LOADING', render=self.render) for oldz, newz in zip(zvalues, newzvalues): # delete the section from output stack renderapi.stack.delete_section(self.args['output_stack'], newz, render=self.render) # generate the tag string to add to output tilespec json file name tagstr = "%s_%s" % (min(zvalues), max(zvalues)) tilespecdir = os.path.join(self.args['image_directory'], self.args['render']['project'], self.args['montage_stack'], 'sections_at_%s' % str(self.args['scale']), 'tilespecs_%s' % tagstr) try: os.makedirs(tilespecdir) except OSError as e: if e.errno != errno.EEXIST: raise pass render_materialize = renderapi.connect(**self.render.make_kwargs( memGB=self.args['memGB_materialize'])) # process for each z mypartial = partial(create_montage_scape_tile_specs, render_materialize, self.args['montage_stack'], self.args['image_directory'], self.args['scale'], self.args['render']['project'], tagstr, self.args['imgformat'], apply_scale=self.args['apply_scale'], level=self.args['level'], pool_size=1, doFilter=self.args['doFilter'], fillWithNoise=self.args['fillWithNoise'], uuid_prefix=self.args["uuid_prefix"], uuid_prefix_length=self.args["uuid_length"], do_mp=False) with renderapi.client.WithPool( self.args['pool_size_materialize']) as pool: pool.map(mypartial, Z) # get all the output tilespec json files tspath = os.path.join(self.args['image_directory'], self.args['render']['project'], self.args['montage_stack'], 'sections_at_%s' % str(self.args['scale']), 'tilespecs_%s' % tagstr) jsonfiles = glob.glob("%s/*.json" % tspath) if not jsonfiles: raise RenderModuleException( 'No tilespecs json files were generated') # create the stack if it doesn't exist if self.output_stack not in self.render.run( renderapi.render.get_stacks_by_owner_project): # stack does not exist # TODO configurable stack metadata self.render.run(renderapi.stack.create_stack, self.output_stack, cycleNumber=5, cycleStepNumber=1, stackResolutionX=1, stackResolutionY=1) # import tilespecs to render self.render.run(renderapi.client.import_jsonfiles_parallel, self.output_stack, jsonfiles) if self.close_stack: # set stack state to complete self.render.run(renderapi.stack.set_stack_state, self.output_stack, state='COMPLETE') self.output({'output_stack': self.output_stack})
def run(self): if "MCRROOT" not in os.environ: raise ValidationError("MCRROOT not set") if self.clone_section_stack: tmp_stack = "{}_zs{}_ze{}_t{}".format( self.args['source_collection']['stack'], self.args['first_section'], self.args['last_section'], time.strftime("%m%d%y_%H%M%S")) zvalues = renderapi.stack.get_z_values_for_stack( self.args['source_collection']['stack'], render=self.render) zs = [z for z in zvalues if \ (z>=self.args['first_section']) \ and (z<=self.args['last_section'])] renderapi.stack.clone_stack( self.args['source_collection']['stack'], tmp_stack, zs=zs, close_stack=True, render=self.render) self.args['source_collection']['stack'] = tmp_stack # the clone stack option should also set the first and last section z appropriately in self.args for solver self.args['first_section'] = min(zs) self.args['last_section'] = max(zs) # Check if the target stack exists and whether we need to overwrite the z target_host = self.args['target_collection']['service_host'] num = target_host[-4:] port = None if num.isdigit(): target_host = target_host[:-5] if target_host.find('http://') < 0: target_host = 'http://%s' % (target_host) port = int(num) list_of_stacks = renderapi.render.get_stacks_by_owner_project( owner=self.args['target_collection']['owner'], project=self.args['target_collection']['project'], host=target_host, port=port, render=self.render) if self.args['target_collection']['stack'] in list_of_stacks: for z in xrange(int(self.args['first_section']), int(self.args['last_section']) + 1): renderapi.stack.delete_section( self.args['target_collection']['stack'], z, host=target_host, port=port, owner=self.args['target_collection']['owner'], project=self.args['target_collection']['project'], render=self.render) # generate a temporary json to feed in to the solver tempjson = tempfile.NamedTemporaryFile(suffix=".json", mode='w', delete=False) tempjson.close() with open(tempjson.name, 'w') as f: json.dump(self.args, f, indent=4) f.close() #assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'], tempjson.name) ret = os.system(cmd) # one successful completion remove the input json file if ret == 0: os.remove(tempjson.name) else: raise RenderModuleException("solve failed with input_json {}", self.args) #try to return the z's solved in the output json try: d = {'zs': zs} self.output(d) except: raise RenderModuleException("unable to output json {}", d) #if you made a tmp stack destroy it if self.clone_section_stack: renderapi.stack.delete_stack(tmp_stack, render=self.render)
def run(self): allzvalues = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['montage_stack']) allzvalues = np.array(allzvalues) Z = [[a, b] for a, b in zip(self.args['old_z'], self.args['new_z']) if a in allzvalues] mypartial = partial( apply_rough_alignment, self.render, self.args['montage_stack'], self.args['prealigned_stack'], self.args['lowres_stack'], self.args['output_stack'], self.args['tilespec_directory'], self.args['scale'], self.args['mask_input_dir'], self.args['update_lowres_with_masks'], self.args['read_masks_from_lowres_stack'], self.args['filter_montage_output_with_masks'], self.args['mask_exts'], apply_scale=self.args['apply_scale'], consolidateTransforms=self.args['consolidate_transforms'], remap_section_ids=self.args['remap_section_ids']) # Create the output stack if it doesn't exist if self.args['output_stack'] not in self.render.run( renderapi.render.get_stacks_by_owner_project): # stack does not exist self.render.run(renderapi.stack.create_stack, self.args['output_stack']) else: # stack exists and has to be set to LOADING state self.render.run(renderapi.stack.set_stack_state, self.args['output_stack'], 'LOADING') lowres_state = self.render.run(renderapi.stack.get_full_stack_metadata, self.args['lowres_stack'])['state'] with renderapi.client.WithPool(self.args['pool_size']) as pool: results = pool.map(mypartial, Z) # raise an exception if all the z values to apply alignment were not if not all([r is None for r in results]): failed_zs = [(result, z) for result, z in zip(results, Z) if result is not None] raise RenderModuleException( "Failed to rough align z values {}".format(failed_zs)) if self.args["close_stack"]: # set stack state to complete self.render.run(renderapi.stack.set_stack_state, self.args['output_stack'], state='COMPLETE') if self.args['update_lowres_with_masks']: # return the lowres stack to original state # if any imports were done self.render.run(renderapi.stack.set_stack_state, self.args['lowres_stack'], state=lowres_state) self.output({ 'zs': np.array(Z), 'output_stack': self.args['output_stack'] })
def run(self, run_lc_bsh=None): run_lc_bsh = self.default_bsh if run_lc_bsh is None else run_lc_bsh outfn = self.args.get('outfile', os.path.abspath(os.path.join( self.args['project_path'], 'lens_correction.json'))) delete_procdir = False procdir = self.args.get('processing_directory', self.args['project_path']) if procdir is None: delete_procdir = True procdir = tempfile.mkdtemp() # command line argument for beanshell script bsh_call = [ "xvfb-run", "-a", self.args['fiji_path'], "-Xms{}g".format(self.args['heap_size']), "-Xmx{}g".format(self.args['heap_size']), "-Xincgc", "-Dman=" + self.args['manifest_path'], "-Ddir=" + self.args['project_path'], "-Dgrid=" + str(self.args['grid_size']), "-Disig=" + str(self.args['SIFT_params']['initialSigma']), "-Dsteps=" + str(self.args['SIFT_params']['steps']), "-Dminos=" + str(self.args['SIFT_params']['minOctaveSize']), "-Dmaxos=" + str(self.args['SIFT_params']['maxOctaveSize']), "-Dfdsize=" + str(self.args['SIFT_params']['fdSize']), "-Dfdbins=" + str(self.args['SIFT_params']['fdBins']), "-Drod=" + str(self.args['align_params']['rod']), "-Dmaxe=" + str(self.args['align_params']['maxEpsilon']), "-Dmir=" + str(self.args['align_params']['minInlierRatio']), "-Dmni=" + str(self.args['align_params']['minNumInliers']), "-Demi=" + str(self.args['align_params']['expectedModelIndex']), "-Dmh=" + str(self.args['align_params']['multipleHypotheses']), "-Dri=" + str(self.args['align_params']['rejectIdentity']), "-Dit=" + str(self.args['align_params']['identityTolerance']), "-Dtaip=" + str(self.args['align_params']['tilesAreInPlace']), "-Ddmi=" + str(self.args['align_params']['desiredModelIndex']), "-Dreg=" + str(self.args['align_params']['regularize']), "-Dmio={}".format( self.args['align_params']['maxIterationsOptimize']), "-Dmpwo={}".format( self.args['align_params']['maxPlateauWidthOptimize']), "-Ddim=" + str(self.args['align_params']['dimension']), "-Dlam=" + str(self.args['align_params']['lambdaVal']), "-Dprocdir={}".format(procdir), "-Doutfn={}".format(outfn), "-Dctrans=" + str(self.args['align_params']['clearTransform']), "-Dvis=" + str(self.args['align_params']['visualize']), "-DthreadsSIFT={}".format(self.args['max_threads_SIFT']), "--", "--no-splash", run_lc_bsh] self.logger.debug('bsh_cmd: {}'.format(bsh_call)) try: subprocess.check_call(bsh_call) except subprocess.CalledProcessError as e: raise(RenderModuleException("{}".format(e))) # self.logger.debug('ret_code: {}'.format(ret)) if delete_procdir: shutil.rmtree(procdir) try: self.output({'output_json': outfn}) except AttributeError as e: # output validation will need to wait for argschema PR self.logger.error(e)
def run(self): zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['output_downsampled_stack']) zrange = range(self.args['minZ'], self.args['maxZ'] + 1) zvalues = list(set(zvalues1).intersection(set(zrange))) zvalues.sort() if len(zvalues) == 0: raise RenderModuleException( ('No valid zvalues found in stack ' 'for given range {} - {}').format(self.args['minZ'], self.args['maxZ'])) # get the boundary polygon for each section mypartial1 = partial(get_poly, self.args['output_downsampled_stack'], self.render) with renderapi.client.WithPool(self.args['pool_size']) as pool: boundary_polygons = pool.map(mypartial1, zvalues) post_polys = {} for poly, z in zip(boundary_polygons, zvalues): post_polys[z] = poly mypartial2 = partial(get_poly, self.args['input_downsampled_stack'], self.render) with renderapi.client.WithPool(self.args['pool_size']) as pool: pre_boundary_polygons = pool.map(mypartial2, zvalues) pre_polys = {} for poly, z in zip(pre_boundary_polygons, zvalues): pre_polys[z] = poly if self.args['output_dir'] is None: self.args['output_dir'] = tempfile.mkdtemp() # compute ious ious = compute_ious(post_polys, zvalues) # compute distortion distortion = [] dio = [] doi = [] for i, z in enumerate(zvalues): di, do, dist = compute_distortion(pre_boundary_polygons[i], boundary_polygons[i], z) dio.append(di) doi.append(do) distortion.append(dist) dist_plt_name = None iou_plt_name = None if self.args['out_file_format'] == 'pdf': # pdf plots dist_plt_name, iou_plt_name = generate_pdf_plots( ious, zrange, self.args['output_dir'], pre_boundary_polygons, boundary_polygons, dio, doi, distortion, zvalues) else: plot_name = generate_bokeh_plots(ious, zrange, self.args['output_dir'], pre_boundary_polygons, boundary_polygons, dio, doi, distortion, zvalues) dist_plt_name = plot_name iou_plt_name = plot_name self.output({ "iou_plot": iou_plt_name, "distortion_plot": dist_plt_name })
def run(self): # inits alltilespecs = [] numtiles = 0 firstts = [] outtilespecs = [] ind = 0 # get tilespecs for z for z in range(self.args['minZ'], self.args['maxZ'] + 1): tilespecs = self.render.run( renderapi.tilespec.get_tile_specs_from_z, self.args['input_stack'], z) alltilespecs.extend(tilespecs) # used for easy creation of tilespecs for output stack firstts.append(tilespecs[0]) numtiles += len(tilespecs) # subsample in the case where the number of tiles is too large if self.args['num_images'] > 0: alltilespecs = randomly_subsample_tilespecs( alltilespecs, self.args['num_images']) stackInfo = renderapi.stack.get_full_stack_metadata( self.args['input_stack'], render=self.render) channel_names = stackInfo['stats'].get('channelNames', []) # Quick fix to have the empty channel logic work when channelNames=[] if len(channel_names) == 0: channel_names = [None] # figure out which of our channels is the default channel if channel_names[0] is not None: try: default_chan = next( (ch.name for ch in firstts[0].channels if ch.ip[0].imageUrl == firstts[0].ip[0].imageUrl)) except StopIteration: raise RenderModuleException( "default channel doesn't match any channel") outdir = self.args['output_directory'] if not os.path.exists(outdir): os.makedirs(outdir) out_images = [] for chan_name in channel_names: chan_str = chan_name if chan_name is not None else "" outImage = outdir + \ "/%s_%s_%s_%d-%d.tif" % (self.args['file_prefix'], self.args['output_stack'], chan_str, self.args['minZ'], self.args['maxZ']) make_median_image(alltilespecs, numtiles, outImage, self.args['pool_size'], chan=chan_name) out_images.append(outImage) for ind, z in enumerate(range(self.args['minZ'], self.args['maxZ'] + 1)): # want many variations on this tilespec so making a # copy by serializing/deserializing ts = renderapi.tilespec.TileSpec(json=firstts[ind].to_dict()) # want a different z for each one ts.z = z if channel_names[0] is None: mml_o = ts.ip[0] mml = renderapi.image_pyramid.MipMap(imageUrl=out_images[0], maskUrl=mml_o.maskUrl) ts.ip[0] = mml else: channels = [] for chan_name, out_image in zip(channel_names, out_images): chan_orig = next( (ch for ch in ts.channels if ch.name == chan_name)) mml = renderapi.image_pyramid.MipMap(imageUrl=out_image) ip = renderapi.image_pyramid.ImagePyramid() ip[0] = mml chan_orig.ip = ip channels.append(chan_orig) # we want the default_chan median to be the # default image pyramid ts.ip = next((med_ch.ip for med_ch in channels if med_ch.name == default_chan)) ts.channels = channels outtilespecs.append(ts) # upload to render renderapi.stack.create_stack(self.args['output_stack'], cycleNumber=2, cycleStepNumber=1, render=self.render) renderapi.stack.set_stack_state(self.args['output_stack'], "LOADING", render=self.render) renderapi.client.import_tilespecs_parallel( self.args['output_stack'], outtilespecs, poolsize=self.args['pool_size'], render=self.render, close_stack=self.args['close_stack'])
def run(self): zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['poststitched_stack']) zrange = range(self.args['minZ'], self.args['maxZ'] + 1) zvalues = list(set(zvalues1).intersection(set(zrange))) if len(zvalues) == 0: raise RenderModuleException( ('No valid zvalues found in stack ' 'for given range {} - {}').format( self.args['minZ'], self.args['maxZ'])) # check if pre or post stitched stack is in LOADING state. # if so, clone them to new stacks status1, new_prestitched = check_status_of_stack( self.render, self.args['prestitched_stack'], zvalues) status2, new_poststitched = check_status_of_stack( self.render, self.args['poststitched_stack'], zvalues) (disconnected_tiles, gap_tiles, seam_centroids, distorted_zs, post_tspecs, matches, stats) = detect_stitching_mistakes( self.render, new_prestitched, new_poststitched, self.args['match_collection'], self.args['match_collection_owner'], self.args['threshold_cutoff'], self.args['residual_threshold'], self.args['neighbors_distance'], self.args['min_cluster_size'], zvalues, pool_size=self.args['pool_size']) # find the indices of sections having holes hole_indices = [i for i, dt in enumerate(disconnected_tiles) if len(dt) > 0] gaps_indices = [i for i, gt in enumerate(gap_tiles) if len(gt) > 0] seams_indices = [i for i, sm in enumerate(seam_centroids) if len(sm) > 0] holes = [zvalues[i] for i in hole_indices] gaps = [zvalues[i] for i in gaps_indices] seams = [zvalues[i] for i in seams_indices] distorted_zs = [z[0] for z in distorted_zs if len(z) > 0] combinedz = list(set(holes + gaps + seams + distorted_zs)) qc_passed_sections = set(zvalues) - set(combinedz) centroids = [seam_centroids[i] for i in seams_indices] print(distorted_zs) self.args['output_html'] = [] if self.args['plot_sections']: self.args['output_html'] = plot_section_maps( self.render, self.args['poststitched_stack'], post_tspecs, matches, disconnected_tiles, gap_tiles, seam_centroids, stats, zvalues, out_html_dir=self.args['out_html_dir']) self.output({'output_html': self.args['output_html'], 'qc_passed_sections': qc_passed_sections, 'hole_sections': holes, 'gap_sections': gaps, 'seam_sections': seams, 'distorted_sections': distorted_zs, 'seam_centroids': np.array(centroids, dtype=object)}) print(self.output) # delete the stacks that were cloned if status1 == 'LOADING': self.render.run(renderapi.stack.delete_stack, new_prestitched) if status2 == 'LOADING': self.render.run(renderapi.stack.delete_stack, new_poststitched)