def run(self): self.logger.debug('Mipmap generation module') """ # get the list of z indices zvalues = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['input_stack']) zvalues = list(set(self.zValues).intersection(set(zvalues))) """ zvalues = self.get_overlapping_inputstack_zvalues() if not zvalues: raise RenderModuleException( "No sections found for stack {} for specified zs".format( self.args['input_stack'])) self.logger.debug("Creating mipmaps...") mipmap_args = make_tilespecs_and_cmds( self.render, self.args['input_stack'], self.args['output_prefix'], zvalues, self.args['levels'], self.args['imgformat'], self.args['convert_to_8bit'], self.args['force_redo'], self.args['pool_size'], self.args['method']) self.output({ "levels": self.args["levels"], "output_prefix": self.args["output_prefix"] })
def run(self): if self.args['transform_json']: with open(self.args['transform_json'], 'r') as f: j = json.load(f) tfj = RegexDict({ k: renderapi.transform.AffineModel(json=v) for (k, v) in j.items() }) else: tfj = RegexDict( get_patch_transforms_from_xml(self.args['transform_xml'])) transform_zs = [int(k.split('_')[0]) for k in tfj] z_overlap = self.get_overlapping_inputstack_zvalues( zValues=transform_zs) new_specs = [] for z in z_overlap: tilespec = renderapi.tilespec.get_tile_specs_from_z( self.args['input_stack'], z, render=self.render) if len(tilespec) != 1: raise RenderModuleException( "expected 1 tilespec for z = %d in stack %s, " "found %d" % (z, self.args['input_stack'], len(tilespec))) tilespec[0].tforms = list(tfj.get_matching('%d_*' % z)) new_specs.append(tilespec[0]) self.output_tilespecs_to_stack(new_specs)
def run(self): if "MCRROOT" not in os.environ: raise ValidationError("MCRROOT not set") if self.args['first_section'] >= self.args['last_section']: raise RenderModuleException( "First section z cannot be greater or equal to last section z") zvalues = renderapi.stack.get_z_values_for_stack( self.args['source_collection']['stack'], render=self.render) self.args['first_section'] = min( zvalues) if self.args['first_section'] > min( zvalues) else self.args['first_section'] self.args['last_section'] = max( zvalues) if self.args['last_section'] > max( zvalues) else self.args['last_section'] # generate a temporary json to feed in to the solver tempjson = tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) tempjson.close() with open(tempjson.name, 'w') as f: json.dump(self.args, f, indent=4) f.close() #assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'], tempjson.name) ret = os.system(cmd) # on successful completion remove the input json file if ret == 0: os.remove(tempjson.name) else: raise RenderModuleException("solve failed with input_json {}", self.args) d = { 'minz': self.args['first_section'], 'maxz': self.args['last_section'] } self.output(d)
def run(self): # get match collections collecs = renderapi.pointmatch.get_matchcollections( owner=self.args['match_owner'], render=self.render) collections = [c['collectionId']['name'] for c in collecs] if (self.args['source_collection'] not in collections) or (self.args['target_collection'] not in collections): raise RenderModuleException( "One of source or target collections does not exist") # get all groupIds from source and target collections source_ids = renderapi.pointmatch.get_match_groupIds( self.args['source_collection'], owner=self.args['match_owner'], render=self.render) target_ids = renderapi.pointmatch.get_match_groupIds( self.args['target_collection'], owner=self.args['match_owner'], render=self.render) # check existence of zvalues ids = [ s for s in self.args['zValues'] if "{}.0".format(str(s)) in source_ids and "{}.0".format(str(s)) in target_ids ] # create a temp collection to copy the data zvalues = [] mypartial = partial(swap_pt_matches, self.render, self.args['source_collection'], self.args['target_collection'], match_owner=self.args['match_owner']) with renderapi.client.WithPool(self.args['pool_size']) as pool: output_bool = pool.map(mypartial, ids) zvalues = [z for z, n in zip(ids, output_bool) if n] self.output({ "source_collection": self.args['source_collection'], "target_collection": self.args['target_collection'], "swapped_zs": zvalues, "nonswapped_zs": set(self.args['zValues']).difference(zvalues) })
def remove_mask_from_tilespec(self, z): tspec = self.get_tilespec(z) if not tspec: return tspec if tspec.ip[0].maskUrl is None: raise RenderModuleException( "Tilespec does not have a mask for z = %d" % (z)) tspec.ip[0].maskUrl = None return tspec
def get_tilespec(self, z): try: tspecs = renderapi.tilespec.get_tile_specs_from_z( self.args['stack'], z, render=self.render) except RenderError: return None if len(tspecs) != 1: raise RenderModuleException( "Expected 1 tilespec for z=%d, found %d" % (z, len(tspecs))) return tspecs[0]
def run(self): # with open(self.args['metafile'], 'r') as f: # meta = json.load(f) meta = json.loads(uri_utils.uri_readbytes(self.args['metafile_uri'])) roidata = meta[0]['metadata'] imgdata = meta[1]['data'] img_coords = { img['img_path']: self.image_coords_from_stage( img['img_meta']['stage_pos'], img['img_meta']['pixel_size_x_move'], img['img_meta']['pixel_size_y_move'], numpy.radians(img['img_meta']['angle'])) for img in imgdata } if not imgdata: raise RenderModuleException( "No relevant image metadata found for metadata at {}".format( self.args['metafile_uri'])) minX, minY = numpy.min(numpy.array(list(img_coords.values())), axis=0) # assume isotropic pixels pixelsize = roidata['calibration']['highmag']['x_nm_per_pix'] imgdir = self.args.get('image_prefix', uri_utils.uri_prefix(self.args['metafile_uri'])) tspecs = [ self.ts_from_imgdata(img, imgdir, img_coords[img['img_path']][0] - minX, img_coords[img['img_path']][1] - minY, minint=self.args['minimum_intensity'], maxint=self.args['maximum_intensity'], width=roidata['camera_info']['width'], height=roidata['camera_info']['height'], z=self.zValues[0], sectionId=self.args.get('sectionId'), scopeId=roidata['temca_id'], cameraId=roidata['camera_info']['camera_id'], pixelsize=pixelsize, maskUrl=self.args['maskUrl_uri']) for img in imgdata ] self.output_tilespecs_to_stack(tspecs) try: self.output({'stack': self.output_stack}) except AttributeError as e: self.logger.error(e)
def add_mask_to_tilespec(self, z, exts): tspec = self.get_tilespec(z) if not tspec: return tspec if tspec.ip[0].maskUrl is not None: raise RenderModuleException( "Tilespec already has a mask for z = %d" % (z)) fnames = [] for ext in exts: fnames += glob.glob( os.path.join(self.args['mask_dir'], '%d_*' % z + os.extsep + ext)) if len(fnames) != 1: raise RenderModuleException( "Expected 1 mask file for z = %d, found %d in %s" % (z, len(fnames), self.args['mask_dir'])) tspec.ip['0'].maskUrl = pathlib.Path(fnames[0]).as_uri() return tspec
def dereference_tforms(tforms, ref_tforms): deref_tforms = [] for tf in tforms: if isinstance(tf, ReferenceTransform): try: mtf = next(mt for mt in ref_tforms if mt.transformId == tf.refId) deref_tforms.append(mtf) except StopIteration as e: raise RenderModuleException(( "reference transform: {} not found in provided refererence transforms {}" .format(tf.refId, ref_tforms))) else: deref_tforms.append(tf) return deref_tforms
def tspecjob(collection, render, tilespecs, estimate=True): result = {} result['tilespecs'] = tilespecs matches = renderapi.pointmatch.get_matches_from_tile_to_tile( collection, tilespecs[0].layout.sectionId, tilespecs[0].tileId, tilespecs[1].layout.sectionId, tilespecs[1].tileId, render=render) if len(matches) != 1: estr = "\n expected 1 matching tile pair, found %d for" % ( len(matches)) estr += '\n %s\n group: %s\n %s\n group: %s' % ( tilespecs[0].tileId, tilespecs[0].layout.sectionId, tilespecs[1].tileId, tilespecs[1].layout.sectionId) raise RenderModuleException(estr) match = matches[0] if estimate: tf = renderapi.transform.RigidModel() src = np.array(match['matches']['p']).transpose() dst = np.array(match['matches']['q']).transpose() if match['pGroupId'] == tilespecs[0].layout.sectionId: src = np.array(match['matches']['q']).transpose() dst = np.array(match['matches']['p']).transpose() ind = np.argwhere(np.isclose(np.array(match['matches']['w']), 1.0)).flatten() tf.estimate(src[ind, :], dst[ind, :]) result['transform'] = tf result['avg_residual'] = { 'z0': tilespecs[0].z, 'z1': tilespecs[1].z, 'avg_residual': avg_residual(match, tilespecs[0], tilespecs[1]) } return result
def run(self): if len(self.zValues) != len(self.args['new_zValues']): raise RenderModuleException( "zValues with length {} cannot be mapped to " "zValues with length {}".format(len(self.zValues), len(self.args['new_zValues']))) for z, newz in zip(self.zValues, self.args['new_zValues']): resolvedtiles = renderapi.resolvedtiles.get_resolved_tiles_from_z( self.input_stack, z, render=self.render) for ts in resolvedtiles.tilespecs: ts.z = newz if self.args.get('remap_sectionId'): ts.layout.sectionId = self.sectionId_from_z(newz) self.output_tilespecs_to_stack( resolvedtiles.tilespecs, sharedTransforms=resolvedtiles.transforms, zValues=[z]) self.output({ "zValues": self.zValues, "output_stack": self.output_stack })
def run(self, run_lc_bsh=None): run_lc_bsh = self.default_bsh if run_lc_bsh is None else run_lc_bsh outfn = self.args.get( 'outfile', os.path.abspath( os.path.join(self.args['project_path'], 'lens_correction.json'))) delete_procdir = False procdir = self.args.get('processing_directory', self.args['project_path']) if procdir is None: delete_procdir = True procdir = tempfile.mkdtemp() # command line argument for beanshell script bsh_call = [ "xvfb-run", "-a", self.args['fiji_path'], "-Xms{}g".format( self.args['heap_size']), "-Xmx{}g".format( self.args['heap_size']), "-Xincgc", "-Dman=" + self.args['manifest_path'], "-Ddir=" + self.args['project_path'], "-Dgrid=" + str(self.args['grid_size']), "-Disig=" + str(self.args['SIFT_params']['initialSigma']), "-Dsteps=" + str(self.args['SIFT_params']['steps']), "-Dminos=" + str(self.args['SIFT_params']['minOctaveSize']), "-Dmaxos=" + str(self.args['SIFT_params']['maxOctaveSize']), "-Dfdsize=" + str(self.args['SIFT_params']['fdSize']), "-Dfdbins=" + str(self.args['SIFT_params']['fdBins']), "-Drod=" + str(self.args['align_params']['rod']), "-Dmaxe=" + str(self.args['align_params']['maxEpsilon']), "-Dmir=" + str(self.args['align_params']['minInlierRatio']), "-Dmni=" + str(self.args['align_params']['minNumInliers']), "-Demi=" + str(self.args['align_params']['expectedModelIndex']), "-Dmh=" + str(self.args['align_params']['multipleHypotheses']), "-Dri=" + str(self.args['align_params']['rejectIdentity']), "-Dit=" + str(self.args['align_params']['identityTolerance']), "-Dtaip=" + str(self.args['align_params']['tilesAreInPlace']), "-Ddmi=" + str(self.args['align_params']['desiredModelIndex']), "-Dreg=" + str(self.args['align_params']['regularize']), "-Dmio={}".format( self.args['align_params']['maxIterationsOptimize']), "-Dmpwo={}".format( self.args['align_params']['maxPlateauWidthOptimize']), "-Ddim=" + str(self.args['align_params']['dimension']), "-Dlam=" + str(self.args['align_params']['lambdaVal']), "-Dprocdir={}".format(procdir), "-Doutfn={}".format(outfn), "-Dctrans=" + str(self.args['align_params']['clearTransform']), "-Dvis=" + str(self.args['align_params']['visualize']), "-DthreadsSIFT={}".format( self.args['max_threads_SIFT']), "--", "--no-splash", run_lc_bsh ] self.logger.debug('bsh_cmd: {}'.format(bsh_call)) try: subprocess.check_call(bsh_call) except subprocess.CalledProcessError as e: raise (RenderModuleException("{}".format(e))) # self.logger.debug('ret_code: {}'.format(ret)) if delete_procdir: shutil.rmtree(procdir) try: self.output({'output_json': outfn}) except AttributeError as e: # output validation will need to wait for argschema PR self.logger.error(e)
def run(self): if "MCRROOT" not in os.environ: raise ValidationError("MCRROOT not set") if self.clone_section_stack: tmp_stack = "{}_zs{}_ze{}_t{}".format( self.args['source_collection']['stack'], self.args['first_section'], self.args['last_section'], time.strftime("%m%d%y_%H%M%S")) zvalues = renderapi.stack.get_z_values_for_stack( self.args['source_collection']['stack'], render=self.render) zs = [z for z in zvalues if \ (z>=self.args['first_section']) \ and (z<=self.args['last_section'])] renderapi.stack.clone_stack( self.args['source_collection']['stack'], tmp_stack, zs=zs, close_stack=True, render=self.render) self.args['source_collection']['stack'] = tmp_stack # the clone stack option should also set the first and last section z appropriately in self.args for solver self.args['first_section'] = min(zs) self.args['last_section'] = max(zs) # Check if the target stack exists and whether we need to overwrite the z target_host = self.args['target_collection']['service_host'] num = target_host[-4:] port = None if num.isdigit(): target_host = target_host[:-5] if target_host.find('http://') < 0: target_host = 'http://%s' % (target_host) port = int(num) list_of_stacks = renderapi.render.get_stacks_by_owner_project( owner=self.args['target_collection']['owner'], project=self.args['target_collection']['project'], host=target_host, port=port, render=self.render) if self.args['target_collection']['stack'] in list_of_stacks: for z in xrange(int(self.args['first_section']), int(self.args['last_section']) + 1): renderapi.stack.delete_section( self.args['target_collection']['stack'], z, host=target_host, port=port, owner=self.args['target_collection']['owner'], project=self.args['target_collection']['project'], render=self.render) # generate a temporary json to feed in to the solver tempjson = tempfile.NamedTemporaryFile(suffix=".json", mode='w', delete=False) tempjson.close() with open(tempjson.name, 'w') as f: json.dump(self.args, f, indent=4) f.close() #assumes that solver_executable is the shell script that sets the LD_LIBRARY path and calls the executable cmd = "%s %s %s" % (self.solver_executable, os.environ['MCRROOT'], tempjson.name) ret = os.system(cmd) # one successful completion remove the input json file if ret == 0: os.remove(tempjson.name) else: raise RenderModuleException("solve failed with input_json {}", self.args) #try to return the z's solved in the output json try: d = {'zs': zs} self.output(d) except: raise RenderModuleException("unable to output json {}", d) #if you made a tmp stack destroy it if self.clone_section_stack: renderapi.stack.delete_stack(tmp_stack, render=self.render)
def run(self): if len(self.args['source_stack']) != len(self.args['target_stack']): raise RenderModuleException( "Count of source and target stacks do not match") list_of_stacks = renderapi.render.get_stacks_by_owner_project( owner=self.render.DEFAULT_OWNER, project=self.render.DEFAULT_PROJECT, host=self.render.DEFAULT_HOST, port=self.render.DEFAULT_PORT, render=self.render) source_stacks = [] target_stacks = [] zvalues = [] for source_stack, target_stack, zVals in zip(self.args['source_stack'], self.args['target_stack'], self.args['zValues']): if source_stack not in list_of_stacks or target_stack not in list_of_stacks: # pragma: no cover continue #if source_stack not in list_of_stacks: # raise RenderModuleException("Source stack {} not in render".format(source_stack)) #if target_stack not in list_of_stacks: # raise RenderModuleException("Target stack {} not in render".format(target_stack)) szvalues = renderapi.stack.get_z_values_for_stack( source_stack, render=self.render) tzvalues = renderapi.stack.get_z_values_for_stack( target_stack, render=self.render) avail_zs = set(szvalues).intersection(tzvalues) final_zs = list(set(zVals).intersection(avail_zs)) #source_stacks.append(source_stack) #target_stacks.append(target_stack) #zvalues.append(final_zs) print(final_zs) #mypartial = partial(swap_section, self.render, source_stack, target_stack) #with renderapi.client.WithPool(self.args['pool_size']) as pool: # output_bool = pool.map(mypartial, final_zs) output_bool = [] for z in final_zs: outbool = swap_section(self.render, source_stack, target_stack, z) output_bool.append(outbool) for z, n in zip(final_zs, output_bool): if n: zvalues.append(z) source_stacks.append(source_stack) target_stacks.append(target_stack) #zs = [z for z, n in zip(final_zs, output_bool) if n] #zvalues.append(zs) if self.args['complete_source_stack']: renderapi.stack.set_stack_state(source_stack, state='COMPLETE', render=self.render) if self.args['complete_target_stack']: renderapi.stack.set_stack_state(target_stack, state='COMPLETE', render=self.render) # delete source stack if specified in input if self.args['delete_source_stack']: # pragma: no cover renderapi.stack.delete_stack(source_stack, render=self.render) #self.swap_zs_from_stacks(source_stack, target_stack, final_zs) self.output({ "source_stacks": source_stacks, "target_stacks": target_stacks, "swapped_zvalues": zvalues })
def run(self): zvalues1 = self.render.run(renderapi.stack.get_z_values_for_stack, self.args['poststitched_stack']) zrange = range(self.args['minZ'], self.args['maxZ']+1) zvalues = list(set(zvalues1).intersection(set(zrange))) if len(zvalues) == 0: raise RenderModuleException('No valid zvalues found in stack for given range {} - {}'.format(self.args['minZ'], self.args['maxZ'])) # check if pre or post stitched stack is in LOADING state. # if so, clone them to new stacks status1, new_prestitched = check_status_of_stack(self.render, self.args['prestitched_stack'], zvalues) status2, new_poststitched = check_status_of_stack(self.render, self.args['poststitched_stack'], zvalues) disconnected_tiles, gap_tiles, seam_centroids, post_tspecs, matches, stats = detect_stitching_mistakes( self.render, new_prestitched, new_poststitched, self.args['match_collection'], self.args['match_collection_owner'], self.args['residual_threshold'], self.args['neighbors_distance'], self.args['min_cluster_size'], zvalues, pool_size=self.args['pool_size']) # find the indices of sections having holes hole_indices = [i for i, dt in enumerate(disconnected_tiles) if len(dt) > 0] gaps_indices = [i for i, gt in enumerate(gap_tiles) if len(gt) > 0] seams_indices = [i for i, sm in enumerate(seam_centroids) if len(sm) > 0] holes = [zvalues[i] for i in hole_indices] gaps = [zvalues[i] for i in gaps_indices] seams = [zvalues[i] for i in seams_indices] combinedz = list(set(holes + gaps + seams)) qc_passed_sections = set(zvalues) - set(combinedz) centroids = [seam_centroids[i] for i in seams_indices] self.args['output_html'] = [] if self.args['plot_sections']: self.args['output_html'] = plot_section_maps(self.render, self.args['poststitched_stack'], post_tspecs, matches, disconnected_tiles, gap_tiles, seam_centroids, stats, zvalues, out_html_dir=self.args['out_html_dir']) self.output({'output_html':self.args['output_html'], 'qc_passed_sections': qc_passed_sections, 'hole_sections': holes, 'gap_sections':gaps, 'seam_sections':seams, 'seam_centroids':np.array(centroids)}) # delete the stacks that were cloned if status1 == 'LOADING': self.render.run(renderapi.stack.delete_stack, new_prestitched) if status2 == 'LOADING': self.render.run(renderapi.stack.delete_stack, new_poststitched)