def run(self): logger.setLevel(self.args['log_level']) resolved_path = os.path.join(self.args['client_mount_or_map'], self.args['fdir'], self.args['resolved_file']) if self.args['backup_copy']: backup(resolved_path) resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(resolved_path)) if self.args['image_directory'] is None: self.args['image_directory'] = pathlib.PurePosixPath( self.args['server_mount'], self.args['fdir']) for t in resolved.tilespecs: for k in ['imageUrl', 'maskUrl']: s = t.ip['0'][k] if s: orig = urllib.parse.unquote(urllib.parse.urlparse(s).path) t.ip['0'][k] = pathlib.PurePosixPath( self.args['image_directory'], os.path.basename(orig)).as_uri() self.args['resolved_file'] = jsongz.dump(resolved.to_dict(), resolved_path, compress=None) logger.info("updated tilespec urls in %s" % resolved_path)
def run(self): self.resolved, self.new_ref_transform, jresult = ( self.solve_resolvedtiles_from_args()) new_path = None if 'outfile' in self.args: fname = self.args['outfile'] if self.args['timestamp']: spf = fname.split(os.extsep, 1) spf[0] += '_%s' % self.new_ref_transform.transformId fname = os.extsep.join(spf) new_path = jsongz.dump(self.resolved.to_dict(), os.path.join(self.args['output_dir'], fname), compress=self.args['compress_output']) new_path = os.path.abspath(new_path) fname = 'output.json' if self.args['timestamp']: fname = 'output_%s.json' % self.new_ref_transform.transformId self.args['output_json'] = os.path.join(self.args['output_dir'], fname) jresult['resolved_tiles'] = new_path self.output(jresult, indent=2) self.logger.info(" wrote solved tilespecs:\n %s" % new_path) return
def test_jsongz(tmpdir, compress, FILE): tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) with open(FILE, 'r') as f: j = json.load(f) tmpfile = os.path.join(tmp_file_dir, "tmp.json") tmpfile = jsongz.dump(j, tmpfile, compress=compress) newj = jsongz.load(tmpfile) assert newj == j shutil.rmtree(tmp_file_dir)
def run(self): if 'metafile' not in self.args: self.args['metafile'] = get_metafile_path(self.args['data_dir']) else: self.args['data_dir'] = os.path.dirname(self.args['metafile']) if not self.args['output_dir']: self.args['output_dir'] = self.args['data_dir'] # read the matches from the metafile matches = meta_to_collection.main([self.args['data_dir']]) montage_filter_matches(matches, self.args['ransacReprojThreshold']) # write to file collection = os.path.join(self.args['output_dir'], "collection.json") collection = jsongz.dump(matches, collection, compress=self.args['compress_output']) # make raw tilespec json rawspecpath, z = make_raw_tilespecs(self.args['metafile'], self.args['output_dir'], matches[0]['pGroupId'], self.args['compress_output']) # get the ref transform tform = get_transform(self.args['metafile'], self.args['ref_transform'], self.args['ref_transform_dict'], self.args['read_transform_from']) # make a resolved tile object input_stack_path = make_resolved(rawspecpath, tform, self.args['output_dir'], self.args['compress_output']) templates = [ os.path.join(self.args['solver_template_dir'], t) for t in self.args['solver_templates'] ] self.results = do_solves(collection, input_stack_path, z, self.args['compress_output'], templates) self.args['output_json'] = os.path.join(self.args['output_dir'], 'montage_results.json') with open(self.args['output_json'], 'w') as f: json.dump(self.results, f, indent=2)
def test_match_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = raw_stack p['pointmatch']['name'] = montage_pointmatches # get the matches and write them to a file sectionData = renderapi.stack.get_stack_sectionData( p['input_stack']['name'], render=render) sections = [sd['sectionId'] for sd in sectionData] matches = [] for s in sections: matches += renderapi.pointmatch.get_matches_with_group( p['pointmatch']['name'], s, render=render) tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) match_file = os.path.join(tmp_file_dir, "matches.json") match_file = jsongz.dump(matches, match_file, compress=compress) p['pointmatch']['db_interface'] = 'file' p['pointmatch']['input_file'] = match_file p['output_mode'] = 'none' tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() for k in ['precision', 'error', 'err']: assert np.all( np.isclose(np.array(tmod.results[k]), np.array(solved_montage.results[k]), atol=1e-7)) assert np.all( np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0), np.linalg.norm(tmod.results['x'], axis=0), atol=1e-7)) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in tmod.resolvedtiles.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def run(self): with open(self.args['metafile'], 'r') as f: meta = json.load(f) roidata = meta[0]['metadata'] imgdata = meta[1]['data'] img_coords = {img['img_path']: self.image_coords_from_stage( img['img_meta']['stage_pos'], img['img_meta']['pixel_size_x_move'], img['img_meta']['pixel_size_y_move'], numpy.radians(img['img_meta']['angle'])) for img in imgdata} # if not imgdata: # raise RenderModuleException( # "No relevant image metadata found for metafile {}".format( # self.args['metafile'])) minX, minY = numpy.min(numpy.array(list(img_coords.values())), axis=0) # assume isotropic pixels pixelsize = roidata['calibration']['highmag']['x_nm_per_pix'] imgdir = self.args.get( 'image_directory', os.path.dirname(self.args['metafile'])) self.render_tspecs = [ self.ts_from_imgdata( img, imgdir, img_coords[img['img_path']][0] - minX, img_coords[img['img_path']][1] - minY, minint=self.args['minimum_intensity'], maxint=self.args['maximum_intensity'], width=roidata['camera_info']['width'], height=roidata['camera_info']['height'], z=self.args['z'], sectionId=self.args.get('sectionId'), scopeId=roidata['temca_id'], cameraId=roidata['camera_info']['camera_id'], pixelsize=pixelsize, maskUrl=self.args['maskUrl']) for img in imgdata] if 'output_path' in self.args: self.args['output_path'] = jsongz.dump( self.tilespecs, self.args['output_path'], self.args['compress_output'])
def test_input_stack_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) resolved = renderapi.resolvedtiles.get_resolved_tiles_from_z( raw_stack, p['first_section'], render=render) tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) input_stack_file = os.path.join(tmp_file_dir, "input_stack.json") input_stack_file = jsongz.dump(resolved.to_dict(), input_stack_file, compress=compress) p['input_stack']['db_interface'] = 'file' p['input_stack']['input_file'] = input_stack_file p['output_mode'] = 'none' p['pointmatch']['name'] = montage_pointmatches tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() for k in ['precision', 'error', 'err']: assert np.all( np.isclose(np.array(tmod.results[k]), np.array(solved_montage.results[k]), atol=1e-7)) assert np.all( np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0), np.linalg.norm(tmod.results['x'], axis=0), atol=1e-7)) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in tmod.resolvedtiles.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def make_resolved(rawspecpath, tform, outputdir, compress): # read in the tilespecs rtj = jsongz.load(rawspecpath) tspecs = [renderapi.tilespec.TileSpec(json=t) for t in rtj] # do not need this anymore os.remove(rawspecpath) # add the reference transform ref = renderapi.transform.ReferenceTransform() ref.refId = tform.transformId for t in tspecs: t.tforms.insert(0, ref) # make a resolved tile object resolved = renderapi.resolvedtiles.ResolvedTiles(tilespecs=tspecs, transformList=[tform]) # write it to file and return the path rpath = os.path.join(outputdir, 'resolvedtiles_input.json') return jsongz.dump(resolved.to_dict(), rpath, compress)
def make_collection_json( template_file, output_dir, thresh, compress, ignore_match_indices=None): with open(template_file, 'r') as f: matches = json.load(f) counts = [] for m in matches['collection']: counts.append({}) ind = np.arange(len(m['matches']['p'][0])) counts[-1]['n_from_gpu'] = ind.size _, _, w, _ = common_utils.pointmatch_filter( m, n_clusters=None, n_cluster_pts=20, ransacReprojThreshold=40.0, model='Similarity') m['matches']['w'] = w.tolist() counts[-1]['n_after_filter'] = np.count_nonzero(w) m = matches['collection'] if ignore_match_indices: m = [match for i, match in enumerate(matches['collection']) if i not in ignore_match_indices] logger.warning("you are ignoring some point matches") collection_file = os.path.join(output_dir, "collection.json") collection_file = jsongz.dump(m, collection_file, compress=compress) return collection_file, counts
def run(self): if 'tilespecs' in self.args: jspecs = self.args['tilespecs'] else: jspecs = jsongz.load(self.args['tilespec_file']) self.tilespecs = np.array( [renderapi.tilespec.TileSpec(json=j) for j in jspecs]) if 'matches' in self.args: self.matches = self.args['matches'] else: self.matches = jsongz.load(self.args['match_file']) remove_weighted_matches(self.matches, weight=0.0) self.tile_width = self.tilespecs[0].width self.tile_height = self.tilespecs[0].height maskUrl = self.tilespecs[0].ip[0].maskUrl # condense coordinates self.coords = condense_coords(self.matches) nc0 = self.coords.shape[0] self.coords = smooth_density(self.coords, self.tile_width, self.tile_height, 10) nc1 = self.coords.shape[0] self.logger.info( "\n smoothing point density reduced points from %d to %d" % (nc0, nc1)) if self.coords.shape[0] == 0: raise MeshLensCorrectionException( "no point matches left after smoothing density, \ probably some sparse areas of matching") # create PSLG self.bbox = create_PSLG(self.tile_width, self.tile_height, maskUrl) # find delaunay with max vertices self.mesh, self.area_triangle_par = \ find_delaunay_with_max_vertices( self.bbox, self.args['nvertex']) # and enforce neighboring matches to vertices self.mesh, self.area_triangle_par = \ force_vertices_with_npoints( self.area_triangle_par, self.bbox, self.coords, 3) nend = self.mesh.points.shape[0] self.logger = logging.getLogger(self.__class__.__name__) self.logger.info("\n aimed for %d mesh points, got %d" % (self.args['nvertex'], nend)) if self.mesh.points.shape[0] < 0.5 * self.args['nvertex']: raise MeshLensCorrectionException("mesh coarser than intended") # prepare the linear algebra and solve self.A, self.weights, self.b, self.lens_dof_start = \ create_A( self.matches, self.tilespecs, self.mesh) self.x0 = create_x0(self.A.shape[1], self.tilespecs) self.reg = create_regularization( self.A.shape[1], len(self.tilespecs), self.args['regularization']['default_lambda'], self.args['regularization']['translation_factor'], self.args['regularization']['lens_lambda']) self.solution, self.errx, self.erry = solve(self.A, self.weights, self.reg, self.x0, self.b) self.transforms = create_transforms(len(self.tilespecs), self.solution) tf_trans, jresult, self.solve_message = report_solution( self.errx, self.erry, self.transforms, self.args['good_solve']) self.logger.info(self.solve_message) # check quality of solution if not all([ self.errx.mean() < self.args['good_solve']['error_mean'], self.erry.mean() < self.args['good_solve']['error_mean'], self.errx.std() < self.args['good_solve']['error_std'], self.erry.std() < self.args['good_solve']['error_std'] ]): raise MeshLensCorrectionException("Solve not good: %s" % self.solve_message) self.logger.debug(self.solve_message) self.new_ref_transform = create_thinplatespline_tf( self.args, self.mesh, self.solution, self.lens_dof_start, self.logger) bbox = self.tilespecs[0].bbox_transformed(tf_limit=0) tbbox = self.new_ref_transform.tform(bbox) bstr = 'new transform corners:\n' for i in range(bbox.shape[0] - 1): bstr += " (%0.1f, %0.1f) -> (%0.1f, %0.1f)\n" % ( bbox[i, 0], bbox[i, 1], tbbox[i, 0], tbbox[i, 1]) self.logger.info(bstr) new_tilespecs = new_specs_with_tf(self.new_ref_transform, self.tilespecs, self.transforms) stage_affine = estimate_stage_affine(self.tilespecs, new_tilespecs) sastr = "affine estimate of tile translations:\n" sastr += " scale: {}\n".format(stage_affine.scale) sastr += " translation: {}\n".format(stage_affine.translation) sastr += " shear: {}\n".format(stage_affine.shear) sastr += " rotation: {}\n".format(np.degrees(stage_affine.rotation)) self.logger.info(sastr) self.resolved = renderapi.resolvedtiles.ResolvedTiles( tilespecs=new_tilespecs, transformList=[self.new_ref_transform]) new_path = None if 'outfile' in self.args: fname = self.args['outfile'] if self.args['timestamp']: spf = fname.split(os.extsep, 1) spf[0] += '_%s' % self.new_ref_transform.transformId fname = os.extsep.join(spf) new_path = jsongz.dump(self.resolved.to_dict(), os.path.join(self.args['output_dir'], fname), compress=self.args['compress_output']) new_path = os.path.abspath(new_path) fname = 'output.json' if self.args['timestamp']: fname = 'output_%s.json' % self.new_ref_transform.transformId self.args['output_json'] = os.path.join(self.args['output_dir'], fname) jresult['resolved_tiles'] = new_path self.output(jresult, indent=2) self.logger.info(" wrote solved tilespecs:\n %s" % new_path) return