def solve_resolvedtiles_from_args(self): """use arguments to run lens correction Returns ------- resolved : renderapi.resolvedtiles.ResolvedTiles new resolvedtiles object with derived lens correction applied new_ref_transform : renderapi.transform.leaf.ThinPlateSplineTransform derived lens correction transform jresult : dict dictionary of solve information """ if 'tilespecs' in self.args: jspecs = self.args['tilespecs'] else: jspecs = jsongz.load(self.args['tilespec_file']) self.tilespecs = np.array( [renderapi.tilespec.TileSpec(json=j) for j in jspecs]) if 'matches' in self.args: self.matches = self.args['matches'] else: self.matches = jsongz.load(self.args['match_file']) return _solve_resolvedtiles( renderapi.resolvedtiles.ResolvedTiles(tilespecs=self.tilespecs, transformList=[]), self.matches, self.args["nvertex"], self.args["regularization"]["default_lambda"], self.args["regularization"]["translation_factor"], self.args["regularization"]["lens_lambda"], self.args["good_solve"], logger=self.logger)
def test_solve_from_file_and_memory(solver_input_args, source): local_args = copy.deepcopy(solver_input_args) metafile = one_file(local_args['data_dir'], '_metadata*.json') templatefile = one_file(local_args['data_dir'], '_template*.json') compress = True with TemporaryDirectory() as output_dir: tspecin = tilespec_input_from_metafile(metafile, local_args['mask_file'], output_dir, local_args['log_level'], compress) gentspecs = GenerateEMTileSpecsModule(input_data=tspecin, args=[]) gentspecs.run() cfile, counts = make_collection_json(templatefile, output_dir, compress, local_args['ransac_thresh']) solver_args = { 'nvertex': local_args['nvertex'], 'regularization': local_args['regularization'], 'output_dir': output_dir, 'compress_output': compress, 'log_level': local_args['log_level'] } if source == 'file': solver_args['tilespec_file'] = gentspecs.args['output_path'] solver_args['match_file'] = cfile solver_args['outfile'] = 'resolvedtiles.json.gz' solver = MeshAndSolveTransform(input_data=solver_args, args=[]) solver.run() with open(solver.args['output_json'], 'r') as f: tspec_path = json.load(f)['resolved_tiles'] tfile = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(tspec_path)) assert (len(tfile.tilespecs) == len(gentspecs.tilespecs) == len( solver.resolved.tilespecs)) if source == 'memory': solver_args['tilespecs'] = gentspecs.tilespecs solver_args['matches'] = jsongz.load(cfile) solver = MeshAndSolveTransform(input_data=solver_args, args=[]) solver.run() assert (len(gentspecs.tilespecs) == len(solver.resolved.tilespecs)) if source == 'fail': solver_args['tilespec_file'] = gentspecs.args['output_path'] solver_args['tilespecs'] = gentspecs.tilespecs solver_args['matches'] = jsongz.load(cfile) with pytest.raises(ValidationError): MeshAndSolveTransform(input_data=solver_args, args=[]) solver_args.pop('tilespecs') solver_args['tilespec_file'] = gentspecs.args['output_path'] solver_args['matches'] = jsongz.load(cfile) solver_args['match_file'] = cfile with pytest.raises(ValidationError): MeshAndSolveTransform(input_data=solver_args, args=[])
def get_transform(self): self.tform = None if 'transform_file' in self.args: self.tform = renderapi.transform.ThinPlateSplineTransform( json=jsongz.load(self.args['transform_file'])) else: for fbase in self.args['resolved_tiles']: fpath = os.path.join(self.args['data_dir'], fbase) if os.path.isfile(fpath): resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(fpath)) self.tform = resolved.transforms[0] break
def test_output_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = raw_stack p['pointmatch']['name'] = montage_pointmatches tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) p['output_stack']['db_interface'] = 'file' p['output_stack']['output_file'] = os.path.join(tmp_file_dir, "resolvedtiles.json") p['output_stack']['compress_output'] = compress tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() solved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(tmod.args['output_stack']['output_file'])) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in solved.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def upload_resolved_file(render_params, stack, resolved_file, close_stack): if resolved_file is None: return resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(resolved_file)) render = renderapi.connect(**render_params) renderapi.stack.create_stack(stack, render=render) renderapi.client.import_tilespecs(stack, resolved.tilespecs, sharedTransforms=resolved.transforms, use_rest=True, render=render) if close_stack: renderapi.stack.set_stack_state(stack, state='COMPLETE', render=render) logger.info("imported %d tilespecs to render from %s" % (len(resolved.tilespecs), resolved_file)) url = ("\nhttp://" + "%s:%d" % (render_params['host'], render_params['port']) + "/render-ws/view/stacks.html?ndvizHost=em-131fs%3A8001" + "&renderStack=%s" % stack + "&renderStackOwner=%s" % render_params['owner'] + "&renderStackProject=%s" % render_params['project']) logger.info(url) return
def run(self): cpath = self.args.get( 'collection_path', os.path.join(self.args['data_dir'], self.args['collection_basename'])) if ((not os.path.isfile(cpath)) & (os.path.splitext(cpath)[-1] == '.json')): cpath += '.gz' self.matches = jsongz.load(cpath) self.get_transform() if self.args['view_all']: inds = np.arange(len(self.matches)) else: inds = [self.args['match_index']] with PdfPages(self.args['pdf_out']) as pdf: for ind in inds: pim, qim, p, q, w, pname, qname = get_ims_and_coords( self.matches[ind], self.args['data_dir']) f, a = plot_ims_and_coords(pim, qim, p, q, w, pname=pname, qname=qname, tform=self.tform, fignum=2) pdf.savefig(f) if self.args['show']: plt.show() print('wrote %s' % os.path.abspath(self.args['pdf_out']))
def run(self): logger.setLevel(self.args['log_level']) resolved_path = os.path.join(self.args['client_mount_or_map'], self.args['fdir'], self.args['resolved_file']) if self.args['backup_copy']: backup(resolved_path) resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(resolved_path)) if self.args['image_directory'] is None: self.args['image_directory'] = pathlib.PurePosixPath( self.args['server_mount'], self.args['fdir']) for t in resolved.tilespecs: for k in ['imageUrl', 'maskUrl']: s = t.ip['0'][k] if s: orig = urllib.parse.unquote(urllib.parse.urlparse(s).path) t.ip['0'][k] = pathlib.PurePosixPath( self.args['image_directory'], os.path.basename(orig)).as_uri() self.args['resolved_file'] = jsongz.dump(resolved.to_dict(), resolved_path, compress=None) logger.info("updated tilespec urls in %s" % resolved_path)
def run(self): matches = jsongz.load(self.args['collection_path']) resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(self.args['resolved_path'])) xy, res, mxy, mres = make_xyres(matches, resolved) if self.args['save_json_path']: with open(self.args['save_json_path'], 'w') as f: json.dump( { 'xy': xy.tolist(), 'res': res.tolist(), 'filtered_xy': mxy.tolist(), 'filtered_res': mres.tolist() }, f, indent=2) pdf = None if self.args['pdf_out']: pdf = PdfPages(self.args['pdf_out']) if self.args['make_plot']: f, a = plt.subplots( 2, 2, clear=True, num=1, sharex=True, sharey=True, figsize=(12, 12)) vmnmx = 5 title = self.args['collection_path'] title += '\n' + self.args['resolved_path'] + '\n' one_plot( f, a[0, 0], xy, res[:, 0], vmin=-vmnmx, vmax=vmnmx, title=title+'x res [px]', fontsize=8) one_plot( f, a[0, 1], xy, res[:, 1], vmin=-vmnmx, vmax=vmnmx, title='y res [px]') one_plot( f, a[1, 0], xy, np.linalg.norm(res, axis=1), vmin=0, vmax=vmnmx*np.sqrt(2), title='mag res [px]') one_plot( f, a[1, 1], mxy, np.linalg.norm(mres, axis=1), vmin=0, vmax=vmnmx*np.sqrt(2), title='filtered') a[0, 0].invert_yaxis() if self.args['show']: plt.show() if pdf: pdf.savefig(f) pdf.close() print('wrote %s' % os.path.abspath(self.args['pdf_out']))
def test_jsongz(tmpdir, compress, FILE): tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) with open(FILE, 'r') as f: j = json.load(f) tmpfile = os.path.join(tmp_file_dir, "tmp.json") tmpfile = jsongz.dump(j, tmpfile, compress=compress) newj = jsongz.load(tmpfile) assert newj == j shutil.rmtree(tmp_file_dir)
def upload_collection_file(render_params, collection, collection_file): if collection is None: return render = renderapi.connect(**render_params) matches = jsongz.load(collection_file) renderapi.pointmatch.import_matches(collection, matches, render=render) logger.info("imported %d pointmatches to render from %s" % (len(matches), collection_file))
def test_make_collection(solver_input_args, compress): ftemplate = glob.glob( os.path.join(solver_input_args['data_dir'], "_template_matches_*.json"))[0] with TemporaryDirectory() as output_dir: cfile, counts = make_collection_json( ftemplate, output_dir, compress, solver_input_args['ransac_thresh']) # for debug purposes, sometimes ignore some matches to see # how the solve does without them m = jsongz.load(cfile) n0 = len(m) cfile, counts = make_collection_json( ftemplate, output_dir, solver_input_args['ransac_thresh'], compress, ignore_match_indices=[0, 1]) m = jsongz.load(cfile) n1 = len(m) assert n0 == (n1 + 2)
def make_resolved(rawspecpath, tform, outputdir, compress): # read in the tilespecs rtj = jsongz.load(rawspecpath) tspecs = [renderapi.tilespec.TileSpec(json=t) for t in rtj] # do not need this anymore os.remove(rawspecpath) # add the reference transform ref = renderapi.transform.ReferenceTransform() ref.refId = tform.transformId for t in tspecs: t.tforms.insert(0, ref) # make a resolved tile object resolved = renderapi.resolvedtiles.ResolvedTiles(tilespecs=tspecs, transformList=[tform]) # write it to file and return the path rpath = os.path.join(outputdir, 'resolvedtiles_input.json') return jsongz.dump(resolved.to_dict(), rpath, compress)
def run(self): self.logger = logging.getLogger(self.__class__.__name__) self.logger.setLevel(self.args['log_level']) utils.logger.setLevel(self.args['log_level']) self.check_for_files() self.output_dir = self.args.get('output_dir', self.args['data_dir']) self.logger.info("destination directory:\n %s" % self.output_dir) tspecin = tilespec_input_from_metafile( self.metafile, self.args['mask_file'], self.output_dir, self.args['log_level'], self.args['compress_output']) gentspecs = GenerateEMTileSpecsModule(input_data=tspecin, args=[]) gentspecs.run() assert os.path.isfile(gentspecs.args['output_path']) self.logger.info( "raw tilespecs written:\n %s" % gentspecs.args['output_path']) collection_path, self.filter_counts = make_collection_json( self.matchfile, self.output_dir, self.args['ransac_thresh'], self.args['compress_output'], self.args['ignore_match_indices']) self.n_from_gpu = np.array( [i['n_from_gpu'] for i in self.filter_counts]).sum() self.n_after_filter = np.array( [i['n_after_filter'] for i in self.filter_counts]).sum() self.logger.info( "filter counts: %0.2f %% kept" % (100 * float(self.n_after_filter) / self.n_from_gpu)) assert os.path.isfile(collection_path) self.logger.info( "filtered collection written:\n %s" % collection_path) solver_args = { 'nvertex': self.args['nvertex'], 'regularization': self.args['regularization'], 'good_solve': self.args['good_solve'], 'tilespecs': gentspecs.tilespecs, 'match_file': collection_path, 'output_dir': self.output_dir, 'outfile': 'resolvedtiles.json.gz', 'compress_output': self.args['compress_output'], 'log_level': self.args['log_level'], 'timestamp': self.args['timestamp']} self.solver = MeshAndSolveTransform(input_data=solver_args, args=[]) self.solver.run() with open(self.solver.args['output_json'], 'r') as f: j = json.load(f) resolved_path = j['resolved_tiles'] resolvedtiles = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(resolved_path)) self.jtform = resolvedtiles.transforms[0].to_dict() self.map1, self.map2, self.mask = utils.maps_from_tform( renderapi.transform.ThinPlateSplineTransform( json=self.jtform), resolvedtiles.tilespecs[0].width, resolvedtiles.tilespecs[0].height, res=32) maskname = os.path.join(self.output_dir, 'mask.png') cv2.imwrite(maskname, self.mask) self.logger.info("wrote:\n %s" % maskname) res = {} res['input'] = {} res['input']['template'] = os.path.abspath(self.matchfile) res['input']['metafile'] = os.path.abspath(self.metafile) res['output'] = {} res['output']['resolved_tiles'] = j.pop('resolved_tiles') res['output']['mask'] = os.path.abspath(maskname) res['output']['collection'] = os.path.abspath(collection_path) res['residual stats'] = j self.args['output_json'] = self.solver.args['output_json'] with open(self.args['output_json'], 'w') as f: json.dump(res, f, indent=2)
def raw_lens_matches_1(render): matches = jsongz.load(RAW_LENS_MATCHES_1) collection = 'raw_lens_matches_1' renderapi.pointmatch.import_matches(collection, matches, render=render) yield collection renderapi.pointmatch.delete_collection(collection, render=render)
def run(self): if 'tilespecs' in self.args: jspecs = self.args['tilespecs'] else: jspecs = jsongz.load(self.args['tilespec_file']) self.tilespecs = np.array( [renderapi.tilespec.TileSpec(json=j) for j in jspecs]) if 'matches' in self.args: self.matches = self.args['matches'] else: self.matches = jsongz.load(self.args['match_file']) remove_weighted_matches(self.matches, weight=0.0) self.tile_width = self.tilespecs[0].width self.tile_height = self.tilespecs[0].height maskUrl = self.tilespecs[0].ip[0].maskUrl # condense coordinates self.coords = condense_coords(self.matches) nc0 = self.coords.shape[0] self.coords = smooth_density(self.coords, self.tile_width, self.tile_height, 10) nc1 = self.coords.shape[0] self.logger.info( "\n smoothing point density reduced points from %d to %d" % (nc0, nc1)) if self.coords.shape[0] == 0: raise MeshLensCorrectionException( "no point matches left after smoothing density, \ probably some sparse areas of matching") # create PSLG self.bbox = create_PSLG(self.tile_width, self.tile_height, maskUrl) # find delaunay with max vertices self.mesh, self.area_triangle_par = \ find_delaunay_with_max_vertices( self.bbox, self.args['nvertex']) # and enforce neighboring matches to vertices self.mesh, self.area_triangle_par = \ force_vertices_with_npoints( self.area_triangle_par, self.bbox, self.coords, 3) nend = self.mesh.points.shape[0] self.logger = logging.getLogger(self.__class__.__name__) self.logger.info("\n aimed for %d mesh points, got %d" % (self.args['nvertex'], nend)) if self.mesh.points.shape[0] < 0.5 * self.args['nvertex']: raise MeshLensCorrectionException("mesh coarser than intended") # prepare the linear algebra and solve self.A, self.weights, self.b, self.lens_dof_start = \ create_A( self.matches, self.tilespecs, self.mesh) self.x0 = create_x0(self.A.shape[1], self.tilespecs) self.reg = create_regularization( self.A.shape[1], len(self.tilespecs), self.args['regularization']['default_lambda'], self.args['regularization']['translation_factor'], self.args['regularization']['lens_lambda']) self.solution, self.errx, self.erry = solve(self.A, self.weights, self.reg, self.x0, self.b) self.transforms = create_transforms(len(self.tilespecs), self.solution) tf_trans, jresult, self.solve_message = report_solution( self.errx, self.erry, self.transforms, self.args['good_solve']) self.logger.info(self.solve_message) # check quality of solution if not all([ self.errx.mean() < self.args['good_solve']['error_mean'], self.erry.mean() < self.args['good_solve']['error_mean'], self.errx.std() < self.args['good_solve']['error_std'], self.erry.std() < self.args['good_solve']['error_std'] ]): raise MeshLensCorrectionException("Solve not good: %s" % self.solve_message) self.logger.debug(self.solve_message) self.new_ref_transform = create_thinplatespline_tf( self.args, self.mesh, self.solution, self.lens_dof_start, self.logger) bbox = self.tilespecs[0].bbox_transformed(tf_limit=0) tbbox = self.new_ref_transform.tform(bbox) bstr = 'new transform corners:\n' for i in range(bbox.shape[0] - 1): bstr += " (%0.1f, %0.1f) -> (%0.1f, %0.1f)\n" % ( bbox[i, 0], bbox[i, 1], tbbox[i, 0], tbbox[i, 1]) self.logger.info(bstr) new_tilespecs = new_specs_with_tf(self.new_ref_transform, self.tilespecs, self.transforms) stage_affine = estimate_stage_affine(self.tilespecs, new_tilespecs) sastr = "affine estimate of tile translations:\n" sastr += " scale: {}\n".format(stage_affine.scale) sastr += " translation: {}\n".format(stage_affine.translation) sastr += " shear: {}\n".format(stage_affine.shear) sastr += " rotation: {}\n".format(np.degrees(stage_affine.rotation)) self.logger.info(sastr) self.resolved = renderapi.resolvedtiles.ResolvedTiles( tilespecs=new_tilespecs, transformList=[self.new_ref_transform]) new_path = None if 'outfile' in self.args: fname = self.args['outfile'] if self.args['timestamp']: spf = fname.split(os.extsep, 1) spf[0] += '_%s' % self.new_ref_transform.transformId fname = os.extsep.join(spf) new_path = jsongz.dump(self.resolved.to_dict(), os.path.join(self.args['output_dir'], fname), compress=self.args['compress_output']) new_path = os.path.abspath(new_path) fname = 'output.json' if self.args['timestamp']: fname = 'output_%s.json' % self.new_ref_transform.transformId self.args['output_json'] = os.path.join(self.args['output_dir'], fname) jresult['resolved_tiles'] = new_path self.output(jresult, indent=2) self.logger.info(" wrote solved tilespecs:\n %s" % new_path) return
def run(self): self.args['sectionId'] = self.get_sectionId_from_metafile_uri( self.args['metafile_uri']) if self.args['output_dir'] is None: self.args['output_dir'] = tempfile.mkdtemp() if self.args['outfile'] is None: outfile = tempfile.NamedTemporaryFile(suffix=".json", delete=False, dir=self.args['output_dir']) outfile.close() self.args['outfile'] = outfile.name out_file = tempfile.NamedTemporaryFile(suffix=".json", delete=False) out_file.close() args_for_input = dict(self.args) metafile = json.loads( uri_utils.uri_readbytes(self.args['metafile_uri'])) self.maskUrl = make_mask( self.args['mask_dir'], metafile[0]['metadata']['camera_info']['width'], metafile[0]['metadata']['camera_info']['height'], self.args['mask_coords'], mask_file=self.args['mask_file'], basename=uri_utils.uri_basename(self.args['metafile_uri']) + '.png') # argschema doesn't like the NumpyArray after processing it once # we don't need it after mask creation self.args['mask_coords'] = None args_for_input['mask_coords'] = None # create a stack with the lens correction tiles ts_example = self.generate_ts_example() mod = GenerateEMTileSpecsModule(input_data=ts_example, args=['--output_json', out_file.name]) mod.run() if self.args['rerun_pointmatch']: # generate tile pairs for this section in the input stack tp_example = self.generate_tilepair_example() tp_mod = TilePairClientModule( input_data=tp_example, args=['--output_json', out_file.name]) tp_mod.run() with open(tp_mod.args['output_json'], 'r') as f: js = json.load(f) self.args['pairJson'] = js['tile_pair_file'] self.logger.setLevel(self.args['log_level']) delete_matches_if_exist(self.render, self.args['render']['owner'], self.args['match_collection'], self.args['sectionId']) args_for_pm = self.get_pm_args() pmgen = GeneratePointMatchesOpenCV( input_data=args_for_pm, args=['--output_json', out_file.name]) pmgen.run() # load these up in memory to pass to actual solver rawtilespecs = renderapi.tilespec.get_tile_specs_from_z( ts_example['output_stack'], ts_example['z'], render=renderapi.connect(**ts_example['render'])) rawtdict = [t.to_dict() for t in rawtilespecs] matches = renderapi.pointmatch.get_matches_within_group( self.args['match_collection'], self.args['sectionId'], render=renderapi.connect(**self.args['render'])) self.logger.setLevel(self.args['log_level']) solver_args = { 'nvertex': self.args['nvertex'], 'regularization': self.args['regularization'], 'good_solve': self.args['good_solve'], 'tilespecs': rawtdict, 'matches': matches, 'output_dir': self.args['output_dir'], 'outfile': 'resolvedtiles.json.gz', 'compress_output': False, 'log_level': self.args['log_level'], 'timestamp': True } meshclass = MeshAndSolveTransform(input_data=solver_args, args=[]) # find the lens correction, write out to new stack meshclass.run() with open(meshclass.args['output_json'], 'r') as f: jout = json.load(f) resolved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(jout['resolved_tiles'])) tform_out = os.path.join(self.args['output_dir'], 'lens_correction_out.json') with open(tform_out, 'w') as f: json.dump(resolved.transforms[0].to_dict(), f) try: self.output({'output_json': tform_out, 'maskUrl': self.maskUrl}) except AttributeError as e: self.logger.error(e)