def test_affine_on_similarity( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = 'sim_out' rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'SimilarityModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() del mod rough_parameters2['input_stack']['name'] = 'sim_out' rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['transformation'] = 'AffineModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() renderapi.stack.delete_stack( rough_parameters2['input_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) del mod
def test_rough_similarity_explicit_depth( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'SimilarityModel' rough_parameters2['matrix_assembly']['depth'] = [0, 1, 2] rough_parameters2['matrix_assembly']['explicit_weight_by_depth'] = \ [0, 0.5, 0.33] mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['output_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) assert len(tin) == len(tout) del mod with pytest.raises(ValidationError): rough_parameters2['matrix_assembly']['depth'] = [0, 1] # not the same length as weights bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[])
def test_apply_list( render, rough_pointmatches, rough_input_stack_3, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack_3 rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'RotationModel' rough_parameters2['transform_apply'] = [0, 1] mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['output_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-10) assert len(tin) == len(tout) rough_parameters2['transform_apply'] = [4] with pytest.raises(IndexError): mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() del mod
def test_output_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = raw_stack p['pointmatch']['name'] = montage_pointmatches tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) p['output_stack']['db_interface'] = 'file' p['output_stack']['output_file'] = os.path.join(tmp_file_dir, "resolvedtiles.json") p['output_stack']['compress_output'] = compress tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() solved = renderapi.resolvedtiles.ResolvedTiles( json=jsongz.load(tmod.args['output_stack']['output_file'])) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in solved.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def test_validation(raw_stack, montage_pointmatches): p = copy.deepcopy(montage_parameters) p['input_stack']['db_interface'] = 'file' p['input_stack']['input_file'] = None p['output_mode'] = 'none' p['pointmatch']['name'] = montage_pointmatches with pytest.raises(ValidationError): tmod = bigfeta.BigFeta(input_data=p, args=[]) del tmod p['input_stack']['db_interface'] = 'mongo' p['input_stack']['name'] = raw_stack p['output_stack']['db_interface'] = 'file' p['output_stack']['output_file'] = None with pytest.raises(ValidationError): tmod = bigfeta.BigFeta(input_data=p, args=[]) del tmod
def test_hdf5_mode_similarity(render, rough_input_stack, rough_pointmatches, tmpdir, chunks): # general parameters parameters = copy.deepcopy(rough_parameters) parameters['hdf5_options']['output_dir'] = str(tmpdir.mkdir('hdf5output')) parameters['input_stack']['name'] = rough_input_stack parameters['pointmatch']['name'] = rough_pointmatches parameters['output_mode'] = 'hdf5' parameters['hdf5_options']['chunks_per_file'] = chunks # check output mode HDF5 mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[]) mod.run() indexfile = os.path.join(parameters['hdf5_options']['output_dir'], 'solution_input.h5') assert os.path.exists(indexfile) del mod # check assemble from file parameters['output_mode'] = 'none' parameters['assemble_from_file'] = indexfile mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) del mod # delete the output stack, just in case renderapi.stack.delete_stack(parameters['output_stack']['name'], render=render) parameters['ingest_from_file'] = indexfile parameters['output_mode'] = 'stack' mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( parameters['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( parameters['output_stack']['name'], render=render) assert len(tin) == len(tout) os.remove(indexfile) del mod renderapi.stack.delete_stack(parameters['output_stack']['name'], render=render)
def solved_montage(render, raw_stack, montage_pointmatches): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = raw_stack p['output_stack']['name'] = 'solver_output_stack' p['pointmatch']['name'] = montage_pointmatches mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() yield mod renderapi.stack.delete_stack('output_stack_name', render=render)
def test_multi_pm(render, split_montage_pointmatches, loading_raw_stack, tmpdir, output_stack_name): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = loading_raw_stack p['output_stack']['name'] = output_stack_name p['pointmatch']['name'] = split_montage_pointmatches mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 200) del mod
def test_poly_validation(output_stack_name): p = copy.deepcopy(montage_parameters) p['regularization'] = { 'default_lambda': 1000.0, 'translation_factor': 1e-5, 'poly_factors': [1e-5, 1000.0, 1e6, 1e3] } p['output_stack']['name'] = output_stack_name p['transformation'] = 'Polynomial2DTransform' p['poly_order'] = 2 with pytest.raises(ValidationError): # because poly_factors should be length 3 bigfeta.BigFeta(input_data=p, args=[])
def test_stack_state(render, montage_pointmatches, output_stack_name, loading_raw_stack, stack_state, tmpdir): renderapi.stack.set_stack_state(loading_raw_stack, stack_state, render=render) p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = loading_raw_stack p['output_stack']['name'] = output_stack_name p['pointmatch']['name'] = montage_pointmatches mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 200) del mod
def test_different_transforms(render, montage_pointmatches, loading_raw_stack, transform, fullsize, output_stack_name, order): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = loading_raw_stack p['output_stack']['name'] = output_stack_name p['pointmatch']['name'] = montage_pointmatches p['transformation'] = transform p['fullsize'] = fullsize p['poly_order'] = order mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 200) del mod
def test_multi_profile_exception( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'SimilarityModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) with pytest.raises(utils.BigFetaException): mod.args['profile_data_load'] = True mod.run() del mod
def test_multi_stack_name_exception( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'SimilarityModel' with pytest.raises(ValidationError): rough_parameters2['input_stack']['name'] = [ rough_parameters2['input_stack']['name']] * 2 # stacks should only have 1 name (so far) bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[])
def test_basic(render, montage_pointmatches, output_stack_name, loading_raw_stack, db_intfc): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = loading_raw_stack p['output_stack']['name'] = output_stack_name p['pointmatch']['name'] = montage_pointmatches p['transformation'] = 'AffineModel' p['fullsize_transform'] = True p['input_stack']['db_interface'] = db_intfc p['output_stack']['db_interface'] = 'render' p['pointmatch']['db_interface'] = db_intfc mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 200) del mod
def test_match_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = raw_stack p['pointmatch']['name'] = montage_pointmatches # get the matches and write them to a file sectionData = renderapi.stack.get_stack_sectionData( p['input_stack']['name'], render=render) sections = [sd['sectionId'] for sd in sectionData] matches = [] for s in sections: matches += renderapi.pointmatch.get_matches_with_group( p['pointmatch']['name'], s, render=render) tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) match_file = os.path.join(tmp_file_dir, "matches.json") match_file = jsongz.dump(matches, match_file, compress=compress) p['pointmatch']['db_interface'] = 'file' p['pointmatch']['input_file'] = match_file p['output_mode'] = 'none' tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() for k in ['precision', 'error', 'err']: assert np.all( np.isclose(np.array(tmod.results[k]), np.array(solved_montage.results[k]), atol=1e-7)) assert np.all( np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0), np.linalg.norm(tmod.results['x'], axis=0), atol=1e-7)) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in tmod.resolvedtiles.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def test_thinplate(render, montage_pointmatches, loading_raw_stack, output_stack_name): p = copy.deepcopy(montage_parameters) p['input_stack']['name'] = loading_raw_stack p['output_stack']['name'] = output_stack_name p['pointmatch']['name'] = montage_pointmatches p['transformation'] = 'ThinPlateSplineTransform' p['regularization'] = { 'default_lambda': 1000.0, 'translation_factor': 1e-5, 'thinplate_factor': 1e-5 } mod = bigfeta.BigFeta(input_data=p, args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-4) assert np.all(np.array(mod.results['error']) < 200) del mod
def test_output_mode_none( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'AffineModel' rough_parameters2['output_mode'] = 'none' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert output_stack_name not in stacks del mod
def do_solve(template_path, args, index): with open(template_path, 'r') as f: template = json.load(f) template['input_stack']['input_file'] = \ args['input_stack']['input_file'] template['pointmatch']['input_file'] = \ args['pointmatch']['input_file'] template['output_stack']['compress_output'] = \ args['output_stack']['compress_output'] template['first_section'] = template['last_section'] = \ args['first_section'] fname = os.path.join( os.path.dirname(args['input_stack']['input_file']), 'resolvedtiles_%s_%d.json' % (template['transformation'], index)) template['output_stack']['output_file'] = fname template['fullsize_transform'] = False aligner = bfa.BigFeta(input_data=template, args=[]) aligner.run() # these numbers only meaningful for fullsize_transform = False # to get results already in memory, on-scope, let's keep it that way # otherwise, we'll need a separate calculation that loads tilespecs # and matches to calculate residuals, costing more time res = { 'output': os.path.basename(aligner.args['output_stack']['output_file']), 'collection': os.path.basename(aligner.args['pointmatch']['input_file']), 'x': { 'mean': aligner.results['err'][0][0], 'stdev': aligner.results['err'][0][1] }, 'y': { 'mean': aligner.results['err'][1][0], 'stdev': aligner.results['err'][1][1] }, 'mag': { 'mean': aligner.results['mag'][0], 'stdev': aligner.results['mag'][1] } } return res
def test_input_stack_file(render, raw_stack, montage_pointmatches, tmpdir, solved_montage, compress): p = copy.deepcopy(montage_parameters) resolved = renderapi.resolvedtiles.get_resolved_tiles_from_z( raw_stack, p['first_section'], render=render) tmp_file_dir = str(tmpdir.mkdir('file_test_dir')) input_stack_file = os.path.join(tmp_file_dir, "input_stack.json") input_stack_file = jsongz.dump(resolved.to_dict(), input_stack_file, compress=compress) p['input_stack']['db_interface'] = 'file' p['input_stack']['input_file'] = input_stack_file p['output_mode'] = 'none' p['pointmatch']['name'] = montage_pointmatches tmod = bigfeta.BigFeta(input_data=p, args=[]) tmod.run() for k in ['precision', 'error', 'err']: assert np.all( np.isclose(np.array(tmod.results[k]), np.array(solved_montage.results[k]), atol=1e-7)) assert np.all( np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0), np.linalg.norm(tmod.results['x'], axis=0), atol=1e-7)) orig_ids = np.array( [t.tileId for t in solved_montage.resolvedtiles.tilespecs]) for t in tmod.resolvedtiles.tilespecs: i = np.argwhere(orig_ids == t.tileId).flatten()[0] assert np.all( np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M, t.tforms[-1].M, atol=1e-7)) del tmod shutil.rmtree(tmp_file_dir)
def test_rough_rotation( render, rough_pointmatches, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches rough_parameters2['transformation'] = 'RotationModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['output_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-10) assert len(tin) == len(tout) del mod
def test_rough_similarity_split( render, split_rough_pointmatches, rough_input_stack_2, output_stack_name, pm_db_intfc): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack_2 rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = split_rough_pointmatches rough_parameters2['pointmatch']['db_interface'] = pm_db_intfc rough_parameters2['transformation'] = 'SimilarityModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['output_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) assert len(tin) == len(tout) del mod
def test_missing_match( # this case was covered elsewhere, but one more test render, rough_pointmatches_missing, rough_input_stack, output_stack_name): rough_parameters2 = copy.deepcopy(rough_parameters) rough_parameters2['input_stack']['name'] = rough_input_stack rough_parameters2['output_stack']['name'] = output_stack_name rough_parameters2['pointmatch']['name'] = rough_pointmatches_missing rough_parameters2['transformation'] = 'SimilarityModel' mod = bigfeta.BigFeta( input_data=copy.deepcopy(rough_parameters2), args=[]) mod.run() tin = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['input_stack']['name'], render=render) tout = renderapi.tilespec.get_tile_specs_from_stack( rough_parameters2['output_stack']['name'], render=render) assert np.all(np.array(mod.results['precision']) < 1e-7) assert np.all(np.array(mod.results['error']) < 1e6) assert len(tin) == len(tout) del mod
def run(self): self.module = bigfeta.BigFeta(input_data=self.args, args=[]) self.module.run() self.output( {"stack": self.args['output_stack']['name'][0]})
def test_petsc_solver(render, rough_input_stack, rough_pointmatches, output_directory, transform): # setup the solver to write out hdf5 files parameters = copy.deepcopy(rough_parameters) parameters['hdf5_options']['output_dir'] = output_directory parameters['input_stack']['name'] = rough_input_stack parameters['pointmatch']['name'] = rough_pointmatches parameters['output_mode'] = 'hdf5' parameters['hdf5_options']['chunks_per_file'] = 10 parameters['transformation'] = transform if transform == 'AffineModel': parameters['regularization'] = { "default_lamda": 1e3, "translation_factor": 1.0e-10 } mod = bigfeta.BigFeta(input_data=parameters, args=[]) mod.run() indexfile = os.path.join(parameters['hdf5_options']['output_dir'], 'solution_input.h5') assert os.path.isfile(indexfile) # call the petsc solver outfile = os.path.join(parameters['hdf5_options']['output_dir'], 'solution_output.h5') # /tmp is automatically bound cmd = ['singularity', 'run', './bigfeta/distributed/bin/petsc_solver.simf'] cmd += ['-input', indexfile] cmd += ['-output', outfile] # this is a direct PaStiX solve cmd += ['-ksp_type', 'preonly', '-pc_type', 'lu'] subprocess.call(cmd) assert os.path.isfile(outfile) # read the petsc result back into render parameters['ingest_from_file'] = outfile parameters['output_mode'] = 'stack' parameters['output_stack']['name'] = 'from_petsc' mod = bigfeta.BigFeta(input_data=parameters, args=[]) mod.run() petsc_solved = renderapi.tilespec.get_tile_specs_from_stack( parameters['output_stack']['name'][0], render=render) # have scipy solve the same thing parameters['ingest_from_file'] = '' parameters['output_mode'] = 'stack' parameters['output_stack']['name'] = 'from_scipy' smod = bigfeta.BigFeta(input_data=parameters, args=[]) smod.run() scipy_solved = renderapi.tilespec.get_tile_specs_from_stack( parameters['output_stack']['name'][0], render=render) assert len(petsc_solved) == len(scipy_solved) assert np.all( np.isclose(mod.results['precision'], smod.results['precision'], rtol=10.0, atol=1e-10)) assert np.all( np.isclose(mod.results['error'], smod.results['error'], rtol=0.01, atol=1.0)) assert np.all( np.isclose(mod.results['err'], smod.results['err'], rtol=0.01, atol=0.2)) ptids = np.array([t.tileId for t in petsc_solved]) stids = np.array([t.tileId for t in scipy_solved]) in1d = np.intersect1d(ptids, stids) for tid in in1d: p = np.argwhere(ptids == tid).flatten()[0] s = np.argwhere(stids == tid).flatten()[0] ptf = petsc_solved[p].tforms[-1] stf = scipy_solved[s].tforms[-1] assert np.isclose(ptf.rotation, stf.rotation, rtol=10.0, atol=0.05) assert np.all(np.isclose(ptf.scale, stf.scale, rtol=0.001, atol=0.001)) assert np.isclose(ptf.shear, stf.shear, rtol=0.1, atol=0.001) assert np.all( np.isclose(ptf.translation, stf.translation, rtol=0.1, atol=0.5))