def test_affine_on_similarity(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = 'sim_out'
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'SimilarityModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
del mod
rough_parameters2['input_stack']['name'] = 'sim_out'
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['transformation'] = 'AffineModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
renderapi.stack.delete_stack(
rough_parameters2['input_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
del mod
def test_rough_similarity_explicit_depth(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'SimilarityModel'
rough_parameters2['matrix_assembly']['depth'] = [0, 1, 2]
rough_parameters2['matrix_assembly']['explicit_weight_by_depth'] = \
[0, 0.5, 0.33]
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['output_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
assert len(tin) == len(tout)
del mod
with pytest.raises(ValidationError):
rough_parameters2['matrix_assembly']['depth'] = [0, 1]
# not the same length as weights
bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
def test_apply_list(
render,
rough_pointmatches,
rough_input_stack_3,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack_3
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'RotationModel'
rough_parameters2['transform_apply'] = [0, 1]
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['output_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-10)
assert len(tin) == len(tout)
rough_parameters2['transform_apply'] = [4]
with pytest.raises(IndexError):
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
del mod
def test_output_file(render, raw_stack, montage_pointmatches, tmpdir,
solved_montage, compress):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = raw_stack
p['pointmatch']['name'] = montage_pointmatches
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
p['output_stack']['db_interface'] = 'file'
p['output_stack']['output_file'] = os.path.join(tmp_file_dir,
"resolvedtiles.json")
p['output_stack']['compress_output'] = compress
tmod = bigfeta.BigFeta(input_data=p, args=[])
tmod.run()
solved = renderapi.resolvedtiles.ResolvedTiles(
json=jsongz.load(tmod.args['output_stack']['output_file']))
orig_ids = np.array(
[t.tileId for t in solved_montage.resolvedtiles.tilespecs])
for t in solved.tilespecs:
i = np.argwhere(orig_ids == t.tileId).flatten()[0]
assert np.all(
np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M,
t.tforms[-1].M,
atol=1e-7))
del tmod
shutil.rmtree(tmp_file_dir)
def test_validation(raw_stack, montage_pointmatches):
p = copy.deepcopy(montage_parameters)
p['input_stack']['db_interface'] = 'file'
p['input_stack']['input_file'] = None
p['output_mode'] = 'none'
p['pointmatch']['name'] = montage_pointmatches
with pytest.raises(ValidationError):
tmod = bigfeta.BigFeta(input_data=p, args=[])
del tmod
p['input_stack']['db_interface'] = 'mongo'
p['input_stack']['name'] = raw_stack
p['output_stack']['db_interface'] = 'file'
p['output_stack']['output_file'] = None
with pytest.raises(ValidationError):
tmod = bigfeta.BigFeta(input_data=p, args=[])
del tmod
def test_hdf5_mode_similarity(render, rough_input_stack, rough_pointmatches,
tmpdir, chunks):
# general parameters
parameters = copy.deepcopy(rough_parameters)
parameters['hdf5_options']['output_dir'] = str(tmpdir.mkdir('hdf5output'))
parameters['input_stack']['name'] = rough_input_stack
parameters['pointmatch']['name'] = rough_pointmatches
parameters['output_mode'] = 'hdf5'
parameters['hdf5_options']['chunks_per_file'] = chunks
# check output mode HDF5
mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[])
mod.run()
indexfile = os.path.join(parameters['hdf5_options']['output_dir'],
'solution_input.h5')
assert os.path.exists(indexfile)
del mod
# check assemble from file
parameters['output_mode'] = 'none'
parameters['assemble_from_file'] = indexfile
mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
del mod
# delete the output stack, just in case
renderapi.stack.delete_stack(parameters['output_stack']['name'],
render=render)
parameters['ingest_from_file'] = indexfile
parameters['output_mode'] = 'stack'
mod = bigfeta.BigFeta(input_data=copy.deepcopy(parameters), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
parameters['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
parameters['output_stack']['name'], render=render)
assert len(tin) == len(tout)
os.remove(indexfile)
del mod
renderapi.stack.delete_stack(parameters['output_stack']['name'],
render=render)
def solved_montage(render, raw_stack, montage_pointmatches):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = raw_stack
p['output_stack']['name'] = 'solver_output_stack'
p['pointmatch']['name'] = montage_pointmatches
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
yield mod
renderapi.stack.delete_stack('output_stack_name', render=render)
def test_multi_pm(render, split_montage_pointmatches, loading_raw_stack,
tmpdir, output_stack_name):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = loading_raw_stack
p['output_stack']['name'] = output_stack_name
p['pointmatch']['name'] = split_montage_pointmatches
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 200)
del mod
def test_poly_validation(output_stack_name):
p = copy.deepcopy(montage_parameters)
p['regularization'] = {
'default_lambda': 1000.0,
'translation_factor': 1e-5,
'poly_factors': [1e-5, 1000.0, 1e6, 1e3]
}
p['output_stack']['name'] = output_stack_name
p['transformation'] = 'Polynomial2DTransform'
p['poly_order'] = 2
with pytest.raises(ValidationError):
# because poly_factors should be length 3
bigfeta.BigFeta(input_data=p, args=[])
def test_stack_state(render, montage_pointmatches, output_stack_name,
loading_raw_stack, stack_state, tmpdir):
renderapi.stack.set_stack_state(loading_raw_stack,
stack_state,
render=render)
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = loading_raw_stack
p['output_stack']['name'] = output_stack_name
p['pointmatch']['name'] = montage_pointmatches
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 200)
del mod
def test_different_transforms(render, montage_pointmatches, loading_raw_stack,
transform, fullsize, output_stack_name, order):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = loading_raw_stack
p['output_stack']['name'] = output_stack_name
p['pointmatch']['name'] = montage_pointmatches
p['transformation'] = transform
p['fullsize'] = fullsize
p['poly_order'] = order
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 200)
del mod
def test_multi_profile_exception(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'SimilarityModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
with pytest.raises(utils.BigFetaException):
mod.args['profile_data_load'] = True
mod.run()
del mod
def test_multi_stack_name_exception(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'SimilarityModel'
with pytest.raises(ValidationError):
rough_parameters2['input_stack']['name'] = [
rough_parameters2['input_stack']['name']] * 2
# stacks should only have 1 name (so far)
bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
def test_basic(render, montage_pointmatches, output_stack_name,
loading_raw_stack, db_intfc):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = loading_raw_stack
p['output_stack']['name'] = output_stack_name
p['pointmatch']['name'] = montage_pointmatches
p['transformation'] = 'AffineModel'
p['fullsize_transform'] = True
p['input_stack']['db_interface'] = db_intfc
p['output_stack']['db_interface'] = 'render'
p['pointmatch']['db_interface'] = db_intfc
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 200)
del mod
def test_match_file(render, raw_stack, montage_pointmatches, tmpdir,
solved_montage, compress):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = raw_stack
p['pointmatch']['name'] = montage_pointmatches
# get the matches and write them to a file
sectionData = renderapi.stack.get_stack_sectionData(
p['input_stack']['name'], render=render)
sections = [sd['sectionId'] for sd in sectionData]
matches = []
for s in sections:
matches += renderapi.pointmatch.get_matches_with_group(
p['pointmatch']['name'], s, render=render)
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
match_file = os.path.join(tmp_file_dir, "matches.json")
match_file = jsongz.dump(matches, match_file, compress=compress)
p['pointmatch']['db_interface'] = 'file'
p['pointmatch']['input_file'] = match_file
p['output_mode'] = 'none'
tmod = bigfeta.BigFeta(input_data=p, args=[])
tmod.run()
for k in ['precision', 'error', 'err']:
assert np.all(
np.isclose(np.array(tmod.results[k]),
np.array(solved_montage.results[k]),
atol=1e-7))
assert np.all(
np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0),
np.linalg.norm(tmod.results['x'], axis=0),
atol=1e-7))
orig_ids = np.array(
[t.tileId for t in solved_montage.resolvedtiles.tilespecs])
for t in tmod.resolvedtiles.tilespecs:
i = np.argwhere(orig_ids == t.tileId).flatten()[0]
assert np.all(
np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M,
t.tforms[-1].M,
atol=1e-7))
del tmod
shutil.rmtree(tmp_file_dir)
def test_thinplate(render, montage_pointmatches, loading_raw_stack,
output_stack_name):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = loading_raw_stack
p['output_stack']['name'] = output_stack_name
p['pointmatch']['name'] = montage_pointmatches
p['transformation'] = 'ThinPlateSplineTransform'
p['regularization'] = {
'default_lambda': 1000.0,
'translation_factor': 1e-5,
'thinplate_factor': 1e-5
}
mod = bigfeta.BigFeta(input_data=p, args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-4)
assert np.all(np.array(mod.results['error']) < 200)
del mod
def test_output_mode_none(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'AffineModel'
rough_parameters2['output_mode'] = 'none'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
stacks = renderapi.render.get_stacks_by_owner_project(render=render)
assert output_stack_name not in stacks
del mod
def do_solve(template_path, args, index):
with open(template_path, 'r') as f:
template = json.load(f)
template['input_stack']['input_file'] = \
args['input_stack']['input_file']
template['pointmatch']['input_file'] = \
args['pointmatch']['input_file']
template['output_stack']['compress_output'] = \
args['output_stack']['compress_output']
template['first_section'] = template['last_section'] = \
args['first_section']
fname = os.path.join(
os.path.dirname(args['input_stack']['input_file']),
'resolvedtiles_%s_%d.json' % (template['transformation'], index))
template['output_stack']['output_file'] = fname
template['fullsize_transform'] = False
aligner = bfa.BigFeta(input_data=template, args=[])
aligner.run()
# these numbers only meaningful for fullsize_transform = False
# to get results already in memory, on-scope, let's keep it that way
# otherwise, we'll need a separate calculation that loads tilespecs
# and matches to calculate residuals, costing more time
res = {
'output':
os.path.basename(aligner.args['output_stack']['output_file']),
'collection':
os.path.basename(aligner.args['pointmatch']['input_file']),
'x': {
'mean': aligner.results['err'][0][0],
'stdev': aligner.results['err'][0][1]
},
'y': {
'mean': aligner.results['err'][1][0],
'stdev': aligner.results['err'][1][1]
},
'mag': {
'mean': aligner.results['mag'][0],
'stdev': aligner.results['mag'][1]
}
}
return res
def test_input_stack_file(render, raw_stack, montage_pointmatches, tmpdir,
solved_montage, compress):
p = copy.deepcopy(montage_parameters)
resolved = renderapi.resolvedtiles.get_resolved_tiles_from_z(
raw_stack, p['first_section'], render=render)
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
input_stack_file = os.path.join(tmp_file_dir, "input_stack.json")
input_stack_file = jsongz.dump(resolved.to_dict(),
input_stack_file,
compress=compress)
p['input_stack']['db_interface'] = 'file'
p['input_stack']['input_file'] = input_stack_file
p['output_mode'] = 'none'
p['pointmatch']['name'] = montage_pointmatches
tmod = bigfeta.BigFeta(input_data=p, args=[])
tmod.run()
for k in ['precision', 'error', 'err']:
assert np.all(
np.isclose(np.array(tmod.results[k]),
np.array(solved_montage.results[k]),
atol=1e-7))
assert np.all(
np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0),
np.linalg.norm(tmod.results['x'], axis=0),
atol=1e-7))
orig_ids = np.array(
[t.tileId for t in solved_montage.resolvedtiles.tilespecs])
for t in tmod.resolvedtiles.tilespecs:
i = np.argwhere(orig_ids == t.tileId).flatten()[0]
assert np.all(
np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M,
t.tforms[-1].M,
atol=1e-7))
del tmod
shutil.rmtree(tmp_file_dir)
def test_rough_rotation(
render,
rough_pointmatches,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches
rough_parameters2['transformation'] = 'RotationModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['output_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-10)
assert len(tin) == len(tout)
del mod
def test_rough_similarity_split(
render,
split_rough_pointmatches,
rough_input_stack_2,
output_stack_name,
pm_db_intfc):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack_2
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = split_rough_pointmatches
rough_parameters2['pointmatch']['db_interface'] = pm_db_intfc
rough_parameters2['transformation'] = 'SimilarityModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['output_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
assert len(tin) == len(tout)
del mod
def test_missing_match(
# this case was covered elsewhere, but one more test
render,
rough_pointmatches_missing,
rough_input_stack,
output_stack_name):
rough_parameters2 = copy.deepcopy(rough_parameters)
rough_parameters2['input_stack']['name'] = rough_input_stack
rough_parameters2['output_stack']['name'] = output_stack_name
rough_parameters2['pointmatch']['name'] = rough_pointmatches_missing
rough_parameters2['transformation'] = 'SimilarityModel'
mod = bigfeta.BigFeta(
input_data=copy.deepcopy(rough_parameters2), args=[])
mod.run()
tin = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['input_stack']['name'], render=render)
tout = renderapi.tilespec.get_tile_specs_from_stack(
rough_parameters2['output_stack']['name'], render=render)
assert np.all(np.array(mod.results['precision']) < 1e-7)
assert np.all(np.array(mod.results['error']) < 1e6)
assert len(tin) == len(tout)
del mod
def run(self):
self.module = bigfeta.BigFeta(input_data=self.args, args=[])
self.module.run()
self.output(
{"stack": self.args['output_stack']['name'][0]})
def test_petsc_solver(render, rough_input_stack, rough_pointmatches,
output_directory, transform):
# setup the solver to write out hdf5 files
parameters = copy.deepcopy(rough_parameters)
parameters['hdf5_options']['output_dir'] = output_directory
parameters['input_stack']['name'] = rough_input_stack
parameters['pointmatch']['name'] = rough_pointmatches
parameters['output_mode'] = 'hdf5'
parameters['hdf5_options']['chunks_per_file'] = 10
parameters['transformation'] = transform
if transform == 'AffineModel':
parameters['regularization'] = {
"default_lamda": 1e3,
"translation_factor": 1.0e-10
}
mod = bigfeta.BigFeta(input_data=parameters, args=[])
mod.run()
indexfile = os.path.join(parameters['hdf5_options']['output_dir'],
'solution_input.h5')
assert os.path.isfile(indexfile)
# call the petsc solver
outfile = os.path.join(parameters['hdf5_options']['output_dir'],
'solution_output.h5')
# /tmp is automatically bound
cmd = ['singularity', 'run', './bigfeta/distributed/bin/petsc_solver.simf']
cmd += ['-input', indexfile]
cmd += ['-output', outfile]
# this is a direct PaStiX solve
cmd += ['-ksp_type', 'preonly', '-pc_type', 'lu']
subprocess.call(cmd)
assert os.path.isfile(outfile)
# read the petsc result back into render
parameters['ingest_from_file'] = outfile
parameters['output_mode'] = 'stack'
parameters['output_stack']['name'] = 'from_petsc'
mod = bigfeta.BigFeta(input_data=parameters, args=[])
mod.run()
petsc_solved = renderapi.tilespec.get_tile_specs_from_stack(
parameters['output_stack']['name'][0], render=render)
# have scipy solve the same thing
parameters['ingest_from_file'] = ''
parameters['output_mode'] = 'stack'
parameters['output_stack']['name'] = 'from_scipy'
smod = bigfeta.BigFeta(input_data=parameters, args=[])
smod.run()
scipy_solved = renderapi.tilespec.get_tile_specs_from_stack(
parameters['output_stack']['name'][0], render=render)
assert len(petsc_solved) == len(scipy_solved)
assert np.all(
np.isclose(mod.results['precision'],
smod.results['precision'],
rtol=10.0,
atol=1e-10))
assert np.all(
np.isclose(mod.results['error'],
smod.results['error'],
rtol=0.01,
atol=1.0))
assert np.all(
np.isclose(mod.results['err'],
smod.results['err'],
rtol=0.01,
atol=0.2))
ptids = np.array([t.tileId for t in petsc_solved])
stids = np.array([t.tileId for t in scipy_solved])
in1d = np.intersect1d(ptids, stids)
for tid in in1d:
p = np.argwhere(ptids == tid).flatten()[0]
s = np.argwhere(stids == tid).flatten()[0]
ptf = petsc_solved[p].tforms[-1]
stf = scipy_solved[s].tforms[-1]
assert np.isclose(ptf.rotation, stf.rotation, rtol=10.0, atol=0.05)
assert np.all(np.isclose(ptf.scale, stf.scale, rtol=0.001, atol=0.001))
assert np.isclose(ptf.shear, stf.shear, rtol=0.1, atol=0.001)
assert np.all(
np.isclose(ptf.translation, stf.translation, rtol=0.1, atol=0.5))
