def run(self):
logger.setLevel(self.args['log_level'])
resolved_path = os.path.join(self.args['client_mount_or_map'],
self.args['fdir'],
self.args['resolved_file'])
if self.args['backup_copy']:
backup(resolved_path)
resolved = renderapi.resolvedtiles.ResolvedTiles(
json=jsongz.load(resolved_path))
if self.args['image_directory'] is None:
self.args['image_directory'] = pathlib.PurePosixPath(
self.args['server_mount'], self.args['fdir'])
for t in resolved.tilespecs:
for k in ['imageUrl', 'maskUrl']:
s = t.ip['0'][k]
if s:
orig = urllib.parse.unquote(urllib.parse.urlparse(s).path)
t.ip['0'][k] = pathlib.PurePosixPath(
self.args['image_directory'],
os.path.basename(orig)).as_uri()
self.args['resolved_file'] = jsongz.dump(resolved.to_dict(),
resolved_path,
compress=None)
logger.info("updated tilespec urls in %s" % resolved_path)
def run(self):
self.resolved, self.new_ref_transform, jresult = (
self.solve_resolvedtiles_from_args())
new_path = None
if 'outfile' in self.args:
fname = self.args['outfile']
if self.args['timestamp']:
spf = fname.split(os.extsep, 1)
spf[0] += '_%s' % self.new_ref_transform.transformId
fname = os.extsep.join(spf)
new_path = jsongz.dump(self.resolved.to_dict(),
os.path.join(self.args['output_dir'],
fname),
compress=self.args['compress_output'])
new_path = os.path.abspath(new_path)
fname = 'output.json'
if self.args['timestamp']:
fname = 'output_%s.json' % self.new_ref_transform.transformId
self.args['output_json'] = os.path.join(self.args['output_dir'], fname)
jresult['resolved_tiles'] = new_path
self.output(jresult, indent=2)
self.logger.info(" wrote solved tilespecs:\n %s" % new_path)
return
def test_jsongz(tmpdir, compress, FILE):
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
with open(FILE, 'r') as f:
j = json.load(f)
tmpfile = os.path.join(tmp_file_dir, "tmp.json")
tmpfile = jsongz.dump(j, tmpfile, compress=compress)
newj = jsongz.load(tmpfile)
assert newj == j
shutil.rmtree(tmp_file_dir)
def run(self):
if 'metafile' not in self.args:
self.args['metafile'] = get_metafile_path(self.args['data_dir'])
else:
self.args['data_dir'] = os.path.dirname(self.args['metafile'])
if not self.args['output_dir']:
self.args['output_dir'] = self.args['data_dir']
# read the matches from the metafile
matches = meta_to_collection.main([self.args['data_dir']])
montage_filter_matches(matches, self.args['ransacReprojThreshold'])
# write to file
collection = os.path.join(self.args['output_dir'], "collection.json")
collection = jsongz.dump(matches,
collection,
compress=self.args['compress_output'])
# make raw tilespec json
rawspecpath, z = make_raw_tilespecs(self.args['metafile'],
self.args['output_dir'],
matches[0]['pGroupId'],
self.args['compress_output'])
# get the ref transform
tform = get_transform(self.args['metafile'],
self.args['ref_transform'],
self.args['ref_transform_dict'],
self.args['read_transform_from'])
# make a resolved tile object
input_stack_path = make_resolved(rawspecpath, tform,
self.args['output_dir'],
self.args['compress_output'])
templates = [
os.path.join(self.args['solver_template_dir'], t)
for t in self.args['solver_templates']
]
self.results = do_solves(collection, input_stack_path, z,
self.args['compress_output'], templates)
self.args['output_json'] = os.path.join(self.args['output_dir'],
'montage_results.json')
with open(self.args['output_json'], 'w') as f:
json.dump(self.results, f, indent=2)
def test_match_file(render, raw_stack, montage_pointmatches, tmpdir,
solved_montage, compress):
p = copy.deepcopy(montage_parameters)
p['input_stack']['name'] = raw_stack
p['pointmatch']['name'] = montage_pointmatches
# get the matches and write them to a file
sectionData = renderapi.stack.get_stack_sectionData(
p['input_stack']['name'], render=render)
sections = [sd['sectionId'] for sd in sectionData]
matches = []
for s in sections:
matches += renderapi.pointmatch.get_matches_with_group(
p['pointmatch']['name'], s, render=render)
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
match_file = os.path.join(tmp_file_dir, "matches.json")
match_file = jsongz.dump(matches, match_file, compress=compress)
p['pointmatch']['db_interface'] = 'file'
p['pointmatch']['input_file'] = match_file
p['output_mode'] = 'none'
tmod = bigfeta.BigFeta(input_data=p, args=[])
tmod.run()
for k in ['precision', 'error', 'err']:
assert np.all(
np.isclose(np.array(tmod.results[k]),
np.array(solved_montage.results[k]),
atol=1e-7))
assert np.all(
np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0),
np.linalg.norm(tmod.results['x'], axis=0),
atol=1e-7))
orig_ids = np.array(
[t.tileId for t in solved_montage.resolvedtiles.tilespecs])
for t in tmod.resolvedtiles.tilespecs:
i = np.argwhere(orig_ids == t.tileId).flatten()[0]
assert np.all(
np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M,
t.tforms[-1].M,
atol=1e-7))
del tmod
shutil.rmtree(tmp_file_dir)
def run(self):
with open(self.args['metafile'], 'r') as f:
meta = json.load(f)
roidata = meta[0]['metadata']
imgdata = meta[1]['data']
img_coords = {img['img_path']: self.image_coords_from_stage(
img['img_meta']['stage_pos'],
img['img_meta']['pixel_size_x_move'],
img['img_meta']['pixel_size_y_move'],
numpy.radians(img['img_meta']['angle'])) for img in imgdata}
# if not imgdata:
# raise RenderModuleException(
# "No relevant image metadata found for metafile {}".format(
# self.args['metafile']))
minX, minY = numpy.min(numpy.array(list(img_coords.values())), axis=0)
# assume isotropic pixels
pixelsize = roidata['calibration']['highmag']['x_nm_per_pix']
imgdir = self.args.get(
'image_directory', os.path.dirname(self.args['metafile']))
self.render_tspecs = [
self.ts_from_imgdata(
img, imgdir,
img_coords[img['img_path']][0] - minX,
img_coords[img['img_path']][1] - minY,
minint=self.args['minimum_intensity'],
maxint=self.args['maximum_intensity'],
width=roidata['camera_info']['width'],
height=roidata['camera_info']['height'],
z=self.args['z'],
sectionId=self.args.get('sectionId'),
scopeId=roidata['temca_id'],
cameraId=roidata['camera_info']['camera_id'],
pixelsize=pixelsize,
maskUrl=self.args['maskUrl']) for img in imgdata]
if 'output_path' in self.args:
self.args['output_path'] = jsongz.dump(
self.tilespecs,
self.args['output_path'],
self.args['compress_output'])
def test_input_stack_file(render, raw_stack, montage_pointmatches, tmpdir,
solved_montage, compress):
p = copy.deepcopy(montage_parameters)
resolved = renderapi.resolvedtiles.get_resolved_tiles_from_z(
raw_stack, p['first_section'], render=render)
tmp_file_dir = str(tmpdir.mkdir('file_test_dir'))
input_stack_file = os.path.join(tmp_file_dir, "input_stack.json")
input_stack_file = jsongz.dump(resolved.to_dict(),
input_stack_file,
compress=compress)
p['input_stack']['db_interface'] = 'file'
p['input_stack']['input_file'] = input_stack_file
p['output_mode'] = 'none'
p['pointmatch']['name'] = montage_pointmatches
tmod = bigfeta.BigFeta(input_data=p, args=[])
tmod.run()
for k in ['precision', 'error', 'err']:
assert np.all(
np.isclose(np.array(tmod.results[k]),
np.array(solved_montage.results[k]),
atol=1e-7))
assert np.all(
np.isclose(np.linalg.norm(solved_montage.results['x'], axis=0),
np.linalg.norm(tmod.results['x'], axis=0),
atol=1e-7))
orig_ids = np.array(
[t.tileId for t in solved_montage.resolvedtiles.tilespecs])
for t in tmod.resolvedtiles.tilespecs:
i = np.argwhere(orig_ids == t.tileId).flatten()[0]
assert np.all(
np.isclose(solved_montage.resolvedtiles.tilespecs[i].tforms[-1].M,
t.tforms[-1].M,
atol=1e-7))
del tmod
shutil.rmtree(tmp_file_dir)
def make_resolved(rawspecpath, tform, outputdir, compress):
# read in the tilespecs
rtj = jsongz.load(rawspecpath)
tspecs = [renderapi.tilespec.TileSpec(json=t) for t in rtj]
# do not need this anymore
os.remove(rawspecpath)
# add the reference transform
ref = renderapi.transform.ReferenceTransform()
ref.refId = tform.transformId
for t in tspecs:
t.tforms.insert(0, ref)
# make a resolved tile object
resolved = renderapi.resolvedtiles.ResolvedTiles(tilespecs=tspecs,
transformList=[tform])
# write it to file and return the path
rpath = os.path.join(outputdir, 'resolvedtiles_input.json')
return jsongz.dump(resolved.to_dict(), rpath, compress)
def make_collection_json(
template_file,
output_dir,
thresh,
compress,
ignore_match_indices=None):
with open(template_file, 'r') as f:
matches = json.load(f)
counts = []
for m in matches['collection']:
counts.append({})
ind = np.arange(len(m['matches']['p'][0]))
counts[-1]['n_from_gpu'] = ind.size
_, _, w, _ = common_utils.pointmatch_filter(
m,
n_clusters=None,
n_cluster_pts=20,
ransacReprojThreshold=40.0,
model='Similarity')
m['matches']['w'] = w.tolist()
counts[-1]['n_after_filter'] = np.count_nonzero(w)
m = matches['collection']
if ignore_match_indices:
m = [match for i, match in enumerate(matches['collection'])
if i not in ignore_match_indices]
logger.warning("you are ignoring some point matches")
collection_file = os.path.join(output_dir, "collection.json")
collection_file = jsongz.dump(m, collection_file, compress=compress)
return collection_file, counts
def run(self):
if 'tilespecs' in self.args:
jspecs = self.args['tilespecs']
else:
jspecs = jsongz.load(self.args['tilespec_file'])
self.tilespecs = np.array(
[renderapi.tilespec.TileSpec(json=j) for j in jspecs])
if 'matches' in self.args:
self.matches = self.args['matches']
else:
self.matches = jsongz.load(self.args['match_file'])
remove_weighted_matches(self.matches, weight=0.0)
self.tile_width = self.tilespecs[0].width
self.tile_height = self.tilespecs[0].height
maskUrl = self.tilespecs[0].ip[0].maskUrl
# condense coordinates
self.coords = condense_coords(self.matches)
nc0 = self.coords.shape[0]
self.coords = smooth_density(self.coords, self.tile_width,
self.tile_height, 10)
nc1 = self.coords.shape[0]
self.logger.info(
"\n smoothing point density reduced points from %d to %d" %
(nc0, nc1))
if self.coords.shape[0] == 0:
raise MeshLensCorrectionException(
"no point matches left after smoothing density, \
probably some sparse areas of matching")
# create PSLG
self.bbox = create_PSLG(self.tile_width, self.tile_height, maskUrl)
# find delaunay with max vertices
self.mesh, self.area_triangle_par = \
find_delaunay_with_max_vertices(
self.bbox,
self.args['nvertex'])
# and enforce neighboring matches to vertices
self.mesh, self.area_triangle_par = \
force_vertices_with_npoints(
self.area_triangle_par,
self.bbox,
self.coords,
3)
nend = self.mesh.points.shape[0]
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info("\n aimed for %d mesh points, got %d" %
(self.args['nvertex'], nend))
if self.mesh.points.shape[0] < 0.5 * self.args['nvertex']:
raise MeshLensCorrectionException("mesh coarser than intended")
# prepare the linear algebra and solve
self.A, self.weights, self.b, self.lens_dof_start = \
create_A(
self.matches,
self.tilespecs,
self.mesh)
self.x0 = create_x0(self.A.shape[1], self.tilespecs)
self.reg = create_regularization(
self.A.shape[1], len(self.tilespecs),
self.args['regularization']['default_lambda'],
self.args['regularization']['translation_factor'],
self.args['regularization']['lens_lambda'])
self.solution, self.errx, self.erry = solve(self.A, self.weights,
self.reg, self.x0, self.b)
self.transforms = create_transforms(len(self.tilespecs), self.solution)
tf_trans, jresult, self.solve_message = report_solution(
self.errx, self.erry, self.transforms, self.args['good_solve'])
self.logger.info(self.solve_message)
# check quality of solution
if not all([
self.errx.mean() < self.args['good_solve']['error_mean'],
self.erry.mean() < self.args['good_solve']['error_mean'],
self.errx.std() < self.args['good_solve']['error_std'],
self.erry.std() < self.args['good_solve']['error_std']
]):
raise MeshLensCorrectionException("Solve not good: %s" %
self.solve_message)
self.logger.debug(self.solve_message)
self.new_ref_transform = create_thinplatespline_tf(
self.args, self.mesh, self.solution, self.lens_dof_start,
self.logger)
bbox = self.tilespecs[0].bbox_transformed(tf_limit=0)
tbbox = self.new_ref_transform.tform(bbox)
bstr = 'new transform corners:\n'
for i in range(bbox.shape[0] - 1):
bstr += " (%0.1f, %0.1f) -> (%0.1f, %0.1f)\n" % (
bbox[i, 0], bbox[i, 1], tbbox[i, 0], tbbox[i, 1])
self.logger.info(bstr)
new_tilespecs = new_specs_with_tf(self.new_ref_transform,
self.tilespecs, self.transforms)
stage_affine = estimate_stage_affine(self.tilespecs, new_tilespecs)
sastr = "affine estimate of tile translations:\n"
sastr += " scale: {}\n".format(stage_affine.scale)
sastr += " translation: {}\n".format(stage_affine.translation)
sastr += " shear: {}\n".format(stage_affine.shear)
sastr += " rotation: {}\n".format(np.degrees(stage_affine.rotation))
self.logger.info(sastr)
self.resolved = renderapi.resolvedtiles.ResolvedTiles(
tilespecs=new_tilespecs, transformList=[self.new_ref_transform])
new_path = None
if 'outfile' in self.args:
fname = self.args['outfile']
if self.args['timestamp']:
spf = fname.split(os.extsep, 1)
spf[0] += '_%s' % self.new_ref_transform.transformId
fname = os.extsep.join(spf)
new_path = jsongz.dump(self.resolved.to_dict(),
os.path.join(self.args['output_dir'],
fname),
compress=self.args['compress_output'])
new_path = os.path.abspath(new_path)
fname = 'output.json'
if self.args['timestamp']:
fname = 'output_%s.json' % self.new_ref_transform.transformId
self.args['output_json'] = os.path.join(self.args['output_dir'], fname)
jresult['resolved_tiles'] = new_path
self.output(jresult, indent=2)
self.logger.info(" wrote solved tilespecs:\n %s" % new_path)
return