def _write_spotmatch_fiducial_config_file(filename):
"""
writes the spotmatch fiducial config file from desimeter metrology
Args:
filename : path to output file
"""
metrology = load_metrology()
selection = (metrology["DEVICE_TYPE"] == "FIF") | (metrology["DEVICE_TYPE"]
== "GIF")
locations = np.unique(metrology["LOCATION"][selection])
with open(filename, "w") as ofile:
for location in locations:
selection = (metrology["LOCATION"] == location)
pinid = metrology["PINHOLE_ID"][selection]
xfp = metrology["X_FP"][selection]
yfp = metrology["Y_FP"][selection]
mx = np.mean(xfp)
my = np.mean(yfp)
dx = xfp - mx
dy = yfp - my
ofile.write("{} {:4.3f} {:4.3f} 000.0 0.001 {:05d} 8\n".format(
location, 0., 0., 0))
for i in range(dx.size):
ofile.write("{} {:4.3f} {:4.3f} 000.0 0.001 {:05d} 2\n".format(
location, dx[i], dy[i], pinid[i]))
print("wrote", filename)
def test_spotmatch(self):
if shutil.which("match_positions") is None :
print("cannot test spotmatch because match_positions is not in PATH.")
return
measured_spots=None
expected_spots=None
print("testing spotmatch")
fvc2fp = FVC2FP.read(fvc2fp_filename())
metrology = load_metrology()
spots = metrology[(metrology["DEVICE_TYPE"]=="POS")|(metrology["DEVICE_TYPE"]=="FIF")|(metrology["DEVICE_TYPE"]=="GIF")]
xfp = spots["X_FP"]
yfp = spots["Y_FP"]
selection = (spots["DEVICE_TYPE"]=="POS")
# pos move error
xfp[selection] += 0.01 * np.random.normal(size=xfp[selection].size)
yfp[selection] += 0.01 * np.random.normal(size=yfp[selection].size)
# measurement error
xfp += 0.005 * np.random.normal(size=xfp.size)
yfp += 0.005 * np.random.normal(size=yfp.size)
xpix,ypix = fvc2fp.fp2fvc(xfp,yfp)
print("test: number of spots sent to spotmatch=",len(xpix))
res = spotmatch(xpix,ypix,expected_x_fp=None,expected_y_fp=None,expected_location=None)
def _write_spotmatch_targets_file(x_fp, y_fp, location, filename, fvc2fp=None):
"""
writes the spotmatch targets file from desimeter metrology and a transform.
Args:
x_fp : 1D numpy array with targets focal plane coordinates X in mm
y_fp : 1D numpy array with targets focal plane coordinates Y in mm
location : 1D numpy array with location index ( = petal_loc*10000+device_loc)
filename : path to output file
Optionnally:
fvc2fp : a instance of desimeter.transform.fvc2fp.FVC2FP (default is default of desimeter)
example:
1000 179.640 5616.148 12.000 0.001 4
1001 336.988 5615.164 12.000 0.001 4
1002 259.003 5479.802 12.000 0.001 4
1003 493.786 5613.850 12.000 0.001 4
1004 415.894 5478.573 12.000 0.001 4
1005 650.448 5612.610 12.000 0.001 4
1006 572.602 5477.142 12.000 0.001 4
...
"""
if fvc2fp is None:
fvc2fp = FVC2FP.read(fvc2fp_filename())
print("use default fvc2fp")
x_fp = np.array(x_fp)
y_fp = np.array(y_fp)
location = np.array(location)
flags = np.repeat(4, x_fp.size)
# add fiducials centers
metrology = load_metrology()
selection = (metrology["DEVICE_TYPE"] == "FIF") | (metrology["DEVICE_TYPE"]
== "GIF")
fid_locations = np.unique(metrology["LOCATION"][selection])
for fid_location in fid_locations:
selection = (metrology["LOCATION"] == fid_location)
mx = np.mean(metrology["X_FP"][selection])
my = np.mean(metrology["Y_FP"][selection])
x_fp = np.append(x_fp, mx)
y_fp = np.append(y_fp, my)
location = np.append(location, fid_location)
flags = np.append(flags, 8)
xpix, ypix = fvc2fp.fp2fvc(x_fp, y_fp)
with open(filename, "w") as ofile:
for i in range(xpix.size):
ofile.write("{} {:4.3f} {:4.3f} 12.000 0.001 {}\n".format(
location[i], xpix[i], ypix[i], flags[i]))
print("wrote", filename)
def test_gfa2fp(self):
metrology = io.load_metrology()
ii = (metrology['DEVICE_TYPE'] == 'GFA')
metrology = metrology[ii]
nx, ny = 2048, 1032
xgfa_ref = np.array([0.0, nx-1, nx-1, 0])
ygfa_ref = np.array([0.0, 0, ny-1, ny-1])
# do not test against reference
# because the transform accounts for the
# z offset of the sensors
"""
for p in range(10):
ii = (metrology['PETAL_LOC'] == p)
if np.count_nonzero(ii) == 0:
print('ERROR: missing GFA metrology for PETAL_LOC {}'.format(p))
continue
xfp_ref = np.asarray(metrology['X_FP'][ii])
yfp_ref = np.asarray(metrology['Y_FP'][ii])
#- Current agreement is not very good, because the GFA metrology
#- is rectangular in 3D space but these transforms are treating
#- them as rectangular in 2D space.
xgfa, ygfa = fp2gfa(p, xfp_ref, yfp_ref)
self.assertLess(np.max(np.abs(xgfa-xgfa_ref)), 2)
self.assertLess(np.max(np.abs(ygfa-ygfa_ref)), 2)
xfp, yfp = gfa2fp(p, xgfa_ref, ygfa_ref)
dxfp = xfp-xfp_ref
dyfp = yfp-yfp_ref
self.assertLess(np.max(np.abs(dxfp)), 0.050)
self.assertLess(np.max(np.abs(dyfp)), 0.050)
drfp = 1000*np.sqrt(dxfp**2 + dyfp**2)
print('PETAL_LOC {} GFA corner max(dr) = {:.1f} um'.format(
p, np.max(drfp)))
"""
#- Round trip tests should be rock solid though
xfp, yfp = gfa2fp(0, xgfa_ref, ygfa_ref)
xgfa, ygfa = fp2gfa(0, xfp, yfp)
self.assertTrue(np.allclose(xgfa, xgfa_ref))
self.assertTrue(np.allclose(ygfa, ygfa_ref))
def get_expected_pos(args, log):
if args.expected_positions is not None:
log.info("reading expected positions in {}".format(
args.expected_positions))
expected_pos = Table.read(
args.expected_positions) # auto-magically guess format
if (not "X_FP"
in expected_pos.keys()) and "X_FP_EXP" in expected_pos.keys():
log.warning("Rename X_FP_EXP,Y_FP_EXP -> X_FP,Y_FP")
expected_pos.rename_column("X_FP_EXP", "X_FP")
expected_pos.rename_column("Y_FP_EXP", "Y_FP")
if not "X_FP" in expected_pos.keys():
if "EXP_Q_0" in expected_pos.keys():
log.info("EXP_Q_0,EXP_S_0 -> X_FP,Y_FP")
x, y = qs2xy(q=expected_pos["EXP_Q_0"],
s=expected_pos["EXP_S_0"])
expected_pos["X_FP"] = x
expected_pos["Y_FP"] = y
bad_expected_pos = (np.isnan(x) | np.isnan(y))
if np.sum(bad_expected_pos) > 0:
expected_pos["X_FP"][bad_expected_pos] = -99999.
expected_pos["Y_FP"][bad_expected_pos] = -99999.
else:
log.error(
"No EXP_Q_0 nor X_FP in expected positions file {}".format(
args.expected_positions))
else:
log.info(
"since no input expected positions, use metrology to match the fibers to the positioner centers"
)
expected_pos = load_metrology()
if not "LOCATION" in expected_pos.keys():
# add useful location keyword
expected_pos["LOCATION"] = np.array(
expected_pos["PETAL_LOC"]) * 1000 + np.array(
expected_pos["DEVICE_LOC"])
if "PINHOLE_ID" in expected_pos.dtype.names:
# exclude pinhole because here we want to match fibers
ii = np.where(expected_pos["PINHOLE_ID"] == 0)[0]
expected_pos = expected_pos[:][ii]
return expected_pos
def _write_spotmatch_device_centers_file(filename, fvc2fp=None):
"""
writes the spotmatch device centers file from desimeter metrology and a transform.
Args:
filename : path to output file
Optionnally:
fvc2fp : a instance of desimeter.transform.fvc2fp.FVC2FP (default is default of desimeter)
"""
if fvc2fp is None:
fvc2fp = FVC2FP.read(fvc2fp_filename())
print("use default fvc2fp")
metrology = load_metrology()
selection = (metrology["DEVICE_TYPE"] == "POS")
x_fp = metrology["X_FP"][selection]
y_fp = metrology["Y_FP"][selection]
location = metrology["LOCATION"][selection]
flags = np.repeat(4, x_fp.size)
# add fiducials centers
selection = (metrology["DEVICE_TYPE"] == "FIF") | (metrology["DEVICE_TYPE"]
== "GIF")
fid_locations = np.unique(metrology["LOCATION"][selection])
for fid_location in fid_locations:
selection = (metrology["LOCATION"] == fid_location)
mx = np.mean(metrology["X_FP"][selection])
my = np.mean(metrology["Y_FP"][selection])
x_fp = np.append(x_fp, mx)
y_fp = np.append(y_fp, my)
location = np.append(location, fid_location)
flags = np.append(flags, 8)
xpix, ypix = fvc2fp.fp2fvc(x_fp, y_fp)
with open(filename, "w") as ofile:
for i in range(xpix.size):
ofile.write("{} {:4.3f} {:4.3f} 12.000 0.001 {}\n".format(
location[i], xpix[i], ypix[i], flags[i]))
print("wrote", filename)
def _write_spotmatch_reference_pos_file(filename, fvc2fp=None):
"""
writes the spotmatch reference pos file from metrology and a transform
Args:
filename : path to output file
Optionnally:
fvc2fp : a instance of desimeter.transform.fvc2fp.FVC2FP (default is default of desimeter)
example:
1541 4179.496 2557.798 13.670 515 1.897 pinhole
1541 4177.541 2572.562 17.823 1027 1.090 pinhole
1541 4191.178 2578.614 13.016 259 2.055 pinhole
1541 4175.588 2587.326 12.784 771 1.851 pinhole
1542 5090.489 2633.933 13.025 771 2.115 pinhole
1542 5099.186 2646.061 12.641 1027 2.115 pinhole
"""
metrology = load_metrology()
if fvc2fp is None:
fvc2fp = FVC2FP.read(fvc2fp_filename())
selection = (metrology["DEVICE_TYPE"] == "FIF") | (metrology["DEVICE_TYPE"]
== "GIF")
xpix, ypix = fvc2fp.fp2fvc(metrology["X_FP"][selection],
metrology["Y_FP"][selection])
locations = metrology["LOCATION"][selection]
pinhole_ids = metrology["PINHOLE_ID"][selection]
num = {1: 515, 2: 1027, 3: 259, 4: 771} # the code wants that
with open(filename, "w") as ofile:
for i in range(xpix.size):
if pinhole_ids[i] in num.keys():
ofile.write(
" {} {:4.3f} {:4.3f} 12.000 {} 2.000 pinhole\n".format(
locations[i], xpix[i], ypix[i], num[pinhole_ids[i]]))
print("wrote", filename)
def fp2gfa(petal_loc, xfp, yfp):
"""
Transforms from focal plane mm to GFA pixel coordinates
Args:
petal_loc (int): Petal location 0-9
xfp, yfp: CS5 focal plane mm
Returns xgfa, ygfa pixel coordinates with (0,0) as center of corner pixel
"""
global _gfa_transforms
if _gfa_transforms is None:
metrology = io.load_metrology()
_gfa_transforms = fit_gfa2fp(metrology)
log = get_logger()
if petal_loc not in _gfa_transforms:
log.error('PETAL_LOC {} GFA metrology missing'.format(petal_loc))
xgfa, ygfa = _gfa_transforms[petal_loc].apply_inverse(xfp, yfp)
return xgfa, ygfa
def gfa2fp(petal_loc, xgfa, ygfa):
"""
Transforms from GFA pixel coordinates to focal plane mm
Args:
petal_loc (int): Petal location 0-9
xgfa, ygfa: GFA pixel coordinates, (0,0) is corner pixel center
Returns CS5 xfp, yfp in mm
"""
global _gfa_transforms
if _gfa_transforms is None:
metrology = io.load_metrology()
_gfa_transforms = fit_gfa2fp(metrology)
log = get_logger()
if petal_loc not in _gfa_transforms:
log.error('PETAL_LOC {} GFA metrology missing'.format(petal_loc))
xfp, yfp = _gfa_transforms[petal_loc].apply(xgfa, ygfa)
return xfp, yfp
def fit(self,
spots,
metrology=None,
update_spots=False,
zbfit=True,
fixed_scale=False,
fixed_rotation=False):
"""
TODO: document
"""
log = get_logger()
if metrology is not None:
self.metrology = metrology
else:
self.metrology = load_metrology()
#- Trim spots to just fiducial spots (not posioners, not unmatchs spots)
ii = (spots['LOCATION'] >= 0) & (spots['PINHOLE_ID'] > 0)
fidspots = spots[ii]
#- trim metrology to just the ones that have spots
fidspots_pinloc = fidspots['LOCATION'] * 10 + fidspots['PINHOLE_ID']
metro_pinloc = self.metrology['LOCATION'] * 10 + self.metrology[
'PINHOLE_ID']
jj = np.in1d(metro_pinloc, fidspots_pinloc)
metrology = self.metrology[jj]
#- Sort so that they match each other
fidspots.sort(keys=('LOCATION', 'PINHOLE_ID'))
metrology.sort(keys=('LOCATION', 'PINHOLE_ID'))
assert np.all(fidspots['LOCATION'] == metrology['LOCATION'])
assert np.all(fidspots['PINHOLE_ID'] == metrology['PINHOLE_ID'])
#- Get reduced coordinates
rxpix, rypix = self._reduce_xyfvc(fidspots['XPIX'], fidspots['YPIX'])
rxfp, ryfp = self._reduce_xyfp(metrology['X_FP'], metrology['Y_FP'])
if fixed_rotation:
fixed_rotation_value = self.rotation
log.info(
"Use fixed rotation = {:5.4f}".format(fixed_rotation_value))
else:
fixed_rotation_value = None
if fixed_scale:
fixed_scale_value = self.scale
log.info("Use fixed scale = {:5.4f}".format(fixed_scale_value))
else:
fixed_scale_value = None
res = fit_scale_rotation_offset(rxpix,
rypix,
rxfp,
ryfp,
fitzb=zbfit,
zbpolids=self.zbpolids,
zbcoeffs=self.zbcoeffs,
fixed_scale=fixed_scale_value,
fixed_rotation=fixed_rotation_value)
self.scale = res[0]
self.rotation = res[1]
self.offset_x = res[2]
self.offset_y = res[3]
if zbfit:
self.zbpolids = res[4]
self.zbcoeffs = res[5]
#- Goodness of fit
xfp_fidmeas, yfp_fidmeas = self.fvc2fp(fidspots['XPIX'],
fidspots['YPIX'])
dx = (metrology['X_FP'] - xfp_fidmeas)
dy = (metrology['Y_FP'] - yfp_fidmeas)
dr = np.sqrt(dx**2 + dy**2)
log.info(
'Mean, median, RMS distance = {:.1f}, {:.1f}, {:.1f} um'.format(
1000 * np.mean(dr), 1000 * np.median(dr),
1000 * np.sqrt(np.mean(dr**2))))
if update_spots:
xfp_meas, yfp_meas = self.fvc2fp(spots['XPIX'], spots['YPIX'])
spots["X_FP"] = xfp_meas
spots["Y_FP"] = yfp_meas
#- the metrology table is in a different order than the original
#- spots table, which is also a superset of the fidicual spots
#- matched to the metrology, so find the sorting of the metrology
#- that will match the order that they appear in the spots table
iifid = (spots['LOCATION'] > 0) & (spots['PINHOLE_ID'] > 0)
fidspots_pinloc = (spots['LOCATION'] * 10 +
spots['PINHOLE_ID'])[iifid]
metro_pinloc = metrology['LOCATION'] * 10 + metrology['PINHOLE_ID']
ii = np.argsort(np.argsort(fidspots_pinloc))
jj = np.argsort(metro_pinloc)
kk = jj[ii]
#- Check that we got that dizzying array of argsorts right
assert np.all(
spots['LOCATION'][iifid] == metrology['LOCATION'][kk])
assert np.all(
spots['PINHOLE_ID'][iifid] == metrology['PINHOLE_ID'][kk])
#- Update the spots table with metrology columns
#- TODO: used masked arrays in addition to default=0
spots["X_FP_METRO"] = np.zeros(len(spots))
spots["Y_FP_METRO"] = np.zeros(len(spots))
spots["Z_FP_METRO"] = np.zeros(len(spots))
spots["X_FP_METRO"][iifid] = metrology['X_FP'][kk]
spots["Y_FP_METRO"][iifid] = metrology['Y_FP'][kk]
spots["Z_FP_METRO"][iifid] = metrology['Z_FP'][kk]
def spotmatch(xpix,
ypix,
expected_x_fp=None,
expected_y_fp=None,
expected_location=None,
verbose=0,
match_radius_pixels=70):
"""
Wrapper to spotmatch in desimeter. Calls the C executable 'match_positions' that has to be in the path. All inputs
to match_positions are generated in this call using desimeter metrology table, the default fvc2fp transform and the inputs
Args :
xpix : 1D numpy array with FVC image X pixel coordinates of detected spots
ypix : 1D numpy array with FVC image Y pixel coordinates of detected spots
Optionnally :
expected_x_fp : 1D numpy array with expected targets focal plane coordinates X in mm
expected_y_fp : 1D numpy array with expected targets focal plane coordinates Y in mm
expected_location : 1D numpy array with targets location index ( = petal_loc*10000+device_loc)
verbose : 0 or 1
match_radius_pixels : match radius in pixels
returns :
an astropy.table.Table object with at least the columns
LOCATION,XPIX,YPIX,FLAG,ERR,SPOTMATCH_DEVICE_TYPE,DEVICE_ID,DEVICE_TYPE,DEVICE_LOC,PETAL_LOC,PINHOLE_ID
and possibly other columns from the input metrology table
LOCATION = petal_loc*10000+device_loc for matched spots
or a negative number if not matched
PINHOLE_ID = 0 for positioners (DEVICE_TYPE="POS")
PINHOLE_ID = 99 for fiducial centers (DEVICE_TYPE="FIF" or DEVICE_TYPE="GIF")
PINHOLE_ID = 10, 11, 12, or 13 for fiducial pinholes (DEVICE_TYPE="FIF" or DEVICE_TYPE="GIF")
the pinholes in a fiducial are not matched to the metrology, so only the fiducial centers (PINHOLE_ID = 99)
can be used to fit the transform.
"""
if shutil.which("match_positions") is None:
raise RuntimeError(
"match_positions is not in PATH. You need to install spotmatch first. It's in https://desi.lbl.gov/trac/browser/code/online/FVC/spotmatch."
)
image_rows = 6000
image_cols = 6000
target_x_dir = 1
target_y_dir = 1
target_x0 = 0
target_y0 = 0
fid_x_dir = -1
fid_y_dir = 1
tmp_dir = tempfile.gettempdir()
fvc2fp = FVC2FP.read(fvc2fp_filename())
exp_pixel_scale = _compute_pixel_scale(fvc2fp)
fiducial_config_filename = os.path.join(
tmp_dir, "desimeter_spotmatch_fiducials.txt")
_write_spotmatch_fiducial_config_file(fiducial_config_filename)
if expected_x_fp is None:
print(
"input expected = None, so we use the centers of positioners as the expected spots"
)
metrology = load_metrology()
spots = metrology[metrology["DEVICE_TYPE"] ==
"POS"] # select positioners
expected_x_fp = spots["X_FP"]
expected_y_fp = spots["Y_FP"]
expected_location = spots["LOCATION"]
targets_filename = os.path.join(tmp_dir, "desimeter_spotmatch_targets.txt")
_write_spotmatch_targets_file(expected_x_fp, expected_y_fp,
expected_location, targets_filename, fvc2fp)
measured_pos_filename = os.path.join(
tmp_dir, "desimeter_spotmatch_input_centroids.txt")
_write_spotmatch_measured_pos_file(xpix, ypix, measured_pos_filename)
reference_pos_filename = os.path.join(
tmp_dir, "desimeter_spotmatch_pinhole_references.txt")
_write_spotmatch_reference_pos_file(reference_pos_filename, fvc2fp=None)
device_centers_filename = os.path.join(
tmp_dir, "desimeter_spotmatch_device_centers.txt")
_write_spotmatch_device_centers_file(device_centers_filename, fvc2fp=None)
# example
# match_positions -verbose 1 -image_rows 6000 -image_cols 6000 -target_x_dir 1 -target_y_dir 1 -target_x0 0.0 -target_y0 0.0 -fid_x_dir -1 -fid_y_dir 1 -exp_pixel_scale 0.073 -match_radius 50 -reduced_pos_file ./match_centers.tmp -fiducial_config_file ./match_fiducials.tmp -target_pos_file ./match_targets.tmp -measured_pos_file ./match_centroids.tmp -pos_save_file ./measured_pos.tmp -reference_pos_file ./pinhole_references_20200410.dat
match_centers_filename = os.path.join(
tmp_dir, "desimeter_spotmatch_output_centers.txt")
saved_pos_filename = os.path.join(tmp_dir,
"desimeter_spotmatch_saved_output.txt")
positioner_reach_in_mm = 6.6 # see https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=5708
positioner_reach_in_pixels = positioner_reach_in_mm / exp_pixel_scale
cmd = "match_positions"
cmd += " -verbose {}".format(verbose)
cmd += " -image_rows {}".format(image_rows)
cmd += " -image_cols {}".format(image_cols)
cmd += " -target_x0 {}".format(target_x0)
cmd += " -target_y0 {}".format(target_y0)
cmd += " -target_y_dir {}".format(target_y_dir)
cmd += " -target_x_dir {}".format(target_x_dir)
cmd += " -target_y_dir {}".format(target_y_dir)
cmd += " -fid_x_dir {}".format(fid_x_dir)
cmd += " -fid_y_dir {}".format(fid_y_dir)
cmd += " -exp_pixel_scale {}".format(exp_pixel_scale)
cmd += " -match_radius {}".format(match_radius_pixels)
cmd += " -reduced_pos_file {}".format(match_centers_filename)
cmd += " -fiducial_config_file {}".format(fiducial_config_filename)
cmd += " -target_pos_file {}".format(targets_filename)
cmd += " -measured_pos_file {}".format(measured_pos_filename)
cmd += " -pos_save_file {}".format(saved_pos_filename)
cmd += " -reference_pos_file {}".format(reference_pos_filename)
cmd += " -device_centers_file {}".format(device_centers_filename)
position_reach_option = " -positioner_reach {:4.3f}".format(
positioner_reach_in_pixels)
cmd += position_reach_option
print(cmd)
status, output = subprocess.getstatusoutput(cmd)
print(output)
if status != 0:
if output.find(
"unexpected command-line argument [ -positioner_reach ]") >= 0:
print(
"WARNING, -positioner_reach is not an argument, we rerun without it, but this means you have in the path an old version of spotmatch"
)
new_cmd = cmd.replace(position_reach_option, "")
print(new_cmd)
status, output = subprocess.getstatusoutput(new_cmd)
print(output)
if status != 0:
raise RuntimeError(
"Error {} from match_positions called in desimeter".format(status))
location = []
xpix = []
ypix = []
mag = []
flag = []
err = []
device_type = []
print("reading", match_centers_filename)
with open(match_centers_filename) as ifile:
for line in ifile.readlines():
vals = line.strip().split()
if len(vals) != 7: continue
location.append(int(vals[0]))
xpix.append(float(vals[1]))
ypix.append(float(vals[2]))
mag.append(float(vals[3]))
flag.append(float(vals[4]))
err.append(float(vals[5]))
device_type.append(vals[6])
print("read {} entries in {}".format(len(xpix), match_centers_filename))
res = Table()
res["LOCATION"] = location
res["XPIX"] = xpix
res["YPIX"] = ypix
res["FLAG"] = flag
res["ERR"] = err
nspots = len(location)
res["SPOTMATCH_DEVICE_TYPE"] = device_type
res["DEVICE_ID"] = np.repeat("unknown", nspots)
res["DEVICE_TYPE"] = np.repeat("unknown", nspots)
res["BUS_ID"] = np.repeat("unknown", nspots)
# use metrology to get more info
metrology = load_metrology()
locmap = {loc: index for index, loc in enumerate(metrology["LOCATION"])}
matched_index = np.where(res["LOCATION"] >= 0)[0]
metrology_index = [locmap[loc] for loc in res["LOCATION"][matched_index]]
keys_to_copy = []
for k in [
"DEVICE_TYPE", "DEVICE_LOC", "DEVICE_ID", "PETAL_LOC", "PETAL_ID",
"BUS_ID"
]:
if k in metrology.dtype.names:
keys_to_copy.append(k)
if not k in res.dtype.names:
res[k] = np.repeat(-1, nspots)
res[k][matched_index] = metrology[k][metrology_index]
# special treatment for pinhole ids
res["PINHOLE_ID"] = np.repeat(0, nspots)
fiducials = (res["SPOTMATCH_DEVICE_TYPE"] == "fiducial")
# dummy pinhole numbers. 99 for center, 10,11,12,13 for pinholes
res["PINHOLE_ID"][fiducials] = 99
for loc in np.unique(res["LOCATION"][fiducials]):
ii = (res["LOCATION"] == loc) & (res["SPOTMATCH_DEVICE_TYPE"]
== "pinhole")
res["PINHOLE_ID"][ii] = 10 + np.arange(np.sum(ii))
return res
def findfiducials(spots, input_transform=None, pinhole_max_separation_mm=1.5):
global metrology_pinholes_table
global metrology_fiducials_table
log = get_logger()
log.debug(
"load input tranformation we will use to go from FP to FVC pixels")
if input_transform is None:
input_transform = fvc2fp_filename()
log.info("loading input tranform from {}".format(input_transform))
input_tx = FVC2FP.read_jsonfile(input_transform)
xpix = np.array([
2000.,
])
ypix = np.array([
0.,
])
xfp1, yfp1 = input_tx.fvc2fp(xpix, ypix)
xfp2, yfp2 = input_tx.fvc2fp(xpix + 1, ypix)
pixel2fp = np.hypot(xfp2 - xfp1, yfp2 - yfp1)[0] # mm
pinhole_max_separation_pixels = pinhole_max_separation_mm / pixel2fp
log.info(
"with pixel2fp = {:4.3f} mm, pinhole max separation = {:4.3f} pixels ".
format(pixel2fp, pinhole_max_separation_pixels))
if metrology_pinholes_table is None:
metrology_table = load_metrology()
log.debug("keep only the pinholes")
metrology_pinholes_table = metrology_table[:][
(metrology_table["DEVICE_TYPE"] == "FIF") |
(metrology_table["DEVICE_TYPE"] == "GIF")]
# use input transform to convert X_FP,Y_FP to XPIX,YPIX
xpix, ypix = input_tx.fp2fvc(metrology_pinholes_table["X_FP"],
metrology_pinholes_table["Y_FP"])
metrology_pinholes_table["XPIX"] = xpix
metrology_pinholes_table["YPIX"] = ypix
log.debug("define fiducial location as the most central dot")
central_pinholes = []
for loc in np.unique(metrology_pinholes_table["LOCATION"]):
ii = np.where(metrology_pinholes_table["LOCATION"] == loc)[0]
mx = np.mean(metrology_pinholes_table["XPIX"][ii])
my = np.mean(metrology_pinholes_table["YPIX"][ii])
k = np.argmin((metrology_pinholes_table["XPIX"][ii] - mx)**2 +
(metrology_pinholes_table["YPIX"][ii] - my)**2)
central_pinholes.append(ii[k])
metrology_fiducials_table = metrology_pinholes_table[:][
central_pinholes]
# find fiducials candidates
log.info("select spots with at least two close neighbors (in pixel units)")
nspots = spots["XPIX"].size
xy = np.array([spots["XPIX"], spots["YPIX"]]).T
tree = KDTree(xy)
measured_spots_distances, measured_spots_indices = tree.query(
xy, k=4, distance_upper_bound=pinhole_max_separation_pixels)
number_of_neighbors = np.sum(
measured_spots_distances < pinhole_max_separation_pixels, axis=1)
fiducials_candidates_indices = np.where(
number_of_neighbors >= 4)[0] # including self, so at least 3 pinholes
log.debug("number of fiducials=", fiducials_candidates_indices.size)
# match candidates to fiducials from metrology
log.info(
"first match {} fiducials candidates to metrology ({}) with iterative fit"
.format(fiducials_candidates_indices.size,
len(metrology_fiducials_table)))
x1 = spots["XPIX"][fiducials_candidates_indices]
y1 = spots["YPIX"][fiducials_candidates_indices]
x2 = metrology_fiducials_table["XPIX"]
y2 = metrology_fiducials_table["YPIX"]
nloop = 20
saved_median_distance = 0
for loop in range(nloop):
indices_2, distances = match_same_system(x1, y1, x2, y2)
mdist = np.median(distances[indices_2 >= 0])
if loop < nloop - 1:
maxdistance = max(10, 3. * 1.4 * mdist)
else: # final iteration
maxdistance = 10 # pixel
selection = np.where((indices_2 >= 0) & (distances < maxdistance))[0]
log.info("iter #{} median_dist={} max_dist={} matches={}".format(
loop, mdist, maxdistance, selection.size))
corr21 = SimpleCorr()
corr21.fit(x2[indices_2[selection]], y2[indices_2[selection]],
x1[selection], y1[selection])
x2, y2 = corr21.apply(x2, y2)
if np.abs(saved_median_distance - mdist) < 0.0001:
break # no more improvement
saved_median_distance = mdist
# use same coord system match (note we now match the otherway around)
indices_1, distances = match_same_system(x2, y2, x1, y1)
maxdistance = 10. # FVC pixels
selection = np.where((indices_1 >= 0) & (distances < maxdistance))[0]
fiducials_candidates_indices = fiducials_candidates_indices[
indices_1[selection]]
matching_known_fiducials_indices = selection
log.debug(
"mean distance = {:4.2f} pixels for {} matched and {} known fiducials".
format(np.mean(distances[distances < maxdistance]),
fiducials_candidates_indices.size,
metrology_fiducials_table["XPIX"].size))
log.debug("now matching pinholes ...")
nspots = spots["XPIX"].size
for k in ['LOCATION', 'PETAL_LOC', 'DEVICE_LOC', 'PINHOLE_ID']:
if k not in spots.dtype.names:
spots.add_column(Column(np.zeros(nspots, dtype=int)), name=k)
spots["LOCATION"][:] = -1
spots["PETAL_LOC"][:] = -1
spots["DEVICE_LOC"][:] = -1
spots["PINHOLE_ID"][:] = 0
for index1, index2 in zip(fiducials_candidates_indices,
matching_known_fiducials_indices):
location = metrology_fiducials_table["LOCATION"][index2]
# get indices of all pinholes for this matched fiducial
# note we now use the full pinholes metrology table
pi1 = measured_spots_indices[index1][
measured_spots_distances[index1] < pinhole_max_separation_pixels]
pi2 = np.where(metrology_pinholes_table["LOCATION"] == location)[0]
x1 = spots["XPIX"][pi1]
y1 = spots["YPIX"][pi1]
x2 = metrology_pinholes_table["XPIX"][pi2]
y2 = metrology_pinholes_table["YPIX"][pi2]
indices_2, distances = match_arbitrary_translation_dilatation(
x1, y1, x2, y2)
metrology_pinhole_ids = metrology_pinholes_table["PINHOLE_ID"][pi2]
pinhole_ids = np.zeros(x1.size, dtype=int)
matched = (indices_2 >= 0)
pinhole_ids[matched] = metrology_pinhole_ids[indices_2[matched]]
spots["LOCATION"][pi1[matched]] = location
spots["PINHOLE_ID"][pi1[matched]] = pinhole_ids[matched]
if np.sum(pinhole_ids == 0) > 0:
log.warning(
"only matched pinholes {} for {} detected at LOCATION {} xpix~{} ypix~{}"
.format(pinhole_ids[pinhole_ids > 0], x1.size, location,
int(np.mean(x1)), int(np.mean(y1))))
# check duplicates
if np.unique(
pinhole_ids[pinhole_ids > 0]).size != np.sum(pinhole_ids > 0):
xfp = np.mean(metrology_pinholes_table[pi2]["X_FP"])
yfp = np.mean(metrology_pinholes_table[pi2]["Y_FP"])
log.warning(
"duplicate(s) pinhole ids in {} at LOCATION={} xpix~{} ypix~{} xfp~{} yfp~{}"
.format(pinhole_ids, location, int(np.mean(x1)),
int(np.mean(y1)), int(xfp), int(yfp)))
bc = np.bincount(pinhole_ids[pinhole_ids > 0])
duplicates = np.where(bc > 1)[0]
for duplicate in duplicates:
log.warning(
"Unmatch ambiguous pinhole id = {}".format(duplicate))
selection = (spots["LOCATION"]
== location) & (spots["PINHOLE_ID"] == duplicate)
spots["PINHOLE_ID"][selection] = 0
ii = (spots["LOCATION"] >= 0)
spots["PETAL_LOC"][ii] = spots["LOCATION"][ii] // 1000
spots["DEVICE_LOC"][ii] = spots["LOCATION"][ii] % 1000
n_matched_pinholes = np.sum(spots["PINHOLE_ID"] > 0)
n_matched_fiducials = np.sum(spots["PINHOLE_ID"] == 4)
log.info("matched {} pinholes from {} fiducials".format(
n_matched_pinholes, n_matched_fiducials))
return spots
def fvc_proc(args, log):
"""Process an FVC image with options specified by args and output to log.
"""
errcode = preproc(args)
if errcode:
return errcode
spots_list, extnames = get_spots_list(args, log)
if spots_list is None:
return 13
for seqid, spots in enumerate(spots_list):
if args.min_spots is not None:
if len(spots) < args.min_spots:
log.error("not enough spots, exiting")
continue
if args.max_spots is not None:
if len(spots) > args.max_spots:
log.error("too many spots, exiting")
continue
outfile = args.outfile
if args.sequence:
if extnames is not None:
outfile = outfile.replace(".csv", "-%s.csv" % extnames[seqid])
else:
outfile = outfile.replace(".csv", "-F{:04d}.csv".format(seqid))
if args.nomatch:
# write spots
spots.write(outfile, format="csv", overwrite=True)
print("wrote {}".format(outfile))
continue
if args.use_spotmatch:
spots = spotmatch(
spots["XPIX"],
spots["YPIX"],
match_radius_pixels=args.spotmatch_match_radius_pixels)
# drop pinholes, keep only center
spots = spots[(spots["PINHOLE_ID"] == 0) |
(spots["PINHOLE_ID"] == 99)]
n_matched_fiducials = np.sum(spots['PINHOLE_ID'] == 99)
else:
spots = findfiducials(
spots,
input_transform=args.input_transform,
pinhole_max_separation_mm=args.pinhole_max_separation_mm)
n_matched_fiducials = np.sum(spots['PINHOLE_ID'] == 4)
if n_matched_fiducials < 3:
log.error('Fewer than three matched fiducials; exiting early.')
return 13
tx = FVC2FP.read_jsonfile(fvc2fp_filename())
if args.use_spotmatch:
metrology = load_metrology()
# keep only the center of fiducials
selection = (metrology["DEVICE_TYPE"]
== "FIF") | (metrology["DEVICE_TYPE"] == "GIF")
for loc in np.unique(metrology["LOCATION"][selection]):
# all the pinholes at that location
ii = np.where(metrology["LOCATION"] == loc)[0]
# replace first entry by mean of pinholes
metrology["X_FP"][ii[0]] = np.mean(metrology["X_FP"][ii])
metrology["Y_FP"][ii[0]] = np.mean(metrology["Y_FP"][ii])
# set a dummy pinhole id = 10 just to make sure it's not interpreted as an existing pinhole
metrology["PINHOLE_ID"][ii[0]] = 99
# drop the others
metrology.remove_rows(ii[1:])
else:
metrology = None
tx.fit(spots,
metrology=metrology,
update_spots=True,
zbfit=(args.zbfit),
fixed_scale=args.fixed_scale,
fixed_rotation=args.fixed_rotation)
expected_pos = get_expected_pos(args, log)
# select spots that are not already matched
selection = (spots["LOCATION"] == -1)
if args.use_spotmatch:
spots = spotmatch(
spots["XPIX"],
spots["YPIX"],
expected_x_fp=expected_pos["X_FP"],
expected_y_fp=expected_pos["Y_FP"],
expected_location=expected_pos["LOCATION"],
fvc2fp=tx,
match_radius_pixels=args.spotmatch_match_radius_pixels)
#spots = spotmatch(spots["XPIX"],spots["YPIX"],expected_x_fp=expected_pos["X_FP"],expected_y_fp=expected_pos["Y_FP"],expected_location=expected_pos["LOCATION"],fvc2fp=None,match_radius_pixels=args.spotmatch_match_radius_pixels)
# add info
nspots = len(spots["LOCATION"])
for k in [
"X_FP", "Y_FP", "X_FP_EXP", "Y_FP_EXP", "X_FP_METRO",
"Y_FP_METRO"
]:
if k not in spots.dtype.names:
spots[k] = np.zeros(nspots, dtype=float)
spots["X_FP"], spots["Y_FP"] = tx.fvc2fp(spots["XPIX"],
spots["YPIX"])
is_matched = (spots["LOCATION"] >= 0)
loc2i = {loc: i for i, loc in enumerate(spots["LOCATION"])}
ii = []
jj = []
for j, loc in enumerate(expected_pos["LOCATION"]):
if loc in loc2i:
ii.append(loc2i[loc])
jj.append(j)
spots["X_FP_EXP"][ii] = expected_pos["X_FP"][jj]
spots["Y_FP_EXP"][ii] = expected_pos["Y_FP"][jj]
metrology = load_metrology()
ii = []
jj = []
for j, loc in enumerate(metrology["LOCATION"]):
if loc in loc2i:
ii.append(loc2i[loc])
jj.append(j)
spots["X_FP_METRO"][ii] = metrology["X_FP"][jj]
spots["Y_FP_METRO"][ii] = metrology["Y_FP"][jj]
else:
# match
indices_of_expected_pos, distances = match_same_system(
spots["X_FP"][selection], spots["Y_FP"][selection],
expected_pos["X_FP"], expected_pos["Y_FP"])
is_matched = (distances < args.max_match_distance) & (
indices_of_expected_pos >= 0)
ii = np.where(selection)[0]
selection[ii] &= is_matched
indices_of_expected_pos = indices_of_expected_pos[is_matched]
distances = distances[is_matched]
# add columns after matching fibers
for k1, k2 in zip(["X_FP", "Y_FP"], ["X_FP_EXP", "Y_FP_EXP"]):
if k2 not in spots.keys(): spots[k2] = np.zeros(len(spots))
spots[k2][selection] = expected_pos[k1][
indices_of_expected_pos]
for k in [
"EXP_Q_0", "EXP_S_0", "PETAL_LOC", "DEVICE_LOC",
"DEVICE_ID", "DEVICE_TYPE", "LOCATION"
]:
if k in expected_pos.keys():
if k not in spots.keys():
if k in ["DEVICE_ID", "DEVICE_TYPE"]:
spots[k] = np.repeat("None ", len(spots))
else:
spots[k] = np.zeros(len(spots))
spots[k][selection] = expected_pos[k][
indices_of_expected_pos]
if args.expected_positions is not None and (
args.turbulence_correction
or args.turbulence_correction_with_pol):
if args.turbulence_correction_with_pol:
log.info("Turbulence correction (local polynomial fit) ...")
else:
log.info("Turbulence correction (gaussian processes) ...")
selection = (spots["LOCATION"] >= 0) & (spots["X_FP_EXP"] != 0) & (
spots["Y_FP_EXP"] != 0) # matched
if args.turbulence_correction_with_pol:
new_x, new_y = correct_with_pol(spots["X_FP"][selection],
spots["Y_FP"][selection],
spots["X_FP_EXP"][selection],
spots["Y_FP_EXP"][selection])
else:
new_x, new_y = correct(spots["X_FP"][selection],
spots["Y_FP"][selection],
spots["X_FP_EXP"][selection],
spots["Y_FP_EXP"][selection])
rms_before = np.sqrt(
np.mean((spots["X_FP"][selection] -
spots["X_FP_EXP"][selection])**2 +
(spots["Y_FP"][selection] -
spots["Y_FP_EXP"][selection])**2))
rms_after = np.sqrt(
np.mean((new_x - spots["X_FP_EXP"][selection])**2 +
(new_y - spots["Y_FP_EXP"][selection])**2))
log.info(
"rms(measured-expected)={:5.4f}mm rms(corrected-expected)={:5.4f}mm "
.format(rms_before, rms_after))
if rms_before < rms_after:
log.warning("turbulence correction does not reduce the rms?")
# apply it anyway because there might be a good reason for this
spots["X_FP"][selection] = new_x
spots["Y_FP"][selection] = new_y
if "X_FP_METRO" in spots.dtype.names:
# for spots with metrology X_FP_EXP=X_FP_METRO
selection = (spots["X_FP_METRO"] != 0)
spots["X_FP_EXP"][selection] = spots["X_FP_METRO"][selection]
selection = (spots["Y_FP_METRO"] != 0)
spots["Y_FP_EXP"][selection] = spots["Y_FP_METRO"][selection]
# write transfo
if args.output_transform is not None:
if not args.output_transform.endswith(".json"):
print(
"error, can only write json files, so please choose an output filename end ing with .json"
)
else:
tx.write_jsonfile(args.output_transform)
print("wrote transform in {}".format(args.output_transform))
if args.field_model is not None:
log.info("Reading field model in {}".format(args.field_model))
with open(args.field_model) as file:
fm = FieldModel.fromjson(file.read())
spots["RA"] = np.zeros(len(spots), dtype=float)
spots["DEC"] = np.zeros(len(spots), dtype=float)
ii = (spots["X_FP"] != 0) & (spots["Y_FP"] != 0)
ra, dec = fm.fp2radec(spots["X_FP"][ii], spots["Y_FP"][ii])
spots["RA"][ii] = ra
spots["DEC"][ii] = dec
# write spots
spots.write(outfile, format="csv", overwrite=True)
print("wrote {}".format(outfile))
return 0
