def test_simple_corr(self):
print("Testing simple corr")
nn=12
x1=np.random.uniform(size=nn)-0.5
y1=np.random.uniform(size=nn)-0.5
# arb. angle
a=10./180.*np.pi
ca=np.cos(a)
sa=np.sin(a)
# arb. dilatation
xscale = 12.
yscale = 13.
x3 = xscale*ca*x1 + yscale*sa*y1
y3 = -xscale*sa*x1 + yscale*ca*y1
# arb. translation
x3 += 33.
y3 += 11.
corr31=SimpleCorr()
corr31.fit(x3,y3,x1,y1)
x3b,y3b=corr31.apply(x3,y3)
dist=np.sqrt((x3b-x1)**2+(y3b-y1)**2)
assert(np.all(dist<1e-6))
print("Testing inverse")
x1b,y1b=corr31.apply_inverse(x1,y1)
dist=np.sqrt((x1b-x3)**2+(y1b-y3)**2)
assert(np.all(dist<1e-6))
def fit_gfa2fp(metrology):
"""
Fit GFA pix -> FP mm scale, rotation, xyoffsets for each GFA
Returns dict keyed by PETAL_LOC, with dictionaries of transform coeffs.
"""
#- HARDCODE: GFA pixel dimensions
nx, ny = 2048, 1032
#- Trim to just GFA entries without altering input table
gfarows = (metrology['DEVICE_TYPE'] == 'GFA')
metrology = metrology[gfarows]
metrology.sort(['PETAL_LOC', 'PINHOLE_ID'])
#- Metrology corners start at (0,0) for middle of pixel
#- Thankfully this is consistent with gfa_reduce, desimeter fvc spots,
#- and the Unified Metrology Table in DESI-5421
xgfa = np.array([0, nx-1, nx-1, 0])
ygfa = np.array([0, 0, ny-1, ny-1])
gfa_transforms = dict()
for p in range(10):
ii = (metrology['PETAL_LOC'] == p)
if np.count_nonzero(ii) > 0:
xfp = np.asarray(metrology['X_FP'][ii])
yfp = np.asarray(metrology['Y_FP'][ii])
zfp = np.asarray(metrology['Z_FP'][ii])
#- fit transform
corr = SimpleCorr()
corr.fit(xgfa, ygfa, xfp, yfp)
#- measure norm of plane
x01 = np.array( [ xfp[1]-xfp[0], yfp[1]-yfp[0], zfp[1]-zfp[0] ] )
x01 /= np.sqrt(np.sum(x01**2))
x12 = np.array( [ xfp[2]-xfp[1], yfp[2]-yfp[1], zfp[2]-zfp[1] ] )
x12 /= np.sqrt(np.sum(x12**2))
norm_vector= np.cross(x01,x12)
# I checked the sign of all components
# The guide CCDs are about 2.23 mm below the focal surface
# because of the filter of thickness 5.03+-0.01 mm
# and refractive index 1.805 to 1.791 from 578nm to 706nm
# see DESI-5336, https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=5336
# there is a correction to apply because the focal surface is curved
#- compute correction to apply
delta_z = 2.23 # mm
delta_x = delta_z*norm_vector[0]/norm_vector[2]
delta_y = delta_z*norm_vector[1]/norm_vector[2]
#- apply correction to offsets
corr.dx += delta_x
corr.dy += delta_y
gfa_transforms[p] = corr
return gfa_transforms
def fit_gfa2fp(metrology):
"""
Fit GFA pix -> FP mm scale, rotation, xyoffsets for each GFA
Returns dict keyed by PETAL_LOC, with dictionaries of transform coeffs.
"""
#- HARDCODE: GFA pixel dimensions
nx, ny = 2048, 1032
#- Trim to just GFA entries without altering input table
gfarows = (metrology['DEVICE_TYPE'] == 'GFA')
metrology = metrology[gfarows]
metrology.sort(['PETAL_LOC', 'PINHOLE_ID'])
#- Metrology corners start at (0,0) for middle of pixel
#- Thankfully this is consistent with gfa_reduce, desimeter fvc spots,
#- and the Unified Metrology Table in DESI-5421
xgfa = np.array([0, nx-1, nx-1, 0])
ygfa = np.array([0, 0, ny-1, ny-1])
gfa_transforms = dict()
for p in range(10):
ii = (metrology['PETAL_LOC'] == p)
if np.count_nonzero(ii) > 0:
xfp = np.asarray(metrology['X_FP'][ii])
yfp = np.asarray(metrology['Y_FP'][ii])
zfp = np.asarray(metrology['Z_FP'][ii])
#- fit transform
corr = SimpleCorr()
corr.fit(xgfa, ygfa, xfp, yfp)
#- measure norm of plane
x01 = np.array( [ xfp[1]-xfp[0], yfp[1]-yfp[0], zfp[1]-zfp[0] ] )
x01 /= np.sqrt(np.sum(x01**2))
x12 = np.array( [ xfp[2]-xfp[1], yfp[2]-yfp[1], zfp[2]-zfp[1] ] )
x12 /= np.sqrt(np.sum(x12**2))
norm_vector= np.cross(x01,x12)
# I checked the sign of all components
#- compute correction to apply
delta_z = 2.23 # mm
delta_x = delta_z*norm_vector[0]/norm_vector[2]
delta_y = delta_z*norm_vector[1]/norm_vector[2]
#- apply correction to offsets
corr.dx += delta_x
corr.dy += delta_y
gfa_transforms[p] = corr
return gfa_transforms
def findfiducials(spots, input_transform=None, pinhole_max_separation_mm=1.5):
global metrology_pinholes_table
global metrology_fiducials_table
log = get_logger()
log.debug(
"load input tranformation we will use to go from FP to FVC pixels")
if input_transform is None:
input_transform = fvc2fp_filename()
log.info("loading input tranform from {}".format(input_transform))
input_tx = FVC2FP.read_jsonfile(input_transform)
xpix = np.array([
2000.,
])
ypix = np.array([
0.,
])
xfp1, yfp1 = input_tx.fvc2fp(xpix, ypix)
xfp2, yfp2 = input_tx.fvc2fp(xpix + 1, ypix)
pixel2fp = np.hypot(xfp2 - xfp1, yfp2 - yfp1)[0] # mm
pinhole_max_separation_pixels = pinhole_max_separation_mm / pixel2fp
log.info(
"with pixel2fp = {:4.3f} mm, pinhole max separation = {:4.3f} pixels ".
format(pixel2fp, pinhole_max_separation_pixels))
if metrology_pinholes_table is None:
metrology_table = load_metrology()
log.debug("keep only the pinholes")
metrology_pinholes_table = metrology_table[:][
(metrology_table["DEVICE_TYPE"] == "FIF") |
(metrology_table["DEVICE_TYPE"] == "GIF")]
# use input transform to convert X_FP,Y_FP to XPIX,YPIX
xpix, ypix = input_tx.fp2fvc(metrology_pinholes_table["X_FP"],
metrology_pinholes_table["Y_FP"])
metrology_pinholes_table["XPIX"] = xpix
metrology_pinholes_table["YPIX"] = ypix
log.debug("define fiducial location as the most central dot")
central_pinholes = []
for loc in np.unique(metrology_pinholes_table["LOCATION"]):
ii = np.where(metrology_pinholes_table["LOCATION"] == loc)[0]
mx = np.mean(metrology_pinholes_table["XPIX"][ii])
my = np.mean(metrology_pinholes_table["YPIX"][ii])
k = np.argmin((metrology_pinholes_table["XPIX"][ii] - mx)**2 +
(metrology_pinholes_table["YPIX"][ii] - my)**2)
central_pinholes.append(ii[k])
metrology_fiducials_table = metrology_pinholes_table[:][
central_pinholes]
# find fiducials candidates
log.info("select spots with at least two close neighbors (in pixel units)")
nspots = spots["XPIX"].size
xy = np.array([spots["XPIX"], spots["YPIX"]]).T
tree = KDTree(xy)
measured_spots_distances, measured_spots_indices = tree.query(
xy, k=4, distance_upper_bound=pinhole_max_separation_pixels)
number_of_neighbors = np.sum(
measured_spots_distances < pinhole_max_separation_pixels, axis=1)
fiducials_candidates_indices = np.where(
number_of_neighbors >= 4)[0] # including self, so at least 3 pinholes
log.debug("number of fiducials=", fiducials_candidates_indices.size)
# match candidates to fiducials from metrology
log.info(
"first match {} fiducials candidates to metrology ({}) with iterative fit"
.format(fiducials_candidates_indices.size,
len(metrology_fiducials_table)))
x1 = spots["XPIX"][fiducials_candidates_indices]
y1 = spots["YPIX"][fiducials_candidates_indices]
x2 = metrology_fiducials_table["XPIX"]
y2 = metrology_fiducials_table["YPIX"]
nloop = 20
saved_median_distance = 0
for loop in range(nloop):
indices_2, distances = match_same_system(x1, y1, x2, y2)
mdist = np.median(distances[indices_2 >= 0])
if loop < nloop - 1:
maxdistance = max(10, 3. * 1.4 * mdist)
else: # final iteration
maxdistance = 10 # pixel
selection = np.where((indices_2 >= 0) & (distances < maxdistance))[0]
log.info("iter #{} median_dist={} max_dist={} matches={}".format(
loop, mdist, maxdistance, selection.size))
corr21 = SimpleCorr()
corr21.fit(x2[indices_2[selection]], y2[indices_2[selection]],
x1[selection], y1[selection])
x2, y2 = corr21.apply(x2, y2)
if np.abs(saved_median_distance - mdist) < 0.0001:
break # no more improvement
saved_median_distance = mdist
# use same coord system match (note we now match the otherway around)
indices_1, distances = match_same_system(x2, y2, x1, y1)
maxdistance = 10. # FVC pixels
selection = np.where((indices_1 >= 0) & (distances < maxdistance))[0]
fiducials_candidates_indices = fiducials_candidates_indices[
indices_1[selection]]
matching_known_fiducials_indices = selection
log.debug(
"mean distance = {:4.2f} pixels for {} matched and {} known fiducials".
format(np.mean(distances[distances < maxdistance]),
fiducials_candidates_indices.size,
metrology_fiducials_table["XPIX"].size))
log.debug("now matching pinholes ...")
nspots = spots["XPIX"].size
for k in ['LOCATION', 'PETAL_LOC', 'DEVICE_LOC', 'PINHOLE_ID']:
if k not in spots.dtype.names:
spots.add_column(Column(np.zeros(nspots, dtype=int)), name=k)
spots["LOCATION"][:] = -1
spots["PETAL_LOC"][:] = -1
spots["DEVICE_LOC"][:] = -1
spots["PINHOLE_ID"][:] = 0
for index1, index2 in zip(fiducials_candidates_indices,
matching_known_fiducials_indices):
location = metrology_fiducials_table["LOCATION"][index2]
# get indices of all pinholes for this matched fiducial
# note we now use the full pinholes metrology table
pi1 = measured_spots_indices[index1][
measured_spots_distances[index1] < pinhole_max_separation_pixels]
pi2 = np.where(metrology_pinholes_table["LOCATION"] == location)[0]
x1 = spots["XPIX"][pi1]
y1 = spots["YPIX"][pi1]
x2 = metrology_pinholes_table["XPIX"][pi2]
y2 = metrology_pinholes_table["YPIX"][pi2]
indices_2, distances = match_arbitrary_translation_dilatation(
x1, y1, x2, y2)
metrology_pinhole_ids = metrology_pinholes_table["PINHOLE_ID"][pi2]
pinhole_ids = np.zeros(x1.size, dtype=int)
matched = (indices_2 >= 0)
pinhole_ids[matched] = metrology_pinhole_ids[indices_2[matched]]
spots["LOCATION"][pi1[matched]] = location
spots["PINHOLE_ID"][pi1[matched]] = pinhole_ids[matched]
if np.sum(pinhole_ids == 0) > 0:
log.warning(
"only matched pinholes {} for {} detected at LOCATION {} xpix~{} ypix~{}"
.format(pinhole_ids[pinhole_ids > 0], x1.size, location,
int(np.mean(x1)), int(np.mean(y1))))
# check duplicates
if np.unique(
pinhole_ids[pinhole_ids > 0]).size != np.sum(pinhole_ids > 0):
xfp = np.mean(metrology_pinholes_table[pi2]["X_FP"])
yfp = np.mean(metrology_pinholes_table[pi2]["Y_FP"])
log.warning(
"duplicate(s) pinhole ids in {} at LOCATION={} xpix~{} ypix~{} xfp~{} yfp~{}"
.format(pinhole_ids, location, int(np.mean(x1)),
int(np.mean(y1)), int(xfp), int(yfp)))
bc = np.bincount(pinhole_ids[pinhole_ids > 0])
duplicates = np.where(bc > 1)[0]
for duplicate in duplicates:
log.warning(
"Unmatch ambiguous pinhole id = {}".format(duplicate))
selection = (spots["LOCATION"]
== location) & (spots["PINHOLE_ID"] == duplicate)
spots["PINHOLE_ID"][selection] = 0
ii = (spots["LOCATION"] >= 0)
spots["PETAL_LOC"][ii] = spots["LOCATION"][ii] // 1000
spots["DEVICE_LOC"][ii] = spots["LOCATION"][ii] % 1000
n_matched_pinholes = np.sum(spots["PINHOLE_ID"] > 0)
n_matched_fiducials = np.sum(spots["PINHOLE_ID"] == 4)
log.info("matched {} pinholes from {} fiducials".format(
n_matched_pinholes, n_matched_fiducials))
return spots
def average_coordinates(tables,xkey,ykey) :
"""
Average x,y coordinates given by xkey and ykey from a list of astropy tables
and return an astropy table with the average coordinates.
This function includes a match and a transformation per table.
The tables do not necessarily have the same number of entries (in case of false detections).
Args
tables: list of astropy.Table objects with same columns
Returns astropy.Table with same columns as input
"""
table1=None
x1=None
y1=None
indices=None
xx=[]
yy=[]
for table in tables :
x2=np.array(table[xkey])
y2=np.array(table[ykey])
if x1 is None :
table1=table
x1=x2
y1=y2
indices=np.arange(len(x1),dtype=int)
else :
# match the two sets of spots
indices_1 = np.arange(len(x1),dtype=int)
indices_2, distances = match_same_system(x1,y1,x2,y2)
ok=np.where((indices_2>=0)&(distances<5.))[0]
indices_1 = indices_1[ok]
indices_2 = indices_2[ok]
distances = distances[ok]
x1=x1[indices_1]
y1=y1[indices_1]
indices=indices[indices_1]
x2=x2[indices_2]
y2=y2[indices_2]
n=len(xx)
for i in range(n) :
xx[i]=xx[i][indices_1]
yy[i]=yy[i][indices_1]
# adjust a possible transfo between the two FVC images
corr=SimpleCorr()
corr.fit(x2,y2,x1,y1)
x2,y2=corr.apply(x2,y2)
xx.append(x2)
yy.append(y2)
xx=np.vstack(xx)
yy=np.vstack(yy)
xrms=np.std(xx,axis=0)
yrms=np.std(yy,axis=0)
mx=np.mean(xx,axis=0)
my=np.mean(yy,axis=0)
table1[xkey][indices]=mx
table1[ykey][indices]=my
print("number of entries found in all tables= {}".format(indices.size))
print("rms({})= {:4.3f}".format(xkey,np.median(xrms)))
print("rms({})= {:4.3f}".format(ykey,np.median(yrms)))
return table1
def findfiducials(spots, input_transform=None, separation=8.):
global metrology_pinholes_table
global metrology_fiducials_table
log = get_logger()
log.debug(
"load input tranformation we will use to go from FP to FVC pixels")
if input_transform is None:
input_transform = resource_filename('desimeter',
"data/single-lens-fvc2fp.json")
log.info("loading input tranform from {}".format(input_transform))
try:
input_tx = FVCFP_ZhaoBurge.read_jsonfile(input_transform)
except AssertionError as e:
log.warning(
"Failed to read input transfo as Zhao Burge, try polynomial...")
input_tx = FVCFP_Polynomial.read_jsonfile(input_transform)
if metrology_pinholes_table is None:
filename = resource_filename('desimeter', "data/fp-metrology.csv")
if not os.path.isfile(filename):
log.error("cannot find {}".format(filename))
raise IOError("cannot find {}".format(filename))
log.info("reading metrology in {}".format(filename))
metrology_table = Table.read(filename, format="csv")
log.debug("keep only the pinholes")
metrology_pinholes_table = metrology_table[:][
metrology_table["PINHOLE_ID"] > 0]
# use input transform to convert X_FP,Y_FP to XPIX,YPIX
xpix, ypix = input_tx.fp2fvc(metrology_pinholes_table["X_FP"],
metrology_pinholes_table["Y_FP"])
metrology_pinholes_table["XPIX"] = xpix
metrology_pinholes_table["YPIX"] = ypix
log.debug("define fiducial location as central dot")
metrology_fiducials_table = metrology_pinholes_table[:][
metrology_pinholes_table["PINHOLE_ID"] == 4]
# find fiducials candidates
log.info("select spots with at least two close neighbors (in pixel units)")
xy = np.array([spots["XPIX"], spots["YPIX"]]).T
tree = KDTree(xy)
measured_spots_distances, measured_spots_indices = tree.query(
xy, k=4, distance_upper_bound=separation)
number_of_neighbors = np.sum(measured_spots_distances < separation, axis=1)
fiducials_candidates_indices = np.where(
number_of_neighbors >= 3)[0] # including self, so at least 3 pinholes
# match candidates to fiducials from metrology
log.info(
"first match {} fiducials candidates to metrology ({}) with iterative fit"
.format(fiducials_candidates_indices.size,
len(metrology_fiducials_table)))
x1 = spots["XPIX"][fiducials_candidates_indices]
y1 = spots["YPIX"][fiducials_candidates_indices]
x2 = metrology_fiducials_table["XPIX"] # do I need to do this?
y2 = metrology_fiducials_table["YPIX"]
nloop = 20
saved_median_distance = 0
for loop in range(nloop):
indices_2, distances = match_same_system(x1, y1, x2, y2)
mdist = np.median(distances[indices_2 >= 0])
if loop < nloop - 1:
maxdistance = max(10, 3. * 1.4 * mdist)
else: # final iteration
maxdistance = 10 # pixel
selection = np.where((indices_2 >= 0) & (distances < maxdistance))[0]
log.info("iter #{} median_dist={} max_dist={} matches={}".format(
loop, mdist, maxdistance, selection.size))
corr21 = SimpleCorr()
corr21.fit(x2[indices_2[selection]], y2[indices_2[selection]],
x1[selection], y1[selection])
x2, y2 = corr21.apply(x2, y2)
if np.abs(saved_median_distance - mdist) < 0.0001:
break # no more improvement
saved_median_distance = mdist
# use same coord system match (note we now match the otherway around)
indices_1, distances = match_same_system(x2, y2, x1, y1)
maxdistance = 10. # FVC pixels
selection = np.where((indices_1 >= 0) & (distances < maxdistance))[0]
fiducials_candidates_indices = fiducials_candidates_indices[
indices_1[selection]]
matching_known_fiducials_indices = selection
log.debug(
"mean distance = {:4.2f} pixels for {} matched and {} known fiducials".
format(np.mean(distances[distances < maxdistance]),
fiducials_candidates_indices.size,
metrology_fiducials_table["XPIX"].size))
log.debug("now matching pinholes ...")
nspots = spots["XPIX"].size
if 'LOCATION' not in spots.dtype.names:
spots.add_column(Column(np.zeros(nspots, dtype=int)), name='LOCATION')
if 'PINHOLE_ID' not in spots.dtype.names:
spots.add_column(Column(np.zeros(nspots, dtype=int)),
name='PINHOLE_ID')
for index1, index2 in zip(fiducials_candidates_indices,
matching_known_fiducials_indices):
location = metrology_fiducials_table["LOCATION"][index2]
# get indices of all pinholes for this matched fiducial
# note we now use the full pinholes metrology table
pi1 = measured_spots_indices[index1][
measured_spots_distances[index1] < separation]
pi2 = np.where(metrology_pinholes_table["LOCATION"] == location)[0]
x1 = spots["XPIX"][pi1]
y1 = spots["YPIX"][pi1]
x2 = metrology_pinholes_table["XPIX"][pi2]
y2 = metrology_pinholes_table["YPIX"][pi2]
indices_2, distances = match_arbitrary_translation_dilatation(
x1, y1, x2, y2)
metrology_pinhole_ids = metrology_pinholes_table["PINHOLE_ID"][pi2]
pinhole_ids = np.zeros(x1.size, dtype=int)
matched = (indices_2 >= 0)
pinhole_ids[matched] = metrology_pinhole_ids[indices_2[matched]]
spots["LOCATION"][pi1[matched]] = location
spots["PINHOLE_ID"][pi1[matched]] = pinhole_ids[matched]
if np.sum(pinhole_ids == 0) > 0:
log.warning(
"only matched pinholes {} for {} detected at LOCATION {} xpix~{} ypix~{}"
.format(pinhole_ids[pinhole_ids > 0], x1.size, location,
int(np.mean(x1)), int(np.mean(y1))))
# check duplicates
if np.unique(
pinhole_ids[pinhole_ids > 0]).size != np.sum(pinhole_ids > 0):
xfp = np.mean(metrology_pinholes_table[pi2]["X_FP"])
yfp = np.mean(metrology_pinholes_table[pi2]["Y_FP"])
log.warning(
"duplicate(s) pinhole ids in {} at LOCATION={} xpix~{} ypix~{} xfp~{} yfp~{}"
.format(pinhole_ids, location, int(np.mean(x1)),
int(np.mean(y1)), int(xfp), int(yfp)))
bc = np.bincount(pinhole_ids[pinhole_ids > 0])
duplicates = np.where(bc > 1)[0]
for duplicate in duplicates:
log.warning(
"Unmatch ambiguous pinhole id = {}".format(duplicate))
selection = (spots["LOCATION"]
== location) & (spots["PINHOLE_ID"] == duplicate)
spots["PINHOLE_ID"][selection] = 0
spots["PETAL_LOC"] = spots["LOCATION"] // 1000
spots["DEVICE_LOC"] = spots["LOCATION"] % 1000
n_matched_pinholes = np.sum(spots["PINHOLE_ID"] > 0)
n_matched_fiducials = np.sum(spots["PINHOLE_ID"] == 4)
log.info("matched {} pinholes from {} fiducials".format(
n_matched_pinholes, n_matched_fiducials))
return spots