def testShiftRotateXYZ10(self):
"""
Shift by 10 in all directions.
"""
rot = [0, 0, 0]
shift = [10, 10, 10]
x = self.x - 10
y = self.y - 10
z = self.z - 10
xr, yr, zr = shiftRotateXYZ(x, y, z, shift + rot)
np.testing.assert_allclose(xr, self.x, rtol=1e-15, atol=1e-15)
np.testing.assert_allclose(yr, self.y, rtol=1e-15, atol=1e-15)
np.testing.assert_allclose(zr, self.z, rtol=1e-15, atol=1e-15)
def testShiftRotateXY90Z(self):
"""
Rotation along z by 90 degrees.
"""
rot = [0, 0, np.deg2rad(90)]
shift = [0, 0, 0]
x = self.x + 1.
y = self.y
z = self.z
xr, yr, zr = shiftRotateXYZ(x, y, z, shift + rot)
np.testing.assert_allclose(xr, y, rtol=1e-15, atol=1e-15)
np.testing.assert_allclose(yr, x, rtol=1e-15, atol=1e-15)
np.testing.assert_allclose(zr, z, rtol=1e-15, atol=1e-15)
def testShiftRotateXYZ0(self):
"""
No rotation nor shift.
"""
rot = [0, 0, 0]
shift = [0, 0, 0]
x = self.x + 1.
y = self.y + 1.
z = self.z
xr, yr, zr = shiftRotateXYZ(x, y, z, shift + rot)
np.testing.assert_array_equal(xr, x)
np.testing.assert_array_equal(yr, y)
np.testing.assert_array_equal(zr, z)
def maskScan(fn, n=512, rot=0):
"""
"""
orgData, cleanData = loadLeicaData(fn, n=None, numpy=False)
x = orgData[0]
y = orgData[1]
z = orgData[2]
xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
xga, yga, zga = maskScanXYZ(xr,
yr,
zr,
n=512,
outMaskThreshold=4,
inMaskThreshold=3,
inMaskClip=5)
return xga, yga, zga
def processNewPTXData(lines,
xyzi=None,
dts=None,
plotTest=True,
rot=None,
sampleSize=None,
iFilter=False,
nFilter=True,
rFilter=True,
filterClose=True,
filterParaboloid=True,
ellipse=[-8., 50., 49., 49., 0.],
residual_threshold=0.008):
"""
"""
"this is the processing we see works with 2019 data"
if rot is None:
rot = -90.0
if ellipse is None:
warnings.warn("No ellipse given. Will use default values.")
ellipse = [-8., 50., 49., 49., 0.]
print("ProcessNewPTXData with: ", ellipse)
if plotTest and lines is not None:
# make some plots that ensure how we are doing
# our radial filtering
tryEllipticalOffsets(lines, ellipse)
if lines is not None:
# Get the actual float values from the file contents.
x, y, z, i = getRawXYZ(lines, sampleSize=sampleSize)
else:
x, y, z, i = xyzi
print("Starting with %d lines of data" % len(x))
# First remove all the zero data.
print("Filtering measurements with zero intensity.")
mask = i != 0.0
intensity = i[mask]
x = x[mask]
y = y[mask]
z = z[mask]
if dts is not None:
dts = dts[mask]
printMaskStats(mask)
# Remove aggregious jumps in data?
if nFilter:
print("Neighbor filter.")
# TBF: document where our tolerance comes from
x, y, z, mask = neighborFilter(x, y, z, 0.122)
intensity = intensity[mask]
print("Now we have %d lines of data" % len(x))
if dts is not None:
dts = dts[mask]
# We only want data that has a decent intesity.
meanI = np.mean(intensity)
stdI = np.std(intensity)
print("Intensity: max=%5.2f, min=%5.2f, mean=%5.2f, std=%5.2f" %
(np.max(intensity), np.min(intensity), meanI, stdI))
if False: # iFilter
mask = np.logical_and(intensity > 0.75, intensity < 0.85)
intensity = intensity[mask]
x = x[mask]
y = y[mask]
z = z[mask]
if dts is not None:
dts = dts[mask]
numFilteredOut = len(x) - len(intensity)
percent = (float(numFilteredOut) / float(len(x))) * 100.
print("Filtered out %d points of %d (%5.2f%%) via intensity" %
(numFilteredOut, len(x), percent))
print(("Now we have %d lines of data" % len(x)))
assert len(x) == len(y)
assert len(y) == len(z)
assert len(z) == len(intensity)
# Save the point cloud thus far for the paraboloid filter.
x_p = copy(x)
y_p = copy(y)
z_p = copy(z)
i_p = copy(intensity)
# We want as much as possible of the dish.
# Since the dish will be rotated, it will look like an ellipse to the TLS.
# This filter is later superseded by the paraboloid filter, but it is
# necessary to find a best fit paraboloid.
if rFilter:
orgNum = len(x)
print('Elliptical filter.')
print(
"The ellipse has semi-major axis {0:.2f} m, semi-minor axis {1:.2f} m and angle {2:.2f} degrees"
.format(ellipse[2], ellipse[3], ellipse[4]))
x, y, z, dts, intensity = ellipticalFilter(x,
y,
z,
ellipse[0],
ellipse[1],
ellipse[2],
ellipse[3],
ellipse[4],
dts=dts,
intensity=intensity)
newNum = len(x)
print(
"Elliptical filter removed {0} points outside the ellipse".format(
orgNum - newNum))
print("Now we have %d lines of data" % len(x))
# The scanner is upside down.
z = -z
# Filter points below the dish.
print("z filter.")
zLimit = -80
x, y, z, dts, intensity = zLimitFilter(x,
y,
z,
zLimit=zLimit,
dts=dts,
intensity=intensity)
if filterClose:
tooClose = 10.
print("Removing points closer than {0:.2f} m from the scanner.".format(
tooClose))
# Removes points that are closer than 10 m from the scanner.
x, y, z, dts, intensity = nearFilter(x,
y,
z,
tol=tooClose,
dts=dts,
intensity=intensity)
# Rotate the data for the paraboloid filter.
print("Rotating about z by {0:5.2f} degrees.".format(rot))
xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
if filterParaboloid:
print("Paraboloid filter.")
perc = 0.01
print("Fitting a paraboloid to {}% of the data.".format(perc * 100.))
sampleSize = int(len(xr) * perc)
lsIdx = random.sample(range(len(xr)), sampleSize)
xs = xr[lsIdx]
ys = yr[lsIdx]
zs = zr[lsIdx]
# Fit a parabola to the data.
pMask, _ = paraboloidFilter(xs, ys, zs, threshold=3)
# Fit the paraboloid again, discarding the already filtered values.
print("Fitting paraboloid again to the un-filtered values.")
xs = xs[pMask]
ys = ys[pMask]
zs = zs[pMask]
pMask, paraFit = paraboloidFilter(xs, ys, zs, threshold=3)
print(
"Applying paraboloid filter to the data before the elliptical filter."
)
x = x_p
y = y_p
z = z_p
i = i_p
z = -z
x, y, z = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
print("Number of data points: {}".format(len(z)))
pMask = paraboloidMask(x, y, z, paraFit, threshold=2)
print("Applying paraboloid filter to the data.")
printMaskStats(pMask)
x = x[pMask]
y = y[pMask]
z = z[pMask]
i = i[pMask]
# Rotate and shift the range measurements.
#cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0))
#xrp, yrp, zrp = shiftRotateXYZ(x, y, z, cor)
xrp, yrp, zrp = align2Paraboloid(x, y, z, paraFit)
# Use a circle to mask data outside of the primary reflector.
r = 51
xc = np.mean((np.nanmax(xrp) - r, np.nanmin(xrp) + r))
yc = np.nanmax(yrp) - r
rMask = radialMask(xrp, yrp, r, xc=xc, yc=yc)
x = x[rMask]
y = y[rMask]
z = z[rMask]
i = i[rMask]
print("Applying circular mask to the paraboloid filtered data.")
printMaskStats(rMask)
#cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0))
#xr, yr, zr = shiftRotateXYZ(x, y, z, cor)
## Rotate and shift the range measurements.
#cor = np.hstack((-1*c[1:4],c[4:6],0))
#xr, yr, zr = shiftRotateXYZ(x, y, z, cor)
## Build a parabola using the best fit parameters.
#zp = paraboloid(xr, yr, c[0])
## Compute the residuals between the parabola and the rotated range measurements.
#diff = zr - zp
## Only keep range measurements whose residuals are less than residual_threshold.
##print("Removing points with residuals larger than {}".format(residual_threshold))
##mask = abs(diff) < residual_threshold
#mdiff = sigma_clip(diff, 3)
#mask = ~mdiff.mask
#percent = mask.sum()/len(mask) * 100.
#print("Parabola filter will remove {0:.2f}% of the data.".format(100. - percent))
#print("Keeping {} out of {} points.".format(mask.sum(), len(mask)))
## Use the rotated range measurements for the last filter.
#xr = xr[mask]
#yr = yr[mask]
#zr = zr[mask]
#x = x[mask]
#y = y[mask]
#z = z[mask]
#intensity = intensity[mask]
## Only keep range measurements within a circle of radius r.
#r = 51 # m
#xc = np.nanmin(xr) + r
#yc = np.nanmin(yr) + r
#circular_mask = (xr - xc)**2 + (yr - yc)**2 <= r**2.
#x = x[circular_mask]
#y = y[circular_mask]
#z = z[circular_mask]
#intensity = intensity[circular_mask]
#print("Keeping {} points inside a circle of radius {} m and center ({},{}) m.".format(circular_mask.sum(), r, xc, yc))
# Rotate around the z axis since the scanner coordinate system does not match the telescope's.
#if rot is not None or rot != 0.0:
# print("Rotating about z by %5.2f degrees" % rot)
# xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
# Create a Nx3 matrix for the coordinates.
xyz = np.c_[x, y, z]
if plotTest:
print("Plotting.")
xlim = (-100, 100)
ylim = (-100, 100)
fig, ax = scatter3dPlot(xr,
yr,
zr,
"Sampling of processed data",
xlim=xlim,
ylim=ylim,
sample=1)
# take a look at the x y orientation
f = plt.figure()
ax = f.gca()
ax.plot(xr, yr, 'bo', [0], [0], '*')
plt.xlabel("x")
plt.ylabel("y")
plt.title("X Y orientation of data")
print("XYZ output of ProcessNewPTXData has {0} lines of data.".format(
len(xyz)))
print(
"Intensity output of ProcessNewPTXData has {0} lines of data.".format(
len(intensity)))
return xyz, dts, i
def maskXYZ(x,
y,
z,
n=512,
guess=[60., 0., 0., 0., 0., 0.],
bounds=None,
radialMask=True,
maskRadius=40.,
poly_order=2,
**kwargs):
"""
"""
xf = x[~np.isnan(z)]
yf = y[~np.isnan(z)]
zf = z[~np.isnan(z)]
# Use masked arrays on the input data to avoid warnings.
x = np.ma.masked_invalid(x)
y = np.ma.masked_invalid(y)
z = np.ma.masked_invalid(z)
# Fit a parabola to the data.
fitresult = fitLeicaData(xf, yf, zf, guess, bounds=bounds, weights=None)
# Subtract the fitted parabola from the data.
# The difference should be flat.
c = fitresult.x
zp = paraboloid(x, y, c[0])
cor = np.hstack((-1 * c[1:4], c[4:6], 0))
xrr, yrr, zrr = shiftRotateXYZ(x, y, z, cor)
diff = zrr - zp
if radialMask:
xc = midPoint(xrr)
yc = midPoint(yrr)
mask = (((xrr - xc)**2. + (yrr - yc)**2.) < maskRadius**2.)
mdiff = np.ma.masked_where(~mask, diff)
else:
mdiff = diff
# Mask any pixels which deviate from the noise.
# This should mask out retroreflectors, misaligned panels and sub-scans where the TLS moved due to wind.
mcdiff = sigma_clip(mdiff)
# Apply the mask to the original data and repeat once more.
# In the end also fit a parabola to each row in the data, and mask outliers.
# This allows to find more subtle features in the data.
xf = np.ma.masked_where(mcdiff.mask, x)
yf = np.ma.masked_where(mcdiff.mask, y)
zf = np.ma.masked_where(mcdiff.mask, z)
masked_fitresult = fitLeicaData(xf.compressed(),
yf.compressed(),
zf.compressed(),
guess,
bounds=bounds,
weights=None)
c = masked_fitresult.x
zp = paraboloid(x, y, c[0])
cor = np.hstack((-1 * c[1:4], c[4:6], 0))
xrrm, yrrm, zrrm = shiftRotateXYZ(x, y, z, cor)
masked_diff = zrrm - zp
mcdiff2 = masked_diff
# Final mask.
map_mask = np.zeros((n, n), dtype=bool)
xl = np.linspace(0, n, n)
# Loop over rows fitting a parabola and masking any pixels that deviate from noise.
for i in range(0, n):
yl = mcdiff2[i]
if len(xl[~yl.mask]) > 3:
poly_c = np.polyfit(xl[~yl.mask], yl[~yl.mask], poly_order)
poly_f = np.poly1d(poly_c)
res = np.ma.masked_invalid(yl - poly_f(xl))
res_sc = sigma_clip(res, **kwargs)
map_mask[i] = res_sc.mask
else:
map_mask[i] = True
# Prepare output.
origData = (x, y, z)
origMaskedData = (np.ma.masked_where(map_mask,
x), np.ma.masked_where(map_mask, y),
np.ma.masked_where(map_mask, z))
rotatedData = (xrrm, yrrm, np.ma.masked_where(map_mask, zrrm))
fitResidual = np.ma.masked_where(map_mask, zrrm - zp)
parabolaFit = (xrrm, yrrm, zp)
outData = {
'origData': origData,
'origMasked': origMaskedData,
'rotated': rotatedData,
'fitResidual': fitResidual,
'parabolaFit': parabolaFit,
'parabolaFitCoeffs': c
}
return outData