def trySmoothGPUMulti(N=10):
if N is None:
N = 10
gpuPath1 = "/home/sandboxes/pmargani/LASSI/gpus/versions/gpuMulti"
gpuPath2 = "/home/sandboxes/pmargani/LASSI/gpus/versions/gpuMulti2"
gpuPaths = [gpuPath1, gpuPath2]
inFile1 = "/home/scratch/pmargani/LASSI/scannerData/Clean9.ptx.csv"
inFile2 = "/home/scratch/pmargani/LASSI/scannerData/Clean9.ptx.copy.csv"
inFiles = [inFile1, inFile2]
host1 = settings.GPU_HOST
host2 = settings.GPU_HOST_2
hosts = [host1, host2]
outFile = 'outFile'
smoothGPUMulti(gpuPaths, hosts, inFiles, outFile, N)
# did it work?
outfiles = []
xyzs = []
for i, gpuPath in enumerate(gpuPaths):
for dim in ['x', 'y', 'z']:
dimFile = "%s.%d.%s.csv" % (outFile, (i + 1), dim)
dimPath = os.path.join(gpuPath, dimFile)
outfiles.append(dimPath)
print(dimPath)
assert os.path.isfile(dimPath)
print(("GPUs created file: ", dimPath))
loadFile = os.path.join(gpuPath, "%s.%d" % (outFile, (i + 1)))
xyz = loadLeicaDataFromGpus(loadFile)
xyzs.append(xyz)
x1, y1, z1 = xyzs[0]
x2, y2, z2 = xyzs[1]
x = np.concatenate((x1, x2))
y = np.concatenate((y1, y2))
z = np.concatenate((z1, z2))
scatter3dPlot(x, y, z, "testSmoothGPUMulti")
def previewData(ptxFile, sample=None):
"""
Loads a PTX file and displays its contents as a 3D scatter plot.
:param ptxFile: PTX file to display.
:param sample: Percentage of the data to display.
"""
print("opening file", ptxFile)
with open(ptxFile, 'r') as f:
ls = f.readlines()
print("reading data ...")
x, y, z, i = getRawXYZ(ls)
print("plotting preview ...")
if sample is None:
sample = 1.0
scatter3dPlot(x, y, z, "preview", sample=sample)
def testRegridXYZ(self):
"Make sure resampling happens sensibly"
# construct some cartesian data
M = 100
width = 10.
maxMin = width / 2.
x = np.linspace(-maxMin, maxMin, M)
y = np.linspace(-maxMin, maxMin, M)
xm, ym = np.meshgrid(x, y)
z = xm * ym
if self.plots:
scatter3dPlot(xm, ym, z, "org")
N = 10
x2, y2, z2 = regridXYZ(xm, ym, z, n=N)
self.assertEquals(x2.shape, (N, N))
self.assertEquals(y2.shape, (N, N))
self.assertEquals(z2.shape, (N, N))
# check the x, y grid - based of min and maxs
self.assertEquals(x2[0, 0], xm[0, 0])
self.assertEquals(y2[0, 0], ym[0, 0])
# spot check the regridded data
self.assertEquals(z[0, 0], 25.0)
# make sure the residuals are low
for i in range(N):
for j in range(N):
# print i, j, x2[i, j], y2[i, j], np.abs((x2[i,j] * y2[i,j]) - z2[i, j])
diff = np.abs((x2[i, j] * y2[i, j]) - z2[i, j])
self.assertTrue(diff < 1e-2)
if self.plots:
scatter3dPlot(x2, y2, z2, "regrid")
# now, how do these max values work?
xmin = ymin = -1.0
x2, y2, z2 = regridXYZ(xm, ym, z, n=N, xmin=xmin, ymin=ymin)
if self.plots:
scatter3dPlot(x2, y2, z2, "regrid")
self.assertEquals(x2.shape, (N, N))
self.assertEquals(y2.shape, (N, N))
self.assertEquals(z2.shape, (N, N))
# check the x, y grid
self.assertEquals(x2[0, 0], xmin)
self.assertEquals(y2[0, 0], ymin)
def weightSmooth(fpath, xyz):
N = 512
# convert to shperical
# s = 1.0
# print "working with %f percent of data" % s
x, y, z = splitXYZ(xyz)
# x, y, z = sampleXYZData(x, y, z, s)
scatter3dPlot(x, y, z, "org sample of data", sample=1.0)
xOrg = copy(x)
yOrg = copy(y)
zOrg = copy(z)
# sigma2 = <r>**2 - <r**2>
################# FIRST: calculate <r>
smoothedXYZfiles = smooth(fpath)
basename = os.path.basename(fpath)
gpuPath = os.path.join(GPU_PATH, basename)
print("loading GPU data from", gpuPath)
xs, ys, zs = loadLeicaDataFromGpus(gpuPath)
scatter3dPlot(xs, ys, zs, "sample of smoothed data", sample=1.0)
# we need the smoothed r, <r>, so convert this
print("Converting to spherical ...")
rs, els, azs = cart2sph(xs, ys, zs)
################ Second: calcualte <r**2>
# convert the original xyz to spherical
r, el, az = cart2sph(x, y, z)
# square the r term, then go back to cartesian
r2 = r**2
x2, y2, z2 = sph2cart(el, az, r2)
# write this to a file for smoothing
xyz2 = aggregateXYZ(x2, y2, z2)
f2 = basename + ".rs.csv"
basepath = os.path.dirname(fpath)
fpath2 = os.path.join(basepath, f2)
print("Saving r**2 data in xyz to file", fpath2)
np.savetxt(fpath2, xyz2, delimiter=',')
# now smooth this data
smoothedXYZ2files = smooth(fpath2)
# retrieve the xyz data
gpuPath2 = os.path.join(GPU_PATH, f2)
print("loading GPU data from", gpuPath2)
xs2, ys2, zs2 = loadLeicaDataFromGpus(gpuPath2)
# convert back to shperical to get <r**2>
rs2, els2, azs2 = cart2sph(xs2, ys2, zs2)
####################: Finally: Calculate Variance
# make sure small negative numerical errors are dealt with
sigma2 = np.abs(rs2 - (rs**2))
print("sigma squared: ", sigma2.shape, np.nanmin(sigma2),
np.nanmax(sigma2), np.nanmean(sigma2), np.nanstd(sigma2))
# not normalized weights
Ws_Not_Norm = 1 / sigma2
# make sure NaNs induced by zeros in sigma2 are dealt with
Ws_Not_Norm[sigma2 == 0.0] = 0.0
# normalize weights, making sure Nans in sum are dealt with
ws = (Ws_Not_Norm) / np.sum(Ws_Not_Norm[np.logical_not(np.isnan(sigma2))])
print("Computed weights: ", ws.shape, ws)
# plot weights
sigma2.shape = (N, N)
imagePlot(sigma2, "sigma^2")
imagePlot(np.log(np.abs(sigma2)), "log sigma^2")
wsc = copy(ws)
wsc.shape = (N, N)
imagePlot(wsc, "weights")
imagePlot(np.log(np.abs(wsc)), "log weights")
return smoothedXYZfiles, ws
def processNewPTXData(lines,
xyzi=None,
dts=None,
plotTest=True,
rot=None,
sampleSize=None,
iFilter=False,
nFilter=True,
rFilter=True,
filterClose=True,
filterParaboloid=True,
ellipse=[-8., 50., 49., 49., 0.],
residual_threshold=0.008):
"""
"""
"this is the processing we see works with 2019 data"
if rot is None:
rot = -90.0
if ellipse is None:
warnings.warn("No ellipse given. Will use default values.")
ellipse = [-8., 50., 49., 49., 0.]
print("ProcessNewPTXData with: ", ellipse)
if plotTest and lines is not None:
# make some plots that ensure how we are doing
# our radial filtering
tryEllipticalOffsets(lines, ellipse)
if lines is not None:
# Get the actual float values from the file contents.
x, y, z, i = getRawXYZ(lines, sampleSize=sampleSize)
else:
x, y, z, i = xyzi
print("Starting with %d lines of data" % len(x))
# First remove all the zero data.
print("Filtering measurements with zero intensity.")
mask = i != 0.0
intensity = i[mask]
x = x[mask]
y = y[mask]
z = z[mask]
if dts is not None:
dts = dts[mask]
printMaskStats(mask)
# Remove aggregious jumps in data?
if nFilter:
print("Neighbor filter.")
# TBF: document where our tolerance comes from
x, y, z, mask = neighborFilter(x, y, z, 0.122)
intensity = intensity[mask]
print("Now we have %d lines of data" % len(x))
if dts is not None:
dts = dts[mask]
# We only want data that has a decent intesity.
meanI = np.mean(intensity)
stdI = np.std(intensity)
print("Intensity: max=%5.2f, min=%5.2f, mean=%5.2f, std=%5.2f" %
(np.max(intensity), np.min(intensity), meanI, stdI))
if False: # iFilter
mask = np.logical_and(intensity > 0.75, intensity < 0.85)
intensity = intensity[mask]
x = x[mask]
y = y[mask]
z = z[mask]
if dts is not None:
dts = dts[mask]
numFilteredOut = len(x) - len(intensity)
percent = (float(numFilteredOut) / float(len(x))) * 100.
print("Filtered out %d points of %d (%5.2f%%) via intensity" %
(numFilteredOut, len(x), percent))
print(("Now we have %d lines of data" % len(x)))
assert len(x) == len(y)
assert len(y) == len(z)
assert len(z) == len(intensity)
# Save the point cloud thus far for the paraboloid filter.
x_p = copy(x)
y_p = copy(y)
z_p = copy(z)
i_p = copy(intensity)
# We want as much as possible of the dish.
# Since the dish will be rotated, it will look like an ellipse to the TLS.
# This filter is later superseded by the paraboloid filter, but it is
# necessary to find a best fit paraboloid.
if rFilter:
orgNum = len(x)
print('Elliptical filter.')
print(
"The ellipse has semi-major axis {0:.2f} m, semi-minor axis {1:.2f} m and angle {2:.2f} degrees"
.format(ellipse[2], ellipse[3], ellipse[4]))
x, y, z, dts, intensity = ellipticalFilter(x,
y,
z,
ellipse[0],
ellipse[1],
ellipse[2],
ellipse[3],
ellipse[4],
dts=dts,
intensity=intensity)
newNum = len(x)
print(
"Elliptical filter removed {0} points outside the ellipse".format(
orgNum - newNum))
print("Now we have %d lines of data" % len(x))
# The scanner is upside down.
z = -z
# Filter points below the dish.
print("z filter.")
zLimit = -80
x, y, z, dts, intensity = zLimitFilter(x,
y,
z,
zLimit=zLimit,
dts=dts,
intensity=intensity)
if filterClose:
tooClose = 10.
print("Removing points closer than {0:.2f} m from the scanner.".format(
tooClose))
# Removes points that are closer than 10 m from the scanner.
x, y, z, dts, intensity = nearFilter(x,
y,
z,
tol=tooClose,
dts=dts,
intensity=intensity)
# Rotate the data for the paraboloid filter.
print("Rotating about z by {0:5.2f} degrees.".format(rot))
xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
if filterParaboloid:
print("Paraboloid filter.")
perc = 0.01
print("Fitting a paraboloid to {}% of the data.".format(perc * 100.))
sampleSize = int(len(xr) * perc)
lsIdx = random.sample(range(len(xr)), sampleSize)
xs = xr[lsIdx]
ys = yr[lsIdx]
zs = zr[lsIdx]
# Fit a parabola to the data.
pMask, _ = paraboloidFilter(xs, ys, zs, threshold=3)
# Fit the paraboloid again, discarding the already filtered values.
print("Fitting paraboloid again to the un-filtered values.")
xs = xs[pMask]
ys = ys[pMask]
zs = zs[pMask]
pMask, paraFit = paraboloidFilter(xs, ys, zs, threshold=3)
print(
"Applying paraboloid filter to the data before the elliptical filter."
)
x = x_p
y = y_p
z = z_p
i = i_p
z = -z
x, y, z = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
print("Number of data points: {}".format(len(z)))
pMask = paraboloidMask(x, y, z, paraFit, threshold=2)
print("Applying paraboloid filter to the data.")
printMaskStats(pMask)
x = x[pMask]
y = y[pMask]
z = z[pMask]
i = i[pMask]
# Rotate and shift the range measurements.
#cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0))
#xrp, yrp, zrp = shiftRotateXYZ(x, y, z, cor)
xrp, yrp, zrp = align2Paraboloid(x, y, z, paraFit)
# Use a circle to mask data outside of the primary reflector.
r = 51
xc = np.mean((np.nanmax(xrp) - r, np.nanmin(xrp) + r))
yc = np.nanmax(yrp) - r
rMask = radialMask(xrp, yrp, r, xc=xc, yc=yc)
x = x[rMask]
y = y[rMask]
z = z[rMask]
i = i[rMask]
print("Applying circular mask to the paraboloid filtered data.")
printMaskStats(rMask)
#cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0))
#xr, yr, zr = shiftRotateXYZ(x, y, z, cor)
## Rotate and shift the range measurements.
#cor = np.hstack((-1*c[1:4],c[4:6],0))
#xr, yr, zr = shiftRotateXYZ(x, y, z, cor)
## Build a parabola using the best fit parameters.
#zp = paraboloid(xr, yr, c[0])
## Compute the residuals between the parabola and the rotated range measurements.
#diff = zr - zp
## Only keep range measurements whose residuals are less than residual_threshold.
##print("Removing points with residuals larger than {}".format(residual_threshold))
##mask = abs(diff) < residual_threshold
#mdiff = sigma_clip(diff, 3)
#mask = ~mdiff.mask
#percent = mask.sum()/len(mask) * 100.
#print("Parabola filter will remove {0:.2f}% of the data.".format(100. - percent))
#print("Keeping {} out of {} points.".format(mask.sum(), len(mask)))
## Use the rotated range measurements for the last filter.
#xr = xr[mask]
#yr = yr[mask]
#zr = zr[mask]
#x = x[mask]
#y = y[mask]
#z = z[mask]
#intensity = intensity[mask]
## Only keep range measurements within a circle of radius r.
#r = 51 # m
#xc = np.nanmin(xr) + r
#yc = np.nanmin(yr) + r
#circular_mask = (xr - xc)**2 + (yr - yc)**2 <= r**2.
#x = x[circular_mask]
#y = y[circular_mask]
#z = z[circular_mask]
#intensity = intensity[circular_mask]
#print("Keeping {} points inside a circle of radius {} m and center ({},{}) m.".format(circular_mask.sum(), r, xc, yc))
# Rotate around the z axis since the scanner coordinate system does not match the telescope's.
#if rot is not None or rot != 0.0:
# print("Rotating about z by %5.2f degrees" % rot)
# xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)])
# Create a Nx3 matrix for the coordinates.
xyz = np.c_[x, y, z]
if plotTest:
print("Plotting.")
xlim = (-100, 100)
ylim = (-100, 100)
fig, ax = scatter3dPlot(xr,
yr,
zr,
"Sampling of processed data",
xlim=xlim,
ylim=ylim,
sample=1)
# take a look at the x y orientation
f = plt.figure()
ax = f.gca()
ax.plot(xr, yr, 'bo', [0], [0], '*')
plt.xlabel("x")
plt.ylabel("y")
plt.title("X Y orientation of data")
print("XYZ output of ProcessNewPTXData has {0} lines of data.".format(
len(xyz)))
print(
"Intensity output of ProcessNewPTXData has {0} lines of data.".format(
len(intensity)))
return xyz, dts, i