def trySmoothGPUMulti(N=10): if N is None: N = 10 gpuPath1 = "/home/sandboxes/pmargani/LASSI/gpus/versions/gpuMulti" gpuPath2 = "/home/sandboxes/pmargani/LASSI/gpus/versions/gpuMulti2" gpuPaths = [gpuPath1, gpuPath2] inFile1 = "/home/scratch/pmargani/LASSI/scannerData/Clean9.ptx.csv" inFile2 = "/home/scratch/pmargani/LASSI/scannerData/Clean9.ptx.copy.csv" inFiles = [inFile1, inFile2] host1 = settings.GPU_HOST host2 = settings.GPU_HOST_2 hosts = [host1, host2] outFile = 'outFile' smoothGPUMulti(gpuPaths, hosts, inFiles, outFile, N) # did it work? outfiles = [] xyzs = [] for i, gpuPath in enumerate(gpuPaths): for dim in ['x', 'y', 'z']: dimFile = "%s.%d.%s.csv" % (outFile, (i + 1), dim) dimPath = os.path.join(gpuPath, dimFile) outfiles.append(dimPath) print(dimPath) assert os.path.isfile(dimPath) print(("GPUs created file: ", dimPath)) loadFile = os.path.join(gpuPath, "%s.%d" % (outFile, (i + 1))) xyz = loadLeicaDataFromGpus(loadFile) xyzs.append(xyz) x1, y1, z1 = xyzs[0] x2, y2, z2 = xyzs[1] x = np.concatenate((x1, x2)) y = np.concatenate((y1, y2)) z = np.concatenate((z1, z2)) scatter3dPlot(x, y, z, "testSmoothGPUMulti")
def previewData(ptxFile, sample=None): """ Loads a PTX file and displays its contents as a 3D scatter plot. :param ptxFile: PTX file to display. :param sample: Percentage of the data to display. """ print("opening file", ptxFile) with open(ptxFile, 'r') as f: ls = f.readlines() print("reading data ...") x, y, z, i = getRawXYZ(ls) print("plotting preview ...") if sample is None: sample = 1.0 scatter3dPlot(x, y, z, "preview", sample=sample)
def testRegridXYZ(self): "Make sure resampling happens sensibly" # construct some cartesian data M = 100 width = 10. maxMin = width / 2. x = np.linspace(-maxMin, maxMin, M) y = np.linspace(-maxMin, maxMin, M) xm, ym = np.meshgrid(x, y) z = xm * ym if self.plots: scatter3dPlot(xm, ym, z, "org") N = 10 x2, y2, z2 = regridXYZ(xm, ym, z, n=N) self.assertEquals(x2.shape, (N, N)) self.assertEquals(y2.shape, (N, N)) self.assertEquals(z2.shape, (N, N)) # check the x, y grid - based of min and maxs self.assertEquals(x2[0, 0], xm[0, 0]) self.assertEquals(y2[0, 0], ym[0, 0]) # spot check the regridded data self.assertEquals(z[0, 0], 25.0) # make sure the residuals are low for i in range(N): for j in range(N): # print i, j, x2[i, j], y2[i, j], np.abs((x2[i,j] * y2[i,j]) - z2[i, j]) diff = np.abs((x2[i, j] * y2[i, j]) - z2[i, j]) self.assertTrue(diff < 1e-2) if self.plots: scatter3dPlot(x2, y2, z2, "regrid") # now, how do these max values work? xmin = ymin = -1.0 x2, y2, z2 = regridXYZ(xm, ym, z, n=N, xmin=xmin, ymin=ymin) if self.plots: scatter3dPlot(x2, y2, z2, "regrid") self.assertEquals(x2.shape, (N, N)) self.assertEquals(y2.shape, (N, N)) self.assertEquals(z2.shape, (N, N)) # check the x, y grid self.assertEquals(x2[0, 0], xmin) self.assertEquals(y2[0, 0], ymin)
def weightSmooth(fpath, xyz): N = 512 # convert to shperical # s = 1.0 # print "working with %f percent of data" % s x, y, z = splitXYZ(xyz) # x, y, z = sampleXYZData(x, y, z, s) scatter3dPlot(x, y, z, "org sample of data", sample=1.0) xOrg = copy(x) yOrg = copy(y) zOrg = copy(z) # sigma2 = <r>**2 - <r**2> ################# FIRST: calculate <r> smoothedXYZfiles = smooth(fpath) basename = os.path.basename(fpath) gpuPath = os.path.join(GPU_PATH, basename) print("loading GPU data from", gpuPath) xs, ys, zs = loadLeicaDataFromGpus(gpuPath) scatter3dPlot(xs, ys, zs, "sample of smoothed data", sample=1.0) # we need the smoothed r, <r>, so convert this print("Converting to spherical ...") rs, els, azs = cart2sph(xs, ys, zs) ################ Second: calcualte <r**2> # convert the original xyz to spherical r, el, az = cart2sph(x, y, z) # square the r term, then go back to cartesian r2 = r**2 x2, y2, z2 = sph2cart(el, az, r2) # write this to a file for smoothing xyz2 = aggregateXYZ(x2, y2, z2) f2 = basename + ".rs.csv" basepath = os.path.dirname(fpath) fpath2 = os.path.join(basepath, f2) print("Saving r**2 data in xyz to file", fpath2) np.savetxt(fpath2, xyz2, delimiter=',') # now smooth this data smoothedXYZ2files = smooth(fpath2) # retrieve the xyz data gpuPath2 = os.path.join(GPU_PATH, f2) print("loading GPU data from", gpuPath2) xs2, ys2, zs2 = loadLeicaDataFromGpus(gpuPath2) # convert back to shperical to get <r**2> rs2, els2, azs2 = cart2sph(xs2, ys2, zs2) ####################: Finally: Calculate Variance # make sure small negative numerical errors are dealt with sigma2 = np.abs(rs2 - (rs**2)) print("sigma squared: ", sigma2.shape, np.nanmin(sigma2), np.nanmax(sigma2), np.nanmean(sigma2), np.nanstd(sigma2)) # not normalized weights Ws_Not_Norm = 1 / sigma2 # make sure NaNs induced by zeros in sigma2 are dealt with Ws_Not_Norm[sigma2 == 0.0] = 0.0 # normalize weights, making sure Nans in sum are dealt with ws = (Ws_Not_Norm) / np.sum(Ws_Not_Norm[np.logical_not(np.isnan(sigma2))]) print("Computed weights: ", ws.shape, ws) # plot weights sigma2.shape = (N, N) imagePlot(sigma2, "sigma^2") imagePlot(np.log(np.abs(sigma2)), "log sigma^2") wsc = copy(ws) wsc.shape = (N, N) imagePlot(wsc, "weights") imagePlot(np.log(np.abs(wsc)), "log weights") return smoothedXYZfiles, ws
def processNewPTXData(lines, xyzi=None, dts=None, plotTest=True, rot=None, sampleSize=None, iFilter=False, nFilter=True, rFilter=True, filterClose=True, filterParaboloid=True, ellipse=[-8., 50., 49., 49., 0.], residual_threshold=0.008): """ """ "this is the processing we see works with 2019 data" if rot is None: rot = -90.0 if ellipse is None: warnings.warn("No ellipse given. Will use default values.") ellipse = [-8., 50., 49., 49., 0.] print("ProcessNewPTXData with: ", ellipse) if plotTest and lines is not None: # make some plots that ensure how we are doing # our radial filtering tryEllipticalOffsets(lines, ellipse) if lines is not None: # Get the actual float values from the file contents. x, y, z, i = getRawXYZ(lines, sampleSize=sampleSize) else: x, y, z, i = xyzi print("Starting with %d lines of data" % len(x)) # First remove all the zero data. print("Filtering measurements with zero intensity.") mask = i != 0.0 intensity = i[mask] x = x[mask] y = y[mask] z = z[mask] if dts is not None: dts = dts[mask] printMaskStats(mask) # Remove aggregious jumps in data? if nFilter: print("Neighbor filter.") # TBF: document where our tolerance comes from x, y, z, mask = neighborFilter(x, y, z, 0.122) intensity = intensity[mask] print("Now we have %d lines of data" % len(x)) if dts is not None: dts = dts[mask] # We only want data that has a decent intesity. meanI = np.mean(intensity) stdI = np.std(intensity) print("Intensity: max=%5.2f, min=%5.2f, mean=%5.2f, std=%5.2f" % (np.max(intensity), np.min(intensity), meanI, stdI)) if False: # iFilter mask = np.logical_and(intensity > 0.75, intensity < 0.85) intensity = intensity[mask] x = x[mask] y = y[mask] z = z[mask] if dts is not None: dts = dts[mask] numFilteredOut = len(x) - len(intensity) percent = (float(numFilteredOut) / float(len(x))) * 100. print("Filtered out %d points of %d (%5.2f%%) via intensity" % (numFilteredOut, len(x), percent)) print(("Now we have %d lines of data" % len(x))) assert len(x) == len(y) assert len(y) == len(z) assert len(z) == len(intensity) # Save the point cloud thus far for the paraboloid filter. x_p = copy(x) y_p = copy(y) z_p = copy(z) i_p = copy(intensity) # We want as much as possible of the dish. # Since the dish will be rotated, it will look like an ellipse to the TLS. # This filter is later superseded by the paraboloid filter, but it is # necessary to find a best fit paraboloid. if rFilter: orgNum = len(x) print('Elliptical filter.') print( "The ellipse has semi-major axis {0:.2f} m, semi-minor axis {1:.2f} m and angle {2:.2f} degrees" .format(ellipse[2], ellipse[3], ellipse[4])) x, y, z, dts, intensity = ellipticalFilter(x, y, z, ellipse[0], ellipse[1], ellipse[2], ellipse[3], ellipse[4], dts=dts, intensity=intensity) newNum = len(x) print( "Elliptical filter removed {0} points outside the ellipse".format( orgNum - newNum)) print("Now we have %d lines of data" % len(x)) # The scanner is upside down. z = -z # Filter points below the dish. print("z filter.") zLimit = -80 x, y, z, dts, intensity = zLimitFilter(x, y, z, zLimit=zLimit, dts=dts, intensity=intensity) if filterClose: tooClose = 10. print("Removing points closer than {0:.2f} m from the scanner.".format( tooClose)) # Removes points that are closer than 10 m from the scanner. x, y, z, dts, intensity = nearFilter(x, y, z, tol=tooClose, dts=dts, intensity=intensity) # Rotate the data for the paraboloid filter. print("Rotating about z by {0:5.2f} degrees.".format(rot)) xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)]) if filterParaboloid: print("Paraboloid filter.") perc = 0.01 print("Fitting a paraboloid to {}% of the data.".format(perc * 100.)) sampleSize = int(len(xr) * perc) lsIdx = random.sample(range(len(xr)), sampleSize) xs = xr[lsIdx] ys = yr[lsIdx] zs = zr[lsIdx] # Fit a parabola to the data. pMask, _ = paraboloidFilter(xs, ys, zs, threshold=3) # Fit the paraboloid again, discarding the already filtered values. print("Fitting paraboloid again to the un-filtered values.") xs = xs[pMask] ys = ys[pMask] zs = zs[pMask] pMask, paraFit = paraboloidFilter(xs, ys, zs, threshold=3) print( "Applying paraboloid filter to the data before the elliptical filter." ) x = x_p y = y_p z = z_p i = i_p z = -z x, y, z = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)]) print("Number of data points: {}".format(len(z))) pMask = paraboloidMask(x, y, z, paraFit, threshold=2) print("Applying paraboloid filter to the data.") printMaskStats(pMask) x = x[pMask] y = y[pMask] z = z[pMask] i = i[pMask] # Rotate and shift the range measurements. #cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0)) #xrp, yrp, zrp = shiftRotateXYZ(x, y, z, cor) xrp, yrp, zrp = align2Paraboloid(x, y, z, paraFit) # Use a circle to mask data outside of the primary reflector. r = 51 xc = np.mean((np.nanmax(xrp) - r, np.nanmin(xrp) + r)) yc = np.nanmax(yrp) - r rMask = radialMask(xrp, yrp, r, xc=xc, yc=yc) x = x[rMask] y = y[rMask] z = z[rMask] i = i[rMask] print("Applying circular mask to the paraboloid filtered data.") printMaskStats(rMask) #cor = np.hstack((-1*paraFit[1:4],paraFit[4:6],0)) #xr, yr, zr = shiftRotateXYZ(x, y, z, cor) ## Rotate and shift the range measurements. #cor = np.hstack((-1*c[1:4],c[4:6],0)) #xr, yr, zr = shiftRotateXYZ(x, y, z, cor) ## Build a parabola using the best fit parameters. #zp = paraboloid(xr, yr, c[0]) ## Compute the residuals between the parabola and the rotated range measurements. #diff = zr - zp ## Only keep range measurements whose residuals are less than residual_threshold. ##print("Removing points with residuals larger than {}".format(residual_threshold)) ##mask = abs(diff) < residual_threshold #mdiff = sigma_clip(diff, 3) #mask = ~mdiff.mask #percent = mask.sum()/len(mask) * 100. #print("Parabola filter will remove {0:.2f}% of the data.".format(100. - percent)) #print("Keeping {} out of {} points.".format(mask.sum(), len(mask))) ## Use the rotated range measurements for the last filter. #xr = xr[mask] #yr = yr[mask] #zr = zr[mask] #x = x[mask] #y = y[mask] #z = z[mask] #intensity = intensity[mask] ## Only keep range measurements within a circle of radius r. #r = 51 # m #xc = np.nanmin(xr) + r #yc = np.nanmin(yr) + r #circular_mask = (xr - xc)**2 + (yr - yc)**2 <= r**2. #x = x[circular_mask] #y = y[circular_mask] #z = z[circular_mask] #intensity = intensity[circular_mask] #print("Keeping {} points inside a circle of radius {} m and center ({},{}) m.".format(circular_mask.sum(), r, xc, yc)) # Rotate around the z axis since the scanner coordinate system does not match the telescope's. #if rot is not None or rot != 0.0: # print("Rotating about z by %5.2f degrees" % rot) # xr, yr, zr = shiftRotateXYZ(x, y, z, [0, 0, 0, 0, 0, np.deg2rad(rot)]) # Create a Nx3 matrix for the coordinates. xyz = np.c_[x, y, z] if plotTest: print("Plotting.") xlim = (-100, 100) ylim = (-100, 100) fig, ax = scatter3dPlot(xr, yr, zr, "Sampling of processed data", xlim=xlim, ylim=ylim, sample=1) # take a look at the x y orientation f = plt.figure() ax = f.gca() ax.plot(xr, yr, 'bo', [0], [0], '*') plt.xlabel("x") plt.ylabel("y") plt.title("X Y orientation of data") print("XYZ output of ProcessNewPTXData has {0} lines of data.".format( len(xyz))) print( "Intensity output of ProcessNewPTXData has {0} lines of data.".format( len(intensity))) return xyz, dts, i