def run(oldpath, newpath): """ Runs the 15th basic test suite. Tests: creating a raster using the 'new' non-spatial mode """ inputSPD = os.path.join(oldpath, INPUT_SPD) outputDEM = os.path.join(newpath, OUTPUT_DEM) info = generic.getLidarFileInfo(inputSPD) header = info.header dataFiles = lidarprocessor.DataFiles() dataFiles.input1 = lidarprocessor.LidarFile(inputSPD, lidarprocessor.READ) xMin, yMax, ncols, nrows = spatial.getGridInfoFromHeader(header, BINSIZE) outImage = numpy.zeros((nrows, ncols)) otherArgs = lidarprocessor.OtherArgs() otherArgs.outImage = outImage otherArgs.xMin = xMin otherArgs.yMax = yMax controls = lidarprocessor.Controls() controls.setSpatialProcessing(False) lidarprocessor.doProcessing(processChunk, dataFiles, otherArgs=otherArgs, controls=controls) iw = spatial.ImageWriter(outputDEM, tlx=xMin, tly=yMax, binSize=BINSIZE) iw.setLayer(outImage) iw.close() utils.compareImageFiles(os.path.join(oldpath, OUTPUT_DEM), outputDEM)
def run(oldpath, newpath): """ Runs the first basic test suite. Tests: Importing LAS Creating spatial index creating a raster """ inputLas = os.path.join(oldpath, INPUT_LAS) info = generic.getLidarFileInfo(inputLas) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputLas, importedSPD, epsg=28356, pulseIndex='FIRST_RETURN', buildPulses=True) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD) indexedSPD = os.path.join(newpath, INDEXED_SPD) createGridSpatialIndex(importedSPD, indexedSPD, binSize=2.0, tempDir=newpath) utils.compareLiDARFiles(os.path.join(oldpath, INDEXED_SPD), indexedSPD) outputDEM = os.path.join(newpath, OUTPUT_DEM) rasterize([indexedSPD], outputDEM, ['Z'], function="numpy.ma.min", atype='POINT') utils.compareImageFiles(os.path.join(oldpath, OUTPUT_DEM), outputDEM)
def run(oldpath, newpath): """ Runs the 10th basic test suite. Tests: Importing SPDV3 Creating spatial index creating a raster """ inputLas = os.path.join(oldpath, INPUT_SPDV3) info = generic.getLidarFileInfo(inputLas) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputLas, importedSPD, scaling=SCALINGS) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD, windowSize=WINDOWSIZE) indexedSPD = os.path.join(newpath, INDEXED_SPD) createGridSpatialIndex(importedSPD, indexedSPD, binSize=2.0, tempDir=newpath) utils.compareLiDARFiles(os.path.join(oldpath, INDEXED_SPD), indexedSPD, windowSize=WINDOWSIZE) outputDEM = os.path.join(newpath, OUTPUT_DEM) rasterize([indexedSPD], outputDEM, ['Z'], function="numpy.ma.min", atype='POINT', windowSize=WINDOWSIZE) utils.compareImageFiles(os.path.join(oldpath, OUTPUT_DEM), outputDEM)
def run(oldpath, newpath): """ Runs the 5th basic test suite. Tests: Exporting to LAS """ inputSPD = os.path.join(oldpath, IMPORTED_SPD) info = generic.getLidarFileInfo(inputSPD) exportedLAS = os.path.join(newpath, EXPORTED_LAS) translate(info, inputSPD, exportedLAS, spatial=False) # now run lasdiff # maybe need to do better with parsing output # not sure if this is valid comparisons with rounding errors etc # I've changed this from comparing against ORIG_LAS as there were # too many spurious errors. Compare to known good version instead. # Hope this ok. origLAS = os.path.join(oldpath, EXPORTED_LAS) result = subprocess.check_output(['lasdiff', '-i', exportedLAS, origLAS], stderr=subprocess.STDOUT) nLines = len(result.split(b'\n')) if nLines > 7: print(result) msg = ('Seems to be more output from lasdiff than expected, ' + 'likely to be a difference in file') raise utils.TestingDataMismatch(msg)
def run(oldpath, newpath): """ Runs the 19th basic test suite. Tests: Exporting to PulseWAVES """ inputSPD = os.path.join(oldpath, IMPORTED_SPD) info = generic.getLidarFileInfo(inputSPD) exportedPW = os.path.join(newpath, EXPORTED_PULSEWAVES) translate(info, inputSPD, exportedPW) # no pulsediff that we can use to compare. Simply # see if file identifies as pulsewaves. Probably can do better. info = generic.getLidarFileInfo(exportedPW) if info.getDriverName() != 'PulseWaves': msg = 'output file was not correctly identified as PulseWaves' raise utils.TestingDataMismatch(msg)
def run(oldpath, newpath): """ Runs the 8th basic test suite. Tests: Importing Riegl Creating spatial index Create an image file updating resulting file """ inputRiegl = os.path.join(oldpath, INPUT_RIEGL) info = generic.getLidarFileInfo(inputRiegl) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputRiegl, importedSPD, scalings=SCALINGS, internalrotation=True) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD, windowSize=WINDOWSIZE) indexedSPD = os.path.join(newpath, INDEXED_SPD) createGridSpatialIndex(importedSPD, indexedSPD, binSize=1.0, tempDir=newpath) utils.compareLiDARFiles(os.path.join(oldpath, INDEXED_SPD), indexedSPD, windowSize=WINDOWSIZE) outputRaster = os.path.join(newpath, OUTPUT_RASTER) rasterize([indexedSPD], outputRaster, ['Z'], function="numpy.ma.min", atype='POINT', windowSize=WINDOWSIZE) utils.compareImageFiles(os.path.join(oldpath, OUTPUT_RASTER), outputRaster) outputUpdate = os.path.join(newpath, UPDATED_SPD) shutil.copyfile(indexedSPD, outputUpdate) dataFiles = lidarprocessor.DataFiles() dataFiles.input1 = lidarprocessor.LidarFile(outputUpdate, lidarprocessor.UPDATE) controls = lidarprocessor.Controls() progress = cuiprogress.GDALProgressBar() controls.setProgress(progress) controls.setWindowSize(WINDOWSIZE) controls.setSpatialProcessing(True) lidarprocessor.doProcessing(updatePointFunc, dataFiles, controls=controls) utils.compareLiDARFiles(os.path.join(oldpath, UPDATED_SPD), outputUpdate, windowSize=WINDOWSIZE)
def prepareInputFiles(infiles, otherargs, index=None): """ Prepare input files for calculation of canopy metrics """ dataFiles = lidarprocessor.DataFiles() if index is not None: dataFiles.inFiles = [ lidarprocessor.LidarFile(infiles[index], lidarprocessor.READ) ] else: dataFiles.inFiles = [ lidarprocessor.LidarFile(fname, lidarprocessor.READ) for fname in infiles ] otherargs.lidardriver = [] otherargs.proj = [] nFiles = len(dataFiles.inFiles) for i in range(nFiles): info = generic.getLidarFileInfo(dataFiles.inFiles[i].fname) if info.getDriverName() == 'riegl': if otherargs.externaltransformfn is not None: if index is not None: externaltransform = numpy.loadtxt( otherargs.externaltransformfn[index], ndmin=2, delimiter=" ", dtype=numpy.float32) else: externaltransform = numpy.loadtxt( otherargs.externaltransformfn[i], ndmin=2, delimiter=" ", dtype=numpy.float32) dataFiles.inFiles[i].setLiDARDriverOption( "ROTATION_MATRIX", externaltransform) elif "ROTATION_MATRIX" in info.header: dataFiles.inFiles[i].setLiDARDriverOption( "ROTATION_MATRIX", info.header["ROTATION_MATRIX"]) else: msg = 'Input file %s has no valid pitch/roll/yaw data' % dataFiles.inFiles[ i].fname raise generic.LiDARInvalidData(msg) otherargs.lidardriver.append(info.getDriverName()) if "SPATIAL_REFERENCE" in info.header.keys(): if len(info.header["SPATIAL_REFERENCE"]) > 0: otherargs.proj.append(info.header["SPATIAL_REFERENCE"]) else: otherargs.proj.append(None) else: otherargs.proj.append(None) return dataFiles
def run(oldpath, newpath): """ Runs the 18th basic test suite. Tests: Importing PulseWaves """ inputPW = os.path.join(oldpath, INPUT_PULSEWAVES) info = generic.getLidarFileInfo(inputPW) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputPW, importedSPD) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD)
def run(oldpath, newpath): """ Runs the 21st basic test suite. Tests: Importing LVIS Binary """ inputLVIS = os.path.join(oldpath, INPUT_LVIS) info = generic.getLidarFileInfo(inputLVIS) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputLVIS, importedSPD) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD)
def run(oldpath, newpath): """ Runs the 14th basic test suite. Tests: Importing time sequential ascii """ inputASCII = os.path.join(oldpath, INPUT_ASCII) info = generic.getLidarFileInfo(inputASCII) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputASCII, importedSPD, COLTYPES, constCols=CONST_COLS) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD)
def run(oldpath, newpath): """ Runs the 24th basic test suite. Tests: Importing Riegl RDB """ inputRiegl = os.path.join(oldpath, INPUT_RIEGL) info = generic.getLidarFileInfo(inputRiegl) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputRiegl, importedSPD, scalings=SCALINGS) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD, windowSize=WINDOWSIZE)
def run(oldpath, newpath): """ Runs the 20th basic test suite. Tests: Importing an LVIS ASCII file """ inputASC = os.path.join(oldpath, INPUT_ASCII) info = generic.getLidarFileInfo(inputASC) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputASC, importedSPD, colTypes=COLTYPES, pulseCols=PULSECOLS, scaling=SCALING) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD)
def deleteEmptyTiles(fnameList): """ After creating all the tiles, some of them will have ended up being empty. Delete them. """ newFileList = [] for (fname, subExtent) in fnameList: info = generic.getLidarFileInfo(fname) header = info.header translator = info.getHeaderTranslationDict() numPointsField = translator[generic.HEADER_NUMBER_OF_POINTS] if header[numPointsField] > 0: newFileList.append((fname, subExtent)) else: if os.path.exists(fname): os.remove(fname) return newFileList
def run(): """ Main function. Looks at the command line arguments and prints the info """ cmdargs = getCmdargs() info = generic.getLidarFileInfo(cmdargs.input, cmdargs.verbose) print('Driver Name:', info.getDriverName()) print('') for key, val in sorted(info.__dict__.items()): if isinstance(val, dict): print(key.upper()) # TODO: we need to be uppercase?? for hkey, hval in sorted(val.items()): print(" ", hkey.upper(), hval) else: print(key.upper(), val)
def run(oldpath, newpath): """ Runs the first basic test suite. Tests: Importing Creating spatial index creating a raster from 2 files at a different resolution """ inputLas = os.path.join(oldpath, INPUT2_LAS) info = generic.getLidarFileInfo(inputLas) importedSPD = os.path.join(newpath, IMPORTED_SPD) translate(info, inputLas, importedSPD, epsg=28356, pulseIndex='FIRST_RETURN', buildPulses=True) utils.compareLiDARFiles(os.path.join(oldpath, IMPORTED_SPD), importedSPD) indexedSPD1 = os.path.join(oldpath, INDEXED_SPD_1) indexedSPD2 = os.path.join(newpath, INDEXED_SPD_2) createGridSpatialIndex(importedSPD, indexedSPD2, binSize=2.0, tempDir=newpath) utils.compareLiDARFiles(os.path.join(oldpath, INDEXED_SPD_2), indexedSPD2) outputDEM = os.path.join(newpath, OUTPUT_DEM) rasterize([indexedSPD1, indexedSPD2], outputDEM, ['Z'], binSize=3.0, function="numpy.ma.min", atype='POINT', footprint=lidarprocessor.UNION) utils.compareImageFiles(os.path.join(oldpath, OUTPUT_DEM), outputDEM)
def run(): """ Main function. Looks at the command line arguments and calls the appropiate translation function. """ cmdargs = getCmdargs() wktStr = None if cmdargs.wktfile is not None: wktStr = open(cmdargs.wktfile).read() # first determine the format of the input file # I'm not sure this is possible in all situations, but assume it is for now info = generic.getLidarFileInfo(cmdargs.input) inFormat = info.getDriverName() if inFormat == 'LAS' and cmdargs.format == 'SPDV4': las2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.spatial, cmdargs.extent, cmdargs.scaling, cmdargs.epsg, cmdargs.binsize, cmdargs.buildpulses, cmdargs.pulseindex, cmdargs.null, cmdargs.constcol, cmdargs.lasscalings) elif inFormat == 'SPDV3' and cmdargs.format == 'SPDV4': spdv32spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.spatial, cmdargs.extent, cmdargs.scaling, cmdargs.null, cmdargs.constcol) elif inFormat == 'riegl RXP' and cmdargs.format == 'SPDV4': rieglrxp2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.scaling, cmdargs.internalrotation, cmdargs.magneticdeclination, cmdargs.externalrotationfn, cmdargs.null, cmdargs.constcol, epsg=cmdargs.epsg, wkt=wktStr) elif inFormat == 'riegl RDB' and cmdargs.format == 'SPDV4': rieglrdb2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.scaling, cmdargs.null, cmdargs.constcol, epsg=cmdargs.epsg, wkt=wktStr) elif inFormat == 'SPDV4' and cmdargs.format == 'LAS': spdv42las.translate(info, cmdargs.input, cmdargs.output, cmdargs.spatial, cmdargs.extent) elif inFormat == 'ASCII' and cmdargs.format == 'SPDV4': if cmdargs.coltype is None: msg = "must pass --coltypes parameter" raise generic.LiDARInvalidSetting(msg) if cmdargs.pulsecols is not None: pulsecols = cmdargs.pulsecols.split(',') else: pulsecols = None classtrans = None if cmdargs.classtrans is not None: # translate strings to codes for internalCode, strLasCode in cmdargs.classtrans: internalCode = int(internalCode) strLasCodeFull = "CLASSIFICATION_%s" % strLasCode try: lasCode = getattr(lidarprocessor, strLasCodeFull) except AttributeError: msg = 'class %s not understood' % strLasCode raise generic.LiDARInvalidSetting(msg) if classtrans is None: classtrans = [] classtrans.append((internalCode, lasCode)) ascii2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.coltype, pulsecols, cmdargs.range, cmdargs.scaling, classtrans, cmdargs.null, cmdargs.constcol) elif inFormat == 'LVIS Binary' and cmdargs.format == 'SPDV4': lvisbin2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.scaling, cmdargs.null, cmdargs.constcol) elif inFormat == 'LVIS HDF5' and cmdargs.format == 'SPDV4': lvishdf52spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.scaling, cmdargs.null, cmdargs.constcol) elif inFormat == 'PulseWaves' and cmdargs.format == 'SPDV4': pulsewaves2spdv4.translate(info, cmdargs.input, cmdargs.output, cmdargs.range, cmdargs.scaling, cmdargs.null, cmdargs.constcol) elif inFormat == 'SPDV4' and cmdargs.format == 'PULSEWAVES': spdv42pulsewaves.translate(info, cmdargs.input, cmdargs.output) else: msg = 'Cannot convert between formats %s and %s' msg = msg % (inFormat, cmdargs.format) raise generic.LiDARFunctionUnsupported(msg)
def splitFileIntoTiles(infiles, binSize=1.0, blockSize=None, tempDir='.', extent=None, indexType=INDEX_CARTESIAN, pulseIndexMethod=PULSE_INDEX_FIRST_RETURN, footprint=lidarprocessor.UNION, outputFormat='SPDV4', buildPulses=False): """ Takes a filename (or list of filenames) and creates a tempfile for every block (using blockSize). If blockSize isn't set then it is picked using BLOCKSIZE_N_BLOCKS. binSize is the size of the bins to create the spatial index. indexType is one of the INDEX_* constants. pulseIndexMethod is one of the PULSE_INDEX_* constants. footprint is one of lidarprocessor.UNION or lidarprocessor.INTERSECTION and is how to combine extents if there is more than one file. outputFormat is either 'SPDV4' or 'LAS'. 'LAS' outputs only supported when input is 'LAS'. buildPulses relevant for 'LAS' and determines whether to build the pulse structure or not. returns the header of the first input file, the extent used and a list of (fname, extent) tuples that contain the information for each tempfile. """ if isinstance(infiles, basestring): infiles = [infiles] # use the first file for header. Not # clear how to combine headers from multiple inputs # or if one should. # leave setting this in case we grab it when working out the extent. firstHeader = None if extent is None: # work out from headers pixGrid = None for infile in infiles: info = generic.getLidarFileInfo(infile) header = info.header if firstHeader is None: firstHeader = header try: if indexType == INDEX_CARTESIAN: xMax = header['X_MAX'] xMin = header['X_MIN'] yMax = header['Y_MAX'] yMin = header['Y_MIN'] elif indexType == INDEX_SPHERICAL: xMax = header['AZIMUTH_MAX'] xMin = header['AZIMUTH_MIN'] yMax = header['ZENITH_MAX'] yMin = header['ZENITH_MIN'] elif indexType == INDEX_SCAN: xMax = header['SCANLINE_IDX_MAX'] xMin = header['SCANLINE_IDX_MIN'] yMax = header['SCANLINE_MAX'] yMin = header['SCANLINE_MIN'] else: msg = 'unsupported indexing method' raise generic.LiDARSpatialIndexNotAvailable(msg) except KeyError: msg = 'info for creating bounding box not available' raise generic.LiDARFunctionUnsupported(msg) newPixGrid = pixelgrid.PixelGridDefn(xMin=xMin, xMax=xMax, yMin=yMin, yMax=yMax, xRes=binSize, yRes=binSize) if pixGrid is None: pixGrid = newPixGrid elif footprint == lidarprocessor.UNION: pixGrid = pixGrid.union(newPixGrid) elif footprint == lidarprocessor.INTERSECTION: pixGrid = pixGrid.intersection(newPixGrid) else: msg = 'Unsupported footprint option' raise generic.LiDARFunctionUnsupported(msg) # TODO: we treat points as being in the block when they are >= # the min coords and < the max coords. What happens on the bottom # and right margins?? We could possibly miss points that are there. # round the coords to the nearest multiple xMin = numpy.floor(pixGrid.xMin / binSize) * binSize yMin = numpy.floor(pixGrid.yMin / binSize) * binSize xMax = numpy.ceil(pixGrid.xMax / binSize) * binSize yMax = numpy.ceil(pixGrid.yMax / binSize) * binSize extent = Extent(xMin, xMax, yMin, yMax, binSize) else: # ensure that our binSize comes from their exent binSize = extent.binSize # get the first header since we aren't doing the above info = generic.getLidarFileInfo(infiles[0]) firstHeader = info.header if blockSize is None: minAxis = min(extent.xMax - extent.xMin, extent.yMax - extent.yMin) blockSize = min(minAxis / BLOCKSIZE_N_BLOCKS, 200.0) # make it a multiple of binSize blockSize = int(numpy.ceil(blockSize / binSize)) * binSize else: # ensure that their given block size can be evenly divided by # the binSize # the modulo operator doesn't work too well with floats # so we take a different approach a = blockSize / binSize if a != int(a): msg = 'blockSize must be evenly divisible be the binSize' raise generic.LiDARInvalidData(msg) extentList = [] subExtent = Extent(extent.xMin, extent.xMin + blockSize, extent.yMax - blockSize, extent.yMax, binSize) controls = lidarprocessor.Controls() controls.setSpatialProcessing(False) tmpSuffix = '.' + outputFormat.lower() bMoreToDo = True while bMoreToDo: fd, fname = tempfile.mkstemp(suffix=tmpSuffix, dir=tempDir) os.close(fd) userClass = lidarprocessor.LidarFile(fname, generic.CREATE) if outputFormat == 'SPDV4': userClass.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING', False) driver = spdv4.SPDV4File(fname, generic.CREATE, controls, userClass) elif outputFormat == 'LAS': driver = las.LasFile(fname, generic.CREATE, controls, userClass) else: msg = 'Unsupported output format %s' % outputFormat raise generic.LiDARFunctionUnsupported(msg) data = (copy.copy(subExtent), driver) extentList.append(data) # move it along subExtent.xMin += blockSize subExtent.xMax += blockSize if subExtent.xMin >= extent.xMax: # next line down subExtent.xMin = extent.xMin subExtent.xMax = extent.xMin + blockSize subExtent.yMax -= blockSize subExtent.yMin -= blockSize # done? bMoreToDo = subExtent.yMax > extent.yMin # ok now set up to read the input files using lidarprocessor dataFiles = lidarprocessor.DataFiles() dataFiles.inputs = [] for infile in infiles: input = lidarprocessor.LidarFile(infile, lidarprocessor.READ) # must be a better way of doing this, but this is what # translate does. We don't know what formats we are getting ahead of time info = generic.getLidarFileInfo(infile) inFormat = info.getDriverName() if inFormat == 'LAS': input.setLiDARDriverOption('BUILD_PULSES', buildPulses) dataFiles.inputs.append(input) controls = lidarprocessor.Controls() progress = cuiprogress.GDALProgressBar() progress.setLabelText('Splitting...') controls.setProgress(progress) controls.setSpatialProcessing(False) controls.setMessageHandler(lidarprocessor.silentMessageFn) otherArgs = lidarprocessor.OtherArgs() otherArgs.outList = extentList otherArgs.indexType = indexType otherArgs.pulseIndexMethod = pulseIndexMethod lidarprocessor.doProcessing(classifyFunc, dataFiles, controls=controls, otherArgs=otherArgs) # close all the output files and save their names to return newExtentList = [] for subExtent, driver in extentList: fname = driver.fname driver.close() data = (fname, subExtent) newExtentList.append(data) return firstHeader, extent, newExtentList