def validateNode(node, nodeAbsPath, hierarchyStepSize, extension): hrcFile = node + ".hrc" hrc = None if not os.path.isfile(nodeAbsPath + "/" + hrcFile): # Check if there is data in this node in Octtree A (we check if the HRC file for this node exist) raise Exception(nodeAbsPath + "/" + hrcFile + " could not be read") hrc = utils.readHRC(nodeAbsPath + "/" + hrcFile, hierarchyStepSize) for level in range(hierarchyStepSize + 1): hrcLevel = hrc[level] for i in range(len(hrcLevel)): hrcNumPoints = hrcLevel[i] if hrcNumPoints: (childNode, isFile) = utils.getNodeName(level, i, node, hierarchyStepSize, extension) childNodeAbsPath = nodeAbsPath + "/" + childNode if not os.path.exists(childNodeAbsPath): print "Error: could not find ", childNodeAbsPath raise Exception(node + " in " + nodeAbsPath + " is not correct") if isFile: fNumPoints = utils.getPCFileDetails(childNodeAbsPath)[0] if hrcNumPoints != fNumPoints: print "Error: number of points in HRC (" + str( hrcNumPoints ) + ") != number of points in file (" + str(fNumPoints) + ") in " + childNodeAbsPath else: validateNode(node + childNode, childNodeAbsPath, hierarchyStepSize, extension)
def runProcess(processIndex, tasksQueue, resultsQueue, outputFolder, useApprox): kill_received = False while not kill_received: tileAbsPath = None try: # This call will patiently wait until new job is available tileAbsPath = tasksQueue.get() except: # if there is an error we will quit kill_received = True if tileAbsPath == None: # If we receive a None job, it means we can stop kill_received = True else: tFile = open(outputFolder + '/' + os.path.basename(tileAbsPath) + '.wkt', 'w') (tMinX,tMinY,tMaxX,tMaxY) = (None, None, None, None) for tilefile in os.listdir(tileAbsPath): (_, fMinX, fMinY, _, fMaxX, fMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(tileAbsPath + '/' + tilefile) if useApprox: if tMinX == None or tMinX > fMinX: tMinX = fMinX if tMinY == None or tMinY > fMinY: tMinY = fMinY if tMaxX == None or tMaxX < fMaxX: tMaxX = fMaxX if tMaxY == None or tMaxY < fMaxY: tMaxY = fMaxY else: tFile.write('POLYGON ((%f %f, %f %f, %f %f, %f %f, %f %f))\n' % (fMinX, fMaxY, fMinX, fMinY, fMaxX, fMinY, fMaxX, fMaxY, fMinX, fMaxY)) if useApprox and tMinX != None: tFile.write('POLYGON ((%f %f, %f %f, %f %f, %f %f, %f %f))\n' % (tMinX, tMaxY, tMinX, tMinY, tMaxX, tMinY, tMaxX, tMaxY, tMinX, tMaxY)) tFile.close() resultsQueue.put((processIndex, tileAbsPath))
def runProcess(processIndex, tasksQueue, resultsQueue, minX, minY, maxX, maxY, outputFolder, tempFolder, axisTiles): kill_received = False while not kill_received: inputFile = None try: # This call will patiently wait until new job is available inputFile = tasksQueue.get() except: # if there is an error we will quit kill_received = True if inputFile == None: # If we receive a None job, it means we can stop kill_received = True else: # Get number of points and BBOX of this file (fCount, fMinX, fMinY, _, fMaxX, fMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(inputFile) print 'Processing', os.path.basename(inputFile), fCount, fMinX, fMinY, fMaxX, fMaxY # For the four vertices of the BBOX we get in which tile they should go posMinXMinY = getTileIndex(fMinX, fMinY, minX, minY, maxX, maxY, axisTiles) posMinXMaxY = getTileIndex(fMinX, fMaxY, minX, minY, maxX, maxY, axisTiles) posMaxXMinY = getTileIndex(fMaxX, fMinY, minX, minY, maxX, maxY, axisTiles) posMaxXMaxY = getTileIndex(fMaxX, fMaxY, minX, minY, maxX, maxY, axisTiles) if (posMinXMinY == posMinXMaxY) and (posMinXMinY == posMaxXMinY) and (posMinXMinY == posMaxXMaxY): # If they are the same the whole file can be directly copied to the tile tileFolder = outputFolder + '/' + getTileName(*posMinXMinY) if not os.path.isdir(tileFolder): utils.shellExecute('mkdir -p ' + tileFolder) utils.shellExecute('cp ' + inputFile + ' ' + tileFolder) else: # If not, we run PDAL gridder to split the file in pieces that can go to the tiles tGCount = runPDALSplitter(processIndex, inputFile, outputFolder, tempFolder, minX, minY, maxX, maxY, axisTiles) if tGCount != fCount: print 'WARNING: split version of ', inputFile, ' does not have same number of points (', tGCount, 'expected', fCount, ')' resultsQueue.put((processIndex, inputFile, fCount))
def runProcess(processIndex, tasksQueue, resultsQueue, connectionString, srid): connection = psycopg2.connect(connectionString) cursor = connection.cursor() kill_received = False while not kill_received: fileAbsPath = None try: # This call will patiently wait until new job is available fileAbsPath = tasksQueue.get() except: # if there is an error we will quit kill_received = True if fileAbsPath == None: # If we receive a None job, it means we can stop kill_received = True else: (count, minX, minY, minZ, maxX, maxY, maxZ, _, _, _, _, _, _) = utils.getPCFileDetails(fileAbsPath) insertStatement = """INSERT INTO """ + utils.DB_TABLE_RAW + """(filepath,numberpoints,minz,maxz,geom) VALUES (%s, %s, %s, %s, ST_MakeEnvelope(%s, %s, %s, %s, %s))""" insertArgs = [fileAbsPath, int(count), float(minZ), float(maxZ), float(minX), float(minY), float(maxX), float(maxY), int(srid)] cursor.execute(insertStatement, insertArgs) cursor.connection.commit() resultsQueue.put((processIndex, fileAbsPath)) connection.close()
def run(srid, userMail, level, bBox, dbName, dbPass, dbUser, dbHost, dbPort, baseURL, basePath): message = '' statusOk = True outputAbsPath = None timeStamp = datetime.datetime.now().strftime("%H_%M_%S_%f") try: # Get the extent of the bounding boxes anc check they are float values (minX,minY,maxX,maxY) = bBox.replace('"','').replace("'","").split(' ') for v in (minX,minY,maxX,maxY): float(v) # Make connection connectionString = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort) connection = psycopg2.connect(connectionString) cursor = connection.cursor() cursor.execute('SELECT max(level) FROM ' + utils.DB_TABLE_POTREE) maxLevelPotree = cursor.fetchone()[0] # print level,maxLevelPotree if level != '': if int(level) <= maxLevelPotree: dbTable = utils.DB_TABLE_POTREE else: dbTable = utils.DB_TABLE_RAW print 'Specified level (' + level + ') is not available in the potree data. Using raw data' else: dbTable = utils.DB_TABLE_RAW estimatedNumPoints = None if dbTable == utils.DB_TABLE_POTREE: cursor.execute("""SELECT floor(sum(numberpoints * (st_area(st_intersection(geom, qgeom)) / st_area(geom)))) FROM """ + utils.DB_TABLE_RAW + """, (SELECT ST_SetSRID(ST_MakeBox2D(ST_Point(""" + minX + """, """ + minY + """),ST_Point(""" + maxX + """, """ + maxY + """)), """ + str(srid) + """) as qgeom) AS B WHERE geom && qgeom AND st_area(geom) != 0""") estimatedNumPoints = cursor.fetchone()[0] connection.close() outputFileName = '%s_%s_%s_%s_%s.laz' % (timeStamp,minX,minY,maxX,maxY) outputAbsPath = basePath + '/' + outputFileName if os.path.isfile(outputAbsPath): raise Exception('The file already existed!') else: query = 'SELECT filepath FROM ' + dbTable + ' where ST_SetSRID(ST_MakeBox2D(ST_Point(' + minX + ', ' + minY + '),ST_Point(' + maxX + ', ' + maxY + ')), ' + str(srid) + ') && geom' if dbTable == utils.DB_TABLE_POTREE: query += ' AND level = ' + str(level) inputList = outputAbsPath + '.list' connectionStringCommandLine = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort, cline = True) precommand = 'psql ' + connectionStringCommandLine + ' -t -A -c "' + query + '" > ' + inputList print precommand os.system(precommand) command = 'lasmerge -lof ' + inputList + ' -inside ' + minX + ' ' + minY + ' ' + maxX + ' ' + maxY + ' -merged -o ' + outputAbsPath print command os.system(command) except: statusOk = False message = 'There was some error in the file generation: ' + traceback.format_exc() if outputAbsPath != None and os.path.isfile(outputAbsPath) and statusOk: (count, _, _, _, _, _, _, _, _, _, _, _, _) = utils.getPCFileDetails(outputAbsPath) size = utils.getFileSize(outputAbsPath) approxStr = '' if dbTable == utils.DB_TABLE_POTREE: approxStr = """ Note that due to the large extent of your selected area only the """ + '%.4f' % (float(count)/float(estimatedNumPoints)) + """ %% of the points are stored. """ content = """Subject: Data is ready Your selected data is ready. """ + str(count) + """ points were selected and stored in """ + outputAbsPath.replace(basePath, baseURL) + """ with a size of """ + str(size) + """ MB. """ + approxStr + """ Please download your data asap. This data will be deleted after 24 hours. To visualize LAZ data there are a few alternatives. For desktop-based simple visualization you can use LAStools lasview. For web-based visualization you can use http://plas.io/ """ else: content = """Subject: Data is NOT ready Your selection could not be stored. Sorry for the inconveniences.""" content += message mailFileAbsPath = timeStamp + '_error.mail' mailFile = open(mailFileAbsPath, 'w') mailFile.write(content) mailFile.close() os.system('sendmail ' + userMail + ' < ' + mailFileAbsPath)
def runPDALSplitter(processIndex, inputFile, outputFolder, tempFolder, minX, minY, maxX, maxY, axisTiles): pTempFolder = tempFolder + '/' + str(processIndex) if not os.path.isdir(pTempFolder): utils.shellExecute('mkdir -p ' + pTempFolder) # Get the lenght required by the PDAL split filter in order to get "squared" tiles lengthPDAL = (maxX - minX) / float(axisTiles) utils.shellExecute('pdal split -i ' + inputFile + ' -o ' + pTempFolder + '/' + os.path.basename(inputFile) + ' --origin_x ' + str(minX) + ' --origin_y ' + str(minY) + ' --length ' + str(lengthPDAL)) tGCount = 0 for gFile in os.listdir(pTempFolder): (gCount, gFileMinX, gFileMinY, _, gFileMaxX, gFileMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(pTempFolder + '/' + gFile) # This tile should match with some tile. Let's use the central point to see which one pX = gFileMinX + ((gFileMaxX - gFileMinX) / 2.) pY = gFileMinY + ((gFileMaxY - gFileMinY) / 2.) tileFolder = outputFolder + '/' + getTileName(*getTileIndex(pX, pY, minX, minY, maxX, maxY, axisTiles)) if not os.path.isdir(tileFolder): utils.shellExecute('mkdir -p ' + tileFolder) utils.shellExecute('mv ' + pTempFolder + '/' + gFile + ' ' + tileFolder + '/' + gFile) tGCount += gCount return tGCount
def fixHeader(inputFile, outputFile): (_, minX, minY, minZ, maxX, maxY, maxZ, _, _, _, _, _, _) = utils.getPCFileDetails(inputFile) utils.shellExecute('lasinfo -i %s -nc -nv -nco -set_bounding_box %f %f %f %f %f %f' % (outputFile, minX, minY, minZ, maxX, maxY, maxZ))