def validateNode(node, nodeAbsPath, hierarchyStepSize, extension):
    hrcFile = node + ".hrc"
    hrc = None
    if not os.path.isfile(nodeAbsPath + "/" + hrcFile):
        # Check if there is data in this node in Octtree A (we check if the HRC file for this node exist)
        raise Exception(nodeAbsPath + "/" + hrcFile + " could not be read")
    hrc = utils.readHRC(nodeAbsPath + "/" + hrcFile, hierarchyStepSize)
    for level in range(hierarchyStepSize + 1):
        hrcLevel = hrc[level]
        for i in range(len(hrcLevel)):
            hrcNumPoints = hrcLevel[i]
            if hrcNumPoints:
                (childNode, isFile) = utils.getNodeName(level, i, node, hierarchyStepSize, extension)
                childNodeAbsPath = nodeAbsPath + "/" + childNode
                if not os.path.exists(childNodeAbsPath):
                    print "Error: could not find ", childNodeAbsPath
                    raise Exception(node + " in " + nodeAbsPath + " is not correct")
                if isFile:
                    fNumPoints = utils.getPCFileDetails(childNodeAbsPath)[0]
                    if hrcNumPoints != fNumPoints:
                        print "Error: number of points in HRC (" + str(
                            hrcNumPoints
                        ) + ") != number of points in file (" + str(fNumPoints) + ") in " + childNodeAbsPath
                else:
                    validateNode(node + childNode, childNodeAbsPath, hierarchyStepSize, extension)
예제 #2
0
def runProcess(processIndex, tasksQueue, resultsQueue, outputFolder, useApprox):
    kill_received = False
    while not kill_received:
        tileAbsPath = None
        try:
            # This call will patiently wait until new job is available
            tileAbsPath = tasksQueue.get()
        except:
            # if there is an error we will quit
            kill_received = True
        if tileAbsPath == None:
            # If we receive a None job, it means we can stop
            kill_received = True
        else:
            tFile = open(outputFolder + '/' + os.path.basename(tileAbsPath) + '.wkt', 'w')
            (tMinX,tMinY,tMaxX,tMaxY) = (None, None, None, None)
            for tilefile in os.listdir(tileAbsPath):
                (_, fMinX, fMinY, _, fMaxX, fMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(tileAbsPath + '/' + tilefile)
                if useApprox:
                    if tMinX == None or tMinX > fMinX:
                        tMinX = fMinX
                    if tMinY == None or tMinY > fMinY:
                        tMinY = fMinY
                    if tMaxX == None or tMaxX < fMaxX:
                        tMaxX = fMaxX
                    if tMaxY == None or tMaxY < fMaxY:
                        tMaxY = fMaxY
                else:
                    tFile.write('POLYGON ((%f %f, %f %f, %f %f, %f %f, %f %f))\n' % (fMinX, fMaxY, fMinX, fMinY, fMaxX, fMinY, fMaxX, fMaxY, fMinX, fMaxY))
            if useApprox and tMinX != None:
                tFile.write('POLYGON ((%f %f, %f %f, %f %f, %f %f, %f %f))\n' % (tMinX, tMaxY, tMinX, tMinY, tMaxX, tMinY, tMaxX, tMaxY, tMinX, tMaxY))
            tFile.close()
            resultsQueue.put((processIndex, tileAbsPath)) 
def runProcess(processIndex, tasksQueue, resultsQueue, minX, minY, maxX, maxY, outputFolder, tempFolder, axisTiles):
    kill_received = False
    while not kill_received:
        inputFile = None
        try:
            # This call will patiently wait until new job is available
            inputFile = tasksQueue.get()
        except:
            # if there is an error we will quit
            kill_received = True
        if inputFile == None:
            # If we receive a None job, it means we can stop
            kill_received = True
        else:            
            # Get number of points and BBOX of this file
            (fCount, fMinX, fMinY, _, fMaxX, fMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(inputFile)
            print 'Processing', os.path.basename(inputFile), fCount, fMinX, fMinY, fMaxX, fMaxY
            # For the four vertices of the BBOX we get in which tile they should go
            posMinXMinY = getTileIndex(fMinX, fMinY, minX, minY, maxX, maxY, axisTiles)
            posMinXMaxY = getTileIndex(fMinX, fMaxY, minX, minY, maxX, maxY, axisTiles)
            posMaxXMinY = getTileIndex(fMaxX, fMinY, minX, minY, maxX, maxY, axisTiles)
            posMaxXMaxY = getTileIndex(fMaxX, fMaxY, minX, minY, maxX, maxY, axisTiles)

            if (posMinXMinY == posMinXMaxY) and (posMinXMinY == posMaxXMinY) and (posMinXMinY == posMaxXMaxY):
                # If they are the same the whole file can be directly copied to the tile
                tileFolder = outputFolder + '/' + getTileName(*posMinXMinY)
                if not os.path.isdir(tileFolder):
                    utils.shellExecute('mkdir -p ' + tileFolder)
                utils.shellExecute('cp ' + inputFile + ' ' + tileFolder)
            else:
                # If not, we run PDAL gridder to split the file in pieces that can go to the tiles
                tGCount = runPDALSplitter(processIndex, inputFile, outputFolder, tempFolder, minX, minY, maxX, maxY, axisTiles)
                if tGCount != fCount:
                    print 'WARNING: split version of ', inputFile, ' does not have same number of points (', tGCount, 'expected', fCount, ')'
            resultsQueue.put((processIndex, inputFile, fCount))   
def runProcess(processIndex, tasksQueue, resultsQueue, connectionString, srid):
    connection = psycopg2.connect(connectionString)
    cursor = connection.cursor()
    kill_received = False
    while not kill_received:
        fileAbsPath = None
        try:
            # This call will patiently wait until new job is available
            fileAbsPath = tasksQueue.get()
        except:
            # if there is an error we will quit
            kill_received = True
        if fileAbsPath == None:
            # If we receive a None job, it means we can stop
            kill_received = True
        else:            
            (count, minX, minY, minZ, maxX, maxY, maxZ, _, _, _, _, _, _) = utils.getPCFileDetails(fileAbsPath)
            insertStatement = """INSERT INTO """ + utils.DB_TABLE_RAW + """(filepath,numberpoints,minz,maxz,geom) VALUES (%s, %s, %s, %s, ST_MakeEnvelope(%s, %s, %s, %s, %s))"""
            insertArgs = [fileAbsPath, int(count), float(minZ), float(maxZ), float(minX), float(minY), float(maxX), float(maxY), int(srid)]
            cursor.execute(insertStatement, insertArgs)
            cursor.connection.commit()
            resultsQueue.put((processIndex, fileAbsPath))
    connection.close()
def run(srid, userMail, level, bBox, dbName, dbPass, dbUser, dbHost, dbPort, baseURL, basePath):
    message = ''
    statusOk = True
    outputAbsPath = None
    timeStamp = datetime.datetime.now().strftime("%H_%M_%S_%f")

    try:
        # Get the extent of the bounding boxes anc check they are float values
        (minX,minY,maxX,maxY) = bBox.replace('"','').replace("'","").split(' ')
        for v in (minX,minY,maxX,maxY):
            float(v)
        
        # Make connection
        connectionString = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort)
        connection = psycopg2.connect(connectionString)
        cursor = connection.cursor()
        
        cursor.execute('SELECT max(level) FROM ' + utils.DB_TABLE_POTREE)
        maxLevelPotree = cursor.fetchone()[0]

#        print level,maxLevelPotree
        if level != '':
            if int(level) <= maxLevelPotree:
                dbTable = utils.DB_TABLE_POTREE
            else:
                dbTable = utils.DB_TABLE_RAW
                print 'Specified level (' + level + ') is not available in the potree data. Using raw data'
        else:
            dbTable = utils.DB_TABLE_RAW
            
        estimatedNumPoints = None
        if dbTable == utils.DB_TABLE_POTREE:
            cursor.execute("""SELECT 
   floor(sum(numberpoints * (st_area(st_intersection(geom, qgeom)) / st_area(geom)))) 
FROM """ + utils.DB_TABLE_RAW + """, (SELECT ST_SetSRID(ST_MakeBox2D(ST_Point(""" + minX + """, """ + minY + """),ST_Point(""" + maxX + """, """ + maxY + """)), """ + str(srid) + """) as qgeom) AS B 
WHERE geom && qgeom AND st_area(geom) != 0""")
            estimatedNumPoints = cursor.fetchone()[0] 
        
        connection.close()
        
        outputFileName = '%s_%s_%s_%s_%s.laz' % (timeStamp,minX,minY,maxX,maxY)
        outputAbsPath = basePath + '/' + outputFileName
        
        if os.path.isfile(outputAbsPath):
            raise Exception('The file already existed!')
        else:
            query = 'SELECT filepath FROM ' + dbTable + ' where ST_SetSRID(ST_MakeBox2D(ST_Point(' + minX + ', ' + minY + '),ST_Point(' + maxX + ', ' + maxY + ')), ' + str(srid) + ') && geom'
            if dbTable == utils.DB_TABLE_POTREE:
                query += ' AND level = ' + str(level)
            
            inputList = outputAbsPath + '.list'
            connectionStringCommandLine = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort, cline = True)
            precommand = 'psql ' + connectionStringCommandLine + ' -t -A -c "' + query + '" > ' + inputList
            print precommand
            os.system(precommand)
            
            command = 'lasmerge -lof ' + inputList + ' -inside ' + minX + ' ' + minY + ' ' + maxX + ' ' + maxY + ' -merged -o ' + outputAbsPath
            print command
            os.system(command)
    except:
        statusOk = False
        message = 'There was some error in the file generation: ' + traceback.format_exc()
    
    
    if outputAbsPath != None and os.path.isfile(outputAbsPath) and statusOk:
        (count, _, _, _, _, _, _, _, _, _, _, _, _) = utils.getPCFileDetails(outputAbsPath)
        size = utils.getFileSize(outputAbsPath)
        
        approxStr = ''
        if dbTable == utils.DB_TABLE_POTREE:
            approxStr = """
Note that due to the large extent of your selected area only the """ + '%.4f' % (float(count)/float(estimatedNumPoints)) + """ %% of the points are stored.
"""
        
        content = """Subject: Data is ready

Your selected data is ready. """ + str(count) + """ points were selected and stored in """ + outputAbsPath.replace(basePath, baseURL) + """ with a size of """ + str(size) + """ MB.
""" + approxStr + """
Please download your data asap. This data will be deleted after 24 hours.

To visualize LAZ data there are a few alternatives. 

For desktop-based simple visualization you can use LAStools lasview.    

For web-based visualization you can use http://plas.io/

"""
    
    else:
        content = """Subject: Data is NOT ready

Your selection could not be stored. Sorry for the inconveniences."""

    content += message

    mailFileAbsPath = timeStamp + '_error.mail'
    mailFile = open(mailFileAbsPath, 'w')
    mailFile.write(content)
    mailFile.close()
    
    os.system('sendmail ' + userMail + ' < ' + mailFileAbsPath)
def runPDALSplitter(processIndex, inputFile, outputFolder, tempFolder, minX, minY, maxX, maxY, axisTiles):
    pTempFolder = tempFolder + '/' + str(processIndex)
    if not os.path.isdir(pTempFolder):
        utils.shellExecute('mkdir -p ' + pTempFolder)
        
    # Get the lenght required by the PDAL split filter in order to get "squared" tiles
    lengthPDAL = (maxX - minX) /  float(axisTiles)
    
    utils.shellExecute('pdal split -i ' + inputFile + ' -o ' + pTempFolder + '/' + os.path.basename(inputFile) + ' --origin_x ' + str(minX) + ' --origin_y ' + str(minY) + ' --length ' + str(lengthPDAL))
    tGCount = 0
    for gFile in os.listdir(pTempFolder):
        (gCount, gFileMinX, gFileMinY, _, gFileMaxX, gFileMaxY, _, _, _, _, _, _, _) = utils.getPCFileDetails(pTempFolder + '/' + gFile)
        # This tile should match with some tile. Let's use the central point to see which one
        pX = gFileMinX + ((gFileMaxX - gFileMinX) / 2.)
        pY = gFileMinY + ((gFileMaxY - gFileMinY) / 2.)
        tileFolder = outputFolder + '/' + getTileName(*getTileIndex(pX, pY, minX, minY, maxX, maxY, axisTiles))
        if not os.path.isdir(tileFolder):
            utils.shellExecute('mkdir -p ' + tileFolder)
        utils.shellExecute('mv ' + pTempFolder + '/' + gFile + ' ' + tileFolder + '/' + gFile)
        tGCount += gCount
    return tGCount
def fixHeader(inputFile, outputFile):
    (_, minX, minY, minZ, maxX, maxY, maxZ, _, _, _, _, _, _) = utils.getPCFileDetails(inputFile)
    utils.shellExecute('lasinfo -i %s -nc -nv -nco -set_bounding_box %f %f %f %f %f %f' % (outputFile, minX, minY, minZ, maxX, maxY, maxZ))