def getPCFileDetails(absPath, srid=None):
    """ Get the details (count numPoints and extent) of a LAS/LAZ file (using LAStools, hence it is fast)"""
    count = None
    (minX, minY, minZ, maxX, maxY, maxZ) = (None, None, None, None, None, None)
    (scaleX, scaleY, scaleZ) = (None, None, None)
    (offsetX, offsetY, offsetZ) = (None, None, None)

    if srid == None:
        srid = getSRID(absPath)
    command = "lasinfo " + absPath + " -nc -nv -nco"
    for line in utils.shellExecute(command).split("\n"):
        if line.count("min x y z:"):
            [minX, minY, minZ] = line.split(":")[-1].strip().split(" ")
            minX = float(minX)
            minY = float(minY)
            minZ = float(minZ)
        elif line.count("max x y z:"):
            [maxX, maxY, maxZ] = line.split(":")[-1].strip().split(" ")
            maxX = float(maxX)
            maxY = float(maxY)
            maxZ = float(maxZ)
        elif line.count("number of point records:"):
            count = int(line.split(":")[-1].strip())
        elif line.count("scale factor x y z:"):
            [scaleX, scaleY, scaleZ] = line.split(":")[-1].strip().split(" ")
            scaleX = float(scaleX)
            scaleY = float(scaleY)
            scaleZ = float(scaleZ)
        elif line.count("offset x y z:"):
            [offsetX, offsetY, offsetZ] = line.split(":")[-1].strip().split(" ")
            offsetX = float(offsetX)
            offsetY = float(offsetY)
            offsetZ = float(offsetZ)
    return (srid, count, minX, minY, minZ, maxX, maxY, maxZ, scaleX, scaleY, scaleZ, offsetX, offsetY, offsetZ)
def executeSQLFileCount(connectionString, sqlFileAbsPath):
    command = 'psql ' + connectionString + ' < ' + sqlFileAbsPath + ' | wc -l'
    result = utils.shellExecute(command).replace('\n','')
    try:
        result  = int(result) - 4
    except:
        result = None
    return result
Exemple #3
0
def executeSQLFileCount(connectionString, sqlFileAbsPath):
    command = 'sqlplus -s ' + connectionString + ' < ' + sqlFileAbsPath + ' | wc -l'
    result = utils.shellExecute(command).replace('\n','')
    try:
        result  = int(result)
    except:
        result = -1
    return result
Exemple #4
0
def executePDALCount(xmlFile):
    c = 'pdal pipeline ' + xmlFile + ' | wc -l'
    logging.debug(c)
    result = utils.shellExecute(c).replace('\n', '')
    # remove the XML file
    #os.system('rm ' + xmlFile)
    try:
        result = int(result) - 1
    except:
        result = -1
    return result
def executePDALCount(xmlFile):
    c = 'pdal pipeline ' + xmlFile + ' | wc -l'
    logging.debug(c)
    result = utils.shellExecute(c).replace('\n','')
    # remove the XML file
    #os.system('rm ' + xmlFile)
    try:
        result  = int(result) - 1
    except:
        result = -1
    return result
def getPCFileDetails(absPath, srid=None):
    """ Get the details (count numPoints and extent) of a LAS/LAZ file (using LAStools, hence it is fast)"""
    count = None
    (minX, minY, minZ, maxX, maxY, maxZ) = (None, None, None, None, None, None)
    (scaleX, scaleY, scaleZ) = (None, None, None)
    (offsetX, offsetY, offsetZ) = (None, None, None)

    if srid == None:
        srid = getSRID(absPath)
    command = 'lasinfo ' + absPath + ' -nc -nv -nco'
    for line in utils.shellExecute(command).split('\n'):
        if line.count('min x y z:'):
            [minX, minY, minZ] = line.split(':')[-1].strip().split(' ')
            minX = float(minX)
            minY = float(minY)
            minZ = float(minZ)
        elif line.count('max x y z:'):
            [maxX, maxY, maxZ] = line.split(':')[-1].strip().split(' ')
            maxX = float(maxX)
            maxY = float(maxY)
            maxZ = float(maxZ)
        elif line.count('number of point records:'):
            count = int(line.split(':')[-1].strip())
        elif line.count('scale factor x y z:'):
            [scaleX, scaleY, scaleZ] = line.split(':')[-1].strip().split(' ')
            scaleX = float(scaleX)
            scaleY = float(scaleY)
            scaleZ = float(scaleZ)
        elif line.count('offset x y z:'):
            [offsetX, offsetY,
             offsetZ] = line.split(':')[-1].strip().split(' ')
            offsetX = float(offsetX)
            offsetY = float(offsetY)
            offsetZ = float(offsetZ)
    return (srid, count, minX, minY, minZ, maxX, maxY, maxZ, scaleX, scaleY,
            scaleZ, offsetX, offsetY, offsetZ)
#!/usr/bin/env python
################################################################################
#    Created by Oscar Martinez                                                 #
#    [email protected]                                                  #
################################################################################
import matplotlib, sys, numpy, os
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pointcloud import utils

inputFile = os.path.abspath(sys.argv[1])
command = 'cat ' + inputFile + ' | grep "Read "'
lines = utils.shellExecute(command).split('\n')

nums = []
reads = []
sorts = []
writes = []
for line in lines:
    if line != '':
        fields = line.split()
        try:
            num = float(fields[1].replace(',','')) / 1000000.
            read = float(fields[3].replace(',',''))
            sort = float(fields[6].replace(',',''))
            write = float(fields[9].replace(',',''))
            nums.append(num)
            reads.append(read)
            sorts.append(sort)
            writes.append(write)
        except:
################################################################################
#    Created by Oscar Martinez                                                 #
#    [email protected]                                                  #
################################################################################
import os, sys, multiprocessing
from pointcloud import utils
from osgeo import osr
import liblas

#
# This module contains methods that use LAStools, GDAL, lasLib to interact
# with LAS/LAZ in the most efficient manner depending on the exact operation
#

# Check the LAStools is installed and that it is in PATH before libLAS
if utils.shellExecute("lasinfo -version").count("LAStools") == 0:
    raise Exception("LAStools is not found!. Please check that it is in PATH and that it is before libLAS")


def getSRID(absPath):
    """ Gets the SRID of a LAS/LAZ file (using liblas and GDAL, hence it is not fast)"""
    lasHeader = liblas.file.File(absPath, mode="r").header
    osrs = osr.SpatialReference()
    osrs.SetFromUserInput(lasHeader.get_srs().get_wkt())
    # osrs.AutoIdentifyEPSG()
    return osrs.GetAttrValue("AUTHORITY", 1)


def getNumPoints(absPath):
    """ Get the number of points of a LAS/LAZ (using LAStools, hence it is fast)"""
    return getPCFileDetails(absPath)[1]
    def query(self, queryId, iterationId, queriesParameters):
        (eTime, result) = (-1, None)
        queryIndex = int(queryId)
        
        self.qp = queriesParameters.getQueryParameters('psql', queryId)
        logging.debug(self.qp.queryKey)

        zquery = ''    
        if (self.qp.minz != None) or (self.qp.maxz != None):
            zconds = []
            if self.qp.minz != None:
                zconds.append(' -drop_z_below ' + str(self.qp.minz) + ' ')
            if self.qp.maxz != None:
                zconds.append(' -drop_z_above ' + str(self.qp.maxz) + ' ')
            zquery = ' '.join(zconds)
        
        shapeFile = 'query' + str(queryIndex) + '.shp'
         
        if iterationId == 0:
            # We insert the polygon in the DB (to be used by lasclip ot by the DB index query) 
            connection = self.getConnection()
            cursor = connection.cursor()
            cursor.execute("INSERT INTO " + utils.QUERY_TABLE + " VALUES (%s,ST_GeomFromEWKT(%s))", [queryIndex, 'SRID='+str(self.srid)+';'+self.qp.wkt])
            connection.commit() 
            connection.close()
            
            if self.qp.queryType == 'generic':
                # We generate a ShapeFile for lasclip in case of not rectangle or circle
                query = "select ST_SetSRID(geom, " + str(self.srid) + ") from " + utils.QUERY_TABLE + " where id = " + str(queryIndex) + ";"
                connString = ' '.join(('-h',self.dbHost,'-p',self.dbPort,'-u',self.userName,'-P',self.password,self.dbName))
                precommand = 'pgsql2shp -f ' + shapeFile + ' ' + connString + ' "' + query + '"'
                logging.info(precommand)
                os.system(precommand)

        if self.qp.queryType not in ('rectangle', 'circle', 'generic'):
            return (eTime, result)
            
        t0 = time.time()

        if self.dbIndex:
            inputList = 'input' +  str(queryIndex) + '.list'
            connString = self.getConnectionString(False, True)
            query = 'SELECT filepath FROM ' + self.lasIndexTableName + ',' + utils.QUERY_TABLE + ' where ST_Intersects( ' + utils.QUERY_TABLE + '.geom, ' + self.lasIndexTableName + '.geom ) and ' + utils.QUERY_TABLE + '.id = ' + str(queryIndex)
            prec = 'psql ' + connString + ' -t -A -c "' + query + '" > ' + inputList
            logging.info(prec)
            os.system(prec) 
        else:
            inputList = DEFAULT_INPUT_FILE_LIST_FILE_NAME
        
        if self.qp.queryType == 'rectangle':
            command = 'lasmerge -lof ' + inputList + ' -inside ' + str(self.qp.minx) + ' ' + str(self.qp.miny) + ' ' + str(self.qp.maxx) + ' ' + str(self.qp.maxy) + zquery
        elif self.qp.queryType == 'circle':
            command = 'lasmerge -lof ' + inputList + ' -inside_circle ' + str(self.qp.cx) + ' ' + str(self.qp.cy) + ' ' + str(self.qp.rad) + zquery
        elif self.qp.queryType == 'generic':
            command = 'lasclip.exe -lof ' + inputList + ' -poly ' + shapeFile + ' ' + zquery
            if not self.isSingle :
                command += ' -merged'                    
        
        if self.qp.queryMethod == 'disk': 
            outputFile = 'output' +  str(queryIndex) + '.' + self.outputExtension
            command += ' -o ' + outputFile
            logging.debug(command)
            os.system(command)
            eTime = time.time() - t0
            npointscommand = "lasinfo " + outputFile+ " -nc -nv -nco 2>&1 | grep 'number of point records:'"
            try:
                result  = int(utils.shellExecute(npointscommand).split()[-1])
            except:
                result = None
        elif self.qp.queryMethod == 'stream':
            
            cols = []
            for c in self.qp.columns:
                cols.append(self.DM_LASTOOLS[c])
            
            command += ' -stdout -otxt -oparse ' + ''.join(cols) + ' | wc -l'
            logging.debug(command)
            result = utils.shellExecute(command).replace('\n','')
            eTime = time.time() - t0
            try:
                result  = int(result)
            except:
                result = None                   
        else: # Statistical query
            try:
                outputFile = 'output' +  str(queryIndex) + '.' + self.outputExtension
                command += ' -o ' + outputFile
                logging.debug(command)
                os.system(command)
                
                options = ' -nv -nmm '
                for i in range(len(self.qp.statistics)):
                    if self.qp.statistics[i] == 'avg':
                        options += ' -histo ' + self.qp.columns[i] + " 10000000 "
                
                statcommand = "lasinfo -i " + outputFile +  options + " | grep 'min \|max\|average'"
                logging.info(statcommand)
                lines  = utils.shellExecute(statcommand).split('\n')
                results= []
                colIs = {'x':4, 'y':5, 'z':6}
                for i in range(len(self.qp.statistics)):
                    for line in lines:
                        if line.count(self.qp.statistics[i]) and line.count(self.qp.columns[i]):
                            if self.qp.statistics[i] == 'avg':
                                results.append(line.split()[-1])
                            else:
                                results.append(line.split()[colIs[self.qp.columns[i]]])
                result = ','.join(results)
            except Exception, err:
                print traceback.format_exc()
                result = None
            eTime = time.time() - t0
#!/usr/bin/env python
################################################################################
#    Created by Oscar Martinez                                                 #
#    [email protected]                                                  #
################################################################################
import matplotlib, sys, numpy, os
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pointcloud import utils

inputFile = os.path.abspath(sys.argv[1])
command = 'cat ' + inputFile + ' | grep "Read "'
lines = utils.shellExecute(command).split('\n')

nums = []
reads = []
sorts = []
writes = []
for line in lines:
    if line != '':
        fields = line.split()
        try:
            num = float(fields[1].replace(',', '')) / 1000000.
            read = float(fields[3].replace(',', ''))
            sort = float(fields[6].replace(',', ''))
            write = float(fields[9].replace(',', ''))
            nums.append(num)
            reads.append(read)
            sorts.append(sort)
            writes.append(write)
        except:
################################################################################
#    Created by Oscar Martinez                                                 #
#    [email protected]                                                  #
################################################################################
import os, sys, multiprocessing
from pointcloud import utils
from osgeo import osr
import liblas

#
# This module contains methods that use LAStools, GDAL, lasLib to interact
# with LAS/LAZ in the most efficient manner depending on the exact operation
#

# Check the LAStools is installed and that it is in PATH before libLAS
if utils.shellExecute('lasinfo -version').count('LAStools') == 0:
    raise Exception(
        "LAStools is not found!. Please check that it is in PATH and that it is before libLAS"
    )


def getSRID(absPath):
    """ Gets the SRID of a LAS/LAZ file (using liblas and GDAL, hence it is not fast)"""
    lasHeader = liblas.file.File(absPath, mode='r').header
    osrs = osr.SpatialReference()
    osrs.SetFromUserInput(lasHeader.get_srs().get_wkt())
    #osrs.AutoIdentifyEPSG()
    return osrs.GetAttrValue('AUTHORITY', 1)


def getNumPoints(absPath):