def getPCFileDetails(absPath, srid=None):
""" Get the details (count numPoints and extent) of a LAS/LAZ file (using LAStools, hence it is fast)"""
count = None
(minX, minY, minZ, maxX, maxY, maxZ) = (None, None, None, None, None, None)
(scaleX, scaleY, scaleZ) = (None, None, None)
(offsetX, offsetY, offsetZ) = (None, None, None)
if srid == None:
srid = getSRID(absPath)
command = "lasinfo " + absPath + " -nc -nv -nco"
for line in utils.shellExecute(command).split("\n"):
if line.count("min x y z:"):
[minX, minY, minZ] = line.split(":")[-1].strip().split(" ")
minX = float(minX)
minY = float(minY)
minZ = float(minZ)
elif line.count("max x y z:"):
[maxX, maxY, maxZ] = line.split(":")[-1].strip().split(" ")
maxX = float(maxX)
maxY = float(maxY)
maxZ = float(maxZ)
elif line.count("number of point records:"):
count = int(line.split(":")[-1].strip())
elif line.count("scale factor x y z:"):
[scaleX, scaleY, scaleZ] = line.split(":")[-1].strip().split(" ")
scaleX = float(scaleX)
scaleY = float(scaleY)
scaleZ = float(scaleZ)
elif line.count("offset x y z:"):
[offsetX, offsetY, offsetZ] = line.split(":")[-1].strip().split(" ")
offsetX = float(offsetX)
offsetY = float(offsetY)
offsetZ = float(offsetZ)
return (srid, count, minX, minY, minZ, maxX, maxY, maxZ, scaleX, scaleY, scaleZ, offsetX, offsetY, offsetZ)
def executeSQLFileCount(connectionString, sqlFileAbsPath):
command = 'psql ' + connectionString + ' < ' + sqlFileAbsPath + ' | wc -l'
result = utils.shellExecute(command).replace('\n','')
try:
result = int(result) - 4
except:
result = None
return result
def executeSQLFileCount(connectionString, sqlFileAbsPath):
command = 'sqlplus -s ' + connectionString + ' < ' + sqlFileAbsPath + ' | wc -l'
result = utils.shellExecute(command).replace('\n','')
try:
result = int(result)
except:
result = -1
return result
def executePDALCount(xmlFile):
c = 'pdal pipeline ' + xmlFile + ' | wc -l'
logging.debug(c)
result = utils.shellExecute(c).replace('\n', '')
# remove the XML file
#os.system('rm ' + xmlFile)
try:
result = int(result) - 1
except:
result = -1
return result
def executePDALCount(xmlFile):
c = 'pdal pipeline ' + xmlFile + ' | wc -l'
logging.debug(c)
result = utils.shellExecute(c).replace('\n','')
# remove the XML file
#os.system('rm ' + xmlFile)
try:
result = int(result) - 1
except:
result = -1
return result
def getPCFileDetails(absPath, srid=None):
""" Get the details (count numPoints and extent) of a LAS/LAZ file (using LAStools, hence it is fast)"""
count = None
(minX, minY, minZ, maxX, maxY, maxZ) = (None, None, None, None, None, None)
(scaleX, scaleY, scaleZ) = (None, None, None)
(offsetX, offsetY, offsetZ) = (None, None, None)
if srid == None:
srid = getSRID(absPath)
command = 'lasinfo ' + absPath + ' -nc -nv -nco'
for line in utils.shellExecute(command).split('\n'):
if line.count('min x y z:'):
[minX, minY, minZ] = line.split(':')[-1].strip().split(' ')
minX = float(minX)
minY = float(minY)
minZ = float(minZ)
elif line.count('max x y z:'):
[maxX, maxY, maxZ] = line.split(':')[-1].strip().split(' ')
maxX = float(maxX)
maxY = float(maxY)
maxZ = float(maxZ)
elif line.count('number of point records:'):
count = int(line.split(':')[-1].strip())
elif line.count('scale factor x y z:'):
[scaleX, scaleY, scaleZ] = line.split(':')[-1].strip().split(' ')
scaleX = float(scaleX)
scaleY = float(scaleY)
scaleZ = float(scaleZ)
elif line.count('offset x y z:'):
[offsetX, offsetY,
offsetZ] = line.split(':')[-1].strip().split(' ')
offsetX = float(offsetX)
offsetY = float(offsetY)
offsetZ = float(offsetZ)
return (srid, count, minX, minY, minZ, maxX, maxY, maxZ, scaleX, scaleY,
scaleZ, offsetX, offsetY, offsetZ)
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import matplotlib, sys, numpy, os matplotlib.use('Agg') import matplotlib.pyplot as plt from pointcloud import utils inputFile = os.path.abspath(sys.argv[1]) command = 'cat ' + inputFile + ' | grep "Read "' lines = utils.shellExecute(command).split('\n') nums = [] reads = [] sorts = [] writes = [] for line in lines: if line != '': fields = line.split() try: num = float(fields[1].replace(',','')) / 1000000. read = float(fields[3].replace(',','')) sort = float(fields[6].replace(',','')) write = float(fields[9].replace(',','')) nums.append(num) reads.append(read) sorts.append(sort) writes.append(write) except:
################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import os, sys, multiprocessing from pointcloud import utils from osgeo import osr import liblas # # This module contains methods that use LAStools, GDAL, lasLib to interact # with LAS/LAZ in the most efficient manner depending on the exact operation # # Check the LAStools is installed and that it is in PATH before libLAS if utils.shellExecute("lasinfo -version").count("LAStools") == 0: raise Exception("LAStools is not found!. Please check that it is in PATH and that it is before libLAS") def getSRID(absPath): """ Gets the SRID of a LAS/LAZ file (using liblas and GDAL, hence it is not fast)""" lasHeader = liblas.file.File(absPath, mode="r").header osrs = osr.SpatialReference() osrs.SetFromUserInput(lasHeader.get_srs().get_wkt()) # osrs.AutoIdentifyEPSG() return osrs.GetAttrValue("AUTHORITY", 1) def getNumPoints(absPath): """ Get the number of points of a LAS/LAZ (using LAStools, hence it is fast)""" return getPCFileDetails(absPath)[1]
def query(self, queryId, iterationId, queriesParameters):
(eTime, result) = (-1, None)
queryIndex = int(queryId)
self.qp = queriesParameters.getQueryParameters('psql', queryId)
logging.debug(self.qp.queryKey)
zquery = ''
if (self.qp.minz != None) or (self.qp.maxz != None):
zconds = []
if self.qp.minz != None:
zconds.append(' -drop_z_below ' + str(self.qp.minz) + ' ')
if self.qp.maxz != None:
zconds.append(' -drop_z_above ' + str(self.qp.maxz) + ' ')
zquery = ' '.join(zconds)
shapeFile = 'query' + str(queryIndex) + '.shp'
if iterationId == 0:
# We insert the polygon in the DB (to be used by lasclip ot by the DB index query)
connection = self.getConnection()
cursor = connection.cursor()
cursor.execute("INSERT INTO " + utils.QUERY_TABLE + " VALUES (%s,ST_GeomFromEWKT(%s))", [queryIndex, 'SRID='+str(self.srid)+';'+self.qp.wkt])
connection.commit()
connection.close()
if self.qp.queryType == 'generic':
# We generate a ShapeFile for lasclip in case of not rectangle or circle
query = "select ST_SetSRID(geom, " + str(self.srid) + ") from " + utils.QUERY_TABLE + " where id = " + str(queryIndex) + ";"
connString = ' '.join(('-h',self.dbHost,'-p',self.dbPort,'-u',self.userName,'-P',self.password,self.dbName))
precommand = 'pgsql2shp -f ' + shapeFile + ' ' + connString + ' "' + query + '"'
logging.info(precommand)
os.system(precommand)
if self.qp.queryType not in ('rectangle', 'circle', 'generic'):
return (eTime, result)
t0 = time.time()
if self.dbIndex:
inputList = 'input' + str(queryIndex) + '.list'
connString = self.getConnectionString(False, True)
query = 'SELECT filepath FROM ' + self.lasIndexTableName + ',' + utils.QUERY_TABLE + ' where ST_Intersects( ' + utils.QUERY_TABLE + '.geom, ' + self.lasIndexTableName + '.geom ) and ' + utils.QUERY_TABLE + '.id = ' + str(queryIndex)
prec = 'psql ' + connString + ' -t -A -c "' + query + '" > ' + inputList
logging.info(prec)
os.system(prec)
else:
inputList = DEFAULT_INPUT_FILE_LIST_FILE_NAME
if self.qp.queryType == 'rectangle':
command = 'lasmerge -lof ' + inputList + ' -inside ' + str(self.qp.minx) + ' ' + str(self.qp.miny) + ' ' + str(self.qp.maxx) + ' ' + str(self.qp.maxy) + zquery
elif self.qp.queryType == 'circle':
command = 'lasmerge -lof ' + inputList + ' -inside_circle ' + str(self.qp.cx) + ' ' + str(self.qp.cy) + ' ' + str(self.qp.rad) + zquery
elif self.qp.queryType == 'generic':
command = 'lasclip.exe -lof ' + inputList + ' -poly ' + shapeFile + ' ' + zquery
if not self.isSingle :
command += ' -merged'
if self.qp.queryMethod == 'disk':
outputFile = 'output' + str(queryIndex) + '.' + self.outputExtension
command += ' -o ' + outputFile
logging.debug(command)
os.system(command)
eTime = time.time() - t0
npointscommand = "lasinfo " + outputFile+ " -nc -nv -nco 2>&1 | grep 'number of point records:'"
try:
result = int(utils.shellExecute(npointscommand).split()[-1])
except:
result = None
elif self.qp.queryMethod == 'stream':
cols = []
for c in self.qp.columns:
cols.append(self.DM_LASTOOLS[c])
command += ' -stdout -otxt -oparse ' + ''.join(cols) + ' | wc -l'
logging.debug(command)
result = utils.shellExecute(command).replace('\n','')
eTime = time.time() - t0
try:
result = int(result)
except:
result = None
else: # Statistical query
try:
outputFile = 'output' + str(queryIndex) + '.' + self.outputExtension
command += ' -o ' + outputFile
logging.debug(command)
os.system(command)
options = ' -nv -nmm '
for i in range(len(self.qp.statistics)):
if self.qp.statistics[i] == 'avg':
options += ' -histo ' + self.qp.columns[i] + " 10000000 "
statcommand = "lasinfo -i " + outputFile + options + " | grep 'min \|max\|average'"
logging.info(statcommand)
lines = utils.shellExecute(statcommand).split('\n')
results= []
colIs = {'x':4, 'y':5, 'z':6}
for i in range(len(self.qp.statistics)):
for line in lines:
if line.count(self.qp.statistics[i]) and line.count(self.qp.columns[i]):
if self.qp.statistics[i] == 'avg':
results.append(line.split()[-1])
else:
results.append(line.split()[colIs[self.qp.columns[i]]])
result = ','.join(results)
except Exception, err:
print traceback.format_exc()
result = None
eTime = time.time() - t0
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import matplotlib, sys, numpy, os matplotlib.use('Agg') import matplotlib.pyplot as plt from pointcloud import utils inputFile = os.path.abspath(sys.argv[1]) command = 'cat ' + inputFile + ' | grep "Read "' lines = utils.shellExecute(command).split('\n') nums = [] reads = [] sorts = [] writes = [] for line in lines: if line != '': fields = line.split() try: num = float(fields[1].replace(',', '')) / 1000000. read = float(fields[3].replace(',', '')) sort = float(fields[6].replace(',', '')) write = float(fields[9].replace(',', '')) nums.append(num) reads.append(read) sorts.append(sort) writes.append(write) except:
################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import os, sys, multiprocessing from pointcloud import utils from osgeo import osr import liblas # # This module contains methods that use LAStools, GDAL, lasLib to interact # with LAS/LAZ in the most efficient manner depending on the exact operation # # Check the LAStools is installed and that it is in PATH before libLAS if utils.shellExecute('lasinfo -version').count('LAStools') == 0: raise Exception( "LAStools is not found!. Please check that it is in PATH and that it is before libLAS" ) def getSRID(absPath): """ Gets the SRID of a LAS/LAZ file (using liblas and GDAL, hence it is not fast)""" lasHeader = liblas.file.File(absPath, mode='r').header osrs = osr.SpatialReference() osrs.SetFromUserInput(lasHeader.get_srs().get_wkt()) #osrs.AutoIdentifyEPSG() return osrs.GetAttrValue('AUTHORITY', 1) def getNumPoints(absPath):