def lasDuplicateFree(directory, output): """ Takes a directory with las [laz] files and an output directory Returns las, [laz] files free from duplicates""" files = getFiles(directory,['laz'],True) checkDirectory(output) for file in files: print file fh = readFileLaspy(file) writeFile(output,file[file.rfind('\\')+1:],fh.header,removeDuplicate(fh).transpose())
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport, createdb, dbtable, srid): opts = 0 childrenQueue = multiprocessing.Queue() ifiles = utils.getFiles(inputFolder) for i in range(len(ifiles)): childrenQueue.put([i, ifiles[i]]) for i in range( int(numcores) ): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO) childrenQueue.put(None) clineConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, True) psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, False) if createdb: os.system('dropdb ' + clineConString) os.system('createdb ' + clineConString) connection = psycopg2.connect(psycopgConString) cursor = connection.cursor() if createdb: cursor.execute('CREATE EXTENSION postgis') connection.commit() q = """ CREATE TABLE """ + dbtable + """ ( id integer, filepath text, num integer, scalex double precision, scaley double precision, scalez double precision, offsetx double precision, offsety double precision, offsetz double precision, geom public.geometry(Geometry,""" + str(srid) + """) )""" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() # q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)" # logging.info(cursor.mogrify(q)) # cursor.execute(q) # connection.commit() print 'numcores', numcores children = [] # We start numcores children processes for i in range(int(numcores)): children.append( multiprocessing.Process(target=runChild, args=(i, childrenQueue, psycopgConString, dbtable, srid))) children[-1].start() # wait for all children to finish their execution for i in range(int(numcores)): children[i].join() q = "create index ON " + dbtable + " using GIST (geom)" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() old_isolation_level = connection.isolation_level connection.set_isolation_level(0) q = "VACUUM FULL ANALYZE " + dbtable logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() connection.set_isolation_level(old_isolation_level) cursor.close()
def getPCFolderDetails(absPath, srid=None, numProc=1): """ Get the details (count numPoints and extent) of a folder with LAS/LAZ files (using LAStools, hence it is fast) It is assumed that all file shave same SRID and scale as first one""" tcount = 0 (tminx, tminy, tminz, tmaxx, tmaxy, tmaxz) = (None, None, None, None, None, None) (tscalex, tscaley, tscalez) = (None, None, None) tsrid = None if os.path.isdir(absPath): inputFiles = utils.getFiles(absPath, recursive=True) else: inputFiles = [absPath] numInputFiles = len(inputFiles) tasksQueue = multiprocessing.Queue() # The queue of tasks detailsQueue = multiprocessing.Queue() # The queue of results/details inputFilesAbsPath = [] inputFilesBoundingCube = [] for i in range(numInputFiles): inputFilesAbsPath.append(None) inputFilesBoundingCube.append(None) tasksQueue.put((i, inputFiles[i])) for i in range(numProc): # we add as many None jobs as numProc to tell them to terminate (queue is FIFO) tasksQueue.put(None) workers = [] # We start numProc users workers for i in range(numProc): workers.append( multiprocessing.Process(target=runProcGetPCFolderDetailsWorker, args=(tasksQueue, detailsQueue, srid)) ) workers[-1].start() for i in range(numInputFiles): sys.stdout.write("\r") ( inputFileIndex, inputFileAbsPath, srid, count, minx, miny, minz, maxx, maxy, maxz, scalex, scaley, scalez, _, _, _, ) = detailsQueue.get() inputFilesAbsPath[inputFileIndex] = inputFileAbsPath inputFilesBoundingCube[inputFileIndex] = (minx, miny, minz, maxx, maxy, maxz) if i == 0: (tscalex, tscaley, tscalez) = (scalex, scaley, scalez) tsrid = srid tcount += count if count: if tminx == None or minx < tminx: tminx = minx if tminy == None or miny < tminy: tminy = miny if tminz == None or minz < tminz: tminz = minz if tmaxx == None or maxx > tmaxx: tmaxx = maxx if tmaxy == None or maxy > tmaxy: tmaxy = maxy if tmaxz == None or maxz > tmaxz: tmaxz = maxz sys.stdout.write("\rCompleted %.02f%%" % (100.0 * float(i) / float(numInputFiles))) sys.stdout.flush() sys.stdout.write("\r") sys.stdout.write("\rCompleted 100.00%!") # wait for all users to finish their execution for i in range(numProc): workers[i].join() print return ( inputFilesAbsPath, inputFilesBoundingCube, tsrid, tcount, (tminx, tminy, tminz, tmaxx, tmaxy, tmaxz), (tscalex, tscaley, tscalez), )
def getPCFolderDetails(absPath, srid=None, numProc=1): """ Get the details (count numPoints and extent) of a folder with LAS/LAZ files (using LAStools, hence it is fast) It is assumed that all file shave same SRID and scale as first one""" tcount = 0 (tminx, tminy, tminz, tmaxx, tmaxy, tmaxz) = (None, None, None, None, None, None) (tscalex, tscaley, tscalez) = (None, None, None) tsrid = None if os.path.isdir(absPath): inputFiles = utils.getFiles(absPath, recursive=True) else: inputFiles = [ absPath, ] numInputFiles = len(inputFiles) tasksQueue = multiprocessing.Queue() # The queue of tasks detailsQueue = multiprocessing.Queue() # The queue of results/details inputFilesAbsPath = [] inputFilesBoundingCube = [] for i in range(numInputFiles): inputFilesAbsPath.append(None) inputFilesBoundingCube.append(None) tasksQueue.put((i, inputFiles[i])) for i in range( numProc ): #we add as many None jobs as numProc to tell them to terminate (queue is FIFO) tasksQueue.put(None) workers = [] # We start numProc users workers for i in range(numProc): workers.append( multiprocessing.Process(target=runProcGetPCFolderDetailsWorker, args=(tasksQueue, detailsQueue, srid))) workers[-1].start() for i in range(numInputFiles): sys.stdout.write('\r') (inputFileIndex, inputFileAbsPath, srid, count, minx, miny, minz, maxx, maxy, maxz, scalex, scaley, scalez, _, _, _) = detailsQueue.get() inputFilesAbsPath[inputFileIndex] = inputFileAbsPath inputFilesBoundingCube[inputFileIndex] = (minx, miny, minz, maxx, maxy, maxz) if i == 0: (tscalex, tscaley, tscalez) = (scalex, scaley, scalez) tsrid = srid tcount += count if count: if tminx == None or minx < tminx: tminx = minx if tminy == None or miny < tminy: tminy = miny if tminz == None or minz < tminz: tminz = minz if tmaxx == None or maxx > tmaxx: tmaxx = maxx if tmaxy == None or maxy > tmaxy: tmaxy = maxy if tmaxz == None or maxz > tmaxz: tmaxz = maxz sys.stdout.write("\rCompleted %.02f%%" % (100. * float(i) / float(numInputFiles))) sys.stdout.flush() sys.stdout.write('\r') sys.stdout.write('\rCompleted 100.00%!') # wait for all users to finish their execution for i in range(numProc): workers[i].join() print return (inputFilesAbsPath, inputFilesBoundingCube, tsrid, tcount, (tminx, tminy, tminz, tmaxx, tmaxy, tmaxz), (tscalex, tscaley, tscalez))
def run(inputFolder, numcores, dbname, dbuser, dbpass, dbhost, dbport, createdb, dbtable, srid): opts = 0 childrenQueue = multiprocessing.Queue() ifiles = utils.getFiles(inputFolder) for i in range(len(ifiles)): childrenQueue.put([i, ifiles[i]]) for i in range(int(numcores)): #we add as many None jobs as numWorkers to tell them to terminate (queue is FIFO) childrenQueue.put(None) clineConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, True) psycopgConString = postgresops.getConnectString(dbname, dbuser, dbpass, dbhost, dbport, False) if createdb: os.system('dropdb ' + clineConString) os.system('createdb ' + clineConString) connection = psycopg2.connect(psycopgConString) cursor = connection.cursor() if createdb: cursor.execute('CREATE EXTENSION postgis') connection.commit() q = """ CREATE TABLE """ + dbtable + """ ( id integer, filepath text, num integer, scalex double precision, scaley double precision, scalez double precision, offsetx double precision, offsety double precision, offsetz double precision, geom public.geometry(Geometry,""" + str(srid) + """) )""" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() # q = "select addgeometrycolumn('" + dbtable + "','geom',28992,'POLYGON',2)" # logging.info(cursor.mogrify(q)) # cursor.execute(q) # connection.commit() print 'numcores',numcores children = [] # We start numcores children processes for i in range(int(numcores)): children.append(multiprocessing.Process(target=runChild, args=(i, childrenQueue, psycopgConString, dbtable, srid))) children[-1].start() # wait for all children to finish their execution for i in range(int(numcores)): children[i].join() q = "create index ON " + dbtable + " using GIST (geom)" logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() old_isolation_level = connection.isolation_level connection.set_isolation_level(0) q = "VACUUM FULL ANALYZE " + dbtable logging.info(cursor.mogrify(q)) cursor.execute(q) connection.commit() connection.set_isolation_level(old_isolation_level) cursor.close()