def processRastersInFolder(fileList,
target_path,
publish_path,
elev_type,
bound_path,
z_min,
z_max,
v_name,
v_unit,
h_name,
h_unit,
h_wkid,
spatial_ref,
runAgain=True):
a = datetime.now()
path = os.path.join(RunUtil.TOOLS_PATH, "A05_B_RevalueRaster.py")
Utility.printArguments([
"fileList", "target_path", "publish_path", "elev_type", "bound_path",
"spatial_ref", "runAgain"
], [(None if fileList is None else len(fileList)), target_path,
publish_path, elev_type, bound_path, spatial_ref, runAgain],
"createLasStatistics")
grouping = PROCESS_CHUNKS
if not runAgain:
grouping = int(PROCESS_CHUNKS / 2)
if grouping <= 1:
grouping = 2
total = len(fileList)
if total > 0:
fileList_repeat = []
procCount = int(os.environ['NUMBER_OF_PROCESSORS'])
if procCount > 4:
procCount = procCount - PROCESS_SPARES
if procCount <= 0:
procCount = 1
arcpy.AddMessage(
"processRastersInFolder: Using {}/{} Processors to process {} files in groups of {}"
.format(procCount, (procCount + PROCESS_SPARES), total, grouping))
processList = []
indx = 0
for f_paths in grouper(fileList, grouping):
f_paths = [x for x in f_paths if x is not None]
f_path = ",".join(f_paths)
indx = indx + len(f_paths)
arcpy.AddMessage(
' processRastersInFolder: Working on {} {}/{}'.format(
elev_type, indx, total))
args = [
f_path, elev_type, target_path, publish_path, bound_path,
str(z_min),
str(z_max), v_name, v_unit, h_name, h_unit,
str(h_wkid), spatial_ref
]
try:
processList.append(
RunUtil.runToolx64_async(path, args, "A05_B", target_path))
# give time for things to wake up
time.sleep(PROCESS_DELAY)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "processRastersInFolder: PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type) + ": " + str(sys.exc_value) + "\n"
arcpy.AddWarning(pymsg)
msgs = "processRastersInFolder: GP ERRORS:\n" + arcpy.GetMessages(
2) + "\n"
arcpy.AddWarning(msgs)
sys.exit(1)
waitForResults = True
first = True
while waitForResults:
if not first:
time.sleep(1)
first = False
# arcpy.AddMessage('processRastersInFolder: Looping LEN Process List = {} ProcCount = {} is greater = {}'.format(len(processList), procCount, (len(processList) >= procCount)))
for i, [p, l] in enumerate(processList):
if p.poll() is not None:
# error log messages are handled in
retCode = RunUtil.endRun_async(path, p, l)
if retCode <> 0:
fileList_repeat.append(f_path)
del processList[i]
waitForResults = (len(processList) >= int(procCount))
# Wait for last subprocesses to complete
arcpy.AddMessage(
" processRastersInFolder: Waiting for process list to clear {} jobs"
.format(len(processList)))
while len(processList) > 0:
for i, [p, l] in enumerate(processList):
if p.poll() is not None:
RunUtil.endRun_async(path, p, l)
del processList[i]
arcpy.AddMessage(
" processRastersInFolder: Waiting for process list to clear {} jobs"
.format(len(processList)))
else:
# arcpy.AddMessage("processRastersInFolder: Waiting for process list to clear {} jobs".format(len(processList)))
time.sleep(PROCESS_DELAY)
if runAgain and len(fileList_repeat) > 0:
# try to clean up any errors along the way
processRastersInFolder(fileList,
target_path,
publish_path,
elev_type,
bound_path,
z_min,
z_max,
v_name,
v_unit,
h_name,
h_unit,
h_wkid,
spatial_ref,
runAgain=False)
try:
shutil.rmtree(os.path.join(target_path, elev_type, "TEMP"), True)
except:
pass
doTime(a, 'processRastersInFolder: All jobs completed.')
def createLasStatistics(fileList,
target_path,
spatial_reference=None,
isClassified=True,
createQARasters=False,
createMissingRasters=True,
overrideBorderPath=None,
runAgain=True):
a = datetime.now()
path = os.path.join(RunUtil.TOOLS_PATH, "A04_B_CreateLASStats.py")
Utility.printArguments([
"fileList", "target_path", "spatial_reference", "isClassified",
"createQARasters", "createMissingRasters", "overrideBorderPath"
], [
fileList, target_path, spatial_reference, isClassified,
createQARasters, createMissingRasters, overrideBorderPath
], "createLasStatistics")
grouping = PROCESS_CHUNKS
if not runAgain:
grouping = int(PROCESS_CHUNKS / 2)
if grouping <= 1:
grouping = 2
total = len(fileList)
if total > 0:
fileList_repeat = []
procCount = int(os.environ['NUMBER_OF_PROCESSORS'])
if procCount > 4:
procCount = procCount - PROCESS_SPARES
arcpy.AddMessage(
"\tUsing {}/{} Processors to process {} files in groups of {}".
format(procCount, (procCount + PROCESS_SPARES), total, grouping))
processList = []
indx = 0
for f_paths in Utility.grouper(fileList, grouping):
f_paths = [x for x in f_paths if x is not None]
f_path = ",".join(f_paths)
indx = indx + len(f_paths)
#arcpy.AddMessage('\t Working on {}/{}: {}'.format(indx, total, f_path))
arcpy.AddMessage('\t Working on {}/{}'.format(indx, total))
args = [
f_path, target_path, spatial_reference,
"{}".format(isClassified), "{}".format(createQARasters),
"{}".format(createMissingRasters), overrideBorderPath
]
try:
processList.append(
RunUtil.runToolx64_async(path, args, "A04_B", target_path))
# give time for things to wake up
time.sleep(PROCESS_DELAY)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = " PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type) + ": " + str(sys.exc_value) + "\n"
arcpy.AddWarning(pymsg)
msgs = "GP ERRORS:\n" + arcpy.GetMessages(2) + "\n"
arcpy.AddWarning(msgs)
sys.exit(1)
waitForResults = True
first = True
while waitForResults:
if not first:
time.sleep(1)
first = False
# arcpy.AddMessage('Looping LEN Process List = {} ProcCount = {} is greater = {}'.format(len(processList), procCount, (len(processList) >= procCount)))
for i, [p, l] in enumerate(processList):
if p.poll() is not None:
# error log messages are handled in
retCode = RunUtil.endRun_async(path, p, l)
if retCode <> 0:
fileList_repeat.append(f_path)
del processList[i]
waitForResults = (len(processList) >= int(procCount))
# Wait for last subprocesses to complete
arcpy.AddMessage("\tWaiting for process list to clear {} jobs".format(
len(processList)))
while len(processList) > 0:
for i, [p, l] in enumerate(processList):
if p.poll() is not None:
retCode = RunUtil.endRun_async(path, p, l)
if retCode <> 0:
fileList_repeat.append(f_path)
del processList[i]
if len(processList) > 0:
arcpy.AddMessage(
"\tWaiting for process list to clear {} jobs".
format(len(processList)))
else:
# arcpy.AddMessage("Waiting for process list to clear {} jobs".format(len(processList)))
time.sleep(PROCESS_DELAY)
# BRUCE's code here
if runAgain and len(fileList_repeat) > 0:
# try to clean up any errors along the way
createLasStatistics(fileList,
target_path,
spatial_reference,
isClassified,
createQARasters,
createMissingRasters,
overrideBorderPath,
runAgain=False)
elif not runAgain and len(fileList_repeat) > 0:
arcpy.AddError("Error processing .las files.")
raise Exception("Error processing .las files.")
doTime(a, 'createLasStatistics: All jobs completed.')