def validateArgs(self):
if not os.path.exists(self.outDir):
os.makedirs(self.outDir)
if not os.path.exists(self.outDir):
raise utilities.TrappedError("Specified Output directory " +
self.outDir +
" not found on file system")
if not os.path.isdir(self.outDir):
raise utilities.TrappedError("Specified Output directory " +
self.outDir + " is not a directory")
if self.logger is None:
self.logger = utilities.logger(self.outDir, self.verbose)
self.writetolog = self.logger.writetolog
# Validate template image.
if not SpatialUtilities.isRaster(self.inputFname):
raise utilities.TrappedError(
"input raster does not appear to be a valid raster")
if self.templateFName != "" and not SpatialUtilities.isRaster(
self.templateFName):
raise utilities.TrappedError(
"template raster does not appear to be a valid raster")
elif self.templateFName != "":
self.templateRaster = SpatialUtilities.SAHMRaster(
self.templateFName)
self.inputRaster = SpatialUtilities.SAHMRaster(self.inputFname)
def pullParamsFromRaster(self, otherRasterFile):
if not os.path.exists(otherRasterFile):
raise utilities.TrappedError("Raster does not appear to be a valid raster.\nThe input file (" + otherRasterFile + ") does not exist on the file system.")
# Get the PARC parameters from the rasterFile.
otherDS = gdal.Open(otherRasterFile, gdalconst.GA_ReadOnly)
self.getParams(otherDS)
def validateArgs(self):
argProblem = ""
if self.logger is None:
self.logger = utilities.logger(outDir, self.verbose)
self.writetolog = self.logger.writetolog
if os.path.isdir(self.inputDir):
pass
elif os.path.exists(self.MDSFile):
if not isMDSFile(self.MDSFile):
argProblem += "The supplied MDS file, " + self.MDSFile + ", does not appear to be in the appropriate format."
else:
argProblem += "Neither an input Directory or MDS File was supplied."
if not os.path.isdir(self.outputDir):
try:
os.mkdir(self.outputDir)
except:
argProblem += 'The supplied output directory, ' + self.outputDir + ", does not exist and could not be created."
if not self.format.lower() in self.driverExt.keys():
argProblem += "The supplied format must be one of " + ", ".join(
self.driverExt.keys())
if argProblem:
raise utilities.TrappedError(
"There was a problem with one or more of the inputs to RasterFormatConverter"
)
def validateArgs(self):
rasterProcessor.validateArgs(self)
validDTypes = [
"Unknown", "Byte", "Int16", "UInt32", "Int32", "CInt16", "CInt32"
]
if not gdal.GetDataTypeName(self.inputRaster.pixelType) in validDTypes:
utilities.TrappedError(
"The input raster is not of a discrete pixel type")
def waitForCondorProcessesToFinish(self, outputs):
errors = []
originalCount = len(outputs)
while outputs:
for process in outputs:
result = self.jobFinished(process)
if result == "finished":
outputs.remove(process)
# cleanup some condor files
success = False
while not success:
try:
for f in [
"log", "stdOut", "stdErr", "CondorSubmit"
]:
fname = process.replace(
".tif", "_" + f + ".txt")
if os.path.exists(fname):
os.remove(fname)
success = True
except:
pass
print str(originalCount -
len(outputs)) + " PARC layers finished"
elif result == "error":
if os.path.exists(process):
os.remove(process)
errors.append(process)
outputs.remove(process)
print str(originalCount - len(outputs)) + \
" PARC layers finished"
if len(errors) > 0:
msg = "There were problems with one or more runs."
for process in errors:
msg += "\n" + process + " did not run correctly"
raise utilities.TrappedError(msg)
def validateArgs(self):
"""
Make sure the user sent us some stuff we can work with
"""
if not os.path.exists(self.out_dir):
raise utilities.TrappedError("Specified Output directory " +
self.out_dir +
" not found on file system")
if not os.path.isdir(self.out_dir):
raise utilities.TrappedError("Specified Output directory " +
self.out_dir + " is not a directory")
if self.logger is None:
self.logger = utilities.logger(self.out_dir, self.verbose)
self.writetolog = self.logger.writetolog
# Validate template image.
if self.template is None:
raise utilities.TrappedError("template raster not provided.")
if not os.path.exists(self.template):
raise utilities.TrappedError("Template file, " + self.template +
", does not exist on file system")
self.templateRaster = SpatialUtilities.SAHMRaster(self.template)
if len(self.templateRaster.Error) != 0:
raise utilities.TrappedError(
"There was a problem with the provided template: \n " +
" " + "\n ".join(self.templateRaster.Error))
# Ensure the template has square pixels.
if abs(
abs(self.templateRaster.xScale) -
abs(self.templateRaster.yScale)) > 1e-6:
raise utilities.TrappedError(
"template image must have square pixels." +
"/n x pixel scale = " +
str(abs(self.templateRaster.xScale)) +
"/n y pixel scale = " +
str(abs(self.templateRaster.yScale)))
# Validate input rasters
if not os.path.exists(self.inputs_CSV):
raise utilities.TrappedError("Inputs CSV, " + self.inputs_CSV +
", does not exist on file system.")
inputs_csv = csv.reader(open(self.inputs_CSV, 'r'))
header = inputs_csv.next()
input_file_errors = ""
output_csv = os.path.join(self.out_dir, "PARC_Files.csv")
output = csv.writer(open(output_csv, "wb"))
output.writerow([
"PARCOutputFile", "Categorical", "Resampling", "Aggregation",
"OriginalFile",
os.path.abspath(self.template),
os.path.abspath(self.out_dir)
])
inputs = []
had_to_shrink = False
for row in inputs_csv:
input_file = row[0]
input_just_file = os.path.splitext(os.path.split(input_file)[1])[0]
if input_just_file == "hdr":
input_file = os.path.split(input_file)[0]
row[0] = input_file
input_just_file = os.path.split(input_file)[1]
if input_just_file in inputs:
input_file_errors += "\n PARC not currently set up to handle identically named inputs."
input_file_errors += "\n\t" + input_just_file + " used multiple times"
else:
inputs.append(input_just_file)
sourceRaster = SpatialUtilities.SAHMRaster(input_file)
if len(sourceRaster.Error) > 0:
input_file_errors += (" " + os.path.split(input_file)[1] +
" had the following errors:\n" + " " +
"\n ".join(sourceRaster.Error)) + "\n"
else:
if not self.ignoreNonOverlap and not self.image_covers_template(
sourceRaster):
input_file_errors += "\n Some part of the template image falls outside of " + input_just_file
input_file_errors += "\n template upper left = (" + str(
self.templateRaster.west) + ", " + str(
self.templateRaster.north) + ")"
input_file_errors += "\n template lower right = (" + str(
self.templateRaster.east) + ", " + str(
self.templateRaster.south) + ")"
upperLeftX, upperLeftY = SpatialUtilities.transformPoint(
sourceRaster.west, sourceRaster.north,
sourceRaster.srs, self.templateRaster.srs)
input_file_errors += "\n image upper left = (" + \
str(sourceRaster.west) + ", " + str(sourceRaster.north) + ")"
input_file_errors += "\n image lower right = (" + str(
sourceRaster.east) + ", " + str(
sourceRaster.south) + ")"
if self.ignoreNonOverlap:
# if this input is smaller in any of the dimensions
if not self.image_covers_template(sourceRaster):
self.shrink_template_extent(sourceRaster)
had_to_shrink = True
if len(row) < 2 or not row[1] in ['0', '1']:
self.writetolog(" " + os.path.split(input_file)[1] +
" categorical either missing or not 0 or 1:" +
"\n Defaulting to 0 (continuous)")
if len(row) < 2:
row.append('0')
else:
row[1] = '0'
resample_methods = [item.lower() for item in self.resample_methods]
if len(row) < 3 or not row[2].lower() in resample_methods:
self.writetolog(
" " + os.path.split(input_file)[1] +
" resample method either missing or not one of " +
", ".join(self.resample_methods) +
"\n Defaulting to 'Bilinear'")
if row[1] == '0':
default = 'Bilinear'
else:
default = 'NearestNeighbor'
if len(row) < 3:
row.append(default)
else:
row[2] = default
lower_agg_methods = [item.lower() for item in self.agg_methods]
if len(row) < 4 or not row[3].lower() in lower_agg_methods:
self.writetolog(
" " + os.path.split(input_file)[1] +
" aggregation method either missing or not one of " +
", ".join(self.agg_methods) + "\n Defaulting to 'Mean'")
if row[1] == '0':
default = 'Mean'
else:
default = 'Majority'
if len(row) < 4:
row.append(default)
else:
row[3] = default
self.inputs.append(row)
# also write the output row, reconfigured to our output file
short_name = SpatialUtilities.getRasterShortName(row[0])
file_name = os.path.abspath(
os.path.join(self.out_dir, short_name + ".tif"))
outputrow = [file_name] + row[1:4] + [
os.path.abspath(row[0]),
os.path.abspath(self.out_dir)
]
output.writerow(outputrow)
del output
if had_to_shrink:
old_template = SpatialUtilities.SAHMRaster(self.template)
new_template_fname = os.path.join(self.out_dir,
os.path.split(self.template)[1])
SpatialUtilities.intermediaryReprojection(
old_template, self.templateRaster, new_template_fname,
gdalconst.GRA_NearestNeighbour, True)
self.template = new_template_fname
self.templateRaster = SpatialUtilities.SAHMRaster(self.template)
output_csv = open(os.path.join(self.out_dir, "PARC_Files.csv"))
first_line, rest_of_lines = output_csv.readline(), output_csv.read(
)
output_csv = open(os.path.join(self.out_dir, "PARC_Files.csv"),
"w")
replacement_line = first_line.split(",")
replacement_line[5] = new_template_fname
replacement_line = ",".join(replacement_line)
output_csv.write(replacement_line)
output_csv.write(rest_of_lines)
output_csv.close()
if input_file_errors != "":
self.writetolog(input_file_errors, False, False)
raise utilities.TrappedError(
"There was one or more problems with " +
"your input rasters: \n" + input_file_errors)
def run(self):
rasterProcessor.run(self)
cellRatio = SpatialUtilities.getTemplateSRSCellSize(
self.inputRaster, self.templateRaster) / self.templateRaster.xScale
targetCellSize, numSourcePerTarget = SpatialUtilities.getAggregateTargetCellSize(
self.inputRaster, self.templateRaster)
if cellRatio > 0.5:
utilities.TrappedError(
"The input raster did not have cells at least half the dimensions of the template raster"
)
shortName = SpatialUtilities.getRasterShortName(self.inputFname)
tmpIntermediaryFname = os.path.join(self.outDir,
shortName + "_tmpRepo.tif")
gdal.Unlink(tmpIntermediaryFname)
SpatialUtilities.intermediaryReprojection(
self.inputRaster, self.templateRaster, tmpIntermediaryFname,
gdalconst.GRA_NearestNeighbour)
self.tmpRaster = SpatialUtilities.SAHMRaster(tmpIntermediaryFname)
#dictionaries to hold our outputRasters
outputs = {}
bSize = 2048 #source pixels
#convert this to the nearest whole number of target pixels
bSize = int(round(bSize / numSourcePerTarget) * numSourcePerTarget)
if bSize == 0:
bSize = int(numSourcePerTarget)
self.tmpRaster.blockSize = bSize
pixelsPerBlock = numSourcePerTarget * numSourcePerTarget
for block in self.tmpRaster.iterBlocks():
uniques = np.unique(block)
X, Y = block.shape
x = X // numSourcePerTarget
y = Y // numSourcePerTarget
ndMask = block.reshape(
(x, numSourcePerTarget, y, numSourcePerTarget))
ndMask = ndMask.transpose([0, 2, 1, 3])
ndMask = ndMask.reshape((x * y, pixelsPerBlock))
ans = np.array(stats.mode(ndMask, 1)[0]).reshape(x, y)
for uniqueVal in uniques:
if not type(uniqueVal) is np.ma.core.MaskedConstant:
ans = ((ndMask == uniqueVal).sum(1) /
(pixelsPerBlock)).reshape(x, y) * 100
else:
ans = ((ndMask.mask).sum(1) /
(pixelsPerBlock)).reshape(x, y) * 100
if not outputs.has_key(uniqueVal):
outputs[uniqueVal] = self.initializeOneOutput(uniqueVal)
outputs[uniqueVal].putBlock(
ans, int(self.tmpRaster.curCol / numSourcePerTarget),
int(self.tmpRaster.curRow / numSourcePerTarget))
self.tmpRaster.close()
gdal.Unlink(tmpIntermediaryFname)
#write our outputs to a PredictorListFile
output_fname = os.path.join(self.outDir, "ouputFileList.csv")
csv_writer = csv.writer(open(output_fname, 'wb'))
csv_writer.writerow(["file", "Resampling", "Aggregation"])
for k, v in outputs.iteritems():
csv_writer.writerow(
[os.path.normpath(v.source), "0", "NearestNeighbor", "Mean"])
del csv_writer
self.outputPredictorsList = output_fname
def validateArgs(self):
"""
Make sure the user sent us some stuff we can work with
"""
if not os.path.exists(self.out_dir):
raise utilities.TrappedError("Specified Output directory " +
self.out_dir +
" not found on file system")
if not os.path.isdir(self.out_dir):
raise utilities.TrappedError("Specified Output directory " +
self.out_dir + " is not a directory")
if self.logger is None:
self.logger = utilities.logger(self.out_dir, self.verbose)
self.writetolog = self.logger.writetolog
# Validate template image.
if self.template is None:
raise utilities.TrappedError("template raster not provided.")
if not os.path.exists(self.template):
raise utilities.TrappedError("Template file, " + self.template +
", does not exist on file system")
self.template_params = self.getRasterParams(self.template)
if len(self.template_params["Error"]) <> 0:
raise utilities.TrappedError(
"There was a problem with the provided template: \n " +
" " + "\n ".join(self.template_params["Error"]))
# Ensure the template has square pixels.
if abs(
abs(self.template_params["xScale"]) -
abs(self.template_params["yScale"])) > 1e-6:
raise utilities.TrappedError(
"template image must have square pixels." +
"/n x pixel scale = " + str(xScale) +
"/n y pixel scale = " + str(yScale))
#Validate input rasters
if not os.path.exists(self.inputs_CSV):
raise utilities.TrappedError("Inputs CSV, " + self.inputs_CSV +
", does not exist on file system.")
inputsCSV = csv.reader(open(self.inputs_CSV, 'r'))
header = inputsCSV.next()
strInputFileErrors = ""
outputCSV = os.path.join(self.out_dir, "PARC_Files.csv")
output = csv.writer(open(outputCSV, "wb"))
output.writerow([
"PARCOutputFile", "Categorical", "Resampling", "Aggregation",
"OriginalFile",
os.path.abspath(self.template),
os.path.abspath(self.out_dir)
])
for row in inputsCSV:
inputFile = row[0]
sourceParams = self.getRasterParams(inputFile)
if len(sourceParams["Error"]) > 0:
strInputFileErrors += (
" " + os.path.split(inputFile)[1] +
" had the following errors:\n" + " " +
"\n ".join(sourceParams["Error"])) + "\n"
else:
pass
if not self.ImageCoversTemplate(sourceParams):
strInputFileErrors += (
"\n Some part of the template image falls outside of "
+ os.path.split(inputFile)[1])
strInputFileErrors += "\n template upper left = (" + str(
self.template_params["gWest"]) + ", " + str(
self.template_params["gNorth"]) + ")"
strInputFileErrors += "\n template lower right = (" + str(
self.template_params["gEast"]) + ", " + str(
self.template_params["gSouth"]) + ")"
strInputFileErrors += "\n image upper left = (" + str(
sourceParams["gWest"]) + ", " + str(
sourceParams["gNorth"]) + ")"
strInputFileErrors += "\n image lower right = (" + str(
sourceParams["gEast"]) + ", " + str(
sourceParams["gSouth"]) + ")"
# strInputFileErrors += "\n points are given in projected coordinates."
# strInputFileErrors += "\n template upper left = (" + str(self.template_params["tWest"]) + ", " + str(self.template_params["tNorth"]) + ")"
# strInputFileErrors += "\n template lower right = (" + str(self.template_params["tEast"]) + ", " + str(self.template_params["tSouth"]) + ")"
# strInputFileErrors += "\n image upper left = (" + str(sourceParams["tWest"]) + ", " + str(sourceParams["tNorth"]) + ")"
# strInputFileErrors += "\n image lower right = (" + str(sourceParams["tEast"]) + ", " + str(sourceParams["tSouth"]) + ")"
# strInputFileErrors += "\n Note: points are given in the template coordinates." + "\n"
#
if len(row) < 2 or not row[1] in ['0', '1']:
self.writetolog(
" " + os.path.split(inputFile)[1] +
" categorical either missing or not 0 or 1:\n Defaulting to 0 (continuous)"
)
if len(row) < 2:
row.append('0')
else:
row[1] = '0'
if len(row) < 3 or not row[2].lower() in [
item.lower() for item in self.resample_methods
]:
self.writetolog(
" " + os.path.split(inputFile)[1] +
" resample method either missing or not one of " +
", ".join(self.resample_methods) +
"\n Defaulting to 'Bilinear'")
if row[1] == '0':
default = 'Bilinear'
else:
default = 'NearestNeighbor'
if len(row) < 3:
row.append(default)
else:
row[2] = default
if len(row) < 4 or not row[3].lower() in [
item.lower() for item in self.agg_methods
]:
self.writetolog(
" " + os.path.split(inputFile)[1] +
" aggregation method either missing or not one of " +
", ".join(self.agg_methods) + "\n Defaulting to 'Mean'")
if row[1] == '0':
default = 'Mean'
else:
default = 'Majority'
if len(row) < 4:
row.append(default)
else:
row[3] = default
self.inputs.append(row)
#also write the output row, reconfigured to our output file
fileName = self.getShortName(row[0])
fileName = os.path.abspath(
os.path.join(self.out_dir, fileName + ".tif"))
outputrow = [fileName] + row[1:4] + [
os.path.abspath(row[0]),
os.path.abspath(self.out_dir)
]
output.writerow(outputrow)
del output
if strInputFileErrors <> "":
self.writetolog(strInputFileErrors)
raise utilities.TrappedError(
"There was one or more problems with your input rasters: \n" +
strInputFileErrors)
