def validateArgs(self): if not os.path.exists(self.outDir): os.makedirs(self.outDir) if not os.path.exists(self.outDir): raise utilities.TrappedError("Specified Output directory " + self.outDir + " not found on file system") if not os.path.isdir(self.outDir): raise utilities.TrappedError("Specified Output directory " + self.outDir + " is not a directory") if self.logger is None: self.logger = utilities.logger(self.outDir, self.verbose) self.writetolog = self.logger.writetolog # Validate template image. if not SpatialUtilities.isRaster(self.inputFname): raise utilities.TrappedError( "input raster does not appear to be a valid raster") if self.templateFName != "" and not SpatialUtilities.isRaster( self.templateFName): raise utilities.TrappedError( "template raster does not appear to be a valid raster") elif self.templateFName != "": self.templateRaster = SpatialUtilities.SAHMRaster( self.templateFName) self.inputRaster = SpatialUtilities.SAHMRaster(self.inputFname)
def pullParamsFromRaster(self, otherRasterFile): if not os.path.exists(otherRasterFile): raise utilities.TrappedError("Raster does not appear to be a valid raster.\nThe input file (" + otherRasterFile + ") does not exist on the file system.") # Get the PARC parameters from the rasterFile. otherDS = gdal.Open(otherRasterFile, gdalconst.GA_ReadOnly) self.getParams(otherDS)
def validateArgs(self): argProblem = "" if self.logger is None: self.logger = utilities.logger(outDir, self.verbose) self.writetolog = self.logger.writetolog if os.path.isdir(self.inputDir): pass elif os.path.exists(self.MDSFile): if not isMDSFile(self.MDSFile): argProblem += "The supplied MDS file, " + self.MDSFile + ", does not appear to be in the appropriate format." else: argProblem += "Neither an input Directory or MDS File was supplied." if not os.path.isdir(self.outputDir): try: os.mkdir(self.outputDir) except: argProblem += 'The supplied output directory, ' + self.outputDir + ", does not exist and could not be created." if not self.format.lower() in self.driverExt.keys(): argProblem += "The supplied format must be one of " + ", ".join( self.driverExt.keys()) if argProblem: raise utilities.TrappedError( "There was a problem with one or more of the inputs to RasterFormatConverter" )
def validateArgs(self): rasterProcessor.validateArgs(self) validDTypes = [ "Unknown", "Byte", "Int16", "UInt32", "Int32", "CInt16", "CInt32" ] if not gdal.GetDataTypeName(self.inputRaster.pixelType) in validDTypes: utilities.TrappedError( "The input raster is not of a discrete pixel type")
def waitForCondorProcessesToFinish(self, outputs): errors = [] originalCount = len(outputs) while outputs: for process in outputs: result = self.jobFinished(process) if result == "finished": outputs.remove(process) # cleanup some condor files success = False while not success: try: for f in [ "log", "stdOut", "stdErr", "CondorSubmit" ]: fname = process.replace( ".tif", "_" + f + ".txt") if os.path.exists(fname): os.remove(fname) success = True except: pass print str(originalCount - len(outputs)) + " PARC layers finished" elif result == "error": if os.path.exists(process): os.remove(process) errors.append(process) outputs.remove(process) print str(originalCount - len(outputs)) + \ " PARC layers finished" if len(errors) > 0: msg = "There were problems with one or more runs." for process in errors: msg += "\n" + process + " did not run correctly" raise utilities.TrappedError(msg)
def validateArgs(self): """ Make sure the user sent us some stuff we can work with """ if not os.path.exists(self.out_dir): raise utilities.TrappedError("Specified Output directory " + self.out_dir + " not found on file system") if not os.path.isdir(self.out_dir): raise utilities.TrappedError("Specified Output directory " + self.out_dir + " is not a directory") if self.logger is None: self.logger = utilities.logger(self.out_dir, self.verbose) self.writetolog = self.logger.writetolog # Validate template image. if self.template is None: raise utilities.TrappedError("template raster not provided.") if not os.path.exists(self.template): raise utilities.TrappedError("Template file, " + self.template + ", does not exist on file system") self.templateRaster = SpatialUtilities.SAHMRaster(self.template) if len(self.templateRaster.Error) != 0: raise utilities.TrappedError( "There was a problem with the provided template: \n " + " " + "\n ".join(self.templateRaster.Error)) # Ensure the template has square pixels. if abs( abs(self.templateRaster.xScale) - abs(self.templateRaster.yScale)) > 1e-6: raise utilities.TrappedError( "template image must have square pixels." + "/n x pixel scale = " + str(abs(self.templateRaster.xScale)) + "/n y pixel scale = " + str(abs(self.templateRaster.yScale))) # Validate input rasters if not os.path.exists(self.inputs_CSV): raise utilities.TrappedError("Inputs CSV, " + self.inputs_CSV + ", does not exist on file system.") inputs_csv = csv.reader(open(self.inputs_CSV, 'r')) header = inputs_csv.next() input_file_errors = "" output_csv = os.path.join(self.out_dir, "PARC_Files.csv") output = csv.writer(open(output_csv, "wb")) output.writerow([ "PARCOutputFile", "Categorical", "Resampling", "Aggregation", "OriginalFile", os.path.abspath(self.template), os.path.abspath(self.out_dir) ]) inputs = [] had_to_shrink = False for row in inputs_csv: input_file = row[0] input_just_file = os.path.splitext(os.path.split(input_file)[1])[0] if input_just_file == "hdr": input_file = os.path.split(input_file)[0] row[0] = input_file input_just_file = os.path.split(input_file)[1] if input_just_file in inputs: input_file_errors += "\n PARC not currently set up to handle identically named inputs." input_file_errors += "\n\t" + input_just_file + " used multiple times" else: inputs.append(input_just_file) sourceRaster = SpatialUtilities.SAHMRaster(input_file) if len(sourceRaster.Error) > 0: input_file_errors += (" " + os.path.split(input_file)[1] + " had the following errors:\n" + " " + "\n ".join(sourceRaster.Error)) + "\n" else: if not self.ignoreNonOverlap and not self.image_covers_template( sourceRaster): input_file_errors += "\n Some part of the template image falls outside of " + input_just_file input_file_errors += "\n template upper left = (" + str( self.templateRaster.west) + ", " + str( self.templateRaster.north) + ")" input_file_errors += "\n template lower right = (" + str( self.templateRaster.east) + ", " + str( self.templateRaster.south) + ")" upperLeftX, upperLeftY = SpatialUtilities.transformPoint( sourceRaster.west, sourceRaster.north, sourceRaster.srs, self.templateRaster.srs) input_file_errors += "\n image upper left = (" + \ str(sourceRaster.west) + ", " + str(sourceRaster.north) + ")" input_file_errors += "\n image lower right = (" + str( sourceRaster.east) + ", " + str( sourceRaster.south) + ")" if self.ignoreNonOverlap: # if this input is smaller in any of the dimensions if not self.image_covers_template(sourceRaster): self.shrink_template_extent(sourceRaster) had_to_shrink = True if len(row) < 2 or not row[1] in ['0', '1']: self.writetolog(" " + os.path.split(input_file)[1] + " categorical either missing or not 0 or 1:" + "\n Defaulting to 0 (continuous)") if len(row) < 2: row.append('0') else: row[1] = '0' resample_methods = [item.lower() for item in self.resample_methods] if len(row) < 3 or not row[2].lower() in resample_methods: self.writetolog( " " + os.path.split(input_file)[1] + " resample method either missing or not one of " + ", ".join(self.resample_methods) + "\n Defaulting to 'Bilinear'") if row[1] == '0': default = 'Bilinear' else: default = 'NearestNeighbor' if len(row) < 3: row.append(default) else: row[2] = default lower_agg_methods = [item.lower() for item in self.agg_methods] if len(row) < 4 or not row[3].lower() in lower_agg_methods: self.writetolog( " " + os.path.split(input_file)[1] + " aggregation method either missing or not one of " + ", ".join(self.agg_methods) + "\n Defaulting to 'Mean'") if row[1] == '0': default = 'Mean' else: default = 'Majority' if len(row) < 4: row.append(default) else: row[3] = default self.inputs.append(row) # also write the output row, reconfigured to our output file short_name = SpatialUtilities.getRasterShortName(row[0]) file_name = os.path.abspath( os.path.join(self.out_dir, short_name + ".tif")) outputrow = [file_name] + row[1:4] + [ os.path.abspath(row[0]), os.path.abspath(self.out_dir) ] output.writerow(outputrow) del output if had_to_shrink: old_template = SpatialUtilities.SAHMRaster(self.template) new_template_fname = os.path.join(self.out_dir, os.path.split(self.template)[1]) SpatialUtilities.intermediaryReprojection( old_template, self.templateRaster, new_template_fname, gdalconst.GRA_NearestNeighbour, True) self.template = new_template_fname self.templateRaster = SpatialUtilities.SAHMRaster(self.template) output_csv = open(os.path.join(self.out_dir, "PARC_Files.csv")) first_line, rest_of_lines = output_csv.readline(), output_csv.read( ) output_csv = open(os.path.join(self.out_dir, "PARC_Files.csv"), "w") replacement_line = first_line.split(",") replacement_line[5] = new_template_fname replacement_line = ",".join(replacement_line) output_csv.write(replacement_line) output_csv.write(rest_of_lines) output_csv.close() if input_file_errors != "": self.writetolog(input_file_errors, False, False) raise utilities.TrappedError( "There was one or more problems with " + "your input rasters: \n" + input_file_errors)
def run(self): rasterProcessor.run(self) cellRatio = SpatialUtilities.getTemplateSRSCellSize( self.inputRaster, self.templateRaster) / self.templateRaster.xScale targetCellSize, numSourcePerTarget = SpatialUtilities.getAggregateTargetCellSize( self.inputRaster, self.templateRaster) if cellRatio > 0.5: utilities.TrappedError( "The input raster did not have cells at least half the dimensions of the template raster" ) shortName = SpatialUtilities.getRasterShortName(self.inputFname) tmpIntermediaryFname = os.path.join(self.outDir, shortName + "_tmpRepo.tif") gdal.Unlink(tmpIntermediaryFname) SpatialUtilities.intermediaryReprojection( self.inputRaster, self.templateRaster, tmpIntermediaryFname, gdalconst.GRA_NearestNeighbour) self.tmpRaster = SpatialUtilities.SAHMRaster(tmpIntermediaryFname) #dictionaries to hold our outputRasters outputs = {} bSize = 2048 #source pixels #convert this to the nearest whole number of target pixels bSize = int(round(bSize / numSourcePerTarget) * numSourcePerTarget) if bSize == 0: bSize = int(numSourcePerTarget) self.tmpRaster.blockSize = bSize pixelsPerBlock = numSourcePerTarget * numSourcePerTarget for block in self.tmpRaster.iterBlocks(): uniques = np.unique(block) X, Y = block.shape x = X // numSourcePerTarget y = Y // numSourcePerTarget ndMask = block.reshape( (x, numSourcePerTarget, y, numSourcePerTarget)) ndMask = ndMask.transpose([0, 2, 1, 3]) ndMask = ndMask.reshape((x * y, pixelsPerBlock)) ans = np.array(stats.mode(ndMask, 1)[0]).reshape(x, y) for uniqueVal in uniques: if not type(uniqueVal) is np.ma.core.MaskedConstant: ans = ((ndMask == uniqueVal).sum(1) / (pixelsPerBlock)).reshape(x, y) * 100 else: ans = ((ndMask.mask).sum(1) / (pixelsPerBlock)).reshape(x, y) * 100 if not outputs.has_key(uniqueVal): outputs[uniqueVal] = self.initializeOneOutput(uniqueVal) outputs[uniqueVal].putBlock( ans, int(self.tmpRaster.curCol / numSourcePerTarget), int(self.tmpRaster.curRow / numSourcePerTarget)) self.tmpRaster.close() gdal.Unlink(tmpIntermediaryFname) #write our outputs to a PredictorListFile output_fname = os.path.join(self.outDir, "ouputFileList.csv") csv_writer = csv.writer(open(output_fname, 'wb')) csv_writer.writerow(["file", "Resampling", "Aggregation"]) for k, v in outputs.iteritems(): csv_writer.writerow( [os.path.normpath(v.source), "0", "NearestNeighbor", "Mean"]) del csv_writer self.outputPredictorsList = output_fname
def validateArgs(self): """ Make sure the user sent us some stuff we can work with """ if not os.path.exists(self.out_dir): raise utilities.TrappedError("Specified Output directory " + self.out_dir + " not found on file system") if not os.path.isdir(self.out_dir): raise utilities.TrappedError("Specified Output directory " + self.out_dir + " is not a directory") if self.logger is None: self.logger = utilities.logger(self.out_dir, self.verbose) self.writetolog = self.logger.writetolog # Validate template image. if self.template is None: raise utilities.TrappedError("template raster not provided.") if not os.path.exists(self.template): raise utilities.TrappedError("Template file, " + self.template + ", does not exist on file system") self.template_params = self.getRasterParams(self.template) if len(self.template_params["Error"]) <> 0: raise utilities.TrappedError( "There was a problem with the provided template: \n " + " " + "\n ".join(self.template_params["Error"])) # Ensure the template has square pixels. if abs( abs(self.template_params["xScale"]) - abs(self.template_params["yScale"])) > 1e-6: raise utilities.TrappedError( "template image must have square pixels." + "/n x pixel scale = " + str(xScale) + "/n y pixel scale = " + str(yScale)) #Validate input rasters if not os.path.exists(self.inputs_CSV): raise utilities.TrappedError("Inputs CSV, " + self.inputs_CSV + ", does not exist on file system.") inputsCSV = csv.reader(open(self.inputs_CSV, 'r')) header = inputsCSV.next() strInputFileErrors = "" outputCSV = os.path.join(self.out_dir, "PARC_Files.csv") output = csv.writer(open(outputCSV, "wb")) output.writerow([ "PARCOutputFile", "Categorical", "Resampling", "Aggregation", "OriginalFile", os.path.abspath(self.template), os.path.abspath(self.out_dir) ]) for row in inputsCSV: inputFile = row[0] sourceParams = self.getRasterParams(inputFile) if len(sourceParams["Error"]) > 0: strInputFileErrors += ( " " + os.path.split(inputFile)[1] + " had the following errors:\n" + " " + "\n ".join(sourceParams["Error"])) + "\n" else: pass if not self.ImageCoversTemplate(sourceParams): strInputFileErrors += ( "\n Some part of the template image falls outside of " + os.path.split(inputFile)[1]) strInputFileErrors += "\n template upper left = (" + str( self.template_params["gWest"]) + ", " + str( self.template_params["gNorth"]) + ")" strInputFileErrors += "\n template lower right = (" + str( self.template_params["gEast"]) + ", " + str( self.template_params["gSouth"]) + ")" strInputFileErrors += "\n image upper left = (" + str( sourceParams["gWest"]) + ", " + str( sourceParams["gNorth"]) + ")" strInputFileErrors += "\n image lower right = (" + str( sourceParams["gEast"]) + ", " + str( sourceParams["gSouth"]) + ")" # strInputFileErrors += "\n points are given in projected coordinates." # strInputFileErrors += "\n template upper left = (" + str(self.template_params["tWest"]) + ", " + str(self.template_params["tNorth"]) + ")" # strInputFileErrors += "\n template lower right = (" + str(self.template_params["tEast"]) + ", " + str(self.template_params["tSouth"]) + ")" # strInputFileErrors += "\n image upper left = (" + str(sourceParams["tWest"]) + ", " + str(sourceParams["tNorth"]) + ")" # strInputFileErrors += "\n image lower right = (" + str(sourceParams["tEast"]) + ", " + str(sourceParams["tSouth"]) + ")" # strInputFileErrors += "\n Note: points are given in the template coordinates." + "\n" # if len(row) < 2 or not row[1] in ['0', '1']: self.writetolog( " " + os.path.split(inputFile)[1] + " categorical either missing or not 0 or 1:\n Defaulting to 0 (continuous)" ) if len(row) < 2: row.append('0') else: row[1] = '0' if len(row) < 3 or not row[2].lower() in [ item.lower() for item in self.resample_methods ]: self.writetolog( " " + os.path.split(inputFile)[1] + " resample method either missing or not one of " + ", ".join(self.resample_methods) + "\n Defaulting to 'Bilinear'") if row[1] == '0': default = 'Bilinear' else: default = 'NearestNeighbor' if len(row) < 3: row.append(default) else: row[2] = default if len(row) < 4 or not row[3].lower() in [ item.lower() for item in self.agg_methods ]: self.writetolog( " " + os.path.split(inputFile)[1] + " aggregation method either missing or not one of " + ", ".join(self.agg_methods) + "\n Defaulting to 'Mean'") if row[1] == '0': default = 'Mean' else: default = 'Majority' if len(row) < 4: row.append(default) else: row[3] = default self.inputs.append(row) #also write the output row, reconfigured to our output file fileName = self.getShortName(row[0]) fileName = os.path.abspath( os.path.join(self.out_dir, fileName + ".tif")) outputrow = [fileName] + row[1:4] + [ os.path.abspath(row[0]), os.path.abspath(self.out_dir) ] output.writerow(outputrow) del output if strInputFileErrors <> "": self.writetolog(strInputFileErrors) raise utilities.TrappedError( "There was one or more problems with your input rasters: \n" + strInputFileErrors)