def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-u",'--user', action="store",dest="user", help="Name of target edb user") parser.add_option("-e","--edb", action="store",dest="edb", help="Name of target edb") parser.add_option("-i","--infile", action="store",dest="infile", help="Input csv file") # parser.add_option("-y","--year", # action="store",dest="year", # help="Only store sources for given year") parser.add_option("-v",dest='loglevel', action="store_const",default=get_loglevel(), help="produce verbose output") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-o", "--outfile", action="store",dest="outfile",default=None, help="Name of outfiles (without extension)") parser.add_option("-d","--delimiter", action="store",dest="delimiter",default="\t", help="Delimiter used in csv-file") parser.add_option("-c","--filterCol", action="store",dest="filterCol", help="Header of column to use as filter") parser.add_option("-f","--filterVal", action="store",dest="filterVal", help="Value to use in filter") # parser.add_option("-g", "--geocodeRasterDir", # action="store",dest="geocodeRasterDir",default=None, # help="Directory with geocode rasters") (options, args) = parser.parse_args() #--------------------Init logger----------------------- # rootLogger = logger.RootLogger(level=options.loglevel) logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) global log # log = rootLogger.getLogger(sys.argv[0]) log = logging.getLogger(parser.prog) #-----------------Validating options------------------- if options.cf is not None: generateCf(path.abspath(options.cf),controlFileTemplate) log.info("Wrote default controlfile") return 1 if options.user is None: log.error("Need to specify -u <user>") return 1 if options.edb is None: log.error("Need to specify -e <edb>") return 1 # if options.year is None: # log.error("Need to specify -y <year>") # return 1 # if len(options.year)!=4: # log.error("Year should be given with four digits") # return 1 if len(args)!=1: log.error("Controlfile should be given as argument") return 1 dmn=Domain() edb=Edb(dmn,options.user,options.edb) if not edb.exists(): log.error("Edb %s does not exist" %options.edb) return 1 log.info("Parsing controlfile") cf=ControlFile(args[0]) cdbPars=re.compile("companydb\.par\.(\w*?):").findall(cf.content) fdbPars=re.compile("facilitydb\.par\.(\w*?):").findall(cf.content) sdbPars=re.compile("sourcedb\.par\.(\w*?):").findall(cf.content) substEmisNr=re.compile("sourcedb\.subst_emis\.([0-9]*)\.emis").findall(cf.content) subgrpEmisNr=re.compile("sourcedb\.subgrp_emis\.([0-9]*)\.emis").findall(cf.content) cdbCols={} cdbDefaults={} for par in cdbPars: cdbCols[par]=cf.findString("companydb.par.%s:" %par) cdbDefaults[par]=cf.findString("companydb.par.%s.default:" %par, optional=True,default=None) fdbCols={} fdbDefaults={} for par in fdbPars: fdbCols[par]=cf.findString("facilitydb.par.%s:" %par) fdbDefaults[par]=cf.findString("facilitydb.par.%s.default:" %par, optional=True,default=None) sdbCols={} sdbDefaults={} for par in sdbPars: sdbCols[par]=cf.findString("sourcedb.par.%s:" %par) sdbDefaults[par]=cf.findString("sourcedb.par.%s.default:" %par, optional=True,default=None) substEmisCols={} substEmisDefaults={} if substEmisNr is not None: for emisNr in substEmisNr: cols={} defaults={} emisPars=re.compile("sourcedb\.subst_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content) emisDefaultPars=re.compile( "sourcedb\.subst_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content) if emisPars is not None: for par in emisPars: cols[par]=cf.findString("sourcedb.subst_emis.%s.%s:" %(emisNr,par)) if emisDefaultPars is not None: for par in emisDefaultPars: defaults[par]=cf.findString("sourcedb.subst_emis.%s.%s.default:" %(emisNr,par), optional=True,default=None) substEmisCols[emisNr]=cols substEmisDefaults[emisNr]=defaults subgrpEmisCols={} subgrpEmisDefaults={} if subgrpEmisNr is not None: for emisNr in subgrpEmisNr: cols={} defaults={} emisPars=re.compile("sourcedb\.subgrp_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content) emisDefaultPars=re.compile( "sourcedb\.subgrp_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content) if emisPars is not None: for par in emisPars: cols[par]=cf.findString("sourcedb.subgrp_emis.%s.%s:" %(emisNr,par)) if emisDefaultPars is not None: for par in emisDefaultPars: defaults[par]=cf.findString("sourcedb.subgrp_emis.%s.%s.default:" %(emisNr,par), optional=True,default=None) subgrpEmisCols[emisNr]=cols subgrpEmisDefaults[emisNr]=defaults log.info("Reading subdb...") subdb=Subdb(edb) subdb.read() log.info("Reading companydb...") companydb=Companydb(edb) companydb.read() log.info("Reading sourcedb...") # source_stream = SourceStream(edb, 'w') source_stream = open(options.outfile, 'w') source_writer = ModelWriter(source_stream,encoding="HP Roman8") log.info("Reading facilitydb...") facilitydb=Facilitydb(edb) facilitydb.read() log.info("Reading subgrpdb") subgrpdb=Subgrpdb(edb) subgrpdb.read() log.info("Reading edb.rsrc") rsrc=Rsrc(edb.rsrcPath()) acCodeTables=[] for i in range(rsrc.numberOfCodeTrees("ac")): acCodeTables.append(CodeTable(rsrc.path,codeType="ac",codeIndex=i+1)) gcCodeTables=[] for i in range(rsrc.numberOfCodeTrees("gc")): gcCodeTables.append(CodeTable(rsrc.path,codeType="gc",codeIndex=i+1)) geocodeRasters=[] rast1=Raster() rast1.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__1.txt") rast2=Raster() rast2.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__2.txt") geocodeRasters.append(rast1) geocodeRasters.append(rast2) log.info("Reading csv-file") table=DataTable() table.read(options.infile,delimiter=options.delimiter,encoding="ISO-8859-15") if options.filterCol is not None: if options.filterCol not in table.colIndex: log.error("Filter column header not found in table") sys.exit(1) invalid=False nFiltered=0 nRows=0 log.info("Processing rows") for rowInd,row in enumerate(table.data): nRows+=1 if options.filterCol is not None: filterVal=row[table.colIndex[options.filterCol]] if options.filterVal!=str(filterVal): nFiltered+=1 continue comp = Company() for par in comp.parOrder: val=cdbDefaults.get(par,None) if par in cdbCols: colId=cdbCols[par] try: tableVal=row[table.colIndex[colId]] except KeyError: log.error( "No column with header %s, columns: %s" %( colId,str(table.listIds()))) if tableVal is not None: val = tableVal if val is not None: #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] comp[par]=val fac = Facility() for par in fac.parOrder: val=fdbDefaults.get(par,None) if par in fdbCols: colId=fdbCols[par] tableVal=row[table.colIndex[colId]] if tableVal is not None: val = tableVal if val is not None: #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] fac[par]=val src = Source() for par in ["X1", "Y1","X2","Y2", "PX","PY","NAME","INFO","INFO2","DATE","CHANGED", "CHIMNEY HEIGHT","GASTEMPERATURE","GAS FLOW", "SEARCHKEY1","SEARCHKEY2","SEARCHKEY3", "SEARCHKEY4","SEARCHKEY5","CHIMNEY OUT","CHIMNEY IN", "HOUSE WIDTH","HOUSE HEIGHT","NOSEGMENTS","BUILD_WIDTHS", "BUILD_HEIGHTS","BUILD_LENGTHS","BUILD_DISTFARWALL", "BUILD_CENTER","GEOCODE","FORMULAMACRO","ALOB"]: val=sdbDefaults.get(par,None) if par in sdbCols: colId=sdbCols[par] tableVal=row[table.colIndex[colId]] if tableVal is not None: val = tableVal if val is not None: #validate code if par=="GEOCODE" and val is not None: gcList=val.split() for codeIndex,code in enumerate(gcList): if not gcCodeTables[codeIndex].hasCode(code): log.error("Invalid geo code %s on row %i" %(code,rowInd)) invalid=True #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] #Store in src object and convert to correct type src._fieldvalues[par] = lazy_parse( src, par, val) gc1=geocodeRasters[0].getVal(src.get_coord()[0],src.get_coord()[1]) gc2=geocodeRasters[1].getVal(src.get_coord()[0],src.get_coord()[1]) src.GEOCODE = [str(int(gc1)) + "." + str(int(gc2))] for emisNr,emis in substEmisCols.items(): substEmis={"unit":None,"ac":None,"substance":None,"emis":None} for par in substEmis.keys(): if par in emis: substEmis[par]=row[table.colIndex[emis[par]]] else: try: substEmis[par]=substEmisDefaults[emisNr][par] except KeyError: log.error( "Need to specify column or default value for subgrp emis %i" %emisNr) substInd=subdb.substIndex(substEmis["substance"]) if substInd is None: log.error("Invalid substance name %s on row %i" %( substEmis["substance"],rowInd)) sys.exit(1) try: unit=rsrc.sub[substEmis["unit"]] except KeyError: log.error("Invalid unit name %s on row %i" %(emis["unit"],rowInd)) sys.exit(1) acList=substEmis["ac"].split('\\')[0].split() for codeIndex,code in enumerate(acList): # if code == "2.A.4.2": # import pdb; pdb.set_trace() refCode = acCodeTables[codeIndex].checkCode(code) if refCode == "-": log.error("Invalid activity code %s on row %i" %(code,rowInd)) sys.exit(1) if refCode != code: acList[codeIndex] = refCode substEmis["ac"] = acList if substEmis["emis"] is not None and substEmis["emis"]!="0": try: emis = src.add_emission() emis.UNIT = substEmis["unit"] emis.ACTCODE = substEmis["ac"] # needs re-formatting emis.EMISSION = float(substEmis["emis"]) emis.SUBSTANCE = substInd emis.auto_adjust_unit(edb) except: # print substEmis # log.error("Invalid substance emission on row %i" %rowInd) invalid=True src.EMISSION=src.EMISSION[:-1] for emis in subgrpEmisCols.values(): subgrpEmis={"unit":None,"ac":None,"name":None,"emis":None} for par in subgrpEmis.keys(): if par in emis: subgrpEmis[par]=row[table.colIndex[emis[par]]] else: try: subgrpEmis[par]=subgrpEmisDefaults[emisNr][par] except KeyError: log.error( "Need to specify column or default value for subgrp emis %i" %emisNr) #validating subgrp name try: subgrp=subgrpdb.getByName(subgrpEmis["name"]) except KeyError: log.error("Invalid subgrp name %s on row %i" %(subgrpEmis["name"],rowInd)) invalid=True #validating subgrp emis unit try: unitFactor=rsrc.subGrpEm[subgrpEmis["unit"]] except KeyError: log.error("Invalid unit %s for subgrp emission on row %i" %( subgrpEmis["unit"],rowInd)) invalid=True #validating subgrp activity code acList=subgrpEmis["ac"].split() for codeIndex,code in enumerate(acList): refCode = acCodeTables[codeIndex].checkCode(code) if refCode == "-": log.error("Invalid activity code %s on row %i" %(code,rowInd)) invalid=True break if refCode != code: acList[codeIndex] = refCode substEmis["ac"] = acList try: src.addSubgrpEmis(subgrp.index,emis=subgrpEmis["emis"],unit=subgrpEmis["unit"], ac=subgrpEmis["ac"]) except: log.error("Invalid subgrp emission on row %i" %rowInd) invalid=True companydb.append(comp,force=True) facilitydb.append(fac,force=True) source_writer.write(src) # sourcedb.append(src) if invalid: log.info("No output written due to validation errors") sys.exit(0) if len(companydb.items)>0: if options.outfile is None: log.info("Writing companydb") else: log.info("Writing company db to file") companydb.write(filename=options.outfile+".companydb") if len(facilitydb.items)>0: if options.outfile is None: log.info("Writing facilitydb") else: log.info("Writing facilitydb to file") facilitydb.write(filename=options.outfile+".facilitydb") # if len(sourcedb.sources)>0: # if options.outfile is None: # log.info("Writing sourcedb") # else: # log.info("Writing sourcedb to file") # sourcedb.write(filename=options.outfile+".sourcedb") if options.filterCol is not None: log.info("Filtered out %i out of %i" %(nFiltered,nRows))
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning("Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning( "Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("--no-progress", action="store_const", dest="progressStream", const=None, default=sys.stdout, help="turn off the progress bar") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--input", action="store", dest="infileName", help="Input raster") parser.add_option("--bbox", action="store", dest="bbox", help="Only read data within bbox," + " --box <\"x1,y1,x2,y2\"") parser.add_option("--reclassify", action="store", dest="classTable", help="Tab-separated table with code " + "and z0 value for each landuse class") parser.add_option("--reclassFromColumn", action="store", dest="reclassFromColumn", help="Header of reclass table column " + "containing values to " + "reclass (default is to use first column of classTable)") parser.add_option("--reclassToColumn", action="store", dest="reclassToColumn", help="Header of reclass table column containing codes," + " default is to use first column (default is to use " + "second column of classTable)") parser.add_option("--resample", action="store", dest="cellFactor", help="Resample grid by dividing cellsize with a" + " factor. Factor <1 results in refinement." + " Reprojection uses temporary refinement") parser.add_option("--resamplingMethod", action="store", dest="resamplingMethod", help="For cellFactor > 1: " + "Choose between 'mean', 'sum', 'majority' or" + " 'count', default is sum," "For cellFactor < 1, choose between: " + "'keepTotal' and 'keepValue', default is 'keepTotal'") parser.add_option("--summarize", action="store_true", dest="summarize", help="Print a summary of the input grid properties") parser.add_option("--bandIndex", action="store", dest="bandIndex", help="Band index to read from", default=1) parser.add_option("--dataType", action="store", dest="dataType", help="Output raster/shape data type", default=None) parser.add_option("--toShape", action="store_true", dest="toShape", help="output as shape file", default=None) parser.add_option("--fieldName", metavar='FIELD', action="store", dest="fieldName", help="write data in shape file to FIELD," + " default is 'value'", default=None) parser.add_option("--filter", action="store", dest="filter", help="Filter out data equal or below" + " limit in shape output", default=None) parser.add_option("-t", "--template", action="store", dest="template", help="Header for output raster when reprojecting") parser.add_option("--fromProj", dest="fromProj", help="Input raster proj4 definition string") parser.add_option("--toProj", dest="toProj", help="Output raster proj4 definition string") (options, args) = parser.parse_args() #------------Setting up logging capabilities ----------- # Setup logging logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) log = logging.getLogger(__name__) #------------Process and validate options--------------- if options.doc: print doc sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") #validate infile path if options.infileName is not None: inFilePath = path.abspath(options.infileName) if not path.exists(inFilePath): log.error("Input raster %s does not exist" % options.infileName) sys.exit(1) else: parser.error("No input data specified") #validate outfile path if options.outfileName is not None: outFilePath = path.abspath(options.outfileName) if options.toShape and ".shp" not in outFilePath: parser.error("Output shape has to to be specified with" + " .shp extension") else: outFilePath = None #Validate fieldName option if options.toShape: if options.fieldName == "": parser.error("fieldName can't be an empty string") fieldName = options.fieldName or "value" elif options.fieldName is not None: parser.error("fieldName option only allowed together" + " with shape output") # Validate filter option and convert filter to numeric value if present if not options.toShape and options.filter is not None: parser.error("Filter option only allowed together with shape output") elif options.toShape: if options.filter is not None: filter = float(options.filter) else: filter = None # read and process reclass table file if options.classTable is not None: reclassFromColumn = options.reclassFromColumn reclassToColumn = options.reclassToColumn if reclassFromColumn is not None and reclassToColumn is not None: desc = [{"id": reclassFromColumn, "type": float}, {"id": reclassToColumn, "type": float}] classTable = datatable.DataTable(desc=desc) classTable.read(options.classTable) else: classTable = datatable.DataTable() classTable.read(options.classTable) reclassFromColumn = classTable.desc[0]["id"] reclassToColumn = classTable.desc[1]["id"] classTable.convertCol(reclassFromColumn, float) classTable.convertCol(reclassToColumn, float) classTable.setKeys([reclassFromColumn]) log.debug("Successfully read landuse class table") classDict = {} for row in classTable.data: classDict[row[classTable.colIndex[reclassFromColumn]] ] = row[classTable.colIndex[reclassToColumn]] if options.cellFactor is not None: cellFactor = float(options.cellFactor) if cellFactor > 1 and \ options.resamplingMethod not in ('sum', 'mean', 'majority', 'count', None): log.error( "Invalid resampling method, valid options for grid " + "coarsening are 'sum' " + "and 'mean' and 'majority', " + "specified %s" % options.resamplingMethod ) sys.exit(1) elif cellFactor < 1 and \ options.resamplingMethod not in ('keepTotal', 'keepValue', None): log.error( "Invalid resampling method, valid options for grid " + "coarsening are 'keepTotal' and 'keepValue'" + ", specified %s" % resamplingMethod) sys.exit(1) # setting default resampling methods if cellFactor > 1 and \ options.resamplingMethod is None: resamplingMethod = 'sum' elif options.resamplingMethod is None: resamplingMethod = 'keepTotal' else: resamplingMethod = options.resamplingMethod if options.resamplingMethod == 'majority' or \ options.resamplingMethod == 'count' and \ options.toProj is not None: log.error( "Resampling method " + "%s not possible to " % options.resamplingMethod + "combine with reprojection") sys.exit(1) # Assure that gdal is present if not __gdal_loaded__: raise OSError("Function readGDAL needs GDAL with python bindings") # register all of the raster drivers gdal.AllRegister() ds = gdal.Open(inFilePath, GA_ReadOnly) if ds is None: print 'Could not open ' + filename sys.exit(1) ncols = ds.RasterXSize nrows = ds.RasterYSize nbands = ds.RasterCount # Info used for georeferencing geoTransform = ds.GetGeoTransform() xul = geoTransform[0] # top left x cellsizeX = geoTransform[1] # w-e pixel resolution rot1 = geoTransform[2] # rotation, 0 if image is "north up" yul = geoTransform[3] # top left y rot2 = geoTransform[4] # rotation, 0 if image is "north up" cellsizeY = geoTransform[5] # n-s pixel resolution proj = osr.SpatialReference(ds.GetProjection()) # Calculate lower left corner xll = xul yll = yul + nrows * cellsizeY # cellsizeY should be a negative value # Rotated rasters not handled...yet if rot1 != 0 or rot2 != 0: print 'Rotated rasters are not supported by pyAirviro.geo.raster' sys.exit(1) if abs(cellsizeX) != abs(cellsizeY): print('Non-homogenous cellsizes are not' + ' supported by pyAirviro.geo.raster') sys.exit(1) bandIndex = int(options.bandIndex) band = ds.GetRasterBand(bandIndex) nodata = band.GetNoDataValue() # If no nodata value is present in raster, set to -9999 for completeness if nodata is None: nodata = -9999 # Read data from a window defined by option --bbox <"x1,y1,x2,y2"> if options.bbox is not None: xur = xll + ncols * cellsizeX try: x1, y1, x2, y2 = map(float, options.bbox.split(",")) except: log.error("Invalid value for option --bbox <\"x1,y1,x2,y2\">") sys.exit(1) # Check if totally outside raster extent if x2 < xll or y2 < yll or x1 > xur or y1 > yul: log.error("Trying to extract outside grid boundaries") sys.exit(1) # Limit bbox to raster extent if x1 < xll: x1 = xll if x2 > xur: x2 = xur if y1 < yll: y1 = yll if y2 > yul: y2 = yul # estimate min and max of rows and cols colmin = int((x1 - xll) / cellsizeX) colmaxdec = (x2 - xll) / float(cellsizeX) rowmin = int((yul - y2) / abs(cellsizeY)) rowmaxdec = (yul - y1) / float(abs(cellsizeY)) if (colmaxdec - int(colmaxdec)) > 0: colmax = int(colmaxdec) else: colmax = int(colmaxdec - 1) if (rowmaxdec - int(rowmaxdec)) > 0: rowmax = int(rowmaxdec) else: rowmax = int(rowmaxdec - 1) nrows = rowmax - rowmin + 1 ncols = colmax - colmin + 1 xll = xll + colmin * cellsizeX yll = yul + (rowmax + 1) * cellsizeY # cellsizeY is negative yul = yll - nrows * cellsizeY else: rowmin = 0 colmin = 0 # process option for resampling if options.cellFactor is not None: cellFactor = float(cellFactor) else: cellFactor = 1 if cellFactor >= 1: procYBlockSize = int(cellFactor) else: procYBlockSize = 1 if options.toProj is None: # Set output raster dimensions and cellsize newNcols = int(ncols / cellFactor) newNrows = int(nrows / cellFactor) newCellsizeX = cellsizeX * cellFactor newCellsizeY = cellsizeY * cellFactor # is a negative value as before newXll = xll newYll = yll newYul = yul newNodata = nodata else: # Create coordinate transform if options.fromProj is None: src_srs = proj else: src_srs = osr.SpatialReference() src_srs.ImportFromProj4(options.fromProj) tgt_srs = osr.SpatialReference() tgt_srs.ImportFromProj4(options.toProj) import pdb;pdb.set_trace() coordTrans = osr.CoordinateTransformation(src_srs, tgt_srs) if options.template is None: # estimate extent from input newNcols = ncols newNrows = nrows newCellsizeX = cellsizeX newCellsizeY = cellsizeY newNodata = nodata newXll, newYll, z = coordTrans.TransformPoint(xll, yll) newXll = int(newXll) newYll = int(newYll) newYul = newYll - newNrows * newCellsizeY else: header = open(options.template, "r").read() newNcols = int( re.compile("ncols\s*([0-9]*)").search(header).group(1)) newNrows = int( re.compile("nrows\s*([0-9]*)").search(header).group(1)) newCellsizeX = float( re.compile("cellsize\s*([0-9]*)").search(header).group(1)) newCellsizeY = -1 * newCellsizeX try: newNodata = float( re.compile( "NODATA_value\s*(.*?)\n" ).search(header).group(1) ) except AttributeError: newNodata = float( re.compile( "nodata_value\s*(.*?)\n" ).search(header).group(1) ) newXll = float( re.compile("xllcorner\s*(.*?)\n").search(header).group(1)) newYll = float( re.compile("yllcorner\s*(.*?)\n").search(header).group(1)) newYul = newYll - newNrows * newCellsizeY # Processing of data is made for blocks of the following size # Important - blocks are set to cover all columns for simpler processing # This might not always be optimal for read/write speed procXBlockSize = ncols # process option for dataType if options.toShape: dataTypes, defaultDataType = ogrDataTypes, 'Real' else: dataTypes, defaultDataType = gdalDataTypes, 'Float32' try: dataType = dataTypes[options.dataType or defaultDataType] except KeyError: log.error( "Unknown datatype choose between: %s" % ",".join(dataTypes.keys())) sys.exit(1) # Create and configure output raster data source if not options.toShape and outFilePath is not None: # Creates a raster dataset with 1 band mem_ds = gdal.GetDriverByName('MEM').Create(outFilePath, newNcols, newNrows, 1, dataType) if mem_ds is None: print "Error: could not create output raster" sys.exit(1) outGeotransform = [newXll, newCellsizeX, 0, newYul, 0, newCellsizeY] mem_ds.SetGeoTransform(outGeotransform) if options.toProj is not None: mem_ds.SetProjection(tgt_srs.ExportToWkt()) else: mem_ds.SetProjection(proj) outBand = mem_ds.GetRasterBand(1) outBand.SetNoDataValue(newNodata) # Set nodata-value if options.toProj is not None: outArray = np.zeros((newNrows, newNcols)) nvals = np.zeros((newNrows, newNcols)) outDef = {"ncols": newNcols, "nrows": newNrows, "xll": newXll, "yll": newYll, "yul": newYll - newNrows * newCellsizeY, "xur": newXll + newNcols * newCellsizeX, "cellsize": newCellsizeX } # Create and inititialize output vector data source if options.toShape: shapeDriver = ogr.GetDriverByName('ESRI Shapefile') if path.exists(outFilePath): shapeDriver.DeleteDataSource(outFilePath) shapeFile = shapeDriver.CreateDataSource(outFilePath) if shapeFile is None: log.error("Could not open output shapefile %s" % outFilePath) sys.exit(1) layer = shapeFile.CreateLayer(outFilePath, geom_type=ogr.wkbPolygon) fieldDefn = ogr.FieldDefn(fieldName, dataType) layer.CreateField(fieldDefn) # inititialize input grid summary inputGridSummary = {"sum": 0, "mean": 0, "nnodata": 0, "nnegative": 0, "xll": xll, "yll": yll, "ncols": ncols, "nrows": nrows, "cellsizeX": cellsizeX, "cellsizeY": cellsizeY, "nodatavalue": nodata} outputGridSummary = {"sum": 0, "mean": 0, "nnodata": 0, "nnegative": 0, "xll": newXll, "yll": newYll, "ncols": newNcols, "nrows": newNrows, "cellsizeX": newCellsizeX, "cellsizeY": newCellsizeY, "nodatavalue": newNodata} # Loop over block of raster (at least one row in each block) rowsOffset = 0 errDict = {} pg = ProgressBar(nrows, options.progressStream) for i in range(0, nrows, int(procYBlockSize)): pg.update(i) data = band.ReadAsArray(xoff=colmin, yoff=rowmin + i, win_xsize=procXBlockSize, win_ysize=procYBlockSize) if options.summarize: inputGridSummary = updateGridSummary(data, inputGridSummary, nodata) if options.classTable is not None: try: data = reclassBlock(data, classDict, errDict) except IOError as e: log.error(str(e)) sys.exit(1) if options.cellFactor is not None: try: data = resampleBlock(data[:, :], cellFactor, resamplingMethod, nodata) except ValueError as err: log.error(err.message) sys.exit(1) except IOError as err: log.error(err.message) sys.exit(1) if options.toProj is not None: blockDef = {"nrows": int(procYBlockSize / cellFactor), "ncols": int(procXBlockSize / cellFactor), "xll": xll + (colmin) * cellsizeX, "yul": yll - (nrows - rowmin - i) * cellsizeY, "cellsize": cellsizeX * cellFactor, "nodata": nodata} reprojectBlock(outArray, data, cellFactor, blockDef, outDef, coordTrans, nvals) if outFilePath is not None: if options.toShape: blockYll = yul + i * newCellsizeY # newCellsizeY is negative blockXll = xll block2vector(data, layer, blockXll, blockYll, newCellsizeX, newCellsizeY, nodata, fieldName, filter) elif options.toProj is None: outBand.WriteArray(data, 0, rowsOffset) # Write block to raster outBand.FlushCache() # Write data to disk if options.summarize: outputGridSummary = updateGridSummary(data, outputGridSummary, nodata) rowsOffset += int(procYBlockSize / cellFactor) # Update offset if options.toShape: shapeFile.Destroy() if not options.toShape and options.toProj is not None: if options.resamplingMethod is not None and resamplingMethod == "mean": outArray = np.where(nvals > 0, outArray / nvals, outArray) outBand.WriteArray(outArray, 0, 0) outBand.FlushCache() # Write data to disk if options.outfileName is not None and not options.toShape: outputDriver = ds.GetDriver() output_ds = outputDriver.CreateCopy(options.outfileName, mem_ds, 0) output_ds = None mem_ds = None pg.finished() if options.summarize: print "\nInput raster summary" printGridSummary(inputGridSummary, prefix="input") print "\nOutput raster summary" printGridSummary(outputGridSummary, prefix="output") if errDict != {}: print "Errors/warnings during processing:" for err, nerr in errDict.items(): print "%s err in %i cells" % (err, nerr)
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("--no-progress", action="store_const", dest="progressStream", const=None, default=sys.stdout, help="turn off the progress bar") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--input", action="store", dest="infileName", help="Input raster") parser.add_option("--bbox", action="store", dest="bbox", help="Only read data within bbox," + " --box <\"x1,y1,x2,y2\"") parser.add_option("--reclassify", action="store", dest="classTable", help="Tab-separated table with code " + "and z0 value for each landuse class") parser.add_option("--reclassFromColumn", action="store", dest="reclassFromColumn", help="Header of reclass table column " + "containing values to " + "reclass (default is to use first column of classTable)") parser.add_option("--reclassToColumn", action="store", dest="reclassToColumn", help="Header of reclass table column containing codes," + " default is to use first column (default is to use " + "second column of classTable)") parser.add_option("--resample", action="store", dest="cellFactor", help="Resample grid by dividing cellsize with a" + " factor. Factor <1 results in refinement." + " Reprojection uses temporary refinement") parser.add_option("--resamplingMethod", action="store", dest="resamplingMethod", help="For cellFactor > 1: " + "Choose between 'mean', 'sum', 'majority' or" + " 'count', default is sum," "For cellFactor < 1, choose between: " + "'keepTotal' and 'keepValue', default is 'keepTotal'") parser.add_option("--summarize", action="store_true", dest="summarize", help="Print a summary of the input grid properties") parser.add_option("--bandIndex", action="store", dest="bandIndex", help="Band index to read from", default=1) parser.add_option("--dataType", action="store", dest="dataType", help="Output raster/shape data type", default=None) parser.add_option("--toShape", action="store_true", dest="toShape", help="output as shape file", default=None) parser.add_option("--fieldName", metavar='FIELD', action="store", dest="fieldName", help="write data in shape file to FIELD," + " default is 'value'", default=None) parser.add_option("--filter", action="store", dest="filter", help="Filter out data equal or below" + " limit in shape output", default=None) parser.add_option("-t", "--template", action="store", dest="template", help="Header for output raster when reprojecting") parser.add_option("--fromProj", dest="fromProj", help="Input raster proj4 definition string") parser.add_option("--toProj", dest="toProj", help="Output raster epsg or proj4 definition string") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- # Setup logging logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) log = logging.getLogger(__name__) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit(0) if len(args) > 0: parser.error("Incorrect number of arguments") # validate infile path if options.infileName is not None: inFilePath = path.abspath(options.infileName) if not path.exists(inFilePath): log.error("Input raster %s does not exist" % options.infileName) sys.exit(1) else: parser.error("No input data specified") # validate outfile path if options.outfileName is not None: outFilePath = path.abspath(options.outfileName) if options.toShape and ".shp" not in outFilePath: parser.error("Output shape has to to be specified with" + " .shp extension") else: outFilePath = None # Validate fieldName option if options.toShape: if options.fieldName == "": parser.error("fieldName can't be an empty string") fieldName = options.fieldName or "value" elif options.fieldName is not None: parser.error("fieldName option only allowed together" + " with shape output") # Validate filter option and convert filter to numeric value if present if not options.toShape and options.filter is not None: parser.error("Filter option only allowed together with shape output") elif options.toShape: if options.filter is not None: filter = float(options.filter) else: filter = None # read and process reclass table file if options.classTable is not None: reclassFromColumn = options.reclassFromColumn reclassToColumn = options.reclassToColumn if reclassFromColumn is not None and reclassToColumn is not None: desc = [{ "id": reclassFromColumn, "type": float }, { "id": reclassToColumn, "type": float }] classTable = datatable.DataTable(desc=desc) classTable.read(options.classTable) else: classTable = datatable.DataTable() classTable.read(options.classTable) reclassFromColumn = classTable.desc[0]["id"] reclassToColumn = classTable.desc[1]["id"] classTable.convertCol(reclassFromColumn, float) classTable.convertCol(reclassToColumn, float) classTable.setKeys([reclassFromColumn]) log.debug("Successfully read landuse class table") classDict = {} for row in classTable.data: classDict[row[classTable.colIndex[reclassFromColumn]]] = row[ classTable.colIndex[reclassToColumn]] if options.cellFactor is not None: cellFactor = float(options.cellFactor) if cellFactor > 1 and \ options.resamplingMethod not in ('sum', 'mean', 'majority', 'count', None): log.error("Invalid resampling method, valid options for grid " + "coarsening are 'sum' " + "and 'mean' and 'majority', " + "specified %s" % options.resamplingMethod) sys.exit(1) elif cellFactor < 1 and \ options.resamplingMethod not in ('keepTotal', 'keepValue', None): log.error("Invalid resampling method, valid options for grid " + "coarsening are 'keepTotal' and 'keepValue'" + ", specified %s" % resamplingMethod) sys.exit(1) # setting default resampling methods if cellFactor > 1 and \ options.resamplingMethod is None: resamplingMethod = 'sum' elif options.resamplingMethod is None: resamplingMethod = 'keepTotal' else: resamplingMethod = options.resamplingMethod if options.resamplingMethod == 'majority' or \ options.resamplingMethod == 'count' and \ options.toProj is not None: log.error("Resampling method " + "%s not possible to " % options.resamplingMethod + "combine with reprojection") sys.exit(1) # Assure that gdal is present if not __gdal_loaded__: raise OSError("Function readGDAL needs GDAL with python bindings") # register all of the raster drivers gdal.AllRegister() ds = gdal.Open(inFilePath, GA_ReadOnly) if ds is None: print 'Could not open ' + inFilePath sys.exit(1) ncols = ds.RasterXSize nrows = ds.RasterYSize nbands = ds.RasterCount # Info used for georeferencing geoTransform = ds.GetGeoTransform() xul = geoTransform[0] # top left x cellsizeX = geoTransform[1] # w-e pixel resolution rot1 = geoTransform[2] # rotation, 0 if image is "north up" yul = geoTransform[3] # top left y rot2 = geoTransform[4] # rotation, 0 if image is "north up" cellsizeY = geoTransform[5] # n-s pixel resolution proj = osr.SpatialReference(ds.GetProjection()) # Calculate lower left corner xll = xul yll = yul + nrows * cellsizeY # cellsizeY should be a negative value # Calculate upper right corner xur = xul + cellsizeX * ncols yur = yul # Rotated rasters not handled...yet if rot1 != 0 or rot2 != 0: print 'Rotated rasters are not supported by pyAirviro.geo.raster' sys.exit(1) if abs(cellsizeX) != abs(cellsizeY): print('Non-homogenous cellsizes are not' + ' supported by pyAirviro.geo.raster') sys.exit(1) bandIndex = int(options.bandIndex) band = ds.GetRasterBand(bandIndex) nodata = band.GetNoDataValue() # If no nodata value is present in raster, set to -9999 for completeness if nodata is None: nodata = -9999 # Read data from a window defined by option --bbox <"x1,y1,x2,y2"> if options.bbox is not None: try: x1, y1, x2, y2 = map(float, options.bbox.split(",")) except: log.error("Invalid value for option --bbox <\"x1,y1,x2,y2\">") sys.exit(1) # Check if totally outside raster extent if x2 < xll or y2 < yll or x1 > xur or y1 > yul: log.error("Trying to extract outside grid boundaries") sys.exit(1) # Limit bbox to raster extent if x1 < xll: x1 = xll if x2 > xur: x2 = xur if y1 < yll: y1 = yll if y2 > yul: y2 = yul # estimate min and max of rows and cols colmin = int((x1 - xll) / cellsizeX) colmaxdec = (x2 - xll) / float(cellsizeX) rowmin = int((yul - y2) / abs(cellsizeY)) rowmaxdec = (yul - y1) / float(abs(cellsizeY)) if (colmaxdec - int(colmaxdec)) > 0: colmax = int(colmaxdec) else: colmax = int(colmaxdec - 1) if (rowmaxdec - int(rowmaxdec)) > 0: rowmax = int(rowmaxdec) else: rowmax = int(rowmaxdec - 1) nrows = rowmax - rowmin + 1 ncols = colmax - colmin + 1 xll = xll + colmin * cellsizeX yll = yul + (rowmax + 1) * cellsizeY # cellsizeY is negative yul = yll - nrows * cellsizeY yur = yul xur = xll + ncols * cellsizeX else: rowmin = 0 colmin = 0 # process option for resampling if options.cellFactor is not None: cellFactor = float(cellFactor) else: cellFactor = 1 if cellFactor >= 1: procYBlockSize = int(cellFactor) else: procYBlockSize = 1 if options.toProj is None: # Set output raster dimensions and cellsize newNcols = int(ncols / cellFactor) newNrows = int(nrows / cellFactor) newCellsizeX = cellsizeX * cellFactor newCellsizeY = cellsizeY * cellFactor # is a negative value as before newXll = xll newYll = yll newYul = yul newNodata = nodata else: # Create coordinate transform if options.fromProj is None: src_srs = proj else: src_srs = osr.SpatialReference() src_srs.ImportFromProj4(options.fromProj) tgt_srs = osr.SpatialReference() if options.toProj.startswith("epsg"): tgt_srs.ImportFromEPSG(int(options.toProj[5:])) else: tgt_srs.ImportFromProj4(options.toProj) coordTrans = osr.CoordinateTransformation(src_srs, tgt_srs) newXur = None newYur = None if options.template is None: # estimate extent from input newNcols = int(ncols / cellFactor) newNrows = int(nrows / cellFactor) newXll, newYll, z = coordTrans.TransformPoint(xll, yll) newXur, newYur, z = coordTrans.TransformPoint(xur, yur) newCellsizeX = (newXur - newXll) / float(newNcols) newCellsizeY = (newYur - newYll) / float(newNrows) newNodata = nodata newXll = newXll newYll = newYll newYul = newYur else: header = open(options.template, "r").read() newNcols = int( re.compile("ncols\s*([0-9]*)").search(header).group(1)) newNrows = int( re.compile("nrows\s*([0-9]*)").search(header).group(1)) newCellsizeX = float( re.compile("cellsize\s*([0-9]*)").search(header).group(1)) newCellsizeY = -1 * newCellsizeX try: newNodata = float( re.compile("NODATA_value\s*(.*?)\n").search(header).group( 1)) except AttributeError: newNodata = float( re.compile("nodata_value\s*(.*?)\n").search(header).group( 1)) newXll = float( re.compile("xllcorner\s*(.*?)\n").search(header).group(1)) newYll = float( re.compile("yllcorner\s*(.*?)\n").search(header).group(1)) newYul = newYll - newNrows * newCellsizeY # Processing of data is made for blocks of the following size # Important - blocks are set to cover all columns for simpler processing # This might not always be optimal for read/write speed procXBlockSize = ncols # process option for dataType if options.toShape: dataTypes, defaultDataType = ogrDataTypes, 'Real' else: dataTypes, defaultDataType = gdalDataTypes, 'Float32' try: dataType = dataTypes[options.dataType or defaultDataType] except KeyError: log.error("Unknown datatype choose between: %s" % ",".join(dataTypes.keys())) sys.exit(1) # Create and configure output raster data source if not options.toShape and outFilePath is not None: # Creates a raster dataset with 1 band mem_ds = gdal.GetDriverByName('MEM').Create(outFilePath, newNcols, newNrows, 1, dataType) if mem_ds is None: print "Error: could not create output raster" sys.exit(1) outGeotransform = [newXll, newCellsizeX, 0, newYul, 0, newCellsizeY] mem_ds.SetGeoTransform(outGeotransform) if options.toProj is not None: mem_ds.SetProjection(tgt_srs.ExportToWkt()) elif isinstance(proj, osr.SpatialReference): mem_ds.SetProjection(proj.ExportToProj4()) else: mem_ds.SetProjection(proj) outBand = mem_ds.GetRasterBand(1) outBand.SetNoDataValue(newNodata) # Set nodata-value if options.toProj is not None: outArray = np.zeros((newNrows, newNcols)) nvals = np.zeros((newNrows, newNcols)) outDef = { "ncols": newNcols, "nrows": newNrows, "xll": newXll, "yll": newYll, "yul": newYll - newNrows * newCellsizeY, "xur": newXll + newNcols * newCellsizeX, "cellsize": newCellsizeX } # Create and inititialize output vector data source if options.toShape: shapeDriver = ogr.GetDriverByName('ESRI Shapefile') if path.exists(outFilePath): shapeDriver.DeleteDataSource(outFilePath) shapeFile = shapeDriver.CreateDataSource(outFilePath) if shapeFile is None: log.error("Could not open output shapefile %s" % outFilePath) sys.exit(1) layer = shapeFile.CreateLayer(outFilePath, geom_type=ogr.wkbPolygon) fieldDefn = ogr.FieldDefn(fieldName, dataType) layer.CreateField(fieldDefn) # inititialize input grid summary inputGridSummary = { "sum": 0, "mean": 0, "nnodata": 0, "nnegative": 0, "xll": xll, "yll": yll, "ncols": ncols, "nrows": nrows, "cellsizeX": cellsizeX, "cellsizeY": cellsizeY, "nodatavalue": nodata } outputGridSummary = { "sum": 0, "mean": 0, "nnodata": 0, "nnegative": 0, "xll": newXll, "yll": newYll, "ncols": newNcols, "nrows": newNrows, "cellsizeX": newCellsizeX, "cellsizeY": newCellsizeY, "nodatavalue": newNodata } # Loop over block of raster (at least one row in each block) rowsOffset = 0 errDict = {} pg = ProgressBar(nrows, options.progressStream) for i in range(0, nrows, int(procYBlockSize)): pg.update(i) data = band.ReadAsArray(xoff=colmin, yoff=rowmin + i, win_xsize=procXBlockSize, win_ysize=procYBlockSize) if options.summarize: inputGridSummary = updateGridSummary(data, inputGridSummary, nodata) if options.classTable is not None: try: data = reclassBlock(data, classDict, errDict) except IOError as e: log.error(str(e)) sys.exit(1) if options.cellFactor is not None: try: data = resampleBlock(data[:, :], cellFactor, resamplingMethod, nodata) except ValueError as err: log.error(err.message) sys.exit(1) except IOError as err: log.error(err.message) sys.exit(1) if options.toProj is not None: blockDef = { "nrows": int(procYBlockSize / cellFactor), "ncols": int(procXBlockSize / cellFactor), "xll": xll + (colmin) * cellsizeX, "yul": yll - (nrows - rowmin - i) * cellsizeY, "cellsize": cellsizeX * cellFactor, "nodata": nodata } reprojectBlock(outArray, data, cellFactor, blockDef, outDef, coordTrans, nvals) if outFilePath is not None: if options.toShape: blockYll = yul + i * newCellsizeY # newCellsizeY is negative blockXll = xll block2vector(data, layer, blockXll, blockYll, newCellsizeX, newCellsizeY, nodata, fieldName, filter) elif options.toProj is None: outBand.WriteArray(data, 0, rowsOffset) # Write block to raster outBand.FlushCache() # Write data to disk if options.summarize: outputGridSummary = updateGridSummary(data, outputGridSummary, nodata) rowsOffset += int(procYBlockSize / cellFactor) # Update offset if options.toShape: shapeFile.Destroy() if not options.toShape and options.toProj is not None: if options.resamplingMethod is not None and resamplingMethod == "mean": outArray = np.where(nvals > 0, outArray / nvals, outArray) outBand.WriteArray(outArray, 0, 0) outBand.FlushCache() # Write data to disk if options.outfileName is not None and not options.toShape: outputDriver = ds.GetDriver() output_ds = outputDriver.CreateCopy(options.outfileName, mem_ds, 0) output_ds = None mem_ds = None pg.finished() if options.summarize: print "\nInput raster summary" printGridSummary(inputGridSummary, prefix="input") print "\nOutput raster summary" printGridSummary(outputGridSummary, prefix="output") if errDict != {}: print "Errors/warnings during processing:" for err, nerr in errDict.items(): print "%s err in %i cells" % (err, nerr)
def main(): #-----------Setting up and using option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option( '-v', action='store_const', dest='loglevel', const=logging.DEBUG, default=get_loglevel(), help='produce verbose output' ) # parser.add_option("-l", "--loglevel", # action="store",dest="loglevel",default=2, # help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") (options, args) = parser.parse_args() #------------Setting up logging capabilities ----------- logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) global log log = logging.getLogger(parser.prog) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") #Opening controlfile cf=ControlFile(fileName=path.abspath(args[0])) domain = Domain() edbName=cf.findString("edb:") userName=cf.findString("user:"******"Edb "+edbName+" does not exist for user "+userName+" in domain "+domain.name) sys.exit() year=cf.findInt("year:") substList=cf.findStringList("substances:") #Remove doubles from substance list substances={'nat':[]} for subst in substList: if subst not in substances['nat']: substances['nat'].append(subst) #Make dictionary for substance,units couples unitDict={} defaultUnit=cf.findString("nationalTotals.defaultUnit:") unitList=cf.findStringList("nationalTotals.units:") for i in range(len(unitList)): unitSubstances=cf.findStringList("nationalTotals.unit."+str(i)+":") for s in unitSubstances: unitDict[s]=unitList[i] for subst in substances['nat']: if subst not in unitDict.keys(): unitDict[subst]=defaultUnit nationalTotalsTablePath=cf.findString("nationalTotalsTable:") nationalTotalsFile=codecs.open(nationalTotalsTablePath,'w',encoding="latin6") nationalTotalsTreePath=cf.findString("nationalTotalsTree:") nationalTotalsTreePath=path.abspath(nationalTotalsTreePath) nationalTotalTreeFile=open(nationalTotalsTreePath,'w') log.info("Reading emissions from CRF") CRFTree=CodeEmisTree(name="CRF") CRFPath=cf.findExistingPath("CRF.path:") CRFTree.parseCRFXML(CRFPath,year) CRFTree.calculateSums(byAttribute="Category") #CRFTree.write(sys.stdout) CRFTable=CRFTree.createTable(keys=["Category","Classification","AWMS"]) HFC={ "HFC-23":11700, "HFC-32":650, "HFC-41":150, "HFC-43-10 mee":1300, "HFC-125":2800, "HFC-134":1000, "HFC-134a":1300, "HFC-152a":140, "HFC-143":300, "HFC-143a":3800, "HFC-227ea":2900, "HFC-236fa":6300, "HFC-245ca":560 } PFC={ "CF4":6500, "C2F6":9200, "C3F8":7000, "C4F10":7000, "c-C4F8":8700, "C5F12":7500, "C6F14":7400 } CRFTable.aggregateCols({"id":"HFC-tot","type":"str","units":"Gg"},HFC) CRFTable.aggregateCols({"id":"PFC-tot","type":"str","units":"Gg"},PFC) CRFTable.sortRows() CRFTableKeys=cf.findStringList("CRF.tableKeys:") CRFTable.keys=CRFTableKeys #CRFTable.write(sys.stdout) log.info("Reading emissions from NFR") NFRTree=CodeEmisTree(name="NFR") NFRPath=cf.findExistingPath("NFR.path:") # pdb.set_trace() NFRTree.parseNFRTable(filename=NFRPath) NFRTree.calculateSums() NFRTable=NFRTree.createTable() NFRTableKeys=cf.findStringList("NFR.tableKeys:") NFRTable.keys=NFRTableKeys colIndex=NFRTable.getColIndex("Code") for row in NFRTable.data: if isinstance(row[colIndex],str): row[colIndex]=row[colIndex].replace(" ","") log.info("Reading emissions from Mobile") MobilePath=cf.findExistingPath("Mobile.path:") MobileTable=DataTable() MobileTable.read(MobilePath,units=True) MobileTable=MobileTable.filtered({"Year":str(year)}) MobileTableKeys=cf.findStringList("Mobile.tableKeys:") MobileTable.keys=MobileTableKeys #Remove biogenic emissions of CO2 #Remove GHG from Bunkers, except for international aviation LTO for row in MobileTable.data: # if "Bio" in row[MobileTable.getColIndex("Codename")]: if "Bio" in row[MobileTable.getColIndex("Fueltype")]: row[MobileTable.getColIndex("CO2")]=None # if row[MobileTable.getColIndex("Localcodename")]=="Bunkers" and row[MobileTable.getColIndex("Localsubcodename")]!="LTO": if row[MobileTable.getColIndex("Subsector")]=="Bunkers" and row[MobileTable.getColIndex("Region")]!="LTO": row[MobileTable.getColIndex("CO2")]=None row[MobileTable.getColIndex("CH4")]=None row[MobileTable.getColIndex("N2O")]=None log.info("Reading emissions from Manual") ManualPath=cf.findExistingPath("Manual.path:") ManualTable=DataTable() ManualTable.read(ManualPath,units=True) ManualTableKeys=cf.findStringList("Manual.tableKeys:") ManualTable.keys=ManualTableKeys srcTables={'NFR':NFRTable, 'CRF':CRFTable, 'Mobile':MobileTable, 'Manual':ManualTable} for tKey in srcTables.keys(): substList=cf.findStringList(tKey+".substances:") substances[tKey]=substList log.info("Reading dataSourceTable") dsPath=cf.findExistingPath("sourceTable:") dsTable=DataTable() dsTable.read(dsPath) acIndex=cf.findInt("nationalTotals.acIndex:") log.info("Reading activity codes from edb") natTree=CodeEmisTree("National totals","Gg") natTree.readActivityCodes(path.join(edb.edbPath(),"edb.rsrc"),acIndex) log.info("Identifying column indices for sources in dataSourceTable") #codeHeaders={"Mobile":"Mobile CRFnodeprefix", # "NFR":"NFR Code", # "CRF":"CRF Code", # "nat":"Code", # "Manual":"Code" # } codeHeaders={"Mobile":"Mobile UNFCCC", "NFR":"NFR Code", "CRF":"CRF Code", "nat":"Code", "Manual":"Code" } codeInd={} for key,colName in codeHeaders.iteritems(): try: codeInd[key]=dsTable.getColIndex(colName) except DataTableException: log.error("Could not find '"+colName+"' in dataSourceTable") sys.exit() log.info("Collecting data from data sources") prevNatCode="" for row in dsTable.data: rowId={} srcRow={} for key in srcTables: rowId[key]=[] srcRow[key]=srcTables[key].ncols*[0] natCode=row[codeInd['nat']] if natCode!=prevNatCode: log.debug("\nProcessing: "+natCode) prevNatCode=natCode #Get row-id for each src table for tKey in srcTables.keys(): colInd=codeInd[tKey] if row[colInd]!=None: for key in srcTables[tKey].keys: try: idPart=row[dsTable.getColIndex(tKey+" "+key)] except ValueError: log.error("No column named '"+tKey+" "+key+"' found in dataSourceTable") sys.exit() rowId[tKey].append(idPart) #If not all key values = None nonNones=[val for val in rowId[tKey] if val !=None] if len(nonNones)==0: rowId[tKey]=None srcRow[tKey]=None else: # if tKey=="CRF": # pdb.set_trace() try: srcRow[tKey]=srcTables[tKey].accumulate(rowId[tKey]) except DataTableException, err: # import pdb; pdb.set_trace() log.error("While trying to get data from "+tKey+" table: "+str(err)) sys.exit() #Creating code tree path for the current code codeParts=natCode.split(".") natPath="" for i in range(1,len(codeParts)+1): natPath+=".".join(codeParts[:i]) if i < len(codeParts): natPath+="/" natNode=natTree.root.find(natPath) if natNode is None: # import pdb; pdb.set_trace() log.error("Could not find national code path : "+natCode) sys.exit() for subst in substances['nat']: log.debug("Substance: "+subst) storedSubstances=[] foundSubstances=[] rec=None #Looping over all sourceTables for tKey in srcTables: st=srcTables[tKey] if rowId[tKey]!=None and subst in substances[tKey]: try: colIndex=st.getColIndex(subst) except DataTableException: log.error("Could not find column %s in table %s" %(subst,tKey)) emis=srcRow[tKey][colIndex] unit=st.desc[colIndex].get('units',None) storedSubstances.append(subst) nodeExists=False if emis!=0 and emis!=None: if subst not in foundSubstances: foundSubstances.append(subst) else: log.error("Substance found in multiple datasources for "+natCode) sys.exit() for node in natNode: if node.tag=="Record" and node.attrib.get("substance",None)==subst: rec=node nodeExists=True break if not nodeExists: rec=ET.SubElement(natNode,"Record") break if unit==None: log.error("No units set for substance "+subst+" in source table "+tKey) sys.exit() if storedSubstances==[]: log.error("No data source specified for substance "+subst) sys.exit() if rec!=None: if emis!=None and emis!=0: try: emis=convertUnits(fromUnit=unit,toUnit=unitDict[subst],val=emis) except TypeError: emis=0 if not nodeExists: rec.attrib["substance"]=subst else: emis+=float(rec.attrib["emission"]) rec.attrib["emission"]=str(emis)