def res2rast(filename): """Read gifmap gridded output from temporary file, return a raster""" lines=open(filename,'r').readlines() trash,xll,xur,yll,yur=lines.pop(0).split() lines.pop(0) trash,ncols,nrows=lines.pop(0).split() ncols=int(ncols) nrows=int(nrows) xll=float(xll) yll=float(yll) xur=float(xur) yur=float(yur) cellsize_x=(xur-xll)/ncols cellsize_y=(yur-yll)/nrows if cellsize_x!=cellsize_y: raise OSError("Non cuadratic cells in gifmap output not supported") rast = Raster(Xll=xll,Yll=yll,Ncols=ncols, Nrows=nrows,Cellsize=cellsize_x, Nodata=-9999) rast.data= numpy.array([map(float,l.split()) for l in lines]) return rast
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning("Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def read(filename,bandIndex=None,dataType=numpy.float): """ Read raster using GDAL drivers @param filename: raster filename @param bandIndex: index of band to read, None gives all bands @param dataType: convert raster to type, default=numpy.float """ #ArgGIS grid: AIG #GeoTIFF: GTiff #GIF #BMP # register all of the drivers gdal.AllRegister() ds = gdal.Open(filename, GA_ReadOnly) if ds is None: print 'Could not open ' + filename sys.exit(1) ncols= ds.RasterXSize nrows = ds.RasterYSize nbands = ds.RasterCount #Info used for georeferencing geoTransform = ds.GetGeoTransform() xul=geoTransform[0] #top left x cellsizeX=geoTransform[1] #w-e pixel resolution rot1=geoTransform[2] #rotation, 0 if image is "north up" yul=geoTransform[3] #top left y rot2=geoTransform[4] #rotation, 0 if image is "north up" cellsizeY=geoTransform[5] #n-s pixel resolution xll=xul yll=yul-nrows*cellsizeY if rot1!=0 or rot2!=0: print 'Rotated rasters are not supported by pyAirviro.geo.raster' sys.exit(1) if abs(cellsizeX)!=abs(cellsizeY): print 'Non-homogenous cellsizes are not supported by pyAirviro.geo.raster' sys.exit(1) #Creates a list with band indices to be read rasters=[] if bandIndex is None: bandRange=range(1,nbands+1) else: if isinstance(bandIndex,list): bandRange=bandIndex else: bandRange=[bandIndex] for bi in bandRange: band = ds.GetRasterBand(bi) nodata=band.GetNoDataValue() rast=Raster(Xll=xll,Yll=yll,Ncols=ncols,Nrows=nrows,Cellsize=abs(cellsizeX),Nodata=nodata) xBlockSize,yBlockSize=band.GetBlockSize() data =None for i in range(0, nrows, yBlockSize): if i + yBlockSize < nrows: numRows = yBlockSize else: numRows=nrows-i #last block is not complete in y direction colBlock=None for j in range(0, ncols, xBlockSize): if j + xBlockSize < ncols: numCols = xBlockSize else: numCols=ncols-j #last block is not complete in x direction dataBlock = band.ReadAsArray(j, i, numCols, numRows) #Read whole array dataBlock = dataBlock.astype(dataType) # Process block here if large file ,before # reading the next block if colBlock is None: colBlock=dataBlock else: colBlock=numpy.concatenate((colBlock,dataBlock),1) if data is None: data=colBlock else: data=numpy.concatenate((data,colBlock),0) rast.data=data rasters.append(rast) return rasters
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning( "Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-u", "--user", action="store", dest="user", default=None, help="Specify user manually") parser.add_option("-e", "--edb", action="store", dest="edb", default=None, help="Name of target edb") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-f", "--force", action="store_true",dest="force",default=False, help="To start the process without confirming the domain") (options, args) = parser.parse_args() # Setup logging logging.configure(terminal_level=logging.DEBUG) log = logging.getLogger(__name__) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") if options.edb ==None: parser.error("Need to specify edb using flag -e") if options.user ==None: parser.error("Need to specify user using flag -u") dmn = Domain() if not options.force: answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?") if answer!="y": sys.exit("Interrupted by user") if not dmn.edbExistForUser(options.edb,options.user): log.error("Edb "+options.edb+" does not exist for user "+ options.user+" in domain "+dmn.name) sys.exit() #---Creating edb and rsrc objects------------------ edb=Edb(dmn,options.user,options.edb) rsrc=Rsrc(edb.rsrcPath()) #Opening controlfile #---retrieving data from control file---- cf=ControlFile(fileName=path.abspath(args[0])) substances=cf.findStringList("substances:") outputDir=cf.findExistingPath("outputDir:") acIndex=cf.findInt("acIndex:") macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:")) fromProj=cf.findString("fromProj:") toProj=cf.findString("toProj:") try: fromProj=transcoord.proj4Dict[fromProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj) try: toProj=transcoord.proj4Dict[toProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj) formats = cf.findStringList("formats:") units = cf.findString("units:") writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True) edb_xll=cf.findInt("edb_xll:") edb_yll=cf.findInt("edb_yll:") edb_ncols=cf.findInt("edb_ncols:") edb_nrows=cf.findInt("edb_nrows:") edb_cellsize=cf.findFloat("edb_cellsize:") if fromProj!=toProj: out_xll=cf.findFloat("out_xll:") out_yll=cf.findFloat("out_yll:") out_ncols=cf.findInt("out_ncols:") out_nrows=cf.findInt("out_nrows:") out_cellsize=cf.findFloat("out_cellsize:") #----------------------------------------- #Finds index to search units unitIndex=None for key,unit in rsrc.search.iteritems(): if isinstance(key,int): if rsrc.search[key]==units: unitIndex=key break if unitIndex is None: log.error("Search units: %s not defined in edb.rsrc" %units) sys.exit() macro = ControlFile(fileName=macroFileName,removeComments=False) #preparing export macro macro.setParam("general.database:",dmn.name) xmin=edb_xll xmax=edb_xll+edb_ncols*edb_cellsize ymin=edb_yll ymax=edb_yll+edb_nrows*edb_cellsize macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("edb.user:"******"edb.edb:",edb.name) macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("USER :"******"EDB :",edb.name) macro.setParam("GRID :", "%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize)) macro.setParam("edb.unit:",unitIndex) macro.setParam("UNIT :",unitIndex) # macro.setParam("NOACTCODE :",acIndex) macro.setParam("NOACTCODE :",len(rsrc.ac)) #Get activity code tree acTree=codeemistree.CodeEmisTree("Activity codes",units=units) acTree.readActivityCodes(rsrc.path,acIndex) substDict=dmn.listSubstanceIndices() edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols, Nrows=edb_nrows,Cellsize=edb_cellsize, Nodata=-9999,init=0) if fromProj!=toProj: outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999) else: outRastTemplate=Raster() outRastTemplate.assign(edbRast) for node in acTree.root.getiterator(): if node.tag=="root" or node.tag=="Record": continue ac=node.tag log.debug("Activity code: "+ac) #Finds row index for activity codes in macro #Add a row with the current ac #If there are many ac already given, these are #replaced by the current ac macroLines=macro.content.split("\n") actCodeInd=None geoCodeInd=None for lineInd,line in enumerate(macroLines): if "NOACTCODE" in line: actCodeInd=lineInd if "NOGEOCODE" in line: geoCodeInd=lineInd if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] else: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] macro.content="\n".join(macroLines) macro.write() #boolean raster marking where there is data for any of the substances if 'CLRTAP' in formats: dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999,init=0) rasterDict={} substancesWithData=[] for substance in substances: log.debug("Substance %s" %substance) substanceIndex=substDict[substance] macro.setParam("ELEMENT :",substanceIndex) macro.write() command="xrepedb -i "+macro.name (returnCode,errMsg,outMsg)=utilities.execute(command) tmp=outMsg.split()[10:-2] tmp.sort() if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00': print "ERROR: The field for "+substance+" is empty!" continue # pdb.set_trace() emisRast=string2rast(outMsg,edbRast) emisSum=emisRast.sum() outRast=Raster() outRast.assign(outRastTemplate) rec=ET.SubElement(node,"Record") rec.attrib["substance"]=substance rec.attrib["emission"]=str(emisSum) if emisSum>0 and writeGrids: if substance not in substancesWithData: substancesWithData.append(substance) if fromProj!=toProj: exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir()) else: exportRast=emisRast if 'CLRTAP' in formats: dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data) rasterDict[substance]=exportRast categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) if 'ESRI Ascii grid' in formats: fileName = path.join(categoryDirPath, substance+ ".asc") exportRast.write(fileName) log.debug("Grid for " + substance + "written to outputDir for category: " + ac) summaryTable=acTree.createTable(writeAll=True) summaryTable.sortRows() tableFile=open(path.join(outputDir,"summaryTable.txt"),'w') summaryTable.write(tableFile) if len(rasterDict)>0 and 'CLRTAP' in formats: #creating substance header in the same order as the substances in the template header = "i\tj\t" #headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"] for s in substancesWithData: header += s + "\t" #remove the tab after the last column and add a newline instead header = header[: - 1]+ "\n" #Creating file for EMEP-data fileName = "CLRTAP_" + ac + ".txt" categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) fid = open(path.join(categoryDirPath, fileName), 'w') fid.writelines(header) sum=0 #Writing indexes and data for all non-zero elements for row in range(dataMarker.nrows): for col in range(dataMarker.ncols): if dataMarker.data[row, col] > 0: (i, j) = dataMarker.getCentreCoords(row, col) fid.write(str(i) + "\t" + str(j) + "\t") for substWithData in substancesWithData[:-1]: fid.write(str(rasterDict[substWithData].data[row, col]) + "\t") sum+=rasterDict[substWithData].data[row, col] fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n") sum+=rasterDict[substancesWithData[-1]].data[row, col] fid.close() log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName)) log.info("Finished")