def res2rast(filename):
"""Read gifmap gridded output from temporary file, return a raster"""
lines=open(filename,'r').readlines()
trash,xll,xur,yll,yur=lines.pop(0).split()
lines.pop(0)
trash,ncols,nrows=lines.pop(0).split()
ncols=int(ncols)
nrows=int(nrows)
xll=float(xll)
yll=float(yll)
xur=float(xur)
yur=float(yur)
cellsize_x=(xur-xll)/ncols
cellsize_y=(yur-yll)/nrows
if cellsize_x!=cellsize_y:
raise OSError("Non cuadratic cells in gifmap output not supported")
rast = Raster(Xll=xll,Yll=yll,Ncols=ncols,
Nrows=nrows,Cellsize=cellsize_x,
Nodata=-9999)
rast.data= numpy.array([map(float,l.split()) for l in lines])
return rast
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d",
"--doc",
action="store_true",
dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=get_loglevel(),
help="Produce verbose output")
parser.add_option("-o",
"--output",
action="store",
dest="outfileName",
default=None,
help="Output file")
parser.add_option("-i",
"--i",
action="store",
dest="infile",
help="Raster to be regionalized")
parser.add_option("-r",
"--regdef",
action="store",
dest="regdef",
help="Raster defining the regions found " +
"in statistics table")
parser.add_option("-s",
"--stat",
action="store",
dest="stat",
help="Tab-separated statistics table, " +
"with 1st column representing geocode and value " +
"representing region sum")
parser.add_option("-c",
"--colname",
action="store",
dest="colname",
help="Header of column in stat table to fetch " +
"region sums from")
(options, args) = parser.parse_args()
# ------------Setting up logging capabilities -----------
rootLogger = logger.RootLogger(int(options.loglevel))
global log
log = rootLogger.getLogger(sys.argv[0])
# ------------Process and validate options---------------
if options.doc:
print __doc__
sys.exit()
if len(args) > 0:
parser.error("Incorrect number of arguments")
regdef = Raster()
if options.regdef is not None:
regdef.read(options.regdef)
else:
log.error("No region definition raster specified")
sys.exit(1)
key = Raster()
if options.infile is not None:
key.read(options.infile)
else:
log.info("No initial distribution raster given, using " +
"homogenious distribution")
key.assign(regdef)
key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata)
if options.stat is None:
log.error("No statistics table specified")
sys.exit(1)
if (regdef.xll != key.xll or regdef.yll != key.yll):
log.error("The raster domain size is differs between key raster " +
"and region raster")
stat = DataTable()
stat.read(options.stat, defaultType=float)
print stat.desc
stat.convertCol(stat.desc[0]["id"], int)
print stat.desc
# Set first column as unique id for rows
stat.setKeys([stat.desc[0]["id"]])
# Remove nodata in rasters
key.nodataToZero()
regdef.nodataToZero()
log.info("Consistency och completeness check for codes " +
"in regdef and statistics")
# Create list of codes in raster
regdefCodes = regdef.unique()
regdefCodes = map(int, regdefCodes)
if 0.0 in regdefCodes:
regdefCodes.remove(0.0)
statRegCodes = [row[0] for row in stat.data]
if options.colname is not None:
try:
col = stat.colIndex[options.colname]
except KeyError:
log.error("No column named " +
"%s found in statistics table" % options.colname)
sys.exit(1)
else:
col = 1
colId = stat.desc[col]["id"]
errFound = False
for code in statRegCodes:
# Assuring that the regional codes are present in regdef
if code not in regdefCodes:
log.error("Input Error: code:" + str(int(code)) +
"in reg totals is not represented in regional raster")
errFound = True
# Assuring that the regional raster IDs are present
# in the regional statistics
for code in regdefCodes:
if code not in statRegCodes:
log.error("Input Error: ID:" + str(code) +
" in region raster is not represented in " +
"regional totals!")
errFound = True
if errFound:
sys.exit(1)
# For all regions, calculate the regional sum,
# set the key values to key/regSum
res = np.zeros(key.data.shape)
log.info("Regionalizing key raster")
for code in regdefCodes:
emis = stat.lookup(colId, code)
log.debug("Statistics for reg %i: %f" % (code, emis))
mask = np.array(regdef.data == code)
regKey = mask * key.data
regSum = np.sum(np.sum(regKey))
if regSum > 0:
res = res + emis * regKey / regSum
else:
log.warning("Distribution key is zero for geocode" +
" %i, using homogenuous distribution for this region" %
code)
res = res + emis * mask / np.sum(np.sum(mask))
key.data = res
# resultRast=(totFractionRast*key)*regionalTotals.regSum(substance)
key.write(options.outfileName)
log.info("Result with sum: %f written" % res.sum())
def read(filename,bandIndex=None,dataType=numpy.float):
"""
Read raster using GDAL drivers
@param filename: raster filename
@param bandIndex: index of band to read, None gives all bands
@param dataType: convert raster to type, default=numpy.float
"""
#ArgGIS grid: AIG
#GeoTIFF: GTiff
#GIF
#BMP
# register all of the drivers
gdal.AllRegister()
ds = gdal.Open(filename, GA_ReadOnly)
if ds is None:
print 'Could not open ' + filename
sys.exit(1)
ncols= ds.RasterXSize
nrows = ds.RasterYSize
nbands = ds.RasterCount
#Info used for georeferencing
geoTransform = ds.GetGeoTransform()
xul=geoTransform[0] #top left x
cellsizeX=geoTransform[1] #w-e pixel resolution
rot1=geoTransform[2] #rotation, 0 if image is "north up"
yul=geoTransform[3] #top left y
rot2=geoTransform[4] #rotation, 0 if image is "north up"
cellsizeY=geoTransform[5] #n-s pixel resolution
xll=xul
yll=yul-nrows*cellsizeY
if rot1!=0 or rot2!=0:
print 'Rotated rasters are not supported by pyAirviro.geo.raster'
sys.exit(1)
if abs(cellsizeX)!=abs(cellsizeY):
print 'Non-homogenous cellsizes are not supported by pyAirviro.geo.raster'
sys.exit(1)
#Creates a list with band indices to be read
rasters=[]
if bandIndex is None:
bandRange=range(1,nbands+1)
else:
if isinstance(bandIndex,list):
bandRange=bandIndex
else:
bandRange=[bandIndex]
for bi in bandRange:
band = ds.GetRasterBand(bi)
nodata=band.GetNoDataValue()
rast=Raster(Xll=xll,Yll=yll,Ncols=ncols,Nrows=nrows,Cellsize=abs(cellsizeX),Nodata=nodata)
xBlockSize,yBlockSize=band.GetBlockSize()
data =None
for i in range(0, nrows, yBlockSize):
if i + yBlockSize < nrows:
numRows = yBlockSize
else:
numRows=nrows-i #last block is not complete in y direction
colBlock=None
for j in range(0, ncols, xBlockSize):
if j + xBlockSize < ncols:
numCols = xBlockSize
else:
numCols=ncols-j #last block is not complete in x direction
dataBlock = band.ReadAsArray(j, i, numCols, numRows)
#Read whole array
dataBlock = dataBlock.astype(dataType)
# Process block here if large file ,before
# reading the next block
if colBlock is None:
colBlock=dataBlock
else:
colBlock=numpy.concatenate((colBlock,dataBlock),1)
if data is None:
data=colBlock
else:
data=numpy.concatenate((data,colBlock),0)
rast.data=data
rasters.append(rast)
return rasters
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d", "--doc",
action="store_true", dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const", const=logging.DEBUG,
dest="loglevel", default=get_loglevel(),
help="Produce verbose output")
parser.add_option("-o", "--output",
action="store", dest="outfileName", default=None,
help="Output file")
parser.add_option("-i", "--i",
action="store", dest="infile",
help="Raster to be regionalized")
parser.add_option("-r", "--regdef",
action="store", dest="regdef",
help="Raster defining the regions found " +
"in statistics table")
parser.add_option("-s", "--stat",
action="store", dest="stat",
help="Tab-separated statistics table, " +
"with 1st column representing geocode and value " +
"representing region sum")
parser.add_option("-c", "--colname",
action="store", dest="colname",
help="Header of column in stat table to fetch " +
"region sums from")
(options, args) = parser.parse_args()
# ------------Setting up logging capabilities -----------
rootLogger = logger.RootLogger(int(options.loglevel))
global log
log = rootLogger.getLogger(sys.argv[0])
# ------------Process and validate options---------------
if options.doc:
print __doc__
sys.exit()
if len(args) > 0:
parser.error("Incorrect number of arguments")
regdef = Raster()
if options.regdef is not None:
regdef.read(options.regdef)
else:
log.error("No region definition raster specified")
sys.exit(1)
key = Raster()
if options.infile is not None:
key.read(options.infile)
else:
log.info("No initial distribution raster given, using " +
"homogenious distribution")
key.assign(regdef)
key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata)
if options.stat is None:
log.error("No statistics table specified")
sys.exit(1)
if (regdef.xll != key.xll or regdef.yll != key.yll):
log.error("The raster domain size is differs between key raster " +
"and region raster")
stat = DataTable()
stat.read(options.stat, defaultType=float)
print stat.desc
stat.convertCol(stat.desc[0]["id"], int)
print stat.desc
# Set first column as unique id for rows
stat.setKeys([stat.desc[0]["id"]])
# Remove nodata in rasters
key.nodataToZero()
regdef.nodataToZero()
log.info("Consistency och completeness check for codes " +
"in regdef and statistics")
# Create list of codes in raster
regdefCodes = regdef.unique()
regdefCodes = map(int, regdefCodes)
if 0.0 in regdefCodes:
regdefCodes.remove(0.0)
statRegCodes = [row[0] for row in stat.data]
if options.colname is not None:
try:
col = stat.colIndex[options.colname]
except KeyError:
log.error("No column named " +
"%s found in statistics table" % options.colname)
sys.exit(1)
else:
col = 1
colId = stat.desc[col]["id"]
errFound = False
for code in statRegCodes:
# Assuring that the regional codes are present in regdef
if code not in regdefCodes:
log.error("Input Error: code:" + str(int(code)) +
"in reg totals is not represented in regional raster")
errFound = True
# Assuring that the regional raster IDs are present
# in the regional statistics
for code in regdefCodes:
if code not in statRegCodes:
log.error("Input Error: ID:" + str(code) +
" in region raster is not represented in " +
"regional totals!")
errFound = True
if errFound:
sys.exit(1)
# For all regions, calculate the regional sum,
# set the key values to key/regSum
res = np.zeros(key.data.shape)
log.info("Regionalizing key raster")
for code in regdefCodes:
emis = stat.lookup(colId, code)
log.debug("Statistics for reg %i: %f" % (code, emis))
mask = np.array(regdef.data == code)
regKey = mask * key.data
regSum = np.sum(np.sum(regKey))
if regSum > 0:
res = res + emis * regKey / regSum
else:
log.warning(
"Distribution key is zero for geocode" +
" %i, using homogenuous distribution for this region" % code)
res = res + emis * mask / np.sum(np.sum(mask))
key.data = res
# resultRast=(totFractionRast*key)*regionalTotals.regSum(substance)
key.write(options.outfileName)
log.info("Result with sum: %f written" % res.sum())
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-u", "--user",
action="store", dest="user", default=None,
help="Specify user manually")
parser.add_option("-e", "--edb",
action="store", dest="edb", default=None,
help="Name of target edb")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-f", "--force",
action="store_true",dest="force",default=False,
help="To start the process without confirming the domain")
(options, args) = parser.parse_args()
# Setup logging
logging.configure(terminal_level=logging.DEBUG)
log = logging.getLogger(__name__)
if options.cf!=None:
generateCf(path.abspath(options.cf))
log.info("Wrote default controlfile")
sys.exit()
if len(args)!=1:
parser.error("Incorrect number of arguments")
if options.edb ==None:
parser.error("Need to specify edb using flag -e")
if options.user ==None:
parser.error("Need to specify user using flag -u")
dmn = Domain()
if not options.force:
answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?")
if answer!="y":
sys.exit("Interrupted by user")
if not dmn.edbExistForUser(options.edb,options.user):
log.error("Edb "+options.edb+" does not exist for user "+
options.user+" in domain "+dmn.name)
sys.exit()
#---Creating edb and rsrc objects------------------
edb=Edb(dmn,options.user,options.edb)
rsrc=Rsrc(edb.rsrcPath())
#Opening controlfile
#---retrieving data from control file----
cf=ControlFile(fileName=path.abspath(args[0]))
substances=cf.findStringList("substances:")
outputDir=cf.findExistingPath("outputDir:")
acIndex=cf.findInt("acIndex:")
macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:"))
fromProj=cf.findString("fromProj:")
toProj=cf.findString("toProj:")
try:
fromProj=transcoord.proj4Dict[fromProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj)
try:
toProj=transcoord.proj4Dict[toProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj)
formats = cf.findStringList("formats:")
units = cf.findString("units:")
writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True)
edb_xll=cf.findInt("edb_xll:")
edb_yll=cf.findInt("edb_yll:")
edb_ncols=cf.findInt("edb_ncols:")
edb_nrows=cf.findInt("edb_nrows:")
edb_cellsize=cf.findFloat("edb_cellsize:")
if fromProj!=toProj:
out_xll=cf.findFloat("out_xll:")
out_yll=cf.findFloat("out_yll:")
out_ncols=cf.findInt("out_ncols:")
out_nrows=cf.findInt("out_nrows:")
out_cellsize=cf.findFloat("out_cellsize:")
#-----------------------------------------
#Finds index to search units
unitIndex=None
for key,unit in rsrc.search.iteritems():
if isinstance(key,int):
if rsrc.search[key]==units:
unitIndex=key
break
if unitIndex is None:
log.error("Search units: %s not defined in edb.rsrc" %units)
sys.exit()
macro = ControlFile(fileName=macroFileName,removeComments=False)
#preparing export macro
macro.setParam("general.database:",dmn.name)
xmin=edb_xll
xmax=edb_xll+edb_ncols*edb_cellsize
ymin=edb_yll
ymax=edb_yll+edb_nrows*edb_cellsize
macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("edb.user:"******"edb.edb:",edb.name)
macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("USER :"******"EDB :",edb.name)
macro.setParam("GRID :",
"%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize))
macro.setParam("edb.unit:",unitIndex)
macro.setParam("UNIT :",unitIndex)
# macro.setParam("NOACTCODE :",acIndex)
macro.setParam("NOACTCODE :",len(rsrc.ac))
#Get activity code tree
acTree=codeemistree.CodeEmisTree("Activity codes",units=units)
acTree.readActivityCodes(rsrc.path,acIndex)
substDict=dmn.listSubstanceIndices()
edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols,
Nrows=edb_nrows,Cellsize=edb_cellsize,
Nodata=-9999,init=0)
if fromProj!=toProj:
outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999)
else:
outRastTemplate=Raster()
outRastTemplate.assign(edbRast)
for node in acTree.root.getiterator():
if node.tag=="root" or node.tag=="Record":
continue
ac=node.tag
log.debug("Activity code: "+ac)
#Finds row index for activity codes in macro
#Add a row with the current ac
#If there are many ac already given, these are
#replaced by the current ac
macroLines=macro.content.split("\n")
actCodeInd=None
geoCodeInd=None
for lineInd,line in enumerate(macroLines):
if "NOACTCODE" in line:
actCodeInd=lineInd
if "NOGEOCODE" in line:
geoCodeInd=lineInd
if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
else:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
macro.content="\n".join(macroLines)
macro.write()
#boolean raster marking where there is data for any of the substances
if 'CLRTAP' in formats:
dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999,init=0)
rasterDict={}
substancesWithData=[]
for substance in substances:
log.debug("Substance %s" %substance)
substanceIndex=substDict[substance]
macro.setParam("ELEMENT :",substanceIndex)
macro.write()
command="xrepedb -i "+macro.name
(returnCode,errMsg,outMsg)=utilities.execute(command)
tmp=outMsg.split()[10:-2]
tmp.sort()
if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00':
print "ERROR: The field for "+substance+" is empty!"
continue
# pdb.set_trace()
emisRast=string2rast(outMsg,edbRast)
emisSum=emisRast.sum()
outRast=Raster()
outRast.assign(outRastTemplate)
rec=ET.SubElement(node,"Record")
rec.attrib["substance"]=substance
rec.attrib["emission"]=str(emisSum)
if emisSum>0 and writeGrids:
if substance not in substancesWithData:
substancesWithData.append(substance)
if fromProj!=toProj:
exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir())
else:
exportRast=emisRast
if 'CLRTAP' in formats:
dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data)
rasterDict[substance]=exportRast
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
if 'ESRI Ascii grid' in formats:
fileName = path.join(categoryDirPath, substance+ ".asc")
exportRast.write(fileName)
log.debug("Grid for " + substance + "written to outputDir for category: " + ac)
summaryTable=acTree.createTable(writeAll=True)
summaryTable.sortRows()
tableFile=open(path.join(outputDir,"summaryTable.txt"),'w')
summaryTable.write(tableFile)
if len(rasterDict)>0 and 'CLRTAP' in formats:
#creating substance header in the same order as the substances in the template
header = "i\tj\t"
#headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"]
for s in substancesWithData:
header += s + "\t"
#remove the tab after the last column and add a newline instead
header = header[: - 1]+ "\n"
#Creating file for EMEP-data
fileName = "CLRTAP_" + ac + ".txt"
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
fid = open(path.join(categoryDirPath, fileName), 'w')
fid.writelines(header)
sum=0
#Writing indexes and data for all non-zero elements
for row in range(dataMarker.nrows):
for col in range(dataMarker.ncols):
if dataMarker.data[row, col] > 0:
(i, j) = dataMarker.getCentreCoords(row, col)
fid.write(str(i) + "\t" + str(j) + "\t")
for substWithData in substancesWithData[:-1]:
fid.write(str(rasterDict[substWithData].data[row, col]) + "\t")
sum+=rasterDict[substWithData].data[row, col]
fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n")
sum+=rasterDict[substancesWithData[-1]].data[row, col]
fid.close()
log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName))
log.info("Finished")
def read(filename,bandIndex=None,dataType=numpy.float):
"""
Read raster using GDAL drivers
@param filename: raster filename
@param bandIndex: index of band to read, None gives all bands
@param dataType: convert raster to type, default=numpy.float
"""
#ArgGIS grid: AIG
#GeoTIFF: GTiff
#GIF
#BMP
# register all of the drivers
gdal.AllRegister()
ds = gdal.Open(filename, GA_ReadOnly)
if ds is None:
print 'Could not open ' + filename
sys.exit(1)
ncols= ds.RasterXSize
nrows = ds.RasterYSize
nbands = ds.RasterCount
#Info used for georeferencing
geoTransform = ds.GetGeoTransform()
xul=geoTransform[0] #top left x
cellsizeX=geoTransform[1] #w-e pixel resolution
rot1=geoTransform[2] #rotation, 0 if image is "north up"
yul=geoTransform[3] #top left y
rot2=geoTransform[4] #rotation, 0 if image is "north up"
cellsizeY=geoTransform[5] #n-s pixel resolution
xll=xul
yll=yul-nrows*cellsizeY
if rot1!=0 or rot2!=0:
print 'Rotated rasters are not supported by pyAirviro.geo.raster'
sys.exit(1)
if abs(cellsizeX)!=abs(cellsizeY):
print 'Non-homogenous cellsizes are not supported by pyAirviro.geo.raster'
sys.exit(1)
#Creates a list with band indices to be read
rasters=[]
if bandIndex is None:
bandRange=range(1,nbands+1)
else:
if isinstance(bandIndex,list):
bandRange=bandIndex
else:
bandRange=[bandIndex]
for bi in bandRange:
band = ds.GetRasterBand(bi)
nodata=band.GetNoDataValue()
rast=Raster(Xll=xll,Yll=yll,Ncols=ncols,Nrows=nrows,Cellsize=abs(cellsizeX),Nodata=nodata)
xBlockSize,yBlockSize=band.GetBlockSize()
data =None
for i in range(0, nrows, yBlockSize):
if i + yBlockSize < nrows:
numRows = yBlockSize
else:
numRows=nrows-i #last block is not complete in y direction
colBlock=None
for j in range(0, ncols, xBlockSize):
if j + xBlockSize < ncols:
numCols = xBlockSize
else:
numCols=ncols-j #last block is not complete in x direction
dataBlock = band.ReadAsArray(j, i, numCols, numRows)
#Read whole array
dataBlock = dataBlock.astype(dataType)
# Process block here if large file ,before
# reading the next block
if colBlock is None:
colBlock=dataBlock
else:
colBlock=numpy.concatenate((colBlock,dataBlock),1)
if data is None:
data=colBlock
else:
data=numpy.concatenate((data,colBlock),0)
rast.data=data
rasters.append(rast)
return rasters
