def string2rast(outString,rastTemplate): rast=Raster() rast.assign(rastTemplate) lines=outString.split("\n") headerMaxLines=100 lineInd=0 while lines[lineInd]!="1" and lineInd<headerMaxLines: lineInd+=1 if lineInd==headerMaxLines: raise IOError,"Wrong report format specified in macro,cannot find start marker '1' of grid" lineInd+=1 if lines[lineInd]=="right of grid": print("No emissions within grid extent") sys.exit(1) #lineInd+=1 row=rast.nrows-1 col=0 while 1: line=lines[lineInd].split() for val in line: if row>rast.nrows or col>rast.ncols: raise IOError, "Header of ascii grid does not fit the number of values" rast.data[row,col]=float(val) col+=1 if col==rast.ncols: col=0 row-=1 if row<0: break lineInd+=1 return rast
def __init__(self, active=None, substances=None, logfile=None, regdefgc=None, gcDefRaster=None): if active is None or not active: self.active = False else: self.active = True self.substances = substances if logfile is not None: if not path.exists(path.dirname(logfile)) and logfile is not None: print "Error, path for trace logfile does not exist" else: self.log = codecs.open(logfile, "w", "UTF-8") self.regdefgc = regdefgc self.regInfo = {} self.regTot = {} self.regPsInfo = {} self.regPsTot = {} self.natTot = {} self.natPs = {} self.gcRaster = Raster() if self.active: self.gcRaster.read(gcDefRaster)
def res2rast(filename): """Read gifmap gridded output from temporary file, return a raster""" lines=open(filename,'r').readlines() trash,xll,xur,yll,yur=lines.pop(0).split() lines.pop(0) trash,ncols,nrows=lines.pop(0).split() ncols=int(ncols) nrows=int(nrows) xll=float(xll) yll=float(yll) xur=float(xur) yur=float(yur) cellsize_x=(xur-xll)/ncols cellsize_y=(yur-yll)/nrows if cellsize_x!=cellsize_y: raise OSError("Non cuadratic cells in gifmap output not supported") rast = Raster(Xll=xll,Yll=yll,Ncols=ncols, Nrows=nrows,Cellsize=cellsize_x, Nodata=-9999) rast.data= numpy.array([map(float,l.split()) for l in lines]) return rast
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-u",'--user', action="store",dest="user", help="Name of target edb user") parser.add_option("-e","--edb", action="store",dest="edb", help="Name of target edb") parser.add_option("-i","--infile", action="store",dest="infile", help="Input csv file") # parser.add_option("-y","--year", # action="store",dest="year", # help="Only store sources for given year") parser.add_option("-v",dest='loglevel', action="store_const",default=get_loglevel(), help="produce verbose output") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-o", "--outfile", action="store",dest="outfile",default=None, help="Name of outfiles (without extension)") parser.add_option("-d","--delimiter", action="store",dest="delimiter",default="\t", help="Delimiter used in csv-file") parser.add_option("-c","--filterCol", action="store",dest="filterCol", help="Header of column to use as filter") parser.add_option("-f","--filterVal", action="store",dest="filterVal", help="Value to use in filter") # parser.add_option("-g", "--geocodeRasterDir", # action="store",dest="geocodeRasterDir",default=None, # help="Directory with geocode rasters") (options, args) = parser.parse_args() #--------------------Init logger----------------------- # rootLogger = logger.RootLogger(level=options.loglevel) logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) global log # log = rootLogger.getLogger(sys.argv[0]) log = logging.getLogger(parser.prog) #-----------------Validating options------------------- if options.cf is not None: generateCf(path.abspath(options.cf),controlFileTemplate) log.info("Wrote default controlfile") return 1 if options.user is None: log.error("Need to specify -u <user>") return 1 if options.edb is None: log.error("Need to specify -e <edb>") return 1 # if options.year is None: # log.error("Need to specify -y <year>") # return 1 # if len(options.year)!=4: # log.error("Year should be given with four digits") # return 1 if len(args)!=1: log.error("Controlfile should be given as argument") return 1 dmn=Domain() edb=Edb(dmn,options.user,options.edb) if not edb.exists(): log.error("Edb %s does not exist" %options.edb) return 1 log.info("Parsing controlfile") cf=ControlFile(args[0]) cdbPars=re.compile("companydb\.par\.(\w*?):").findall(cf.content) fdbPars=re.compile("facilitydb\.par\.(\w*?):").findall(cf.content) sdbPars=re.compile("sourcedb\.par\.(\w*?):").findall(cf.content) substEmisNr=re.compile("sourcedb\.subst_emis\.([0-9]*)\.emis").findall(cf.content) subgrpEmisNr=re.compile("sourcedb\.subgrp_emis\.([0-9]*)\.emis").findall(cf.content) cdbCols={} cdbDefaults={} for par in cdbPars: cdbCols[par]=cf.findString("companydb.par.%s:" %par) cdbDefaults[par]=cf.findString("companydb.par.%s.default:" %par, optional=True,default=None) fdbCols={} fdbDefaults={} for par in fdbPars: fdbCols[par]=cf.findString("facilitydb.par.%s:" %par) fdbDefaults[par]=cf.findString("facilitydb.par.%s.default:" %par, optional=True,default=None) sdbCols={} sdbDefaults={} for par in sdbPars: sdbCols[par]=cf.findString("sourcedb.par.%s:" %par) sdbDefaults[par]=cf.findString("sourcedb.par.%s.default:" %par, optional=True,default=None) substEmisCols={} substEmisDefaults={} if substEmisNr is not None: for emisNr in substEmisNr: cols={} defaults={} emisPars=re.compile("sourcedb\.subst_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content) emisDefaultPars=re.compile( "sourcedb\.subst_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content) if emisPars is not None: for par in emisPars: cols[par]=cf.findString("sourcedb.subst_emis.%s.%s:" %(emisNr,par)) if emisDefaultPars is not None: for par in emisDefaultPars: defaults[par]=cf.findString("sourcedb.subst_emis.%s.%s.default:" %(emisNr,par), optional=True,default=None) substEmisCols[emisNr]=cols substEmisDefaults[emisNr]=defaults subgrpEmisCols={} subgrpEmisDefaults={} if subgrpEmisNr is not None: for emisNr in subgrpEmisNr: cols={} defaults={} emisPars=re.compile("sourcedb\.subgrp_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content) emisDefaultPars=re.compile( "sourcedb\.subgrp_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content) if emisPars is not None: for par in emisPars: cols[par]=cf.findString("sourcedb.subgrp_emis.%s.%s:" %(emisNr,par)) if emisDefaultPars is not None: for par in emisDefaultPars: defaults[par]=cf.findString("sourcedb.subgrp_emis.%s.%s.default:" %(emisNr,par), optional=True,default=None) subgrpEmisCols[emisNr]=cols subgrpEmisDefaults[emisNr]=defaults log.info("Reading subdb...") subdb=Subdb(edb) subdb.read() log.info("Reading companydb...") companydb=Companydb(edb) companydb.read() log.info("Reading sourcedb...") # source_stream = SourceStream(edb, 'w') source_stream = open(options.outfile, 'w') source_writer = ModelWriter(source_stream,encoding="HP Roman8") log.info("Reading facilitydb...") facilitydb=Facilitydb(edb) facilitydb.read() log.info("Reading subgrpdb") subgrpdb=Subgrpdb(edb) subgrpdb.read() log.info("Reading edb.rsrc") rsrc=Rsrc(edb.rsrcPath()) acCodeTables=[] for i in range(rsrc.numberOfCodeTrees("ac")): acCodeTables.append(CodeTable(rsrc.path,codeType="ac",codeIndex=i+1)) gcCodeTables=[] for i in range(rsrc.numberOfCodeTrees("gc")): gcCodeTables.append(CodeTable(rsrc.path,codeType="gc",codeIndex=i+1)) geocodeRasters=[] rast1=Raster() rast1.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__1.txt") rast2=Raster() rast2.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__2.txt") geocodeRasters.append(rast1) geocodeRasters.append(rast2) log.info("Reading csv-file") table=DataTable() table.read(options.infile,delimiter=options.delimiter,encoding="ISO-8859-15") if options.filterCol is not None: if options.filterCol not in table.colIndex: log.error("Filter column header not found in table") sys.exit(1) invalid=False nFiltered=0 nRows=0 log.info("Processing rows") for rowInd,row in enumerate(table.data): nRows+=1 if options.filterCol is not None: filterVal=row[table.colIndex[options.filterCol]] if options.filterVal!=str(filterVal): nFiltered+=1 continue comp = Company() for par in comp.parOrder: val=cdbDefaults.get(par,None) if par in cdbCols: colId=cdbCols[par] try: tableVal=row[table.colIndex[colId]] except KeyError: log.error( "No column with header %s, columns: %s" %( colId,str(table.listIds()))) if tableVal is not None: val = tableVal if val is not None: #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] comp[par]=val fac = Facility() for par in fac.parOrder: val=fdbDefaults.get(par,None) if par in fdbCols: colId=fdbCols[par] tableVal=row[table.colIndex[colId]] if tableVal is not None: val = tableVal if val is not None: #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] fac[par]=val src = Source() for par in ["X1", "Y1","X2","Y2", "PX","PY","NAME","INFO","INFO2","DATE","CHANGED", "CHIMNEY HEIGHT","GASTEMPERATURE","GAS FLOW", "SEARCHKEY1","SEARCHKEY2","SEARCHKEY3", "SEARCHKEY4","SEARCHKEY5","CHIMNEY OUT","CHIMNEY IN", "HOUSE WIDTH","HOUSE HEIGHT","NOSEGMENTS","BUILD_WIDTHS", "BUILD_HEIGHTS","BUILD_LENGTHS","BUILD_DISTFARWALL", "BUILD_CENTER","GEOCODE","FORMULAMACRO","ALOB"]: val=sdbDefaults.get(par,None) if par in sdbCols: colId=sdbCols[par] tableVal=row[table.colIndex[colId]] if tableVal is not None: val = tableVal if val is not None: #validate code if par=="GEOCODE" and val is not None: gcList=val.split() for codeIndex,code in enumerate(gcList): if not gcCodeTables[codeIndex].hasCode(code): log.error("Invalid geo code %s on row %i" %(code,rowInd)) invalid=True #Too long names are truncated if par=="NAME" and len(val)>45: val=val[:45] #Store in src object and convert to correct type src._fieldvalues[par] = lazy_parse( src, par, val) gc1=geocodeRasters[0].getVal(src.get_coord()[0],src.get_coord()[1]) gc2=geocodeRasters[1].getVal(src.get_coord()[0],src.get_coord()[1]) src.GEOCODE = [str(int(gc1)) + "." + str(int(gc2))] for emisNr,emis in substEmisCols.items(): substEmis={"unit":None,"ac":None,"substance":None,"emis":None} for par in substEmis.keys(): if par in emis: substEmis[par]=row[table.colIndex[emis[par]]] else: try: substEmis[par]=substEmisDefaults[emisNr][par] except KeyError: log.error( "Need to specify column or default value for subgrp emis %i" %emisNr) substInd=subdb.substIndex(substEmis["substance"]) if substInd is None: log.error("Invalid substance name %s on row %i" %( substEmis["substance"],rowInd)) sys.exit(1) try: unit=rsrc.sub[substEmis["unit"]] except KeyError: log.error("Invalid unit name %s on row %i" %(emis["unit"],rowInd)) sys.exit(1) acList=substEmis["ac"].split('\\')[0].split() for codeIndex,code in enumerate(acList): # if code == "2.A.4.2": # import pdb; pdb.set_trace() refCode = acCodeTables[codeIndex].checkCode(code) if refCode == "-": log.error("Invalid activity code %s on row %i" %(code,rowInd)) sys.exit(1) if refCode != code: acList[codeIndex] = refCode substEmis["ac"] = acList if substEmis["emis"] is not None and substEmis["emis"]!="0": try: emis = src.add_emission() emis.UNIT = substEmis["unit"] emis.ACTCODE = substEmis["ac"] # needs re-formatting emis.EMISSION = float(substEmis["emis"]) emis.SUBSTANCE = substInd emis.auto_adjust_unit(edb) except: # print substEmis # log.error("Invalid substance emission on row %i" %rowInd) invalid=True src.EMISSION=src.EMISSION[:-1] for emis in subgrpEmisCols.values(): subgrpEmis={"unit":None,"ac":None,"name":None,"emis":None} for par in subgrpEmis.keys(): if par in emis: subgrpEmis[par]=row[table.colIndex[emis[par]]] else: try: subgrpEmis[par]=subgrpEmisDefaults[emisNr][par] except KeyError: log.error( "Need to specify column or default value for subgrp emis %i" %emisNr) #validating subgrp name try: subgrp=subgrpdb.getByName(subgrpEmis["name"]) except KeyError: log.error("Invalid subgrp name %s on row %i" %(subgrpEmis["name"],rowInd)) invalid=True #validating subgrp emis unit try: unitFactor=rsrc.subGrpEm[subgrpEmis["unit"]] except KeyError: log.error("Invalid unit %s for subgrp emission on row %i" %( subgrpEmis["unit"],rowInd)) invalid=True #validating subgrp activity code acList=subgrpEmis["ac"].split() for codeIndex,code in enumerate(acList): refCode = acCodeTables[codeIndex].checkCode(code) if refCode == "-": log.error("Invalid activity code %s on row %i" %(code,rowInd)) invalid=True break if refCode != code: acList[codeIndex] = refCode substEmis["ac"] = acList try: src.addSubgrpEmis(subgrp.index,emis=subgrpEmis["emis"],unit=subgrpEmis["unit"], ac=subgrpEmis["ac"]) except: log.error("Invalid subgrp emission on row %i" %rowInd) invalid=True companydb.append(comp,force=True) facilitydb.append(fac,force=True) source_writer.write(src) # sourcedb.append(src) if invalid: log.info("No output written due to validation errors") sys.exit(0) if len(companydb.items)>0: if options.outfile is None: log.info("Writing companydb") else: log.info("Writing company db to file") companydb.write(filename=options.outfile+".companydb") if len(facilitydb.items)>0: if options.outfile is None: log.info("Writing facilitydb") else: log.info("Writing facilitydb to file") facilitydb.write(filename=options.outfile+".facilitydb") # if len(sourcedb.sources)>0: # if options.outfile is None: # log.info("Writing sourcedb") # else: # log.info("Writing sourcedb to file") # sourcedb.write(filename=options.outfile+".sourcedb") if options.filterCol is not None: log.info("Filtered out %i out of %i" %(nFiltered,nRows))
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning("Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def read(filename,bandIndex=None,dataType=numpy.float): """ Read raster using GDAL drivers @param filename: raster filename @param bandIndex: index of band to read, None gives all bands @param dataType: convert raster to type, default=numpy.float """ #ArgGIS grid: AIG #GeoTIFF: GTiff #GIF #BMP # register all of the drivers gdal.AllRegister() ds = gdal.Open(filename, GA_ReadOnly) if ds is None: print 'Could not open ' + filename sys.exit(1) ncols= ds.RasterXSize nrows = ds.RasterYSize nbands = ds.RasterCount #Info used for georeferencing geoTransform = ds.GetGeoTransform() xul=geoTransform[0] #top left x cellsizeX=geoTransform[1] #w-e pixel resolution rot1=geoTransform[2] #rotation, 0 if image is "north up" yul=geoTransform[3] #top left y rot2=geoTransform[4] #rotation, 0 if image is "north up" cellsizeY=geoTransform[5] #n-s pixel resolution xll=xul yll=yul-nrows*cellsizeY if rot1!=0 or rot2!=0: print 'Rotated rasters are not supported by pyAirviro.geo.raster' sys.exit(1) if abs(cellsizeX)!=abs(cellsizeY): print 'Non-homogenous cellsizes are not supported by pyAirviro.geo.raster' sys.exit(1) #Creates a list with band indices to be read rasters=[] if bandIndex is None: bandRange=range(1,nbands+1) else: if isinstance(bandIndex,list): bandRange=bandIndex else: bandRange=[bandIndex] for bi in bandRange: band = ds.GetRasterBand(bi) nodata=band.GetNoDataValue() rast=Raster(Xll=xll,Yll=yll,Ncols=ncols,Nrows=nrows,Cellsize=abs(cellsizeX),Nodata=nodata) xBlockSize,yBlockSize=band.GetBlockSize() data =None for i in range(0, nrows, yBlockSize): if i + yBlockSize < nrows: numRows = yBlockSize else: numRows=nrows-i #last block is not complete in y direction colBlock=None for j in range(0, ncols, xBlockSize): if j + xBlockSize < ncols: numCols = xBlockSize else: numCols=ncols-j #last block is not complete in x direction dataBlock = band.ReadAsArray(j, i, numCols, numRows) #Read whole array dataBlock = dataBlock.astype(dataType) # Process block here if large file ,before # reading the next block if colBlock is None: colBlock=dataBlock else: colBlock=numpy.concatenate((colBlock,dataBlock),1) if data is None: data=colBlock else: data=numpy.concatenate((data,colBlock),0) rast.data=data rasters.append(rast) return rasters
def main(): # -----------Setting up and unsing option parser----------------------- parser = OptionParser(usage=usage, version=version) parser.add_option("-d", "--doc", action="store_true", dest="doc", help="Prints more detailed documentation and exit") parser.add_option("-v", action="store_const", const=logging.DEBUG, dest="loglevel", default=get_loglevel(), help="Produce verbose output") parser.add_option("-o", "--output", action="store", dest="outfileName", default=None, help="Output file") parser.add_option("-i", "--i", action="store", dest="infile", help="Raster to be regionalized") parser.add_option("-r", "--regdef", action="store", dest="regdef", help="Raster defining the regions found " + "in statistics table") parser.add_option("-s", "--stat", action="store", dest="stat", help="Tab-separated statistics table, " + "with 1st column representing geocode and value " + "representing region sum") parser.add_option("-c", "--colname", action="store", dest="colname", help="Header of column in stat table to fetch " + "region sums from") (options, args) = parser.parse_args() # ------------Setting up logging capabilities ----------- rootLogger = logger.RootLogger(int(options.loglevel)) global log log = rootLogger.getLogger(sys.argv[0]) # ------------Process and validate options--------------- if options.doc: print __doc__ sys.exit() if len(args) > 0: parser.error("Incorrect number of arguments") regdef = Raster() if options.regdef is not None: regdef.read(options.regdef) else: log.error("No region definition raster specified") sys.exit(1) key = Raster() if options.infile is not None: key.read(options.infile) else: log.info("No initial distribution raster given, using " + "homogenious distribution") key.assign(regdef) key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata) if options.stat is None: log.error("No statistics table specified") sys.exit(1) if (regdef.xll != key.xll or regdef.yll != key.yll): log.error("The raster domain size is differs between key raster " + "and region raster") stat = DataTable() stat.read(options.stat, defaultType=float) print stat.desc stat.convertCol(stat.desc[0]["id"], int) print stat.desc # Set first column as unique id for rows stat.setKeys([stat.desc[0]["id"]]) # Remove nodata in rasters key.nodataToZero() regdef.nodataToZero() log.info("Consistency och completeness check for codes " + "in regdef and statistics") # Create list of codes in raster regdefCodes = regdef.unique() regdefCodes = map(int, regdefCodes) if 0.0 in regdefCodes: regdefCodes.remove(0.0) statRegCodes = [row[0] for row in stat.data] if options.colname is not None: try: col = stat.colIndex[options.colname] except KeyError: log.error("No column named " + "%s found in statistics table" % options.colname) sys.exit(1) else: col = 1 colId = stat.desc[col]["id"] errFound = False for code in statRegCodes: # Assuring that the regional codes are present in regdef if code not in regdefCodes: log.error("Input Error: code:" + str(int(code)) + "in reg totals is not represented in regional raster") errFound = True # Assuring that the regional raster IDs are present # in the regional statistics for code in regdefCodes: if code not in statRegCodes: log.error("Input Error: ID:" + str(code) + " in region raster is not represented in " + "regional totals!") errFound = True if errFound: sys.exit(1) # For all regions, calculate the regional sum, # set the key values to key/regSum res = np.zeros(key.data.shape) log.info("Regionalizing key raster") for code in regdefCodes: emis = stat.lookup(colId, code) log.debug("Statistics for reg %i: %f" % (code, emis)) mask = np.array(regdef.data == code) regKey = mask * key.data regSum = np.sum(np.sum(regKey)) if regSum > 0: res = res + emis * regKey / regSum else: log.warning( "Distribution key is zero for geocode" + " %i, using homogenuous distribution for this region" % code) res = res + emis * mask / np.sum(np.sum(mask)) key.data = res # resultRast=(totFractionRast*key)*regionalTotals.regSum(substance) key.write(options.outfileName) log.info("Result with sum: %f written" % res.sum())
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-i","--infiles", action="store",dest="infiles", help="List of input files") parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-p","--pattern", action="store",dest="pattern", default=r".+(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec).+", help="Regular expression to extract month from filename") parser.add_option("-o","--outfile", action="store",dest="outfile", help="Output file for sum of montly emission rasters") (options, args) = parser.parse_args() #--------------------Init logger----------------------- rootLogger = logger.RootLogger(level=options.loglevel) global log log = rootLogger.getStreamLogger(sys.argv[0]) #-----------------Validating options------------------- if len(args)!=0: log.error("No arguments wanted, only options") pat=re.compile(options.pattern) files=glob.glob(options.infiles) yearRast=None for f in files: monthMatch=pat.match(f) if monthMatch is None: continue month= monthMatch.group(1) rast=Raster() try: rast.read(f) except: rast=res2rast(f) cellsPerKm2=1.0e6/pow(rast.cellsize,2) rast=rast/float(cellsPerKm2) if yearRast is None: yearRast= rast/12.0 else: yearRast+= rast/12.0 monthSum=rast.sum()/12.0 print "%s, %s: %f" %(f,month,monthSum) if options.outfile is not None: yearRast.write(options.outfile)
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-u", "--user", action="store", dest="user", default=None, help="Specify user manually") parser.add_option("-e", "--edb", action="store", dest="edb", default=None, help="Name of target edb") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-f", "--force", action="store_true",dest="force",default=False, help="To start the process without confirming the domain") (options, args) = parser.parse_args() # Setup logging logging.configure(terminal_level=logging.DEBUG) log = logging.getLogger(__name__) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") if options.edb ==None: parser.error("Need to specify edb using flag -e") if options.user ==None: parser.error("Need to specify user using flag -u") dmn = Domain() if not options.force: answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?") if answer!="y": sys.exit("Interrupted by user") if not dmn.edbExistForUser(options.edb,options.user): log.error("Edb "+options.edb+" does not exist for user "+ options.user+" in domain "+dmn.name) sys.exit() #---Creating edb and rsrc objects------------------ edb=Edb(dmn,options.user,options.edb) rsrc=Rsrc(edb.rsrcPath()) #Opening controlfile #---retrieving data from control file---- cf=ControlFile(fileName=path.abspath(args[0])) substances=cf.findStringList("substances:") outputDir=cf.findExistingPath("outputDir:") acIndex=cf.findInt("acIndex:") macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:")) fromProj=cf.findString("fromProj:") toProj=cf.findString("toProj:") try: fromProj=transcoord.proj4Dict[fromProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj) try: toProj=transcoord.proj4Dict[toProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj) formats = cf.findStringList("formats:") units = cf.findString("units:") writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True) edb_xll=cf.findInt("edb_xll:") edb_yll=cf.findInt("edb_yll:") edb_ncols=cf.findInt("edb_ncols:") edb_nrows=cf.findInt("edb_nrows:") edb_cellsize=cf.findFloat("edb_cellsize:") if fromProj!=toProj: out_xll=cf.findFloat("out_xll:") out_yll=cf.findFloat("out_yll:") out_ncols=cf.findInt("out_ncols:") out_nrows=cf.findInt("out_nrows:") out_cellsize=cf.findFloat("out_cellsize:") #----------------------------------------- #Finds index to search units unitIndex=None for key,unit in rsrc.search.iteritems(): if isinstance(key,int): if rsrc.search[key]==units: unitIndex=key break if unitIndex is None: log.error("Search units: %s not defined in edb.rsrc" %units) sys.exit() macro = ControlFile(fileName=macroFileName,removeComments=False) #preparing export macro macro.setParam("general.database:",dmn.name) xmin=edb_xll xmax=edb_xll+edb_ncols*edb_cellsize ymin=edb_yll ymax=edb_yll+edb_nrows*edb_cellsize macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("edb.user:"******"edb.edb:",edb.name) macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("USER :"******"EDB :",edb.name) macro.setParam("GRID :", "%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize)) macro.setParam("edb.unit:",unitIndex) macro.setParam("UNIT :",unitIndex) # macro.setParam("NOACTCODE :",acIndex) macro.setParam("NOACTCODE :",len(rsrc.ac)) #Get activity code tree acTree=codeemistree.CodeEmisTree("Activity codes",units=units) acTree.readActivityCodes(rsrc.path,acIndex) substDict=dmn.listSubstanceIndices() edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols, Nrows=edb_nrows,Cellsize=edb_cellsize, Nodata=-9999,init=0) if fromProj!=toProj: outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999) else: outRastTemplate=Raster() outRastTemplate.assign(edbRast) for node in acTree.root.getiterator(): if node.tag=="root" or node.tag=="Record": continue ac=node.tag log.debug("Activity code: "+ac) #Finds row index for activity codes in macro #Add a row with the current ac #If there are many ac already given, these are #replaced by the current ac macroLines=macro.content.split("\n") actCodeInd=None geoCodeInd=None for lineInd,line in enumerate(macroLines): if "NOACTCODE" in line: actCodeInd=lineInd if "NOGEOCODE" in line: geoCodeInd=lineInd if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] else: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] macro.content="\n".join(macroLines) macro.write() #boolean raster marking where there is data for any of the substances if 'CLRTAP' in formats: dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999,init=0) rasterDict={} substancesWithData=[] for substance in substances: log.debug("Substance %s" %substance) substanceIndex=substDict[substance] macro.setParam("ELEMENT :",substanceIndex) macro.write() command="xrepedb -i "+macro.name (returnCode,errMsg,outMsg)=utilities.execute(command) tmp=outMsg.split()[10:-2] tmp.sort() if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00': print "ERROR: The field for "+substance+" is empty!" continue # pdb.set_trace() emisRast=string2rast(outMsg,edbRast) emisSum=emisRast.sum() outRast=Raster() outRast.assign(outRastTemplate) rec=ET.SubElement(node,"Record") rec.attrib["substance"]=substance rec.attrib["emission"]=str(emisSum) if emisSum>0 and writeGrids: if substance not in substancesWithData: substancesWithData.append(substance) if fromProj!=toProj: exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir()) else: exportRast=emisRast if 'CLRTAP' in formats: dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data) rasterDict[substance]=exportRast categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) if 'ESRI Ascii grid' in formats: fileName = path.join(categoryDirPath, substance+ ".asc") exportRast.write(fileName) log.debug("Grid for " + substance + "written to outputDir for category: " + ac) summaryTable=acTree.createTable(writeAll=True) summaryTable.sortRows() tableFile=open(path.join(outputDir,"summaryTable.txt"),'w') summaryTable.write(tableFile) if len(rasterDict)>0 and 'CLRTAP' in formats: #creating substance header in the same order as the substances in the template header = "i\tj\t" #headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"] for s in substancesWithData: header += s + "\t" #remove the tab after the last column and add a newline instead header = header[: - 1]+ "\n" #Creating file for EMEP-data fileName = "CLRTAP_" + ac + ".txt" categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) fid = open(path.join(categoryDirPath, fileName), 'w') fid.writelines(header) sum=0 #Writing indexes and data for all non-zero elements for row in range(dataMarker.nrows): for col in range(dataMarker.ncols): if dataMarker.data[row, col] > 0: (i, j) = dataMarker.getCentreCoords(row, col) fid.write(str(i) + "\t" + str(j) + "\t") for substWithData in substancesWithData[:-1]: fid.write(str(rasterDict[substWithData].data[row, col]) + "\t") sum+=rasterDict[substWithData].data[row, col] fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n") sum+=rasterDict[substancesWithData[-1]].data[row, col] fid.close() log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName)) log.info("Finished")
class TraceDef: def __init__(self, active=None, substances=None, logfile=None, regdefgc=None, gcDefRaster=None): if active is None or not active: self.active = False else: self.active = True self.substances = substances if logfile is not None: if not path.exists(path.dirname(logfile)) and logfile is not None: print "Error, path for trace logfile does not exist" else: self.log = codecs.open(logfile, "w", "UTF-8") self.regdefgc = regdefgc self.regInfo = {} self.regTot = {} self.regPsInfo = {} self.regPsTot = {} self.natTot = {} self.natPs = {} self.gcRaster = Raster() if self.active: self.gcRaster.read(gcDefRaster) def addRegInfo(self,regCode,par,val): regCode = int(regCode) if regCode not in self.regInfo: self.regInfo[regCode] = {} self.regInfo[regCode][par] = val def addRegTot(self, regCode, par, subst, val): if regCode not in self.regTot: self.regTot[regCode] = {} if par not in self.regTot[regCode]: self.regTot[regCode][par] = {} self.regTot[regCode][par][subst] = val def addRegPsInfo(self,regCode,psInd,psDict): if regCode not in self.regPsInfo: self.regPsInfo[regCode] = {} if psInd not in self.regPsInfo[regCode]: self.regPsInfo[regCode][psInd] = {} ps = self.regPsInfo[regCode][psInd] for key, val in psDict.iteritems(): ps[key] = val def addRegPsTot(self, regCode, psInd, par, subst, val): if regCode not in self.regPsTot: self.regPsTot[regCode] = {} if psInd not in self.regPsTot[regCode]: self.regPsTot[regCode][psInd] = {} if par not in self.regPsTot[regCode][psInd]: self.regPsTot[regCode][psInd][par] = {} self.regPsTot[regCode][psInd][par][subst] = val def write(self): if not self.active: return content = "" for reg in self.regInfo: content += "Region code: %i\n" % reg for key, val in self.regInfo[reg].iteritems(): content += key + ": " + str(val) + "\n" for par, substances in self.regTot[reg].iteritems(): content += par + (20 - len(par)) * " " for subst in self.substances: content += subst + (13 - len(subst)) * " " content += "\n" content += 20 * " " for subst in self.substances: val = self.regTot[reg][par].get(subst, "") if isinstance(val, basestring): content += "%-12s " % val else: content += "%-12e " % val content += "\n" content += "--------Original point-sources---------\n" content += "Index Name" + 46 * " " + "Geocode " for subst in self.substances: content += subst + (13 - len(subst)) * " " content += "\n" if reg in self.regPsInfo: for psInd, props in self.regPsInfo[reg].iteritems(): content += "%-6i" % psInd content += props["NAME"] + (50 - len(props["NAME"])) * " " content += props["GEOCODE"] + (12 - len(props["GEOCODE"])) * " " for subst in self.substances: content += "%-12e " % self.regPsTot[reg][psInd]["Emission"][subst] content += "\n" content += "--------Corrected point-sources---------\n" content += "Index Name" + 46 * " " for subst in self.substances: content += subst +(12 - len(subst)) * " " content += "\n" if reg in self.regPsInfo: for psInd, props in self.regPsInfo[reg].iteritems(): content += "%-6i" % psInd content += props["NAME"] + (50 - len(props["NAME"])) * " " content += props["GEOCODE"] + (12 - len(props["GEOCODE"])) * " " for subst in self.substances: try: content += "%-12e" % self.regPsTot[reg][psInd]["Emission_corr"][subst] except KeyError: pass content += "\n" content += "\n" self.log.write(content)
def main(): # Parse command line arguments parser = argparse.ArgumentParser(description=__doc__) utils.add_standard_command_options(parser) parser.add_argument("controlfile", metavar='CONTROLFILE', action="store", help="Controlfile for topdown processing") parser.add_argument("-t", "--template", metavar='TEMPLATEFILE', action="store",dest="cf",default=None, help="Generate default controlfile") args = parser.parse_args() if args.cf is not None: generateCf(args.cf) log.info("Wrote default controlfile") sys.exit(0) log.info("Starting topdown processing") # Opening controlfile cf = ControlFile(args.controlfile) dmn = Domain() log.info("Reading topdown table") tdTableName = cf.findExistingPath("topDownTable:") tdTable = DataTable() tdTable.keys.append("Code") tdTable.read(tdTableName,delimiter=";") log.info("Reading national totals table") natTotalTableName = cf.findExistingPath("nationalTotalTable:") natTable = DataTable(desc=[{"id": "Code", "type":unicode}, {"id": "description", "type":unicode}]) natTable.keys.append("Code") natTable.read(natTotalTableName, units=True, defaultType=str) notationKeys = ["NE", "NO", "NA", "IE"] log.debug("Remove notation keys from national totals table") for row in natTable.data: for i in range(len(row)): if row[i] in notationKeys: row[i] = None log.debug("Convert all emission columns in national totals to float") for colId in natTable.listIds(): if colId not in ["Code","description"]: natTable.convertCol(colId,float) log.debug("Store units from national totals for each substance in dict") natUnits={} for col in natTable.desc: if col.get("units",None)!=None: natUnits[col["id"]]=col["units"] log.debug("Read remaining data from control file") bottomupEdbName = cf.findString("bottomUpEdb:") topDownEdbName = cf.findString("topDownEdb:") emissionsEdbName = cf.findString("emissionsEdb:") userName = cf.findString("user:"******"year:") #initialize edb objects buEdb = Edb(dmn,userName,bottomupEdbName) tdEdb = Edb(dmn,userName,topDownEdbName) eEdb = Edb(dmn,userName,emissionsEdbName) log.info("Reading/preparing EDB:s") log.info("Reading subdb") subdb = Subdb(eEdb) subdb.read() log.info("Reading subgrpdb") subgrpdb = SubgrpStream(buEdb) subgrpdb.read() log.info("Reading facilitydb") facilityIn = FacilityStream(buEdb) log.info("Reading companydb") companyIn = CompanyStream(buEdb) facilityOut = FacilityStream(eEdb,mode="w") companyOut = CompanyStream(eEdb,mode="w") log.info("Writing company db to result edb") companyOut.write(companyIn.read()) log.info("Writing facility db to result edb") facilityOut.write(facilityIn.read()) if not buEdb.exists(): log.error("Edb " + buEdb.name + " does not exist for user " + userName + " in domain " + dmn.name) sys.exit(1) if not tdEdb.exists(): log.error("Edb " + tdEdb.name + " does not exist for user " + userName + " in domain " + dmn.name) sys.exit(1) if not eEdb.exists(): log.error("Edb " + eEdb.name + " does not exist for user " + userName + " in domain " + dmn.name) sys.exit(1) keys = tdEdb.listGrids() msg = "%i keys found in edb: %s" % (len(keys), tdEdb.name) log.info(msg) # sourcedb from bottom-up edb with SourceStream(buEdb, mode='rb') as source_instream: source_reader = ModelReader(source_instream) bu_sources = list(source_reader) log.info( "%i point sources found in edb: %s" % ( len(bu_sources), buEdb.name) ) # Empty sourcedb of the result edb if cf.findBoolean("emptyEmissionSourcedb:"): eEdb.empty_sourcedb() e_sources = [] log.info("Removed point sources from edb: %s" % (eEdb.name)) else: # sourcedb from emission edb (result edb) with SourceStream(eEdb, mode='rb') as source_instream: source_reader = ModelReader(source_instream) e_sources = list(source_reader) msg = "%i point sources found in edb: %s" % (len(e_sources), eEdb.name) log.info(msg) if not path.exists(eEdb.rsrcPath()): log.error("No edb.rsrc exists for emission edb") sys.exit() else: rsrc = Rsrc(eEdb.rsrcPath()) acIndex = cf.findInt("acIndex:") codeDepth = rsrc.ac[acIndex-1].depth substances = cf.findStringList("substances:") for subst in substances: if subst not in subdb.substIndices: log.error("Substance: " + subst + " not in Airviro substance list") sys.exit() # Initialize trace for debug and additional logging if cf.findBoolean("trace:") == True: log.info("Initializing trace for detailed logging") trace = TraceDef( active=True, substances=cf.findStringList("trace.substances:"), logfile=cf.findString("trace.logfile:"), regdefgc=cf.findIntList("trace.regdef.gc:", optional=True, default=None), gcDefRaster=cf.findExistingPath("trace.gcraster:") ) else: trace = TraceDef(active=False) log.info("Initializing result table") resTablePath = cf.findString("resTable:") resTable = DataTable(desc=[{"id": "Code", "type": unicode}]) resTable.keys.append("Code") for subst in substances: resTable.addCol({"id": subst, "type": float, "unit": "%"}) # Create emission grid template (with geocodes) log.info("Reading emission grid template") eGridTemplatePath = cf.findExistingPath("emisGridTemplatePath:") eGridTemplate = Egrid(eEdb,"name") if eGridTemplatePath[-4:] == ".asc": eGridTemplatePath=eGridTemplatePath[:-4] eGridTemplate.readData(eGridTemplatePath) eGridTemplate.substances = {} eGridTemplate.par["SUBSTANCE"].val = [] dd = {"key": None, "regstat": None, "regdef": None, "bu_sources": bu_sources, "psIndices": [], "units": natUnits, "rsrc": rsrc, "subdb": subdb, "trace": trace, "subgrpdb": subgrpdb } # Process all rows in the topdown table for row in tdTable.data: code = row[tdTable.colIndex["Code"]] active = row[tdTable.colIndex["Active"]] statType = row[tdTable.colIndex["Stat_type"]] if active == "no": continue log.info("Code: "+code) distributed=False # Add '-' to the code to reach max length (fix for a GUI bug) airviroCode = code # while len(airviroCode.split(".")) < codeDepth: # airviroCode += ".-" tdrow = tdTable.data[tdTable.rowIndex([code])] nrow = natTable.data[natTable.rowIndex([code])] # Create a resTable row to fill with data resrow = [None] * resTable.ncols resrow[0] = code # Check if national totals are non-zero nonZero = False for val in nrow: if val != None: if val > 0: nonZero = True break # Filter out indices for pointsources with the current ac # Also including sources coded with sub-codes # This allows to estimate top-down emissions on a higher code-level psIndices = [] for i, ps in enumerate(bu_sources): codeMatch = False for emis in ps.EMISSION: # It is assumed that the first code is used while processing topdown ac = emis.ACTCODE[0] if ac[-1] == ".": ac=ac[:-1] # if ac[:len(code)] == code: if ac == code: codeMatch = True break if not codeMatch: for emis in ps.SUBGRP: # It is assumed that the first code is used while processing topdown ac = emis.ACTCODE[0] if ac[:len(code)] == code: codeMatch = True break if codeMatch: psIndices.append(i) dd["psIndices"] = psIndices keyName = row[tdTable.colIndex["Key"]] #If no distribution key specified and no ps in bottom-up edb - cont. if keyName is None and psIndices == []: log.debug("No key and no point sources found for code: %s, skipping..." % code) resTable.addRow(resrow) continue if psIndices!=[]: msg = "--Found %i pointsources" % len(psIndices) log.info(msg) if keyName is not None: if keyName not in keys: log.error("No such key: " + keyName) sys.exit() msg = "--Key: %s" % keyName log.info(msg) keyGrid = Egrid(tdEdb, keyName) keyGrid.readData() log.debug("Read key: " + keyName + " from topdownEdb") # create emission grid to store distributed emissions eGrid = deepcopy(eGridTemplate) eGrid.name = code.replace(".", "_") eGrid.par["NAME"].val = code eGrid.par["INFO2"].val = "Distribution key: " + keyGrid.par["NAME"].val eGrid.par["ACTIVITYCODE"].val = [airviroCode.split(".")] regstatName = row[tdTable.colIndex["Regstat"]] regdefName = row[tdTable.colIndex["Regdef"]] if regstatName is not None: if regdefName is None: log.error("No region definition given for regional statistics: " + regstatName) sys.exit(1) regstatPath = path.join(dmn.domainPath(), "topdown", "regstat", regstatName) regstat = DataTable() log.info("regstatPath: "+regstatPath) regstat.read(regstatPath, units=True, defaultType=float, delimiter=";") if not "Geocode" in regstat.listIds(): log.error("No Geocode column found in regstat") sys.exit(1) regstat.convertCol("Geocode", int) regstat.keys.append("Geocode") # Making Geocode the primary key # create list of unique geo codes geocodes = [row[regstat.colIndex["Geocode"]] for row in regstat.data] geocodes = unique(geocodes) for colId in regstat.listIds(): if colId.lower() == "year": rows = [] regstat.convertCol(colId, int) # Make it possible to accumulate year regstat.setKeys(regstat.keys + [colId]) # Calculates the total emission for each geocode # in case there are multiple rows for different fuels etc colsToSum = regstat.listIds() colsToSum.remove(colId) colsToSum.remove("Geocode") for gc in geocodes: # sums all numeric values in colsToSum for # rows matching row id [gc,year] #returns an accumulated row and appends it to rows rowId = regstat.dict2RowId({"Geocode": gc, colId: year}) rows.append(regstat.accumulate(rowId, "sum", colsToSum)) regstat.data = rows # replace original rows with accumulated rows regstat.keys.remove(colId) break # dd["regstat"] = regstat regdef = Raster() regdefPath = path.join(dmn.domainPath(), "topdown", "regdef", regdefName) regdef.read(regdefPath) dd["regstat"] = regstat dd["regdef"] = regdef else: dd["regstat"] = None dd["regdef"] = None if dd["regstat"] is not None and len(bu_sources) > 0 and statType == "fixed": log.info("--Regionalizing pointsources") dd = regionalizePS(dd, code) if keyName is not None and nonZero: regionalizedDefault = False # Spatial distribution of emissions for subst in substances: sInd = subdb.substIndices[subst] toUnit = dd["units"][subst] + "/year" ntot = nrow[natTable.colIndex[subst]] pstot = 0 for i in dd["psIndices"]: source = dd["bu_sources"][i] # TODO: should give reference to subgrps to include emis from them pstot += source.get_emis( sInd, toUnit, eEdb, actcodes=[code] ) if ntot is None or ntot == 0: if pstot > 0: # 9999 is used as marker for no national total resrow[resTable.colIndex[subst]] = 9999.0 log.warning( "Nattot is 0 but ps tot is: %f %s" % (pstot, toUnit)) continue nrest = ntot - pstot resrow[resTable.colIndex[subst]] = 100.0 if abs(nrest / ntot) < 0.0001: nrest = 0 log.info( "--Rest is < 0.01 % of national total, rounded to zero" ) continue elif nrest < 0: log.warning( "--National rest is below zero, %4.2f proc for %s" % ( -1 * nrest / ntot * 100, subst) ) dd["trace"].write() # continue log.info( "---Substance: "+subst+ ", rest is: " + str(nrest) + toUnit + " = " + str(nrest / ntot * 100.0) + "%" ) try: keyRast = keyGrid.substances[sInd] except KeyError: keyRast = keyGrid.substances[subdb.substIndices["all"]] dd["key"] = keyRast if dd["regstat"] is not None: if (subst not in regstat.colIndex and sInd not in keyGrid.substances and not regionalizedDefault): dd = regionalizeKey(dd, subst, code) regionalizedDefault = True else: dd = regionalizeKey(dd, subst, code) emisRast = distribute(dd["key"], nrest) emisRast = emisRast * unitConvFac(toUnit, "ton/year") eGrid.addData(emisRast, dd["subdb"].substIndices[subst]) distributed = True else: # resTable is filled # In case all national totals are zero but there are ps for subst in substances: sInd = dd["subdb"].substIndices[subst] toUnit = dd["units"][subst] + "/year" ntot = nrow[natTable.colIndex[subst]] pstot = 0 for i in dd["psIndices"]: source = dd["bu_sources"][i] # subgrps are not used! pstot += source.get_emis(sInd, toUnit, buEdb, actcodes=[code]) if ntot!=0 and ntot is not None: resrow[resTable.colIndex[subst]] = pstot / ntot * 100.0 else: resrow[resTable.colIndex[subst]] = -999.0 if len(dd["psIndices"]) > 0: tmp_sources = (bu_sources[i] for i in dd["psIndices"]) with SourceStream(eEdb, mode='wb') as out_source_stream: source_writer = ModelWriter(out_source_stream) for source in tmp_sources: source_writer.write(source) log.debug("Wrote ps to emission edb") if distributed: eGrid.load() log.debug("Wrote emission grid to emission edb") dd["trace"].write() resTable.addRow(resrow) resTableFile = open(resTablePath,"w") resTable.write(resTableFile) log.info("Finished topdown process")