def main():
#setting up parser
parser = argparse.ArgumentParser(description=__doc__)
utils.add_standard_command_options(parser)
parser.add_argument(
"-e","--edbs",
action="store",dest="edbList",
help="List of 'user/edb' pairs separated by :"
)
parser.add_argument(
"-L","--labels",
action="store",
dest="labels",
help="List of edb labels separated by :"
)
parser.add_argument(
"-s","--substances",
action="store",dest="substances",
help="List of substance names separated by :"
)
parser.add_argument(
"-t","--title",
action="store",dest="title",
help="Report title"
)
parser.add_argument(
"-g","--gc-filter",
action="store",dest="gcfilter",
help="Filter on Geo codes, separated by :"
)
parser.add_argument(
"-o","--outfile",
action="store",dest="outfile",
help="Output filename"
)
parser.add_argument(
"-f","--format",
action="store",dest="format",
help="Output in 'excel','csv' or 'raw' " +
"(Excel-format requires xlwt python module)"
)
parser.add_argument("--substMapping",
action="store",dest="substMapping",
help="File with tab separated mappings of substance names")
parser.add_argument("--markerTable",
action="store",dest="markerTable",
help="Table of codes to be formatted and commented")
parser.add_argument(
"macro",metavar="MACRO",
help="A macro to use"
)
args = parser.parse_args()
if args.markerTable is not None:
keys=["Year","GC","AC","note_1","note_2"]
markerTable = DataTable(keys=keys,desc=[{"id":"Year","type":str},{"id":"GC","type":str},{"id":"AC","type":str},{"id":"note_1","type":str},{"id":"note_2","type":str}])
markerTable.read(args.markerTable)
else:
markerTable=None
substMapping={}
if args.substMapping is not None:
with codecs.open(args.substMapping,encoding="HP Roman8",mode="r") as f:
for line in f:
oldName,newName = line.split(":")
substMapping[oldName.strip()]=newName.strip()
dmn = Domain()
if args.gcfilter is not None:
args.gcfilter = args.gcfilter.split(":")
# Read original macro
with codecs.open(args.macro, encoding="HP Roman8", mode="r") as f:
originalContent = f.read()
# Create a tmp copy of the macro, write content from the original macro
macroTempFile = tempfile.NamedTemporaryFile(
suffix=".sedb",
dir=dmn.tmpDir()
)
tmpMacro = codecs.open(
macroTempFile.name,
encoding="HP Roman8",mode="w"
)
tmpMacro.write(originalContent)
tmpMacro.flush()
# Create a ControlFile obj to simplify reading and modifying macro
macro = ControlFile(macroTempFile.name, removeComments=False)
ebd = macro.findString("edb.edb:")
user = macro.findString("edb.user:"******"edb.reportgeocode:")[-1])
acIndex = int(macro.findString("edb.reportactcode:")[-1])
if args.edbList is None:
ebds = [[user, edb]]
else:
edbs = args.edbList.split(":")
edbs = [e.split("/") for e in edbs]
nedbs = len(edbs)
if args.labels is None:
labels = ["No label"] * len(edbs)
else:
labels = args.labels.split(":")
if len(labels) != nedbs:
log.error("Number of labels specified should match number of edb:s")
sys.exit(1)
if args.substances is None:
log.error("Need to specify substances")
sys.exit(1)
else:
substances = args.substances.split(":")
if args.format not in ('excel','csv','raw'):
log.error(
"Invalid format specifier : %s, should be one of 'excel'" +
", 'csv' or 'raw'" %args.format
)
sys.exit(1)
elif args.format == "excel":
try:
import xlwt
except:
log.error(
"trendReport.py requires python module xlwt to write excel-files")
sys.exit(1)
# first edb
# import pdb; pdb.set_trace()
edb = Edb(dmn, edbs[0][0], edbs[0][1])
# assume same code definitions in all edbs to be processed, read from first
rsrc = edb.rsrc
nrsubstances = len(substances)
unitIndex = int(macro.findString("UNIT :"))
units = rsrc.search[unitIndex]
subdb = Subdb(edb)
subdb.read()
#decode input title using stdin encoding
title=args.title.decode(sys.stdin.encoding)
rawOutput = ""
rawMeta = u"name: %s\nnrmacros: %i\nnrsub: %i\nunit: %s\n" %(
title, nedbs, nrsubstances, units)
emissions = []
for ind, edbUser in enumerate(edbs):
label = labels[ind]
userName = edbUser[0]
edbName = edbUser[1]
macro.setParam("edb.user:"******"edb.edb:", edbName)
macro.setParam("USER :"******"EDB :", edbName)
rawMeta += "macro.%i.edbuser: %s\n" %(ind, userName)
rawMeta += "macro.%i.edbname: %s\n" %(ind, edbName)
rawMeta += "macro.%i.desc: %s\n" %(ind, label)
for subst in substances:
log.info(
"User: %s, edb: %s, substance %s" %(
userName, edbName, subst)
)
substanceIndex = subdb.substIndex(subst)
macro.setParam("ELEMENT :", substanceIndex)
macro.write()
command = "xrepedb -i " + macro.name
log.info("Running xrepedb for substance %s" % subst)
# import pdb; pdb.set_trace()
(returnCode, errMsg, outMsg) = utilities.execute(command)
if returnCode != 0:
log.error("Could not run %s\nstdout: %s\nstderr:%s" %(
command,outMsg,errMsg))
sys.exit(1)
if len(outMsg) < 10:
log.error("Invalid output from xrepedb: %s" % outMsg)
sys.exit(1)
rawOutput += "#MACRO %i \"%s\" \"%s\"\n" % (ind, subst, labels[ind])
rawOutput += outMsg
lines = outMsg.split("\n")[:-1]
for lineInd, line in enumerate(lines):
vals = line.split()
ac = vals[1].split(".")
gc = vals[3].split(".")
if len(ac) == 1:
if ac[0] == "<all>":
acLev1 = "alla"
else:
acLev1 = ac[0]
acLev2 = "alla"
else:
acLev1 = ac[0]
acLev2 = ac[1]
if len(gc) == 1:
if gc[0] == "<all>":
gcLev1 = "alla"
else:
gcLev1 = gc[0]
gcLev2 = "alla"
else:
gcLev1 = gc[0]
gcLev2 = gc[1]
emis = float(vals[4])
if acLev1 == "alla":
acLev1Name = "alla"
acLev2Name = "alla"
else:
node = rsrc.ac[acIndex - 1].root.find(acLev1)
acLev1Name = node.attrib["name"]
if acLev2 == "alla":
acLev2Name = "alla"
else:
node = rsrc.ac[acIndex-1].root.find(
acLev1 + "/" + acLev2
)
acLev2Name = node.attrib["name"]
if gcLev1 == "alla":
gcLev1Name = "alla"
gcLev2Name = "alla"
else:
node = rsrc.gc[gcIndex-1].root.find(gcLev1)
gcLev1Name = node.attrib["name"]
if gcLev2 == "alla":
gcLev2Name = "alla"
else:
node = rsrc.gc[gcIndex - 1].root.find(
gcLev1 + "/" + gcLev2
)
gcLev2Name = node.attrib["name"]
if args.gcfilter is not None:
if gc[0] not in args.gcfilter:
# if args.gcfilter != gcLev1:
continue
emissions.append({"label": label,
"substance": subst,
"ac": '.'.join(ac),
"gc": '.'.join(gc),
"gcLev1": gcLev1Name,
"gcLev2": gcLev2Name,
"acLev1": acLev1Name,
"acLev2": acLev2Name,
"acLev1Code": acLev1,
"acLev2Code": acLev2,
"val": emis,
"edbIndex": ind})
#Close tempfile to automatically remove it
tmpMacro.close()
if args.format == "raw":
outfile = codecs.open(args.outfile,"w","HP Roman8")
outfile.write(rawMeta)
outfile.write(rawOutput)
outfile.close()
elif args.format == "csv":
outfile = open(args.outfile,"w")
desc = [
{'id': 'gc', 'type': unicode},
{'id': 'ac', 'type': unicode},
{'id': 'label', 'type': unicode},
{'id': 'user', 'type': unicode},
{'id': 'edb', 'type': unicode}
]
for subst in substances:
desc.append({'id': subst, 'type': float})
keys = ['gc', 'ac', 'label']
table = DataTable(desc=desc, keys=keys)
log.info("Adding emissions to csv-table")
for emis in emissions:
row = [None] * len(desc)
user = edbs[emis['edbIndex']][0]
edb = edbs[emis['edbIndex']][1]
row[table.colIndex['gc']] = emis['gc']
row[table.colIndex['ac']] = emis['ac']
row[table.colIndex['label']] = emis['label']
row[table.colIndex['user']] = user
row[table.colIndex['edb']] = edb
row[table.colIndex[emis['substance']]] = emis['val']
# data is appended to the correct row, or a new row is added if the
# table keys do not match any existing row
log.debug(
"Adding row for substance %s, gc %s, ac %s" %(
emis['substance'],
emis['gc'],
emis['ac'])
)
table.addRow(row, append=True)
table.write(outfile)
outfile.close()
else:
# Create style objects for excel output
header1Style = xlwt.easyxf(
'font: name Times New Roman,color-index black, bold on',
num_format_str='0.000E+00'
)
markerStyle1 = xlwt.easyxf(
'font: name Times New Roman,color-index red, bold off, italic on',
num_format_str='0.000E+00')
markerStyle2 = xlwt.easyxf(
'font: name Times New Roman,color-index orange, bold off, italic on',
num_format_str='0.000E+00')
normalStyle = xlwt.easyxf(
'font: name Times New Roman,color-index black, bold off',
num_format_str='0.000E+00'
)
excelBook = xlwt.Workbook()
# Creating info sheet
infoWs = excelBook.add_sheet("Info")
infoWs.col(0).width = 256*20
infoWs.col(1).width = 256*25
infoWs.col(2).width = 256*20
infoWs.col(3).width = 256*200
infoWs.write(0,0,u"Rapportnamn:",header1Style)
infoWs.write(0,1,title,header1Style)
infoWs.write(1,0,u"Beskrivning av dataunderlaget",header1Style)
infoWs.write(3,0,u"Makron (specificerar utsökningar ur databasen)",header1Style)
infoWs.write(4,0,u"Etikett",header1Style)
infoWs.write(4,1,u"Ägare till EDB",header1Style)
infoWs.write(4,2,u"EDB (emissiondatabas)",header1Style)
infoWs.write(4,3,u"Beskrivning",header1Style)
for i,edbUser in enumerate(edbs):
userName=edbUser[0]
edbName=edbUser[1]
label=labels[i]
infoWs.write(5+i,0,label)
infoWs.write(5+i,1,userName)
infoWs.write(5+i,2,edbName)
#reading edb description file (if it exists)
edb=Edb(dmn,userName,edbName)
infoWs.write(5+i,3,edb.desc().replace("\n"," "))
#split substances in green house gases and air quality related
ghgList=[s for s in substances if s in ghgs]
aqList=[s for s in substances if s not in ghgs]
#Write air quality headers
firstRow=4
#Add two rows for marker comments
if markerTable is not None:
firstRow+=2
if len(aqList)>0:
aqWs = excelBook.add_sheet(u"Luftföroreningar")
aqWs.col(0).width = 256*25
aqWs.col(1).width = 256*30
aqWs.col(2).width = 256*20
aqWs.col(3).width = 256*15
for col in range(nrsubstances*nedbs):
aqWs.col(col+4).width=256*15
aqWs.write(0,0,u"Rapportnamn:",header1Style)
aqWs.write(0,1,title,header1Style)
aqWs.write(1,0,u"Emissioner av luftföroreningar",header1Style)
aqWs.write(1,1,u"Enhet: "+units,header1Style)
if markerTable is not None:
aqWs.write(2,0,u"OBS! Röd kursiv text anger osäkra värden p.g.a. att en stor del av emissionen är fördelad med schabloner inom kommungruppen. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle1)
aqWs.write(3,0,u"OBS! Orange kursiv text anger osäkra värden p.g.a. att trenden varierar kraftigt och eventuellt felaktigt, ytterligare verifiering krävs. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle2)
aqWs.write(firstRow,0,"Huvudsektor",header1Style)
aqWs.write(firstRow,1,"Undersektor",header1Style)
aqWs.write(firstRow,2,u"Län",header1Style)
aqWs.write(firstRow,3,"Kommun",header1Style)
#Write ghg headers
if len(ghgList)>0:
ghgWs = excelBook.add_sheet(u"Växthusgaser")
ghgWs.col(0).width = 256*25
ghgWs.col(1).width = 256*30
ghgWs.col(2).width = 256*20
ghgWs.col(3).width = 256*15
for col in range(nrsubstances*nedbs):
ghgWs.col(col+4).width=256*15
ghgWs.write(0,0,u"Rapportnamn:",header1Style)
ghgWs.write(0,1,title,header1Style)
ghgWs.write(1,0,u"Emissioner av Växthusgaser",header1Style)
ghgWs.write(2,0,u"CO2-ekv. efter ämnesnamn innebär att emissionen är uttryckt i CO2-ekvivalenter",header1Style)
if markerTable is not None:
ghgWs.write(3,0,u"OBS! Röd kursiv text anger osäkra värden p.g.a. att en stor del av emissionen är fördelad med schabloner inom kommungruppen. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle1)
ghgWs.write(4,0,u"OBS! Orange kursiv text anger osäkra värden p.g.a. att trenden varierar kraftigt och eventuellt felaktigt, ytterligare verifiering krävs. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle2)
ghgWs.write(1,1,u"Enhet: "+units,header1Style)
ghgWs.write(firstRow,0,"Huvudsektor",header1Style)
ghgWs.write(firstRow,1,"Undersektor",header1Style)
ghgWs.write(firstRow,2,u"Län",header1Style)
ghgWs.write(firstRow,3,"Kommun",header1Style)
def getColInd(nmacros, substances,macroInd,subst):
#gets the column index in excel file
sInd=substances.index(subst)
#Including extra columns to write CO2-equivalents
nSubstWithCO2equivalents=0
for s in substances[:sInd+1]:
if s in doubleColumns:
nSubstWithCO2equivalents+=1
return 4 + macroInd+sInd*nmacros+nSubstWithCO2equivalents*(macroInd+1)
#write macro labels and substance headers for air quality sheet
for sInd,subst in enumerate(aqList):
for i,edbUser in enumerate(edbs):
col=getColInd(nedbs,aqList,i,subst)
aqWs.write(firstRow-1,col,labels[i],header1Style)
#If a substance name is given in mapping this is used, otherwise
#The substance bname from the airviro substance list is used
aqWs.write(firstRow,col,substMapping.get(subst,subst),header1Style)
#write macro labels and substance headers for ghg sheet
for sInd,subst in enumerate(ghgList):
for i,edbUser in enumerate(edbs):
col=getColInd(nedbs,ghgList,i,subst)
#If CO2-equivalents are calculated, an extra column is needed
if subst in doubleColumns:
ghgWs.write(firstRow-1,col-1,labels[i],header1Style)
ghgWs.write(firstRow-1,col,labels[i],header1Style)
#If CO2-equivalents are calculated, an extra column is needed
if subst in doubleColumns:
#debug statement
#print "writing subst %s in col %i and %i" %(subst,col-1,col)
ghgWs.write(firstRow,col-1,substMapping.get(subst,subst),header1Style)
ghgWs.write(firstRow,col,substMapping.get(subst,subst)+"CO2-ekv.",header1Style)
elif subst in storedAsCO2equivalents:
#debug statement
#print "writing subst %s in col %i" %(subst,col)
ghgWs.write(firstRow,col,substMapping.get(subst,subst)+"CO2-ekv.",header1Style)
else:
#debug statement
#print "writing subst %s in col %i" %(subst,col)
ghgWs.write(firstRow,col,substMapping.get(subst,subst),header1Style)
#looping over all emissions, writing them to the correct column and row
ghgRow=[]
aqRow=[]
for m in range(nedbs*nrsubstances+4+3*nedbs):
ghgRow.append(firstRow+1)
for m in range(nedbs*nrsubstances+4):
aqRow.append(firstRow+1)
for emis in emissions:
subst = emis["substance"]
emisVal=emis["val"]
edbInd=emis["edbIndex"]
#Check if gc, ac and year can be found in the error list
#debugging marker style
if markerTable is not None:
TableRowInd=markerTable.rowIndices([labels[edbInd],
emis["gc"],
emis["ac"],
"ja","*"])
if len(TableRowInd) >0:
valueStyle=markerStyle1
else:
TableRowInd=markerTable.rowIndices([labels[edbInd],
emis["gc"],
emis["ac"],
"*","ja"])
if len(TableRowInd)>0:
valueStyle=markerStyle2
else:
valueStyle=normalStyle
else:
valueStyle=normalStyle
if subst in ghgList:
col=getColInd(nedbs,ghgList,edbInd,subst)
row=ghgRow[col]
if ghgRow[0]<=+row:
ghgWs.write(row,0,emis["acLev1"],normalStyle)
ghgWs.write(row,1,emis["acLev2"],normalStyle)
ghgWs.write(row,2,emis["gcLev1"],normalStyle)
ghgWs.write(row,3,emis["gcLev2"],normalStyle)
ghgRow[0]+=1
#converts the emission to CO2-ekquivalents
if subst in doubleColumns:
ghgWs.write(row,col-1,float(emisVal),valueStyle)
ghgWs.write(row,col,float(emisVal)*float(ekvFactors[subst]),valueStyle)
else:
ghgWs.write(row,col,float(emisVal),valueStyle)
ghgRow[col]+=1
else:
col=getColInd(nedbs,aqList,edbInd,subst)
row=aqRow[col]
if aqRow[0]<=+row:
aqWs.write(row,0,emis["acLev1"],normalStyle)
aqWs.write(row,1,emis["acLev2"],normalStyle)
aqWs.write(row,2,emis["gcLev1"],normalStyle)
aqWs.write(row,3,emis["gcLev2"],normalStyle)
aqRow[0]+=1
aqWs.write(row,col,float(emisVal),valueStyle)
aqRow[col]+=1
excelBook.save(args.outfile)
log.info("Finished!")
def main():
# Parse command line arguments
parser = argparse.ArgumentParser(description=__doc__)
utils.add_standard_command_options(parser)
parser.add_argument("controlfile", metavar='CONTROLFILE',
action="store",
help="Controlfile for topdown processing")
parser.add_argument("-t", "--template", metavar='TEMPLATEFILE',
action="store",dest="cf",default=None,
help="Generate default controlfile")
args = parser.parse_args()
if args.cf is not None:
generateCf(args.cf)
log.info("Wrote default controlfile")
sys.exit(0)
log.info("Starting topdown processing")
# Opening controlfile
cf = ControlFile(args.controlfile)
dmn = Domain()
log.info("Reading topdown table")
tdTableName = cf.findExistingPath("topDownTable:")
tdTable = DataTable()
tdTable.keys.append("Code")
tdTable.read(tdTableName,delimiter=";")
log.info("Reading national totals table")
natTotalTableName = cf.findExistingPath("nationalTotalTable:")
natTable = DataTable(desc=[{"id": "Code", "type":unicode},
{"id": "description", "type":unicode}])
natTable.keys.append("Code")
natTable.read(natTotalTableName, units=True, defaultType=str)
notationKeys = ["NE", "NO", "NA", "IE"]
log.debug("Remove notation keys from national totals table")
for row in natTable.data:
for i in range(len(row)):
if row[i] in notationKeys:
row[i] = None
log.debug("Convert all emission columns in national totals to float")
for colId in natTable.listIds():
if colId not in ["Code","description"]:
natTable.convertCol(colId,float)
log.debug("Store units from national totals for each substance in dict")
natUnits={}
for col in natTable.desc:
if col.get("units",None)!=None:
natUnits[col["id"]]=col["units"]
log.debug("Read remaining data from control file")
bottomupEdbName = cf.findString("bottomUpEdb:")
topDownEdbName = cf.findString("topDownEdb:")
emissionsEdbName = cf.findString("emissionsEdb:")
userName = cf.findString("user:"******"year:")
#initialize edb objects
buEdb = Edb(dmn,userName,bottomupEdbName)
tdEdb = Edb(dmn,userName,topDownEdbName)
eEdb = Edb(dmn,userName,emissionsEdbName)
log.info("Reading/preparing EDB:s")
log.info("Reading subdb")
subdb = Subdb(eEdb)
subdb.read()
log.info("Reading subgrpdb")
subgrpdb = SubgrpStream(buEdb)
subgrpdb.read()
log.info("Reading facilitydb")
facilityIn = FacilityStream(buEdb)
log.info("Reading companydb")
companyIn = CompanyStream(buEdb)
facilityOut = FacilityStream(eEdb,mode="w")
companyOut = CompanyStream(eEdb,mode="w")
log.info("Writing company db to result edb")
companyOut.write(companyIn.read())
log.info("Writing facility db to result edb")
facilityOut.write(facilityIn.read())
if not buEdb.exists():
log.error("Edb " + buEdb.name + " does not exist for user " + userName +
" in domain " + dmn.name)
sys.exit(1)
if not tdEdb.exists():
log.error("Edb " + tdEdb.name + " does not exist for user " + userName +
" in domain " + dmn.name)
sys.exit(1)
if not eEdb.exists():
log.error("Edb " + eEdb.name + " does not exist for user " + userName +
" in domain " + dmn.name)
sys.exit(1)
keys = tdEdb.listGrids()
msg = "%i keys found in edb: %s" % (len(keys), tdEdb.name)
log.info(msg)
# sourcedb from bottom-up edb
with SourceStream(buEdb, mode='rb') as source_instream:
source_reader = ModelReader(source_instream)
bu_sources = list(source_reader)
log.info(
"%i point sources found in edb: %s" % (
len(bu_sources),
buEdb.name)
)
# Empty sourcedb of the result edb
if cf.findBoolean("emptyEmissionSourcedb:"):
eEdb.empty_sourcedb()
e_sources = []
log.info("Removed point sources from edb: %s" % (eEdb.name))
else:
# sourcedb from emission edb (result edb)
with SourceStream(eEdb, mode='rb') as source_instream:
source_reader = ModelReader(source_instream)
e_sources = list(source_reader)
msg = "%i point sources found in edb: %s" % (len(e_sources), eEdb.name)
log.info(msg)
if not path.exists(eEdb.rsrcPath()):
log.error("No edb.rsrc exists for emission edb")
sys.exit()
else:
rsrc = Rsrc(eEdb.rsrcPath())
acIndex = cf.findInt("acIndex:")
codeDepth = rsrc.ac[acIndex-1].depth
substances = cf.findStringList("substances:")
for subst in substances:
if subst not in subdb.substIndices:
log.error("Substance: " + subst + " not in Airviro substance list")
sys.exit()
# Initialize trace for debug and additional logging
if cf.findBoolean("trace:") == True:
log.info("Initializing trace for detailed logging")
trace = TraceDef(
active=True,
substances=cf.findStringList("trace.substances:"),
logfile=cf.findString("trace.logfile:"),
regdefgc=cf.findIntList("trace.regdef.gc:",
optional=True,
default=None),
gcDefRaster=cf.findExistingPath("trace.gcraster:")
)
else:
trace = TraceDef(active=False)
log.info("Initializing result table")
resTablePath = cf.findString("resTable:")
resTable = DataTable(desc=[{"id": "Code", "type": unicode}])
resTable.keys.append("Code")
for subst in substances:
resTable.addCol({"id": subst, "type": float, "unit": "%"})
# Create emission grid template (with geocodes)
log.info("Reading emission grid template")
eGridTemplatePath = cf.findExistingPath("emisGridTemplatePath:")
eGridTemplate = Egrid(eEdb,"name")
if eGridTemplatePath[-4:] == ".asc":
eGridTemplatePath=eGridTemplatePath[:-4]
eGridTemplate.readData(eGridTemplatePath)
eGridTemplate.substances = {}
eGridTemplate.par["SUBSTANCE"].val = []
dd = {"key": None,
"regstat": None,
"regdef": None,
"bu_sources": bu_sources,
"psIndices": [],
"units": natUnits,
"rsrc": rsrc,
"subdb": subdb,
"trace": trace,
"subgrpdb": subgrpdb
}
# Process all rows in the topdown table
for row in tdTable.data:
code = row[tdTable.colIndex["Code"]]
active = row[tdTable.colIndex["Active"]]
statType = row[tdTable.colIndex["Stat_type"]]
if active == "no":
continue
log.info("Code: "+code)
distributed=False
# Add '-' to the code to reach max length (fix for a GUI bug)
airviroCode = code
# while len(airviroCode.split(".")) < codeDepth:
# airviroCode += ".-"
tdrow = tdTable.data[tdTable.rowIndex([code])]
nrow = natTable.data[natTable.rowIndex([code])]
# Create a resTable row to fill with data
resrow = [None] * resTable.ncols
resrow[0] = code
# Check if national totals are non-zero
nonZero = False
for val in nrow:
if val != None:
if val > 0:
nonZero = True
break
# Filter out indices for pointsources with the current ac
# Also including sources coded with sub-codes
# This allows to estimate top-down emissions on a higher code-level
psIndices = []
for i, ps in enumerate(bu_sources):
codeMatch = False
for emis in ps.EMISSION:
# It is assumed that the first code is used while processing topdown
ac = emis.ACTCODE[0]
if ac[-1] == ".":
ac=ac[:-1]
# if ac[:len(code)] == code:
if ac == code:
codeMatch = True
break
if not codeMatch:
for emis in ps.SUBGRP:
# It is assumed that the first code is used while processing topdown
ac = emis.ACTCODE[0]
if ac[:len(code)] == code:
codeMatch = True
break
if codeMatch:
psIndices.append(i)
dd["psIndices"] = psIndices
keyName = row[tdTable.colIndex["Key"]]
#If no distribution key specified and no ps in bottom-up edb - cont.
if keyName is None and psIndices == []:
log.debug("No key and no point sources found for code: %s, skipping..." % code)
resTable.addRow(resrow)
continue
if psIndices!=[]:
msg = "--Found %i pointsources" % len(psIndices)
log.info(msg)
if keyName is not None:
if keyName not in keys:
log.error("No such key: " + keyName)
sys.exit()
msg = "--Key: %s" % keyName
log.info(msg)
keyGrid = Egrid(tdEdb, keyName)
keyGrid.readData()
log.debug("Read key: " + keyName + " from topdownEdb")
# create emission grid to store distributed emissions
eGrid = deepcopy(eGridTemplate)
eGrid.name = code.replace(".", "_")
eGrid.par["NAME"].val = code
eGrid.par["INFO2"].val = "Distribution key: " + keyGrid.par["NAME"].val
eGrid.par["ACTIVITYCODE"].val = [airviroCode.split(".")]
regstatName = row[tdTable.colIndex["Regstat"]]
regdefName = row[tdTable.colIndex["Regdef"]]
if regstatName is not None:
if regdefName is None:
log.error("No region definition given for regional statistics: " +
regstatName)
sys.exit(1)
regstatPath = path.join(dmn.domainPath(), "topdown", "regstat", regstatName)
regstat = DataTable()
log.info("regstatPath: "+regstatPath)
regstat.read(regstatPath, units=True, defaultType=float, delimiter=";")
if not "Geocode" in regstat.listIds():
log.error("No Geocode column found in regstat")
sys.exit(1)
regstat.convertCol("Geocode", int)
regstat.keys.append("Geocode") # Making Geocode the primary key
# create list of unique geo codes
geocodes = [row[regstat.colIndex["Geocode"]] for row in regstat.data]
geocodes = unique(geocodes)
for colId in regstat.listIds():
if colId.lower() == "year":
rows = []
regstat.convertCol(colId, int)
# Make it possible to accumulate year
regstat.setKeys(regstat.keys + [colId])
# Calculates the total emission for each geocode
# in case there are multiple rows for different fuels etc
colsToSum = regstat.listIds()
colsToSum.remove(colId)
colsToSum.remove("Geocode")
for gc in geocodes:
# sums all numeric values in colsToSum for
# rows matching row id [gc,year]
#returns an accumulated row and appends it to rows
rowId = regstat.dict2RowId({"Geocode": gc, colId: year})
rows.append(regstat.accumulate(rowId, "sum", colsToSum))
regstat.data = rows # replace original rows with accumulated rows
regstat.keys.remove(colId)
break
# dd["regstat"] = regstat
regdef = Raster()
regdefPath = path.join(dmn.domainPath(), "topdown", "regdef", regdefName)
regdef.read(regdefPath)
dd["regstat"] = regstat
dd["regdef"] = regdef
else:
dd["regstat"] = None
dd["regdef"] = None
if dd["regstat"] is not None and len(bu_sources) > 0 and statType == "fixed":
log.info("--Regionalizing pointsources")
dd = regionalizePS(dd, code)
if keyName is not None and nonZero:
regionalizedDefault = False
# Spatial distribution of emissions
for subst in substances:
sInd = subdb.substIndices[subst]
toUnit = dd["units"][subst] + "/year"
ntot = nrow[natTable.colIndex[subst]]
pstot = 0
for i in dd["psIndices"]:
source = dd["bu_sources"][i]
# TODO: should give reference to subgrps to include emis from them
pstot += source.get_emis(
sInd,
toUnit,
eEdb,
actcodes=[code]
)
if ntot is None or ntot == 0:
if pstot > 0:
# 9999 is used as marker for no national total
resrow[resTable.colIndex[subst]] = 9999.0
log.warning(
"Nattot is 0 but ps tot is: %f %s" % (pstot, toUnit))
continue
nrest = ntot - pstot
resrow[resTable.colIndex[subst]] = 100.0
if abs(nrest / ntot) < 0.0001:
nrest = 0
log.info(
"--Rest is < 0.01 % of national total, rounded to zero"
)
continue
elif nrest < 0:
log.warning(
"--National rest is below zero, %4.2f proc for %s" % (
-1 * nrest / ntot * 100,
subst)
)
dd["trace"].write()
# continue
log.info(
"---Substance: "+subst+
", rest is: " + str(nrest) +
toUnit + " = " + str(nrest / ntot * 100.0) + "%"
)
try:
keyRast = keyGrid.substances[sInd]
except KeyError:
keyRast = keyGrid.substances[subdb.substIndices["all"]]
dd["key"] = keyRast
if dd["regstat"] is not None:
if (subst not in regstat.colIndex and
sInd not in keyGrid.substances and not regionalizedDefault):
dd = regionalizeKey(dd, subst, code)
regionalizedDefault = True
else:
dd = regionalizeKey(dd, subst, code)
emisRast = distribute(dd["key"], nrest)
emisRast = emisRast * unitConvFac(toUnit, "ton/year")
eGrid.addData(emisRast, dd["subdb"].substIndices[subst])
distributed = True
else:
# resTable is filled
# In case all national totals are zero but there are ps
for subst in substances:
sInd = dd["subdb"].substIndices[subst]
toUnit = dd["units"][subst] + "/year"
ntot = nrow[natTable.colIndex[subst]]
pstot = 0
for i in dd["psIndices"]:
source = dd["bu_sources"][i]
# subgrps are not used!
pstot += source.get_emis(sInd, toUnit, buEdb,
actcodes=[code])
if ntot!=0 and ntot is not None:
resrow[resTable.colIndex[subst]] = pstot / ntot * 100.0
else:
resrow[resTable.colIndex[subst]] = -999.0
if len(dd["psIndices"]) > 0:
tmp_sources = (bu_sources[i] for i in dd["psIndices"])
with SourceStream(eEdb, mode='wb') as out_source_stream:
source_writer = ModelWriter(out_source_stream)
for source in tmp_sources:
source_writer.write(source)
log.debug("Wrote ps to emission edb")
if distributed:
eGrid.load()
log.debug("Wrote emission grid to emission edb")
dd["trace"].write()
resTable.addRow(resrow)
resTableFile = open(resTablePath,"w")
resTable.write(resTableFile)
log.info("Finished topdown process")