def main(): #-----------Setting up and using option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option( '-v', action='store_const', dest='loglevel', const=logging.DEBUG, default=get_loglevel(), help='produce verbose output' ) # parser.add_option("-l", "--loglevel", # action="store",dest="loglevel",default=2, # help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") (options, args) = parser.parse_args() #------------Setting up logging capabilities ----------- logging.basicConfig( format='%(levelname)s:%(name)s: %(message)s', level=options.loglevel, ) global log log = logging.getLogger(parser.prog) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") #Opening controlfile cf=ControlFile(fileName=path.abspath(args[0])) domain = Domain() edbName=cf.findString("edb:") userName=cf.findString("user:"******"Edb "+edbName+" does not exist for user "+userName+" in domain "+domain.name) sys.exit() year=cf.findInt("year:") substList=cf.findStringList("substances:") #Remove doubles from substance list substances={'nat':[]} for subst in substList: if subst not in substances['nat']: substances['nat'].append(subst) #Make dictionary for substance,units couples unitDict={} defaultUnit=cf.findString("nationalTotals.defaultUnit:") unitList=cf.findStringList("nationalTotals.units:") for i in range(len(unitList)): unitSubstances=cf.findStringList("nationalTotals.unit."+str(i)+":") for s in unitSubstances: unitDict[s]=unitList[i] for subst in substances['nat']: if subst not in unitDict.keys(): unitDict[subst]=defaultUnit nationalTotalsTablePath=cf.findString("nationalTotalsTable:") nationalTotalsFile=codecs.open(nationalTotalsTablePath,'w',encoding="latin6") nationalTotalsTreePath=cf.findString("nationalTotalsTree:") nationalTotalsTreePath=path.abspath(nationalTotalsTreePath) nationalTotalTreeFile=open(nationalTotalsTreePath,'w') log.info("Reading emissions from CRF") CRFTree=CodeEmisTree(name="CRF") CRFPath=cf.findExistingPath("CRF.path:") CRFTree.parseCRFXML(CRFPath,year) CRFTree.calculateSums(byAttribute="Category") #CRFTree.write(sys.stdout) CRFTable=CRFTree.createTable(keys=["Category","Classification","AWMS"]) HFC={ "HFC-23":11700, "HFC-32":650, "HFC-41":150, "HFC-43-10 mee":1300, "HFC-125":2800, "HFC-134":1000, "HFC-134a":1300, "HFC-152a":140, "HFC-143":300, "HFC-143a":3800, "HFC-227ea":2900, "HFC-236fa":6300, "HFC-245ca":560 } PFC={ "CF4":6500, "C2F6":9200, "C3F8":7000, "C4F10":7000, "c-C4F8":8700, "C5F12":7500, "C6F14":7400 } CRFTable.aggregateCols({"id":"HFC-tot","type":"str","units":"Gg"},HFC) CRFTable.aggregateCols({"id":"PFC-tot","type":"str","units":"Gg"},PFC) CRFTable.sortRows() CRFTableKeys=cf.findStringList("CRF.tableKeys:") CRFTable.keys=CRFTableKeys #CRFTable.write(sys.stdout) log.info("Reading emissions from NFR") NFRTree=CodeEmisTree(name="NFR") NFRPath=cf.findExistingPath("NFR.path:") # pdb.set_trace() NFRTree.parseNFRTable(filename=NFRPath) NFRTree.calculateSums() NFRTable=NFRTree.createTable() NFRTableKeys=cf.findStringList("NFR.tableKeys:") NFRTable.keys=NFRTableKeys colIndex=NFRTable.getColIndex("Code") for row in NFRTable.data: if isinstance(row[colIndex],str): row[colIndex]=row[colIndex].replace(" ","") log.info("Reading emissions from Mobile") MobilePath=cf.findExistingPath("Mobile.path:") MobileTable=DataTable() MobileTable.read(MobilePath,units=True) MobileTable=MobileTable.filtered({"Year":str(year)}) MobileTableKeys=cf.findStringList("Mobile.tableKeys:") MobileTable.keys=MobileTableKeys #Remove biogenic emissions of CO2 #Remove GHG from Bunkers, except for international aviation LTO for row in MobileTable.data: # if "Bio" in row[MobileTable.getColIndex("Codename")]: if "Bio" in row[MobileTable.getColIndex("Fueltype")]: row[MobileTable.getColIndex("CO2")]=None # if row[MobileTable.getColIndex("Localcodename")]=="Bunkers" and row[MobileTable.getColIndex("Localsubcodename")]!="LTO": if row[MobileTable.getColIndex("Subsector")]=="Bunkers" and row[MobileTable.getColIndex("Region")]!="LTO": row[MobileTable.getColIndex("CO2")]=None row[MobileTable.getColIndex("CH4")]=None row[MobileTable.getColIndex("N2O")]=None log.info("Reading emissions from Manual") ManualPath=cf.findExistingPath("Manual.path:") ManualTable=DataTable() ManualTable.read(ManualPath,units=True) ManualTableKeys=cf.findStringList("Manual.tableKeys:") ManualTable.keys=ManualTableKeys srcTables={'NFR':NFRTable, 'CRF':CRFTable, 'Mobile':MobileTable, 'Manual':ManualTable} for tKey in srcTables.keys(): substList=cf.findStringList(tKey+".substances:") substances[tKey]=substList log.info("Reading dataSourceTable") dsPath=cf.findExistingPath("sourceTable:") dsTable=DataTable() dsTable.read(dsPath) acIndex=cf.findInt("nationalTotals.acIndex:") log.info("Reading activity codes from edb") natTree=CodeEmisTree("National totals","Gg") natTree.readActivityCodes(path.join(edb.edbPath(),"edb.rsrc"),acIndex) log.info("Identifying column indices for sources in dataSourceTable") #codeHeaders={"Mobile":"Mobile CRFnodeprefix", # "NFR":"NFR Code", # "CRF":"CRF Code", # "nat":"Code", # "Manual":"Code" # } codeHeaders={"Mobile":"Mobile UNFCCC", "NFR":"NFR Code", "CRF":"CRF Code", "nat":"Code", "Manual":"Code" } codeInd={} for key,colName in codeHeaders.iteritems(): try: codeInd[key]=dsTable.getColIndex(colName) except DataTableException: log.error("Could not find '"+colName+"' in dataSourceTable") sys.exit() log.info("Collecting data from data sources") prevNatCode="" for row in dsTable.data: rowId={} srcRow={} for key in srcTables: rowId[key]=[] srcRow[key]=srcTables[key].ncols*[0] natCode=row[codeInd['nat']] if natCode!=prevNatCode: log.debug("\nProcessing: "+natCode) prevNatCode=natCode #Get row-id for each src table for tKey in srcTables.keys(): colInd=codeInd[tKey] if row[colInd]!=None: for key in srcTables[tKey].keys: try: idPart=row[dsTable.getColIndex(tKey+" "+key)] except ValueError: log.error("No column named '"+tKey+" "+key+"' found in dataSourceTable") sys.exit() rowId[tKey].append(idPart) #If not all key values = None nonNones=[val for val in rowId[tKey] if val !=None] if len(nonNones)==0: rowId[tKey]=None srcRow[tKey]=None else: # if tKey=="CRF": # pdb.set_trace() try: srcRow[tKey]=srcTables[tKey].accumulate(rowId[tKey]) except DataTableException, err: # import pdb; pdb.set_trace() log.error("While trying to get data from "+tKey+" table: "+str(err)) sys.exit() #Creating code tree path for the current code codeParts=natCode.split(".") natPath="" for i in range(1,len(codeParts)+1): natPath+=".".join(codeParts[:i]) if i < len(codeParts): natPath+="/" natNode=natTree.root.find(natPath) if natNode is None: # import pdb; pdb.set_trace() log.error("Could not find national code path : "+natCode) sys.exit() for subst in substances['nat']: log.debug("Substance: "+subst) storedSubstances=[] foundSubstances=[] rec=None #Looping over all sourceTables for tKey in srcTables: st=srcTables[tKey] if rowId[tKey]!=None and subst in substances[tKey]: try: colIndex=st.getColIndex(subst) except DataTableException: log.error("Could not find column %s in table %s" %(subst,tKey)) emis=srcRow[tKey][colIndex] unit=st.desc[colIndex].get('units',None) storedSubstances.append(subst) nodeExists=False if emis!=0 and emis!=None: if subst not in foundSubstances: foundSubstances.append(subst) else: log.error("Substance found in multiple datasources for "+natCode) sys.exit() for node in natNode: if node.tag=="Record" and node.attrib.get("substance",None)==subst: rec=node nodeExists=True break if not nodeExists: rec=ET.SubElement(natNode,"Record") break if unit==None: log.error("No units set for substance "+subst+" in source table "+tKey) sys.exit() if storedSubstances==[]: log.error("No data source specified for substance "+subst) sys.exit() if rec!=None: if emis!=None and emis!=0: try: emis=convertUnits(fromUnit=unit,toUnit=unitDict[subst],val=emis) except TypeError: emis=0 if not nodeExists: rec.attrib["substance"]=subst else: emis+=float(rec.attrib["emission"]) rec.attrib["emission"]=str(emis)