def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-u", "--user", action="store", dest="user", default=None, help="Specify user manually") parser.add_option("-e", "--edb", action="store", dest="edb", default=None, help="Name of target edb") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-f", "--force", action="store_true",dest="force",default=False, help="To start the process without confirming the domain") (options, args) = parser.parse_args() # Setup logging logging.configure(terminal_level=logging.DEBUG) log = logging.getLogger(__name__) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") if options.edb ==None: parser.error("Need to specify edb using flag -e") if options.user ==None: parser.error("Need to specify user using flag -u") dmn = Domain() if not options.force: answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?") if answer!="y": sys.exit("Interrupted by user") if not dmn.edbExistForUser(options.edb,options.user): log.error("Edb "+options.edb+" does not exist for user "+ options.user+" in domain "+dmn.name) sys.exit() #---Creating edb and rsrc objects------------------ edb=Edb(dmn,options.user,options.edb) rsrc=Rsrc(edb.rsrcPath()) #Opening controlfile #---retrieving data from control file---- cf=ControlFile(fileName=path.abspath(args[0])) substances=cf.findStringList("substances:") outputDir=cf.findExistingPath("outputDir:") acIndex=cf.findInt("acIndex:") macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:")) fromProj=cf.findString("fromProj:") toProj=cf.findString("toProj:") try: fromProj=transcoord.proj4Dict[fromProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj) try: toProj=transcoord.proj4Dict[toProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj) formats = cf.findStringList("formats:") units = cf.findString("units:") writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True) edb_xll=cf.findInt("edb_xll:") edb_yll=cf.findInt("edb_yll:") edb_ncols=cf.findInt("edb_ncols:") edb_nrows=cf.findInt("edb_nrows:") edb_cellsize=cf.findFloat("edb_cellsize:") if fromProj!=toProj: out_xll=cf.findFloat("out_xll:") out_yll=cf.findFloat("out_yll:") out_ncols=cf.findInt("out_ncols:") out_nrows=cf.findInt("out_nrows:") out_cellsize=cf.findFloat("out_cellsize:") #----------------------------------------- #Finds index to search units unitIndex=None for key,unit in rsrc.search.iteritems(): if isinstance(key,int): if rsrc.search[key]==units: unitIndex=key break if unitIndex is None: log.error("Search units: %s not defined in edb.rsrc" %units) sys.exit() macro = ControlFile(fileName=macroFileName,removeComments=False) #preparing export macro macro.setParam("general.database:",dmn.name) xmin=edb_xll xmax=edb_xll+edb_ncols*edb_cellsize ymin=edb_yll ymax=edb_yll+edb_nrows*edb_cellsize macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("edb.user:"******"edb.edb:",edb.name) macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("USER :"******"EDB :",edb.name) macro.setParam("GRID :", "%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize)) macro.setParam("edb.unit:",unitIndex) macro.setParam("UNIT :",unitIndex) # macro.setParam("NOACTCODE :",acIndex) macro.setParam("NOACTCODE :",len(rsrc.ac)) #Get activity code tree acTree=codeemistree.CodeEmisTree("Activity codes",units=units) acTree.readActivityCodes(rsrc.path,acIndex) substDict=dmn.listSubstanceIndices() edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols, Nrows=edb_nrows,Cellsize=edb_cellsize, Nodata=-9999,init=0) if fromProj!=toProj: outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999) else: outRastTemplate=Raster() outRastTemplate.assign(edbRast) for node in acTree.root.getiterator(): if node.tag=="root" or node.tag=="Record": continue ac=node.tag log.debug("Activity code: "+ac) #Finds row index for activity codes in macro #Add a row with the current ac #If there are many ac already given, these are #replaced by the current ac macroLines=macro.content.split("\n") actCodeInd=None geoCodeInd=None for lineInd,line in enumerate(macroLines): if "NOACTCODE" in line: actCodeInd=lineInd if "NOGEOCODE" in line: geoCodeInd=lineInd if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] else: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] macro.content="\n".join(macroLines) macro.write() #boolean raster marking where there is data for any of the substances if 'CLRTAP' in formats: dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999,init=0) rasterDict={} substancesWithData=[] for substance in substances: log.debug("Substance %s" %substance) substanceIndex=substDict[substance] macro.setParam("ELEMENT :",substanceIndex) macro.write() command="xrepedb -i "+macro.name (returnCode,errMsg,outMsg)=utilities.execute(command) tmp=outMsg.split()[10:-2] tmp.sort() if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00': print "ERROR: The field for "+substance+" is empty!" continue # pdb.set_trace() emisRast=string2rast(outMsg,edbRast) emisSum=emisRast.sum() outRast=Raster() outRast.assign(outRastTemplate) rec=ET.SubElement(node,"Record") rec.attrib["substance"]=substance rec.attrib["emission"]=str(emisSum) if emisSum>0 and writeGrids: if substance not in substancesWithData: substancesWithData.append(substance) if fromProj!=toProj: exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir()) else: exportRast=emisRast if 'CLRTAP' in formats: dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data) rasterDict[substance]=exportRast categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) if 'ESRI Ascii grid' in formats: fileName = path.join(categoryDirPath, substance+ ".asc") exportRast.write(fileName) log.debug("Grid for " + substance + "written to outputDir for category: " + ac) summaryTable=acTree.createTable(writeAll=True) summaryTable.sortRows() tableFile=open(path.join(outputDir,"summaryTable.txt"),'w') summaryTable.write(tableFile) if len(rasterDict)>0 and 'CLRTAP' in formats: #creating substance header in the same order as the substances in the template header = "i\tj\t" #headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"] for s in substancesWithData: header += s + "\t" #remove the tab after the last column and add a newline instead header = header[: - 1]+ "\n" #Creating file for EMEP-data fileName = "CLRTAP_" + ac + ".txt" categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) fid = open(path.join(categoryDirPath, fileName), 'w') fid.writelines(header) sum=0 #Writing indexes and data for all non-zero elements for row in range(dataMarker.nrows): for col in range(dataMarker.ncols): if dataMarker.data[row, col] > 0: (i, j) = dataMarker.getCentreCoords(row, col) fid.write(str(i) + "\t" + str(j) + "\t") for substWithData in substancesWithData[:-1]: fid.write(str(rasterDict[substWithData].data[row, col]) + "\t") sum+=rasterDict[substWithData].data[row, col] fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n") sum+=rasterDict[substancesWithData[-1]].data[row, col] fid.close() log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName)) log.info("Finished")
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-f", "--force", action="store_true",dest="force",default=False, help="To start the process without confirming the domain") parser.add_option("-n", "--nproc", action="store",dest="nproc",default=1, help="Number of gifmap processes to run at a time") (options, args) = parser.parse_args() #------------Setting up logging capabilities ----------- rootLogger=logger.RootLogger(int(options.loglevel)) log=rootLogger.getLogger(sys.argv[0]) #logger=basicLogger(int(options.loglevel),sys.argv[0]) if options.cf!=None: controlfile.generateCf(path.abspath(options.cf),controlFileTemplate) print("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") domainName=os.environ["AVDBNAME"] dmn = domain.Domain(domainName) if not options.force: answer=raw_input("Chosen dbase is: "+domainName+",continue(y/n)?") if answer=="y": dmn=domain.Domain() else: sys.exit("Interrupted by user") #Opening controlfile #---retrieving data from control file---- cf=controlfile.ControlFile(fileName=path.abspath(args[0])) #Get search parameters from control file substances=cf.findStringList("substances:") outputDir=cf.findExistingPath("outputDir:") baseSearches=cf.findStringList("searches:") fromProj=cf.findString("fromProj:") toProj=cf.findString("toProj:") resample = cf.findBoolean("resample:") dmn=domain.Domain(os.environ.get('AVDBNAME')) substDict=dmn.listSubstanceIndices() if resample: #Get proj4 projection definitions from aliases try: fromProj=transcoord.proj4Dict[fromProj] except KeyError: print("Projection %s not found in proj4Dictin transCoord.py" %fromProj) try: toProj=transcoord.proj4Dict[toProj] except KeyError: print("Projection %s not found in proj4Dictin transCoord.py" %toProj) #Get output grind parameters from controlfile out_xll=cf.findFloat("out_xll:") out_yll=cf.findFloat("out_yll:") out_ncols=cf.findInt("out_ncols:") out_nrows=cf.findInt("out_nrows:") out_cellsize=cf.findFloat("out_cellsize:") #Build list with search definitions searches=[] for search_id in baseSearches: prefix=cf.findString("search."+search_id+".prefix:") macroPath=cf.findString("search."+search_id+".macro:") starttime=cf.findString("search."+search_id+".starttime:") endtime=cf.findString("search."+search_id+".endtime:") parVals = cf.findParam("search."+search_id+".par.",findAll=True) parDict={} for p in parVals: ind=p.index(":") par = p[:ind+1] val=p[ind+1:].strip() parDict[par]=val alobVals = cf.findParam("search."+search_id+".alob.",findAll=True) alobDict={} for a in alobVals: ind=a.index(":") key = a[:ind] val=a[ind+1:].strip() alobDict[par]=val macro = controlfile.ControlFile(fileName=macroPath,removeComments=False) unitIndex=macro.findInt("edb.unit:") edbName=macro.findString("edb.edb:") userName=macro.findString("edb.user:"******"FROM :",starttime) macro.setParam("TO :",endtime) for key,val in parDict.iteritems(): macro.setParam(key,val) for key,val in alobDict.iteritems(): macro.setParam(key,"R"+val,addSpace=False) edb=Edb(dmn.name,userName,edbName) rsrc=Rsrc(edb.rsrcPath()) unitName=rsrc.search[unitIndex] for substance in substances: filename=path.join(outputDir,prefix+"_"+substance.replace(" ","_")+".asc") searches.append( {'id':search_id, 'macro':macro, 'prefix':prefix, 'substance':substance, 'unit':unitName, 'filename':filename } ) if resample: summaryTable=datatable.DataTable(desc=[{"id":'search_id',"type":unicode}, {"id":'substance',"type":unicode}, {"id":'macro',"type":unicode}, {"id":'filename',"type":unicode}, {"id":'sum',"type":float}, {"id":'proj_sum',"type":float}, {"id":'unit',"type":unicode}], keys=["search_id","substance"]) else: summaryTable=datatable.DataTable(desc=[{"id":'search_id',"type":unicode}, {"id":'substance',"type":unicode}, {"id":'macro',"type":unicode}, {"id":'filename',"type":unicode}, {"id":'sum',"type":float}, {"id":'unit',"type":unicode}], keys=["search_id","substance"]) #Inititalising parallell run nproc=int(options.nproc) #numer of processes to run in parallell running_proc=0 #counter for number of running processes pids={} #dictionary to store process output and info todo=len(searches) #list of processes to be run searchInd=0 #index of the process def countRunning(pids): """Count the number of running processes""" for pid, props in pids.iteritems(): props['ret_val']=props['proc'].poll() return len([pid for pid in pids if pids[pid]['ret_val'] is None]) while todo+running_proc>0: #loop until all processes have finished if running_proc==nproc or todo==0: print("Running gifmap...") #When nproc processes are started, the program waits here #until one is finished before adding a new one while countRunning(pids)==running_proc: time.sleep(5) #mark first found finished process as done for pid,props in pids.iteritems(): if not props['ret_val'] is None and not props['done']: props['done']=True break command=props["cmd"] if props['ret_val']!=0: errMsg=pids[pid]["proc"].stderr.read() print("Error while running command: %s\n%s" %(pids[pid],errMsg)) sys.exit(1) print("Finished search %s" %props['search_id']) #Find search in search list running_proc-=1 for f_search in searches: if f_search["id"]==props['search_id'] and f_search['substance']==props['substance']: break f_search['done']=True try: f_search['res']=res2rast(f_search['filename']) #Store original result from search f_search['sum']=f_search['res'].sum() except: f_search['res']=None output=pids[pid]["proc"].stdout.read() f_search['sum']=re.compile("#EMIS (\S*).*").search(output).group(1) print "Could not extract result raster from %s" %f_search['filename'] print "Uses total from gifmap log" if resample: outRast = raster.Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999) #Run post-processing of result for finsished processes if f_search['sum']>0: print("Projecting result to target CRS") f_search['res'] = transcoord.transformEmisRaster(f_search['res'],outRast, fromProj,toProj, tmpDir=dmn.tmpDir()) #Store projected result from search f_search['proj_sum']=f_search['res'].sum() else: f_search['proj_sum']=0 #Write result to file if f_search['res'] is not None: f_search['res'].write(f_search['filename']) #Add total emission to summary table if resample: summaryTable.addRow( [f_search['id'], f_search['substance'], f_search['macro'].name, f_search['filename'], f_search['sum'], f_search['proj_sum'], f_search['unit']]) else: summaryTable.addRow( [f_search['id'], f_search['substance'], f_search['macro'].name, f_search['filename'], f_search['sum'], f_search['unit']]) #Add another search process elif todo>0: c_search=searches[searchInd] substance=c_search["substance"] substanceIndex=substDict[substance] c_search['macro'].setParam("ELEMENT :",substanceIndex) c_search['macro'].write() command="gifmap -T -i "+c_search['macro'].name+" -o "+c_search['filename'] p=subprocess.Popen(command,stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True) #Add info and handles to process dictionary pids[p.pid]={"proc":p, "search_id":c_search['id'], "substance":c_search['substance'], "cmd":command, 'ret_val':None, 'done':False} print( "Started search %s, substance %s" %( c_search['id'],c_search['substance'])) running_proc+=1 todo-=1 searchInd+=1 #Each process needs some time to read #the macro before it is changed time.sleep(10) #summaryTable.sortRows() tableFile=open(path.join(outputDir,"summaryTable.txt"),'w') summaryTable.write(tableFile) print("Finished")