def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-u", "--user",
action="store", dest="user", default=None,
help="Specify user manually")
parser.add_option("-e", "--edb",
action="store", dest="edb", default=None,
help="Name of target edb")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-f", "--force",
action="store_true",dest="force",default=False,
help="To start the process without confirming the domain")
(options, args) = parser.parse_args()
# Setup logging
logging.configure(terminal_level=logging.DEBUG)
log = logging.getLogger(__name__)
if options.cf!=None:
generateCf(path.abspath(options.cf))
log.info("Wrote default controlfile")
sys.exit()
if len(args)!=1:
parser.error("Incorrect number of arguments")
if options.edb ==None:
parser.error("Need to specify edb using flag -e")
if options.user ==None:
parser.error("Need to specify user using flag -u")
dmn = Domain()
if not options.force:
answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?")
if answer!="y":
sys.exit("Interrupted by user")
if not dmn.edbExistForUser(options.edb,options.user):
log.error("Edb "+options.edb+" does not exist for user "+
options.user+" in domain "+dmn.name)
sys.exit()
#---Creating edb and rsrc objects------------------
edb=Edb(dmn,options.user,options.edb)
rsrc=Rsrc(edb.rsrcPath())
#Opening controlfile
#---retrieving data from control file----
cf=ControlFile(fileName=path.abspath(args[0]))
substances=cf.findStringList("substances:")
outputDir=cf.findExistingPath("outputDir:")
acIndex=cf.findInt("acIndex:")
macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:"))
fromProj=cf.findString("fromProj:")
toProj=cf.findString("toProj:")
try:
fromProj=transcoord.proj4Dict[fromProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj)
try:
toProj=transcoord.proj4Dict[toProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj)
formats = cf.findStringList("formats:")
units = cf.findString("units:")
writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True)
edb_xll=cf.findInt("edb_xll:")
edb_yll=cf.findInt("edb_yll:")
edb_ncols=cf.findInt("edb_ncols:")
edb_nrows=cf.findInt("edb_nrows:")
edb_cellsize=cf.findFloat("edb_cellsize:")
if fromProj!=toProj:
out_xll=cf.findFloat("out_xll:")
out_yll=cf.findFloat("out_yll:")
out_ncols=cf.findInt("out_ncols:")
out_nrows=cf.findInt("out_nrows:")
out_cellsize=cf.findFloat("out_cellsize:")
#-----------------------------------------
#Finds index to search units
unitIndex=None
for key,unit in rsrc.search.iteritems():
if isinstance(key,int):
if rsrc.search[key]==units:
unitIndex=key
break
if unitIndex is None:
log.error("Search units: %s not defined in edb.rsrc" %units)
sys.exit()
macro = ControlFile(fileName=macroFileName,removeComments=False)
#preparing export macro
macro.setParam("general.database:",dmn.name)
xmin=edb_xll
xmax=edb_xll+edb_ncols*edb_cellsize
ymin=edb_yll
ymax=edb_yll+edb_nrows*edb_cellsize
macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("edb.user:"******"edb.edb:",edb.name)
macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("USER :"******"EDB :",edb.name)
macro.setParam("GRID :",
"%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize))
macro.setParam("edb.unit:",unitIndex)
macro.setParam("UNIT :",unitIndex)
# macro.setParam("NOACTCODE :",acIndex)
macro.setParam("NOACTCODE :",len(rsrc.ac))
#Get activity code tree
acTree=codeemistree.CodeEmisTree("Activity codes",units=units)
acTree.readActivityCodes(rsrc.path,acIndex)
substDict=dmn.listSubstanceIndices()
edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols,
Nrows=edb_nrows,Cellsize=edb_cellsize,
Nodata=-9999,init=0)
if fromProj!=toProj:
outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999)
else:
outRastTemplate=Raster()
outRastTemplate.assign(edbRast)
for node in acTree.root.getiterator():
if node.tag=="root" or node.tag=="Record":
continue
ac=node.tag
log.debug("Activity code: "+ac)
#Finds row index for activity codes in macro
#Add a row with the current ac
#If there are many ac already given, these are
#replaced by the current ac
macroLines=macro.content.split("\n")
actCodeInd=None
geoCodeInd=None
for lineInd,line in enumerate(macroLines):
if "NOACTCODE" in line:
actCodeInd=lineInd
if "NOGEOCODE" in line:
geoCodeInd=lineInd
if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
else:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
macro.content="\n".join(macroLines)
macro.write()
#boolean raster marking where there is data for any of the substances
if 'CLRTAP' in formats:
dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999,init=0)
rasterDict={}
substancesWithData=[]
for substance in substances:
log.debug("Substance %s" %substance)
substanceIndex=substDict[substance]
macro.setParam("ELEMENT :",substanceIndex)
macro.write()
command="xrepedb -i "+macro.name
(returnCode,errMsg,outMsg)=utilities.execute(command)
tmp=outMsg.split()[10:-2]
tmp.sort()
if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00':
print "ERROR: The field for "+substance+" is empty!"
continue
# pdb.set_trace()
emisRast=string2rast(outMsg,edbRast)
emisSum=emisRast.sum()
outRast=Raster()
outRast.assign(outRastTemplate)
rec=ET.SubElement(node,"Record")
rec.attrib["substance"]=substance
rec.attrib["emission"]=str(emisSum)
if emisSum>0 and writeGrids:
if substance not in substancesWithData:
substancesWithData.append(substance)
if fromProj!=toProj:
exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir())
else:
exportRast=emisRast
if 'CLRTAP' in formats:
dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data)
rasterDict[substance]=exportRast
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
if 'ESRI Ascii grid' in formats:
fileName = path.join(categoryDirPath, substance+ ".asc")
exportRast.write(fileName)
log.debug("Grid for " + substance + "written to outputDir for category: " + ac)
summaryTable=acTree.createTable(writeAll=True)
summaryTable.sortRows()
tableFile=open(path.join(outputDir,"summaryTable.txt"),'w')
summaryTable.write(tableFile)
if len(rasterDict)>0 and 'CLRTAP' in formats:
#creating substance header in the same order as the substances in the template
header = "i\tj\t"
#headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"]
for s in substancesWithData:
header += s + "\t"
#remove the tab after the last column and add a newline instead
header = header[: - 1]+ "\n"
#Creating file for EMEP-data
fileName = "CLRTAP_" + ac + ".txt"
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
fid = open(path.join(categoryDirPath, fileName), 'w')
fid.writelines(header)
sum=0
#Writing indexes and data for all non-zero elements
for row in range(dataMarker.nrows):
for col in range(dataMarker.ncols):
if dataMarker.data[row, col] > 0:
(i, j) = dataMarker.getCentreCoords(row, col)
fid.write(str(i) + "\t" + str(j) + "\t")
for substWithData in substancesWithData[:-1]:
fid.write(str(rasterDict[substWithData].data[row, col]) + "\t")
sum+=rasterDict[substWithData].data[row, col]
fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n")
sum+=rasterDict[substancesWithData[-1]].data[row, col]
fid.close()
log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName))
log.info("Finished")
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-f", "--force",
action="store_true",dest="force",default=False,
help="To start the process without confirming the domain")
parser.add_option("-n", "--nproc",
action="store",dest="nproc",default=1,
help="Number of gifmap processes to run at a time")
(options, args) = parser.parse_args()
#------------Setting up logging capabilities -----------
rootLogger=logger.RootLogger(int(options.loglevel))
log=rootLogger.getLogger(sys.argv[0])
#logger=basicLogger(int(options.loglevel),sys.argv[0])
if options.cf!=None:
controlfile.generateCf(path.abspath(options.cf),controlFileTemplate)
print("Wrote default controlfile")
sys.exit()
if len(args)!=1:
parser.error("Incorrect number of arguments")
domainName=os.environ["AVDBNAME"]
dmn = domain.Domain(domainName)
if not options.force:
answer=raw_input("Chosen dbase is: "+domainName+",continue(y/n)?")
if answer=="y":
dmn=domain.Domain()
else:
sys.exit("Interrupted by user")
#Opening controlfile
#---retrieving data from control file----
cf=controlfile.ControlFile(fileName=path.abspath(args[0]))
#Get search parameters from control file
substances=cf.findStringList("substances:")
outputDir=cf.findExistingPath("outputDir:")
baseSearches=cf.findStringList("searches:")
fromProj=cf.findString("fromProj:")
toProj=cf.findString("toProj:")
resample = cf.findBoolean("resample:")
dmn=domain.Domain(os.environ.get('AVDBNAME'))
substDict=dmn.listSubstanceIndices()
if resample:
#Get proj4 projection definitions from aliases
try:
fromProj=transcoord.proj4Dict[fromProj]
except KeyError:
print("Projection %s not found in proj4Dictin transCoord.py" %fromProj)
try:
toProj=transcoord.proj4Dict[toProj]
except KeyError:
print("Projection %s not found in proj4Dictin transCoord.py" %toProj)
#Get output grind parameters from controlfile
out_xll=cf.findFloat("out_xll:")
out_yll=cf.findFloat("out_yll:")
out_ncols=cf.findInt("out_ncols:")
out_nrows=cf.findInt("out_nrows:")
out_cellsize=cf.findFloat("out_cellsize:")
#Build list with search definitions
searches=[]
for search_id in baseSearches:
prefix=cf.findString("search."+search_id+".prefix:")
macroPath=cf.findString("search."+search_id+".macro:")
starttime=cf.findString("search."+search_id+".starttime:")
endtime=cf.findString("search."+search_id+".endtime:")
parVals = cf.findParam("search."+search_id+".par.",findAll=True)
parDict={}
for p in parVals:
ind=p.index(":")
par = p[:ind+1]
val=p[ind+1:].strip()
parDict[par]=val
alobVals = cf.findParam("search."+search_id+".alob.",findAll=True)
alobDict={}
for a in alobVals:
ind=a.index(":")
key = a[:ind]
val=a[ind+1:].strip()
alobDict[par]=val
macro = controlfile.ControlFile(fileName=macroPath,removeComments=False)
unitIndex=macro.findInt("edb.unit:")
edbName=macro.findString("edb.edb:")
userName=macro.findString("edb.user:"******"FROM :",starttime)
macro.setParam("TO :",endtime)
for key,val in parDict.iteritems():
macro.setParam(key,val)
for key,val in alobDict.iteritems():
macro.setParam(key,"R"+val,addSpace=False)
edb=Edb(dmn.name,userName,edbName)
rsrc=Rsrc(edb.rsrcPath())
unitName=rsrc.search[unitIndex]
for substance in substances:
filename=path.join(outputDir,prefix+"_"+substance.replace(" ","_")+".asc")
searches.append(
{'id':search_id,
'macro':macro,
'prefix':prefix,
'substance':substance,
'unit':unitName,
'filename':filename
}
)
if resample:
summaryTable=datatable.DataTable(desc=[{"id":'search_id',"type":unicode},
{"id":'substance',"type":unicode},
{"id":'macro',"type":unicode},
{"id":'filename',"type":unicode},
{"id":'sum',"type":float},
{"id":'proj_sum',"type":float},
{"id":'unit',"type":unicode}],
keys=["search_id","substance"])
else:
summaryTable=datatable.DataTable(desc=[{"id":'search_id',"type":unicode},
{"id":'substance',"type":unicode},
{"id":'macro',"type":unicode},
{"id":'filename',"type":unicode},
{"id":'sum',"type":float},
{"id":'unit',"type":unicode}],
keys=["search_id","substance"])
#Inititalising parallell run
nproc=int(options.nproc) #numer of processes to run in parallell
running_proc=0 #counter for number of running processes
pids={} #dictionary to store process output and info
todo=len(searches) #list of processes to be run
searchInd=0 #index of the process
def countRunning(pids):
"""Count the number of running processes"""
for pid, props in pids.iteritems():
props['ret_val']=props['proc'].poll()
return len([pid for pid in pids if pids[pid]['ret_val'] is None])
while todo+running_proc>0:
#loop until all processes have finished
if running_proc==nproc or todo==0:
print("Running gifmap...")
#When nproc processes are started, the program waits here
#until one is finished before adding a new one
while countRunning(pids)==running_proc:
time.sleep(5)
#mark first found finished process as done
for pid,props in pids.iteritems():
if not props['ret_val'] is None and not props['done']:
props['done']=True
break
command=props["cmd"]
if props['ret_val']!=0:
errMsg=pids[pid]["proc"].stderr.read()
print("Error while running command: %s\n%s" %(pids[pid],errMsg))
sys.exit(1)
print("Finished search %s" %props['search_id'])
#Find search in search list
running_proc-=1
for f_search in searches:
if f_search["id"]==props['search_id'] and f_search['substance']==props['substance']:
break
f_search['done']=True
try:
f_search['res']=res2rast(f_search['filename'])
#Store original result from search
f_search['sum']=f_search['res'].sum()
except:
f_search['res']=None
output=pids[pid]["proc"].stdout.read()
f_search['sum']=re.compile("#EMIS (\S*).*").search(output).group(1)
print "Could not extract result raster from %s" %f_search['filename']
print "Uses total from gifmap log"
if resample:
outRast = raster.Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999)
#Run post-processing of result for finsished processes
if f_search['sum']>0:
print("Projecting result to target CRS")
f_search['res'] = transcoord.transformEmisRaster(f_search['res'],outRast,
fromProj,toProj,
tmpDir=dmn.tmpDir())
#Store projected result from search
f_search['proj_sum']=f_search['res'].sum()
else:
f_search['proj_sum']=0
#Write result to file
if f_search['res'] is not None:
f_search['res'].write(f_search['filename'])
#Add total emission to summary table
if resample:
summaryTable.addRow(
[f_search['id'],
f_search['substance'],
f_search['macro'].name,
f_search['filename'],
f_search['sum'],
f_search['proj_sum'],
f_search['unit']])
else:
summaryTable.addRow(
[f_search['id'],
f_search['substance'],
f_search['macro'].name,
f_search['filename'],
f_search['sum'],
f_search['unit']])
#Add another search process
elif todo>0:
c_search=searches[searchInd]
substance=c_search["substance"]
substanceIndex=substDict[substance]
c_search['macro'].setParam("ELEMENT :",substanceIndex)
c_search['macro'].write()
command="gifmap -T -i "+c_search['macro'].name+" -o "+c_search['filename']
p=subprocess.Popen(command,stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)
#Add info and handles to process dictionary
pids[p.pid]={"proc":p,
"search_id":c_search['id'],
"substance":c_search['substance'],
"cmd":command,
'ret_val':None,
'done':False}
print(
"Started search %s, substance %s" %(
c_search['id'],c_search['substance']))
running_proc+=1
todo-=1
searchInd+=1
#Each process needs some time to read
#the macro before it is changed
time.sleep(10)
#summaryTable.sortRows()
tableFile=open(path.join(outputDir,"summaryTable.txt"),'w')
summaryTable.write(tableFile)
print("Finished")
