def main(): domain = Domain() # Parse command line arguments parser = argparse.ArgumentParser(description=__doc__) utils.add_standard_command_options(parser) parser.add_argument( "-t", "--templatedir", action="store", dest="templatedir", help="Generate grid .asc template in specified dir" ) parser.add_argument( "-i", "--inputdir", action="store", dest="inputdir", help="Directory containing input files" ) parser.add_argument( "-u", "--user", action="store", dest="user", help="User name (to fetch substance group info)" ) parser.add_argument( "-e", "--edb", action="store", dest="edb", help="EDB name (to fetch substance group info)" ) parser.add_argument( "-d", "--dynamicdir", action="store", dest="dynamicdir", help="Directory with grids for dynamic parameters" ) parser.add_argument( "--intensity", action="store_true", dest="intensity", help="If input rasters are given as ton/(year*km2)" ) args = parser.parse_args() if args.templatedir is not None: generateTemplate( path.join(args.templatedir, "grid_template.asc") ) log.info("Wrote default grid template") sys.exit() substances = domain.listSubstanceIndices() log.debug( "Using substance list of current domain: " + domain.name ) if args.user is not None and args.edb is not None: edb = Edb(domain, args.user, args.edb) with SubgrpStream(edb, mode='r') as subgrpstream: subgrp_reader = ModelReader(subgrpstream) subgrps = list(subgrp_reader) else: subgrps = None dirs = glob.glob(args.inputdir) msg = "directories:" + str([path.basename(d) for d in dirs]) log.info(msg) dyndir = args.dynamicdir if dyndir is not None: dyndir = path.abspath(dyndir) dynamic_rasters = glob.glob(path.join(dyndir, "*.txt")) for d in dirs: log.debug("Processing: " + d) raster_paths = glob.glob(path.join(d, "*.txt")) if len(raster_paths) == 0: log.error("No rasters in directory: " + d) sys.exit() log.debug( "Rasters in directory: " + str([path.basename(r) for r in raster_paths]) ) substance_rasters = [] subgrp_rasters = [] dynamic_rasters = [] for rp in raster_paths: gridname = path.basename(rp) nameparts = gridname.split("__") log.debug("Raster prefix: " + nameparts[0]) if len(nameparts) < 3: # unvalid raster name msg = ("Not able to extract prefix (substance, " + "substance group or dynamic) from raster " + "filename %s" % path.basename(rp)) log.error(msg) sys.exit() if nameparts[0] == "subgrp": subgrp_rasters.append(rp) elif nameparts[0] == "substance": substance_name = nameparts[1] try: substance_index = substances[substance_name] except KeyError: try: substance_index = int(substance_name) except: log.error( "Substance: " + substance_name + " not found in subdb of current domain: " + domain.name ) sys.exit(1) substance_rasters.append(rp) elif nameparts[0] == "dynamic": try: dyn_name = gridname.split("__")[1] dyn_level = int(path.basename(rp).split("__")[2]) except: log.error( "Could not extract name of dynamic " + "parameter and level for raster: " + gridname ) sys.exit(1) dynamic_rasters.append(rp) else: log.error( "Prefix of raster: " + nameparts[0] + " is unvalid" ) sys.exit(1) if args.dynamicdir is not None: for rp in dynamic_rasters: gridname = path.splitext(path.basename(rp))[0] nameparts = gridname.split("__") if nameparts[0] == "dynamic": try: dyn_name = gridname.split("__")[1] dyn_level = int(gridname.split("__")[2]) except: log.error( "Could not extract name of dynamic " + "parameter and level for raster: " + gridname ) sys.exit(1) dynamic_rasters.append(rp) if len(subgrp_rasters) > 1: log.error("There can be only one subgrp raster per grid source") sys.exit(1) if len(subgrp_rasters) > 0 and len(substance_rasters) > 0: log.error( "Both subgrp rasters and substance rasters " + "in the same grid/directory is not allowed" ) sys.exit(1) asc_path = path.join(d, "grid_template.asc") if not path.exists(asc_path): msg = ( "Could not find " "%s, using default template .asc file" % asc_path ) log.warning(msg) generateTemplate(asc_path) grid = EmissionGrid() grid.read_meta_from_file(asc_path) rast = raster.Raster() if len(substance_rasters) > 0: log.debug("Reading substance raster: " + substance_rasters[0]) try: rast.read(substance_rasters[0]) except IOError as e: log.error(e) sys.exit(1) rast.nodataToZero() elif len(subgrp_rasters) > 0: log.debug("Reading subgrp raster: " + subgrp_rasters[0]) try: rast.read(subgrp_rasters[0]) except IOError as e: log.error(e) sys.exit(1) rast.nodataToZero() else: log.error( "Not possible to create grid without any substance " + "rasters or subgrp raster" ) sys.exit(1) grid.X = int(rast.xll) grid.Y = int(rast.yll) grid.DX = int(rast.cellsize) grid.DY = int(rast.cellsize) grid.NX = rast.ncols grid.NY = rast.nrows if len(subgrp_rasters) > 0: subgrp_name = path.basename(subgrp_rasters[0]).split("__")[1] if subgrps is None: subgrp_index = 1 else: subgrp_index = next( (s.INDEX for s in subgrps if s.NAME == subgrp_name), None ) if subgrp_index is None: log.warning( "Could not find subgrp named " + "%s in edb, using default index 1" % subgrp_name ) subgrp_index = 1 grid.FUEL = subgrp_index log.debug("Adding subgrp raster to grid") rast.read(subgrp_rasters[0]) rast.nodataToZero() if args.intensity: # Convert from ton/(year*km2) to ton/year rast.data *= rast.cellsize * rast.cellsize / 1.0e6 grid.add_field_from_raster(rast, subgrp_index, subgrp=True) else: for rp in substance_rasters: rast.read(rp) rast.nodataToZero() if args.intensity: # Convert from ton/(year*km2) to ton/year rast.data *= rast.cellsize * rast.cellsize / 1.0e6 subst = path.basename(rp).split("__")[1] try: substance_index = substances[subst] except KeyError: substance_index = int(subst) log.debug( "Adding substance " + subst + " to grid") grid.add_field_from_raster(rast, substance_index) dynamic_raster_dict = {"GEOCODE": [], "ACTIVITYCODE": []} for dr in dynamic_rasters: nameparts = path.basename(dr).split("__") if nameparts[1] == "GEOCODE": dynamic_raster_dict["GEOCODE"].append(dr) elif nameparts[1] == "ACTIVITYCODE": dynamic_raster_dict["ACTIVITYCODE"].append(dr) else: dynamic_raster_dict[nameparts[1]] = [dr] # Sorting the code rasters after code level # Function that returns the code level from the raster name cmpfunc = lambda x: int(path.basename(x)[: -4].split("__")[2]) dynamic_raster_dict["GEOCODE"].sort(key=cmpfunc) dynamic_raster_dict["ACTIVITYCODE"].sort(key=cmpfunc) for dp in dynamic_raster_dict.keys(): if len(dynamic_raster_dict[dp]) > 0: log.debug( "Adding dynamic raster for " + dp + ": " + str(dynamic_raster_dict[dp]) ) grid.addDynamicRasters(dp, dynamic_raster_dict[dp]) grid_dir = path.abspath(d) grid_name = path.basename(grid_dir) grid_path = path.join(grid_dir, grid_name) grid.write_data_to_file(grid_path) msg = "Wrote grid for %s" % path.basename(d) log.info(msg) log.info("Finished successfully")
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-u", "--user", action="store", dest="user", default=None, help="Specify user manually") parser.add_option("-e", "--edb", action="store", dest="edb", default=None, help="Name of target edb") parser.add_option("-t", "--template", action="store",dest="cf",default=None, help="Generate default controlfile") parser.add_option("-f", "--force", action="store_true",dest="force",default=False, help="To start the process without confirming the domain") (options, args) = parser.parse_args() # Setup logging logging.configure(terminal_level=logging.DEBUG) log = logging.getLogger(__name__) if options.cf!=None: generateCf(path.abspath(options.cf)) log.info("Wrote default controlfile") sys.exit() if len(args)!=1: parser.error("Incorrect number of arguments") if options.edb ==None: parser.error("Need to specify edb using flag -e") if options.user ==None: parser.error("Need to specify user using flag -u") dmn = Domain() if not options.force: answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?") if answer!="y": sys.exit("Interrupted by user") if not dmn.edbExistForUser(options.edb,options.user): log.error("Edb "+options.edb+" does not exist for user "+ options.user+" in domain "+dmn.name) sys.exit() #---Creating edb and rsrc objects------------------ edb=Edb(dmn,options.user,options.edb) rsrc=Rsrc(edb.rsrcPath()) #Opening controlfile #---retrieving data from control file---- cf=ControlFile(fileName=path.abspath(args[0])) substances=cf.findStringList("substances:") outputDir=cf.findExistingPath("outputDir:") acIndex=cf.findInt("acIndex:") macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:")) fromProj=cf.findString("fromProj:") toProj=cf.findString("toProj:") try: fromProj=transcoord.proj4Dict[fromProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj) try: toProj=transcoord.proj4Dict[toProj] except KeyError: log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj) formats = cf.findStringList("formats:") units = cf.findString("units:") writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True) edb_xll=cf.findInt("edb_xll:") edb_yll=cf.findInt("edb_yll:") edb_ncols=cf.findInt("edb_ncols:") edb_nrows=cf.findInt("edb_nrows:") edb_cellsize=cf.findFloat("edb_cellsize:") if fromProj!=toProj: out_xll=cf.findFloat("out_xll:") out_yll=cf.findFloat("out_yll:") out_ncols=cf.findInt("out_ncols:") out_nrows=cf.findInt("out_nrows:") out_cellsize=cf.findFloat("out_cellsize:") #----------------------------------------- #Finds index to search units unitIndex=None for key,unit in rsrc.search.iteritems(): if isinstance(key,int): if rsrc.search[key]==units: unitIndex=key break if unitIndex is None: log.error("Search units: %s not defined in edb.rsrc" %units) sys.exit() macro = ControlFile(fileName=macroFileName,removeComments=False) #preparing export macro macro.setParam("general.database:",dmn.name) xmin=edb_xll xmax=edb_xll+edb_ncols*edb_cellsize ymin=edb_yll ymax=edb_yll+edb_nrows*edb_cellsize macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("edb.user:"******"edb.edb:",edb.name) macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax)) macro.setParam("USER :"******"EDB :",edb.name) macro.setParam("GRID :", "%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize)) macro.setParam("edb.unit:",unitIndex) macro.setParam("UNIT :",unitIndex) # macro.setParam("NOACTCODE :",acIndex) macro.setParam("NOACTCODE :",len(rsrc.ac)) #Get activity code tree acTree=codeemistree.CodeEmisTree("Activity codes",units=units) acTree.readActivityCodes(rsrc.path,acIndex) substDict=dmn.listSubstanceIndices() edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols, Nrows=edb_nrows,Cellsize=edb_cellsize, Nodata=-9999,init=0) if fromProj!=toProj: outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999) else: outRastTemplate=Raster() outRastTemplate.assign(edbRast) for node in acTree.root.getiterator(): if node.tag=="root" or node.tag=="Record": continue ac=node.tag log.debug("Activity code: "+ac) #Finds row index for activity codes in macro #Add a row with the current ac #If there are many ac already given, these are #replaced by the current ac macroLines=macro.content.split("\n") actCodeInd=None geoCodeInd=None for lineInd,line in enumerate(macroLines): if "NOACTCODE" in line: actCodeInd=lineInd if "NOGEOCODE" in line: geoCodeInd=lineInd if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] else: macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:] macro.content="\n".join(macroLines) macro.write() #boolean raster marking where there is data for any of the substances if 'CLRTAP' in formats: dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols, Nrows=out_nrows,Cellsize=out_cellsize, Nodata=-9999,init=0) rasterDict={} substancesWithData=[] for substance in substances: log.debug("Substance %s" %substance) substanceIndex=substDict[substance] macro.setParam("ELEMENT :",substanceIndex) macro.write() command="xrepedb -i "+macro.name (returnCode,errMsg,outMsg)=utilities.execute(command) tmp=outMsg.split()[10:-2] tmp.sort() if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00': print "ERROR: The field for "+substance+" is empty!" continue # pdb.set_trace() emisRast=string2rast(outMsg,edbRast) emisSum=emisRast.sum() outRast=Raster() outRast.assign(outRastTemplate) rec=ET.SubElement(node,"Record") rec.attrib["substance"]=substance rec.attrib["emission"]=str(emisSum) if emisSum>0 and writeGrids: if substance not in substancesWithData: substancesWithData.append(substance) if fromProj!=toProj: exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir()) else: exportRast=emisRast if 'CLRTAP' in formats: dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data) rasterDict[substance]=exportRast categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) if 'ESRI Ascii grid' in formats: fileName = path.join(categoryDirPath, substance+ ".asc") exportRast.write(fileName) log.debug("Grid for " + substance + "written to outputDir for category: " + ac) summaryTable=acTree.createTable(writeAll=True) summaryTable.sortRows() tableFile=open(path.join(outputDir,"summaryTable.txt"),'w') summaryTable.write(tableFile) if len(rasterDict)>0 and 'CLRTAP' in formats: #creating substance header in the same order as the substances in the template header = "i\tj\t" #headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"] for s in substancesWithData: header += s + "\t" #remove the tab after the last column and add a newline instead header = header[: - 1]+ "\n" #Creating file for EMEP-data fileName = "CLRTAP_" + ac + ".txt" categoryDirPath = path.join(outputDir, ac) if not path.isdir(categoryDirPath): os.mkdir(categoryDirPath) fid = open(path.join(categoryDirPath, fileName), 'w') fid.writelines(header) sum=0 #Writing indexes and data for all non-zero elements for row in range(dataMarker.nrows): for col in range(dataMarker.ncols): if dataMarker.data[row, col] > 0: (i, j) = dataMarker.getCentreCoords(row, col) fid.write(str(i) + "\t" + str(j) + "\t") for substWithData in substancesWithData[:-1]: fid.write(str(rasterDict[substWithData].data[row, col]) + "\t") sum+=rasterDict[substWithData].data[row, col] fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n") sum+=rasterDict[substancesWithData[-1]].data[row, col] fid.close() log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName)) log.info("Finished")