def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-u",'--user',
action="store",dest="user",
help="Name of target edb user")
parser.add_option("-e","--edb",
action="store",dest="edb",
help="Name of target edb")
parser.add_option("-i","--infile",
action="store",dest="infile",
help="Input csv file")
# parser.add_option("-y","--year",
# action="store",dest="year",
# help="Only store sources for given year")
parser.add_option("-v",dest='loglevel',
action="store_const",default=get_loglevel(),
help="produce verbose output")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-o", "--outfile",
action="store",dest="outfile",default=None,
help="Name of outfiles (without extension)")
parser.add_option("-d","--delimiter",
action="store",dest="delimiter",default="\t",
help="Delimiter used in csv-file")
parser.add_option("-c","--filterCol",
action="store",dest="filterCol",
help="Header of column to use as filter")
parser.add_option("-f","--filterVal",
action="store",dest="filterVal",
help="Value to use in filter")
# parser.add_option("-g", "--geocodeRasterDir",
# action="store",dest="geocodeRasterDir",default=None,
# help="Directory with geocode rasters")
(options, args) = parser.parse_args()
#--------------------Init logger-----------------------
# rootLogger = logger.RootLogger(level=options.loglevel)
logging.basicConfig(
format='%(levelname)s:%(name)s: %(message)s',
level=options.loglevel,
)
global log
# log = rootLogger.getLogger(sys.argv[0])
log = logging.getLogger(parser.prog)
#-----------------Validating options-------------------
if options.cf is not None:
generateCf(path.abspath(options.cf),controlFileTemplate)
log.info("Wrote default controlfile")
return 1
if options.user is None:
log.error("Need to specify -u <user>")
return 1
if options.edb is None:
log.error("Need to specify -e <edb>")
return 1
# if options.year is None:
# log.error("Need to specify -y <year>")
# return 1
# if len(options.year)!=4:
# log.error("Year should be given with four digits")
# return 1
if len(args)!=1:
log.error("Controlfile should be given as argument")
return 1
dmn=Domain()
edb=Edb(dmn,options.user,options.edb)
if not edb.exists():
log.error("Edb %s does not exist" %options.edb)
return 1
log.info("Parsing controlfile")
cf=ControlFile(args[0])
cdbPars=re.compile("companydb\.par\.(\w*?):").findall(cf.content)
fdbPars=re.compile("facilitydb\.par\.(\w*?):").findall(cf.content)
sdbPars=re.compile("sourcedb\.par\.(\w*?):").findall(cf.content)
substEmisNr=re.compile("sourcedb\.subst_emis\.([0-9]*)\.emis").findall(cf.content)
subgrpEmisNr=re.compile("sourcedb\.subgrp_emis\.([0-9]*)\.emis").findall(cf.content)
cdbCols={}
cdbDefaults={}
for par in cdbPars:
cdbCols[par]=cf.findString("companydb.par.%s:" %par)
cdbDefaults[par]=cf.findString("companydb.par.%s.default:" %par,
optional=True,default=None)
fdbCols={}
fdbDefaults={}
for par in fdbPars:
fdbCols[par]=cf.findString("facilitydb.par.%s:" %par)
fdbDefaults[par]=cf.findString("facilitydb.par.%s.default:" %par,
optional=True,default=None)
sdbCols={}
sdbDefaults={}
for par in sdbPars:
sdbCols[par]=cf.findString("sourcedb.par.%s:" %par)
sdbDefaults[par]=cf.findString("sourcedb.par.%s.default:" %par,
optional=True,default=None)
substEmisCols={}
substEmisDefaults={}
if substEmisNr is not None:
for emisNr in substEmisNr:
cols={}
defaults={}
emisPars=re.compile("sourcedb\.subst_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
emisDefaultPars=re.compile(
"sourcedb\.subst_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
if emisPars is not None:
for par in emisPars:
cols[par]=cf.findString("sourcedb.subst_emis.%s.%s:" %(emisNr,par))
if emisDefaultPars is not None:
for par in emisDefaultPars:
defaults[par]=cf.findString("sourcedb.subst_emis.%s.%s.default:" %(emisNr,par),
optional=True,default=None)
substEmisCols[emisNr]=cols
substEmisDefaults[emisNr]=defaults
subgrpEmisCols={}
subgrpEmisDefaults={}
if subgrpEmisNr is not None:
for emisNr in subgrpEmisNr:
cols={}
defaults={}
emisPars=re.compile("sourcedb\.subgrp_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
emisDefaultPars=re.compile(
"sourcedb\.subgrp_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
if emisPars is not None:
for par in emisPars:
cols[par]=cf.findString("sourcedb.subgrp_emis.%s.%s:" %(emisNr,par))
if emisDefaultPars is not None:
for par in emisDefaultPars:
defaults[par]=cf.findString("sourcedb.subgrp_emis.%s.%s.default:" %(emisNr,par),
optional=True,default=None)
subgrpEmisCols[emisNr]=cols
subgrpEmisDefaults[emisNr]=defaults
log.info("Reading subdb...")
subdb=Subdb(edb)
subdb.read()
log.info("Reading companydb...")
companydb=Companydb(edb)
companydb.read()
log.info("Reading sourcedb...")
# source_stream = SourceStream(edb, 'w')
source_stream = open(options.outfile, 'w')
source_writer = ModelWriter(source_stream,encoding="HP Roman8")
log.info("Reading facilitydb...")
facilitydb=Facilitydb(edb)
facilitydb.read()
log.info("Reading subgrpdb")
subgrpdb=Subgrpdb(edb)
subgrpdb.read()
log.info("Reading edb.rsrc")
rsrc=Rsrc(edb.rsrcPath())
acCodeTables=[]
for i in range(rsrc.numberOfCodeTrees("ac")):
acCodeTables.append(CodeTable(rsrc.path,codeType="ac",codeIndex=i+1))
gcCodeTables=[]
for i in range(rsrc.numberOfCodeTrees("gc")):
gcCodeTables.append(CodeTable(rsrc.path,codeType="gc",codeIndex=i+1))
geocodeRasters=[]
rast1=Raster()
rast1.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__1.txt")
rast2=Raster()
rast2.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__2.txt")
geocodeRasters.append(rast1)
geocodeRasters.append(rast2)
log.info("Reading csv-file")
table=DataTable()
table.read(options.infile,delimiter=options.delimiter,encoding="ISO-8859-15")
if options.filterCol is not None:
if options.filterCol not in table.colIndex:
log.error("Filter column header not found in table")
sys.exit(1)
invalid=False
nFiltered=0
nRows=0
log.info("Processing rows")
for rowInd,row in enumerate(table.data):
nRows+=1
if options.filterCol is not None:
filterVal=row[table.colIndex[options.filterCol]]
if options.filterVal!=str(filterVal):
nFiltered+=1
continue
comp = Company()
for par in comp.parOrder:
val=cdbDefaults.get(par,None)
if par in cdbCols:
colId=cdbCols[par]
try:
tableVal=row[table.colIndex[colId]]
except KeyError:
log.error(
"No column with header %s, columns: %s" %(
colId,str(table.listIds())))
if tableVal is not None:
val = tableVal
if val is not None:
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
comp[par]=val
fac = Facility()
for par in fac.parOrder:
val=fdbDefaults.get(par,None)
if par in fdbCols:
colId=fdbCols[par]
tableVal=row[table.colIndex[colId]]
if tableVal is not None:
val = tableVal
if val is not None:
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
fac[par]=val
src = Source()
for par in ["X1", "Y1","X2","Y2",
"PX","PY","NAME","INFO","INFO2","DATE","CHANGED",
"CHIMNEY HEIGHT","GASTEMPERATURE","GAS FLOW",
"SEARCHKEY1","SEARCHKEY2","SEARCHKEY3",
"SEARCHKEY4","SEARCHKEY5","CHIMNEY OUT","CHIMNEY IN",
"HOUSE WIDTH","HOUSE HEIGHT","NOSEGMENTS","BUILD_WIDTHS",
"BUILD_HEIGHTS","BUILD_LENGTHS","BUILD_DISTFARWALL",
"BUILD_CENTER","GEOCODE","FORMULAMACRO","ALOB"]:
val=sdbDefaults.get(par,None)
if par in sdbCols:
colId=sdbCols[par]
tableVal=row[table.colIndex[colId]]
if tableVal is not None:
val = tableVal
if val is not None:
#validate code
if par=="GEOCODE" and val is not None:
gcList=val.split()
for codeIndex,code in enumerate(gcList):
if not gcCodeTables[codeIndex].hasCode(code):
log.error("Invalid geo code %s on row %i" %(code,rowInd))
invalid=True
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
#Store in src object and convert to correct type
src._fieldvalues[par] = lazy_parse(
src, par, val)
gc1=geocodeRasters[0].getVal(src.get_coord()[0],src.get_coord()[1])
gc2=geocodeRasters[1].getVal(src.get_coord()[0],src.get_coord()[1])
src.GEOCODE = [str(int(gc1)) + "." + str(int(gc2))]
for emisNr,emis in substEmisCols.items():
substEmis={"unit":None,"ac":None,"substance":None,"emis":None}
for par in substEmis.keys():
if par in emis:
substEmis[par]=row[table.colIndex[emis[par]]]
else:
try:
substEmis[par]=substEmisDefaults[emisNr][par]
except KeyError:
log.error(
"Need to specify column or default value for subgrp emis %i" %emisNr)
substInd=subdb.substIndex(substEmis["substance"])
if substInd is None:
log.error("Invalid substance name %s on row %i" %(
substEmis["substance"],rowInd))
sys.exit(1)
try:
unit=rsrc.sub[substEmis["unit"]]
except KeyError:
log.error("Invalid unit name %s on row %i" %(emis["unit"],rowInd))
sys.exit(1)
acList=substEmis["ac"].split('\\')[0].split()
for codeIndex,code in enumerate(acList):
# if code == "2.A.4.2":
# import pdb; pdb.set_trace()
refCode = acCodeTables[codeIndex].checkCode(code)
if refCode == "-":
log.error("Invalid activity code %s on row %i" %(code,rowInd))
sys.exit(1)
if refCode != code:
acList[codeIndex] = refCode
substEmis["ac"] = acList
if substEmis["emis"] is not None and substEmis["emis"]!="0":
try:
emis = src.add_emission()
emis.UNIT = substEmis["unit"]
emis.ACTCODE = substEmis["ac"] # needs re-formatting
emis.EMISSION = float(substEmis["emis"])
emis.SUBSTANCE = substInd
emis.auto_adjust_unit(edb)
except:
# print substEmis
# log.error("Invalid substance emission on row %i" %rowInd)
invalid=True
src.EMISSION=src.EMISSION[:-1]
for emis in subgrpEmisCols.values():
subgrpEmis={"unit":None,"ac":None,"name":None,"emis":None}
for par in subgrpEmis.keys():
if par in emis:
subgrpEmis[par]=row[table.colIndex[emis[par]]]
else:
try:
subgrpEmis[par]=subgrpEmisDefaults[emisNr][par]
except KeyError:
log.error(
"Need to specify column or default value for subgrp emis %i" %emisNr)
#validating subgrp name
try:
subgrp=subgrpdb.getByName(subgrpEmis["name"])
except KeyError:
log.error("Invalid subgrp name %s on row %i" %(subgrpEmis["name"],rowInd))
invalid=True
#validating subgrp emis unit
try:
unitFactor=rsrc.subGrpEm[subgrpEmis["unit"]]
except KeyError:
log.error("Invalid unit %s for subgrp emission on row %i" %(
subgrpEmis["unit"],rowInd))
invalid=True
#validating subgrp activity code
acList=subgrpEmis["ac"].split()
for codeIndex,code in enumerate(acList):
refCode = acCodeTables[codeIndex].checkCode(code)
if refCode == "-":
log.error("Invalid activity code %s on row %i" %(code,rowInd))
invalid=True
break
if refCode != code:
acList[codeIndex] = refCode
substEmis["ac"] = acList
try:
src.addSubgrpEmis(subgrp.index,emis=subgrpEmis["emis"],unit=subgrpEmis["unit"],
ac=subgrpEmis["ac"])
except:
log.error("Invalid subgrp emission on row %i" %rowInd)
invalid=True
companydb.append(comp,force=True)
facilitydb.append(fac,force=True)
source_writer.write(src)
# sourcedb.append(src)
if invalid:
log.info("No output written due to validation errors")
sys.exit(0)
if len(companydb.items)>0:
if options.outfile is None:
log.info("Writing companydb")
else:
log.info("Writing company db to file")
companydb.write(filename=options.outfile+".companydb")
if len(facilitydb.items)>0:
if options.outfile is None:
log.info("Writing facilitydb")
else:
log.info("Writing facilitydb to file")
facilitydb.write(filename=options.outfile+".facilitydb")
# if len(sourcedb.sources)>0:
# if options.outfile is None:
# log.info("Writing sourcedb")
# else:
# log.info("Writing sourcedb to file")
# sourcedb.write(filename=options.outfile+".sourcedb")
if options.filterCol is not None:
log.info("Filtered out %i out of %i" %(nFiltered,nRows))
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d",
"--doc",
action="store_true",
dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=get_loglevel(),
help="Produce verbose output")
parser.add_option("-o",
"--output",
action="store",
dest="outfileName",
default=None,
help="Output file")
parser.add_option("-i",
"--i",
action="store",
dest="infile",
help="Raster to be regionalized")
parser.add_option("-r",
"--regdef",
action="store",
dest="regdef",
help="Raster defining the regions found " +
"in statistics table")
parser.add_option("-s",
"--stat",
action="store",
dest="stat",
help="Tab-separated statistics table, " +
"with 1st column representing geocode and value " +
"representing region sum")
parser.add_option("-c",
"--colname",
action="store",
dest="colname",
help="Header of column in stat table to fetch " +
"region sums from")
(options, args) = parser.parse_args()
# ------------Setting up logging capabilities -----------
rootLogger = logger.RootLogger(int(options.loglevel))
global log
log = rootLogger.getLogger(sys.argv[0])
# ------------Process and validate options---------------
if options.doc:
print __doc__
sys.exit()
if len(args) > 0:
parser.error("Incorrect number of arguments")
regdef = Raster()
if options.regdef is not None:
regdef.read(options.regdef)
else:
log.error("No region definition raster specified")
sys.exit(1)
key = Raster()
if options.infile is not None:
key.read(options.infile)
else:
log.info("No initial distribution raster given, using " +
"homogenious distribution")
key.assign(regdef)
key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata)
if options.stat is None:
log.error("No statistics table specified")
sys.exit(1)
if (regdef.xll != key.xll or regdef.yll != key.yll):
log.error("The raster domain size is differs between key raster " +
"and region raster")
stat = DataTable()
stat.read(options.stat, defaultType=float)
print stat.desc
stat.convertCol(stat.desc[0]["id"], int)
print stat.desc
# Set first column as unique id for rows
stat.setKeys([stat.desc[0]["id"]])
# Remove nodata in rasters
key.nodataToZero()
regdef.nodataToZero()
log.info("Consistency och completeness check for codes " +
"in regdef and statistics")
# Create list of codes in raster
regdefCodes = regdef.unique()
regdefCodes = map(int, regdefCodes)
if 0.0 in regdefCodes:
regdefCodes.remove(0.0)
statRegCodes = [row[0] for row in stat.data]
if options.colname is not None:
try:
col = stat.colIndex[options.colname]
except KeyError:
log.error("No column named " +
"%s found in statistics table" % options.colname)
sys.exit(1)
else:
col = 1
colId = stat.desc[col]["id"]
errFound = False
for code in statRegCodes:
# Assuring that the regional codes are present in regdef
if code not in regdefCodes:
log.error("Input Error: code:" + str(int(code)) +
"in reg totals is not represented in regional raster")
errFound = True
# Assuring that the regional raster IDs are present
# in the regional statistics
for code in regdefCodes:
if code not in statRegCodes:
log.error("Input Error: ID:" + str(code) +
" in region raster is not represented in " +
"regional totals!")
errFound = True
if errFound:
sys.exit(1)
# For all regions, calculate the regional sum,
# set the key values to key/regSum
res = np.zeros(key.data.shape)
log.info("Regionalizing key raster")
for code in regdefCodes:
emis = stat.lookup(colId, code)
log.debug("Statistics for reg %i: %f" % (code, emis))
mask = np.array(regdef.data == code)
regKey = mask * key.data
regSum = np.sum(np.sum(regKey))
if regSum > 0:
res = res + emis * regKey / regSum
else:
log.warning("Distribution key is zero for geocode" +
" %i, using homogenuous distribution for this region" %
code)
res = res + emis * mask / np.sum(np.sum(mask))
key.data = res
# resultRast=(totFractionRast*key)*regionalTotals.regSum(substance)
key.write(options.outfileName)
log.info("Result with sum: %f written" % res.sum())
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d", "--doc",
action="store_true", dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const", const=logging.DEBUG,
dest="loglevel", default=get_loglevel(),
help="Produce verbose output")
parser.add_option("-o", "--output",
action="store", dest="outfileName", default=None,
help="Output file")
parser.add_option("-i", "--i",
action="store", dest="infile",
help="Raster to be regionalized")
parser.add_option("-r", "--regdef",
action="store", dest="regdef",
help="Raster defining the regions found " +
"in statistics table")
parser.add_option("-s", "--stat",
action="store", dest="stat",
help="Tab-separated statistics table, " +
"with 1st column representing geocode and value " +
"representing region sum")
parser.add_option("-c", "--colname",
action="store", dest="colname",
help="Header of column in stat table to fetch " +
"region sums from")
(options, args) = parser.parse_args()
# ------------Setting up logging capabilities -----------
rootLogger = logger.RootLogger(int(options.loglevel))
global log
log = rootLogger.getLogger(sys.argv[0])
# ------------Process and validate options---------------
if options.doc:
print __doc__
sys.exit()
if len(args) > 0:
parser.error("Incorrect number of arguments")
regdef = Raster()
if options.regdef is not None:
regdef.read(options.regdef)
else:
log.error("No region definition raster specified")
sys.exit(1)
key = Raster()
if options.infile is not None:
key.read(options.infile)
else:
log.info("No initial distribution raster given, using " +
"homogenious distribution")
key.assign(regdef)
key.data = np.where(regdef.data != regdef.nodata, 1, regdef.nodata)
if options.stat is None:
log.error("No statistics table specified")
sys.exit(1)
if (regdef.xll != key.xll or regdef.yll != key.yll):
log.error("The raster domain size is differs between key raster " +
"and region raster")
stat = DataTable()
stat.read(options.stat, defaultType=float)
print stat.desc
stat.convertCol(stat.desc[0]["id"], int)
print stat.desc
# Set first column as unique id for rows
stat.setKeys([stat.desc[0]["id"]])
# Remove nodata in rasters
key.nodataToZero()
regdef.nodataToZero()
log.info("Consistency och completeness check for codes " +
"in regdef and statistics")
# Create list of codes in raster
regdefCodes = regdef.unique()
regdefCodes = map(int, regdefCodes)
if 0.0 in regdefCodes:
regdefCodes.remove(0.0)
statRegCodes = [row[0] for row in stat.data]
if options.colname is not None:
try:
col = stat.colIndex[options.colname]
except KeyError:
log.error("No column named " +
"%s found in statistics table" % options.colname)
sys.exit(1)
else:
col = 1
colId = stat.desc[col]["id"]
errFound = False
for code in statRegCodes:
# Assuring that the regional codes are present in regdef
if code not in regdefCodes:
log.error("Input Error: code:" + str(int(code)) +
"in reg totals is not represented in regional raster")
errFound = True
# Assuring that the regional raster IDs are present
# in the regional statistics
for code in regdefCodes:
if code not in statRegCodes:
log.error("Input Error: ID:" + str(code) +
" in region raster is not represented in " +
"regional totals!")
errFound = True
if errFound:
sys.exit(1)
# For all regions, calculate the regional sum,
# set the key values to key/regSum
res = np.zeros(key.data.shape)
log.info("Regionalizing key raster")
for code in regdefCodes:
emis = stat.lookup(colId, code)
log.debug("Statistics for reg %i: %f" % (code, emis))
mask = np.array(regdef.data == code)
regKey = mask * key.data
regSum = np.sum(np.sum(regKey))
if regSum > 0:
res = res + emis * regKey / regSum
else:
log.warning(
"Distribution key is zero for geocode" +
" %i, using homogenuous distribution for this region" % code)
res = res + emis * mask / np.sum(np.sum(mask))
key.data = res
# resultRast=(totFractionRast*key)*regionalTotals.regSum(substance)
key.write(options.outfileName)
log.info("Result with sum: %f written" % res.sum())
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d", "--doc",
action="store_true", dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const", const=logging.DEBUG,
dest="loglevel", default=get_loglevel(),
help="Produce verbose output")
parser.add_option("--no-progress",
action="store_const", dest="progressStream",
const=None, default=sys.stdout,
help="turn off the progress bar")
parser.add_option("-o", "--output",
action="store", dest="outfileName", default=None,
help="Output file")
parser.add_option("-i", "--input",
action="store", dest="infileName",
help="Input raster")
parser.add_option("--bbox",
action="store", dest="bbox",
help="Only read data within bbox," +
" --box <\"x1,y1,x2,y2\"")
parser.add_option("--reclassify",
action="store", dest="classTable",
help="Tab-separated table with code " +
"and z0 value for each landuse class")
parser.add_option("--reclassFromColumn",
action="store", dest="reclassFromColumn",
help="Header of reclass table column " +
"containing values to " +
"reclass (default is to use first column of classTable)")
parser.add_option("--reclassToColumn",
action="store", dest="reclassToColumn",
help="Header of reclass table column containing codes," +
" default is to use first column (default is to use " +
"second column of classTable)")
parser.add_option("--resample",
action="store", dest="cellFactor",
help="Resample grid by dividing cellsize with a" +
" factor. Factor <1 results in refinement." +
" Reprojection uses temporary refinement")
parser.add_option("--resamplingMethod",
action="store", dest="resamplingMethod",
help="For cellFactor > 1: " +
"Choose between 'mean', 'sum', 'majority' or" +
" 'count', default is sum,"
"For cellFactor < 1, choose between: " +
"'keepTotal' and 'keepValue', default is 'keepTotal'")
parser.add_option("--summarize",
action="store_true", dest="summarize",
help="Print a summary of the input grid properties")
parser.add_option("--bandIndex",
action="store", dest="bandIndex",
help="Band index to read from",
default=1)
parser.add_option("--dataType",
action="store", dest="dataType",
help="Output raster/shape data type",
default=None)
parser.add_option("--toShape",
action="store_true", dest="toShape",
help="output as shape file",
default=None)
parser.add_option("--fieldName", metavar='FIELD',
action="store", dest="fieldName",
help="write data in shape file to FIELD," +
" default is 'value'",
default=None)
parser.add_option("--filter",
action="store", dest="filter",
help="Filter out data equal or below" +
" limit in shape output",
default=None)
parser.add_option("-t", "--template",
action="store", dest="template",
help="Header for output raster when reprojecting")
parser.add_option("--fromProj", dest="fromProj",
help="Input raster proj4 definition string")
parser.add_option("--toProj", dest="toProj",
help="Output raster proj4 definition string")
(options, args) = parser.parse_args()
#------------Setting up logging capabilities -----------
# Setup logging
logging.basicConfig(
format='%(levelname)s:%(name)s: %(message)s',
level=options.loglevel,
)
log = logging.getLogger(__name__)
#------------Process and validate options---------------
if options.doc:
print doc
sys.exit()
if len(args) > 0:
parser.error("Incorrect number of arguments")
#validate infile path
if options.infileName is not None:
inFilePath = path.abspath(options.infileName)
if not path.exists(inFilePath):
log.error("Input raster %s does not exist" % options.infileName)
sys.exit(1)
else:
parser.error("No input data specified")
#validate outfile path
if options.outfileName is not None:
outFilePath = path.abspath(options.outfileName)
if options.toShape and ".shp" not in outFilePath:
parser.error("Output shape has to to be specified with" +
" .shp extension")
else:
outFilePath = None
#Validate fieldName option
if options.toShape:
if options.fieldName == "":
parser.error("fieldName can't be an empty string")
fieldName = options.fieldName or "value"
elif options.fieldName is not None:
parser.error("fieldName option only allowed together" +
" with shape output")
# Validate filter option and convert filter to numeric value if present
if not options.toShape and options.filter is not None:
parser.error("Filter option only allowed together with shape output")
elif options.toShape:
if options.filter is not None:
filter = float(options.filter)
else:
filter = None
# read and process reclass table file
if options.classTable is not None:
reclassFromColumn = options.reclassFromColumn
reclassToColumn = options.reclassToColumn
if reclassFromColumn is not None and reclassToColumn is not None:
desc = [{"id": reclassFromColumn, "type": float},
{"id": reclassToColumn, "type": float}]
classTable = datatable.DataTable(desc=desc)
classTable.read(options.classTable)
else:
classTable = datatable.DataTable()
classTable.read(options.classTable)
reclassFromColumn = classTable.desc[0]["id"]
reclassToColumn = classTable.desc[1]["id"]
classTable.convertCol(reclassFromColumn, float)
classTable.convertCol(reclassToColumn, float)
classTable.setKeys([reclassFromColumn])
log.debug("Successfully read landuse class table")
classDict = {}
for row in classTable.data:
classDict[row[classTable.colIndex[reclassFromColumn]]
] = row[classTable.colIndex[reclassToColumn]]
if options.cellFactor is not None:
cellFactor = float(options.cellFactor)
if cellFactor > 1 and \
options.resamplingMethod not in ('sum',
'mean',
'majority',
'count',
None):
log.error(
"Invalid resampling method, valid options for grid " +
"coarsening are 'sum' " +
"and 'mean' and 'majority', " +
"specified %s" % options.resamplingMethod
)
sys.exit(1)
elif cellFactor < 1 and \
options.resamplingMethod not in ('keepTotal',
'keepValue',
None):
log.error(
"Invalid resampling method, valid options for grid " +
"coarsening are 'keepTotal' and 'keepValue'" +
", specified %s" % resamplingMethod)
sys.exit(1)
# setting default resampling methods
if cellFactor > 1 and \
options.resamplingMethod is None:
resamplingMethod = 'sum'
elif options.resamplingMethod is None:
resamplingMethod = 'keepTotal'
else:
resamplingMethod = options.resamplingMethod
if options.resamplingMethod == 'majority' or \
options.resamplingMethod == 'count' and \
options.toProj is not None:
log.error(
"Resampling method " +
"%s not possible to " % options.resamplingMethod +
"combine with reprojection")
sys.exit(1)
# Assure that gdal is present
if not __gdal_loaded__:
raise OSError("Function readGDAL needs GDAL with python bindings")
# register all of the raster drivers
gdal.AllRegister()
ds = gdal.Open(inFilePath, GA_ReadOnly)
if ds is None:
print 'Could not open ' + filename
sys.exit(1)
ncols = ds.RasterXSize
nrows = ds.RasterYSize
nbands = ds.RasterCount
# Info used for georeferencing
geoTransform = ds.GetGeoTransform()
xul = geoTransform[0] # top left x
cellsizeX = geoTransform[1] # w-e pixel resolution
rot1 = geoTransform[2] # rotation, 0 if image is "north up"
yul = geoTransform[3] # top left y
rot2 = geoTransform[4] # rotation, 0 if image is "north up"
cellsizeY = geoTransform[5] # n-s pixel resolution
proj = osr.SpatialReference(ds.GetProjection())
# Calculate lower left corner
xll = xul
yll = yul + nrows * cellsizeY # cellsizeY should be a negative value
# Rotated rasters not handled...yet
if rot1 != 0 or rot2 != 0:
print 'Rotated rasters are not supported by pyAirviro.geo.raster'
sys.exit(1)
if abs(cellsizeX) != abs(cellsizeY):
print('Non-homogenous cellsizes are not' +
' supported by pyAirviro.geo.raster')
sys.exit(1)
bandIndex = int(options.bandIndex)
band = ds.GetRasterBand(bandIndex)
nodata = band.GetNoDataValue()
# If no nodata value is present in raster, set to -9999 for completeness
if nodata is None:
nodata = -9999
# Read data from a window defined by option --bbox <"x1,y1,x2,y2">
if options.bbox is not None:
xur = xll + ncols * cellsizeX
try:
x1, y1, x2, y2 = map(float, options.bbox.split(","))
except:
log.error("Invalid value for option --bbox <\"x1,y1,x2,y2\">")
sys.exit(1)
# Check if totally outside raster extent
if x2 < xll or y2 < yll or x1 > xur or y1 > yul:
log.error("Trying to extract outside grid boundaries")
sys.exit(1)
# Limit bbox to raster extent
if x1 < xll:
x1 = xll
if x2 > xur:
x2 = xur
if y1 < yll:
y1 = yll
if y2 > yul:
y2 = yul
# estimate min and max of rows and cols
colmin = int((x1 - xll) / cellsizeX)
colmaxdec = (x2 - xll) / float(cellsizeX)
rowmin = int((yul - y2) / abs(cellsizeY))
rowmaxdec = (yul - y1) / float(abs(cellsizeY))
if (colmaxdec - int(colmaxdec)) > 0:
colmax = int(colmaxdec)
else:
colmax = int(colmaxdec - 1)
if (rowmaxdec - int(rowmaxdec)) > 0:
rowmax = int(rowmaxdec)
else:
rowmax = int(rowmaxdec - 1)
nrows = rowmax - rowmin + 1
ncols = colmax - colmin + 1
xll = xll + colmin * cellsizeX
yll = yul + (rowmax + 1) * cellsizeY # cellsizeY is negative
yul = yll - nrows * cellsizeY
else:
rowmin = 0
colmin = 0
# process option for resampling
if options.cellFactor is not None:
cellFactor = float(cellFactor)
else:
cellFactor = 1
if cellFactor >= 1:
procYBlockSize = int(cellFactor)
else:
procYBlockSize = 1
if options.toProj is None:
# Set output raster dimensions and cellsize
newNcols = int(ncols / cellFactor)
newNrows = int(nrows / cellFactor)
newCellsizeX = cellsizeX * cellFactor
newCellsizeY = cellsizeY * cellFactor # is a negative value as before
newXll = xll
newYll = yll
newYul = yul
newNodata = nodata
else:
# Create coordinate transform
if options.fromProj is None:
src_srs = proj
else:
src_srs = osr.SpatialReference()
src_srs.ImportFromProj4(options.fromProj)
tgt_srs = osr.SpatialReference()
tgt_srs.ImportFromProj4(options.toProj)
import pdb;pdb.set_trace()
coordTrans = osr.CoordinateTransformation(src_srs, tgt_srs)
if options.template is None:
# estimate extent from input
newNcols = ncols
newNrows = nrows
newCellsizeX = cellsizeX
newCellsizeY = cellsizeY
newNodata = nodata
newXll, newYll, z = coordTrans.TransformPoint(xll, yll)
newXll = int(newXll)
newYll = int(newYll)
newYul = newYll - newNrows * newCellsizeY
else:
header = open(options.template, "r").read()
newNcols = int(
re.compile("ncols\s*([0-9]*)").search(header).group(1))
newNrows = int(
re.compile("nrows\s*([0-9]*)").search(header).group(1))
newCellsizeX = float(
re.compile("cellsize\s*([0-9]*)").search(header).group(1))
newCellsizeY = -1 * newCellsizeX
try:
newNodata = float(
re.compile(
"NODATA_value\s*(.*?)\n"
).search(header).group(1)
)
except AttributeError:
newNodata = float(
re.compile(
"nodata_value\s*(.*?)\n"
).search(header).group(1)
)
newXll = float(
re.compile("xllcorner\s*(.*?)\n").search(header).group(1))
newYll = float(
re.compile("yllcorner\s*(.*?)\n").search(header).group(1))
newYul = newYll - newNrows * newCellsizeY
# Processing of data is made for blocks of the following size
# Important - blocks are set to cover all columns for simpler processing
# This might not always be optimal for read/write speed
procXBlockSize = ncols
# process option for dataType
if options.toShape:
dataTypes, defaultDataType = ogrDataTypes, 'Real'
else:
dataTypes, defaultDataType = gdalDataTypes, 'Float32'
try:
dataType = dataTypes[options.dataType or defaultDataType]
except KeyError:
log.error(
"Unknown datatype choose between: %s" % ",".join(dataTypes.keys()))
sys.exit(1)
# Create and configure output raster data source
if not options.toShape and outFilePath is not None:
# Creates a raster dataset with 1 band
mem_ds = gdal.GetDriverByName('MEM').Create(outFilePath,
newNcols,
newNrows,
1,
dataType)
if mem_ds is None:
print "Error: could not create output raster"
sys.exit(1)
outGeotransform = [newXll, newCellsizeX, 0, newYul, 0, newCellsizeY]
mem_ds.SetGeoTransform(outGeotransform)
if options.toProj is not None:
mem_ds.SetProjection(tgt_srs.ExportToWkt())
else:
mem_ds.SetProjection(proj)
outBand = mem_ds.GetRasterBand(1)
outBand.SetNoDataValue(newNodata) # Set nodata-value
if options.toProj is not None:
outArray = np.zeros((newNrows, newNcols))
nvals = np.zeros((newNrows, newNcols))
outDef = {"ncols": newNcols,
"nrows": newNrows,
"xll": newXll,
"yll": newYll,
"yul": newYll - newNrows * newCellsizeY,
"xur": newXll + newNcols * newCellsizeX,
"cellsize": newCellsizeX
}
# Create and inititialize output vector data source
if options.toShape:
shapeDriver = ogr.GetDriverByName('ESRI Shapefile')
if path.exists(outFilePath):
shapeDriver.DeleteDataSource(outFilePath)
shapeFile = shapeDriver.CreateDataSource(outFilePath)
if shapeFile is None:
log.error("Could not open output shapefile %s" % outFilePath)
sys.exit(1)
layer = shapeFile.CreateLayer(outFilePath, geom_type=ogr.wkbPolygon)
fieldDefn = ogr.FieldDefn(fieldName, dataType)
layer.CreateField(fieldDefn)
# inititialize input grid summary
inputGridSummary = {"sum": 0,
"mean": 0,
"nnodata": 0,
"nnegative": 0,
"xll": xll,
"yll": yll,
"ncols": ncols,
"nrows": nrows,
"cellsizeX": cellsizeX,
"cellsizeY": cellsizeY,
"nodatavalue": nodata}
outputGridSummary = {"sum": 0,
"mean": 0,
"nnodata": 0,
"nnegative": 0,
"xll": newXll,
"yll": newYll,
"ncols": newNcols,
"nrows": newNrows,
"cellsizeX": newCellsizeX,
"cellsizeY": newCellsizeY,
"nodatavalue": newNodata}
# Loop over block of raster (at least one row in each block)
rowsOffset = 0
errDict = {}
pg = ProgressBar(nrows, options.progressStream)
for i in range(0, nrows, int(procYBlockSize)):
pg.update(i)
data = band.ReadAsArray(xoff=colmin, yoff=rowmin + i,
win_xsize=procXBlockSize,
win_ysize=procYBlockSize)
if options.summarize:
inputGridSummary = updateGridSummary(data,
inputGridSummary, nodata)
if options.classTable is not None:
try:
data = reclassBlock(data, classDict, errDict)
except IOError as e:
log.error(str(e))
sys.exit(1)
if options.cellFactor is not None:
try:
data = resampleBlock(data[:, :],
cellFactor,
resamplingMethod,
nodata)
except ValueError as err:
log.error(err.message)
sys.exit(1)
except IOError as err:
log.error(err.message)
sys.exit(1)
if options.toProj is not None:
blockDef = {"nrows": int(procYBlockSize / cellFactor),
"ncols": int(procXBlockSize / cellFactor),
"xll": xll + (colmin) * cellsizeX,
"yul": yll - (nrows - rowmin - i) * cellsizeY,
"cellsize": cellsizeX * cellFactor,
"nodata": nodata}
reprojectBlock(outArray,
data,
cellFactor,
blockDef,
outDef,
coordTrans,
nvals)
if outFilePath is not None:
if options.toShape:
blockYll = yul + i * newCellsizeY # newCellsizeY is negative
blockXll = xll
block2vector(data, layer, blockXll, blockYll, newCellsizeX,
newCellsizeY, nodata, fieldName, filter)
elif options.toProj is None:
outBand.WriteArray(data, 0,
rowsOffset) # Write block to raster
outBand.FlushCache() # Write data to disk
if options.summarize:
outputGridSummary = updateGridSummary(data,
outputGridSummary,
nodata)
rowsOffset += int(procYBlockSize / cellFactor) # Update offset
if options.toShape:
shapeFile.Destroy()
if not options.toShape and options.toProj is not None:
if options.resamplingMethod is not None and resamplingMethod == "mean":
outArray = np.where(nvals > 0, outArray / nvals, outArray)
outBand.WriteArray(outArray, 0, 0)
outBand.FlushCache() # Write data to disk
if options.outfileName is not None and not options.toShape:
outputDriver = ds.GetDriver()
output_ds = outputDriver.CreateCopy(options.outfileName, mem_ds, 0)
output_ds = None
mem_ds = None
pg.finished()
if options.summarize:
print "\nInput raster summary"
printGridSummary(inputGridSummary, prefix="input")
print "\nOutput raster summary"
printGridSummary(outputGridSummary, prefix="output")
if errDict != {}:
print "Errors/warnings during processing:"
for err, nerr in errDict.items():
print "%s err in %i cells" % (err, nerr)
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option("-d",
"--doc",
action="store_true",
dest="doc",
help="Prints more detailed documentation and exit")
parser.add_option("-v",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=get_loglevel(),
help="Produce verbose output")
parser.add_option("--no-progress",
action="store_const",
dest="progressStream",
const=None,
default=sys.stdout,
help="turn off the progress bar")
parser.add_option("-o",
"--output",
action="store",
dest="outfileName",
default=None,
help="Output file")
parser.add_option("-i",
"--input",
action="store",
dest="infileName",
help="Input raster")
parser.add_option("--bbox",
action="store",
dest="bbox",
help="Only read data within bbox," +
" --box <\"x1,y1,x2,y2\"")
parser.add_option("--reclassify",
action="store",
dest="classTable",
help="Tab-separated table with code " +
"and z0 value for each landuse class")
parser.add_option("--reclassFromColumn",
action="store",
dest="reclassFromColumn",
help="Header of reclass table column " +
"containing values to " +
"reclass (default is to use first column of classTable)")
parser.add_option("--reclassToColumn",
action="store",
dest="reclassToColumn",
help="Header of reclass table column containing codes," +
" default is to use first column (default is to use " +
"second column of classTable)")
parser.add_option("--resample",
action="store",
dest="cellFactor",
help="Resample grid by dividing cellsize with a" +
" factor. Factor <1 results in refinement." +
" Reprojection uses temporary refinement")
parser.add_option("--resamplingMethod",
action="store",
dest="resamplingMethod",
help="For cellFactor > 1: " +
"Choose between 'mean', 'sum', 'majority' or" +
" 'count', default is sum,"
"For cellFactor < 1, choose between: " +
"'keepTotal' and 'keepValue', default is 'keepTotal'")
parser.add_option("--summarize",
action="store_true",
dest="summarize",
help="Print a summary of the input grid properties")
parser.add_option("--bandIndex",
action="store",
dest="bandIndex",
help="Band index to read from",
default=1)
parser.add_option("--dataType",
action="store",
dest="dataType",
help="Output raster/shape data type",
default=None)
parser.add_option("--toShape",
action="store_true",
dest="toShape",
help="output as shape file",
default=None)
parser.add_option("--fieldName",
metavar='FIELD',
action="store",
dest="fieldName",
help="write data in shape file to FIELD," +
" default is 'value'",
default=None)
parser.add_option("--filter",
action="store",
dest="filter",
help="Filter out data equal or below" +
" limit in shape output",
default=None)
parser.add_option("-t",
"--template",
action="store",
dest="template",
help="Header for output raster when reprojecting")
parser.add_option("--fromProj",
dest="fromProj",
help="Input raster proj4 definition string")
parser.add_option("--toProj",
dest="toProj",
help="Output raster epsg or proj4 definition string")
(options, args) = parser.parse_args()
# ------------Setting up logging capabilities -----------
# Setup logging
logging.basicConfig(
format='%(levelname)s:%(name)s: %(message)s',
level=options.loglevel,
)
log = logging.getLogger(__name__)
# ------------Process and validate options---------------
if options.doc:
print __doc__
sys.exit(0)
if len(args) > 0:
parser.error("Incorrect number of arguments")
# validate infile path
if options.infileName is not None:
inFilePath = path.abspath(options.infileName)
if not path.exists(inFilePath):
log.error("Input raster %s does not exist" % options.infileName)
sys.exit(1)
else:
parser.error("No input data specified")
# validate outfile path
if options.outfileName is not None:
outFilePath = path.abspath(options.outfileName)
if options.toShape and ".shp" not in outFilePath:
parser.error("Output shape has to to be specified with" +
" .shp extension")
else:
outFilePath = None
# Validate fieldName option
if options.toShape:
if options.fieldName == "":
parser.error("fieldName can't be an empty string")
fieldName = options.fieldName or "value"
elif options.fieldName is not None:
parser.error("fieldName option only allowed together" +
" with shape output")
# Validate filter option and convert filter to numeric value if present
if not options.toShape and options.filter is not None:
parser.error("Filter option only allowed together with shape output")
elif options.toShape:
if options.filter is not None:
filter = float(options.filter)
else:
filter = None
# read and process reclass table file
if options.classTable is not None:
reclassFromColumn = options.reclassFromColumn
reclassToColumn = options.reclassToColumn
if reclassFromColumn is not None and reclassToColumn is not None:
desc = [{
"id": reclassFromColumn,
"type": float
}, {
"id": reclassToColumn,
"type": float
}]
classTable = datatable.DataTable(desc=desc)
classTable.read(options.classTable)
else:
classTable = datatable.DataTable()
classTable.read(options.classTable)
reclassFromColumn = classTable.desc[0]["id"]
reclassToColumn = classTable.desc[1]["id"]
classTable.convertCol(reclassFromColumn, float)
classTable.convertCol(reclassToColumn, float)
classTable.setKeys([reclassFromColumn])
log.debug("Successfully read landuse class table")
classDict = {}
for row in classTable.data:
classDict[row[classTable.colIndex[reclassFromColumn]]] = row[
classTable.colIndex[reclassToColumn]]
if options.cellFactor is not None:
cellFactor = float(options.cellFactor)
if cellFactor > 1 and \
options.resamplingMethod not in ('sum',
'mean',
'majority',
'count',
None):
log.error("Invalid resampling method, valid options for grid " +
"coarsening are 'sum' " + "and 'mean' and 'majority', " +
"specified %s" % options.resamplingMethod)
sys.exit(1)
elif cellFactor < 1 and \
options.resamplingMethod not in ('keepTotal',
'keepValue',
None):
log.error("Invalid resampling method, valid options for grid " +
"coarsening are 'keepTotal' and 'keepValue'" +
", specified %s" % resamplingMethod)
sys.exit(1)
# setting default resampling methods
if cellFactor > 1 and \
options.resamplingMethod is None:
resamplingMethod = 'sum'
elif options.resamplingMethod is None:
resamplingMethod = 'keepTotal'
else:
resamplingMethod = options.resamplingMethod
if options.resamplingMethod == 'majority' or \
options.resamplingMethod == 'count' and \
options.toProj is not None:
log.error("Resampling method " +
"%s not possible to " % options.resamplingMethod +
"combine with reprojection")
sys.exit(1)
# Assure that gdal is present
if not __gdal_loaded__:
raise OSError("Function readGDAL needs GDAL with python bindings")
# register all of the raster drivers
gdal.AllRegister()
ds = gdal.Open(inFilePath, GA_ReadOnly)
if ds is None:
print 'Could not open ' + inFilePath
sys.exit(1)
ncols = ds.RasterXSize
nrows = ds.RasterYSize
nbands = ds.RasterCount
# Info used for georeferencing
geoTransform = ds.GetGeoTransform()
xul = geoTransform[0] # top left x
cellsizeX = geoTransform[1] # w-e pixel resolution
rot1 = geoTransform[2] # rotation, 0 if image is "north up"
yul = geoTransform[3] # top left y
rot2 = geoTransform[4] # rotation, 0 if image is "north up"
cellsizeY = geoTransform[5] # n-s pixel resolution
proj = osr.SpatialReference(ds.GetProjection())
# Calculate lower left corner
xll = xul
yll = yul + nrows * cellsizeY # cellsizeY should be a negative value
# Calculate upper right corner
xur = xul + cellsizeX * ncols
yur = yul
# Rotated rasters not handled...yet
if rot1 != 0 or rot2 != 0:
print 'Rotated rasters are not supported by pyAirviro.geo.raster'
sys.exit(1)
if abs(cellsizeX) != abs(cellsizeY):
print('Non-homogenous cellsizes are not' +
' supported by pyAirviro.geo.raster')
sys.exit(1)
bandIndex = int(options.bandIndex)
band = ds.GetRasterBand(bandIndex)
nodata = band.GetNoDataValue()
# If no nodata value is present in raster, set to -9999 for completeness
if nodata is None:
nodata = -9999
# Read data from a window defined by option --bbox <"x1,y1,x2,y2">
if options.bbox is not None:
try:
x1, y1, x2, y2 = map(float, options.bbox.split(","))
except:
log.error("Invalid value for option --bbox <\"x1,y1,x2,y2\">")
sys.exit(1)
# Check if totally outside raster extent
if x2 < xll or y2 < yll or x1 > xur or y1 > yul:
log.error("Trying to extract outside grid boundaries")
sys.exit(1)
# Limit bbox to raster extent
if x1 < xll:
x1 = xll
if x2 > xur:
x2 = xur
if y1 < yll:
y1 = yll
if y2 > yul:
y2 = yul
# estimate min and max of rows and cols
colmin = int((x1 - xll) / cellsizeX)
colmaxdec = (x2 - xll) / float(cellsizeX)
rowmin = int((yul - y2) / abs(cellsizeY))
rowmaxdec = (yul - y1) / float(abs(cellsizeY))
if (colmaxdec - int(colmaxdec)) > 0:
colmax = int(colmaxdec)
else:
colmax = int(colmaxdec - 1)
if (rowmaxdec - int(rowmaxdec)) > 0:
rowmax = int(rowmaxdec)
else:
rowmax = int(rowmaxdec - 1)
nrows = rowmax - rowmin + 1
ncols = colmax - colmin + 1
xll = xll + colmin * cellsizeX
yll = yul + (rowmax + 1) * cellsizeY # cellsizeY is negative
yul = yll - nrows * cellsizeY
yur = yul
xur = xll + ncols * cellsizeX
else:
rowmin = 0
colmin = 0
# process option for resampling
if options.cellFactor is not None:
cellFactor = float(cellFactor)
else:
cellFactor = 1
if cellFactor >= 1:
procYBlockSize = int(cellFactor)
else:
procYBlockSize = 1
if options.toProj is None:
# Set output raster dimensions and cellsize
newNcols = int(ncols / cellFactor)
newNrows = int(nrows / cellFactor)
newCellsizeX = cellsizeX * cellFactor
newCellsizeY = cellsizeY * cellFactor # is a negative value as before
newXll = xll
newYll = yll
newYul = yul
newNodata = nodata
else:
# Create coordinate transform
if options.fromProj is None:
src_srs = proj
else:
src_srs = osr.SpatialReference()
src_srs.ImportFromProj4(options.fromProj)
tgt_srs = osr.SpatialReference()
if options.toProj.startswith("epsg"):
tgt_srs.ImportFromEPSG(int(options.toProj[5:]))
else:
tgt_srs.ImportFromProj4(options.toProj)
coordTrans = osr.CoordinateTransformation(src_srs, tgt_srs)
newXur = None
newYur = None
if options.template is None:
# estimate extent from input
newNcols = int(ncols / cellFactor)
newNrows = int(nrows / cellFactor)
newXll, newYll, z = coordTrans.TransformPoint(xll, yll)
newXur, newYur, z = coordTrans.TransformPoint(xur, yur)
newCellsizeX = (newXur - newXll) / float(newNcols)
newCellsizeY = (newYur - newYll) / float(newNrows)
newNodata = nodata
newXll = newXll
newYll = newYll
newYul = newYur
else:
header = open(options.template, "r").read()
newNcols = int(
re.compile("ncols\s*([0-9]*)").search(header).group(1))
newNrows = int(
re.compile("nrows\s*([0-9]*)").search(header).group(1))
newCellsizeX = float(
re.compile("cellsize\s*([0-9]*)").search(header).group(1))
newCellsizeY = -1 * newCellsizeX
try:
newNodata = float(
re.compile("NODATA_value\s*(.*?)\n").search(header).group(
1))
except AttributeError:
newNodata = float(
re.compile("nodata_value\s*(.*?)\n").search(header).group(
1))
newXll = float(
re.compile("xllcorner\s*(.*?)\n").search(header).group(1))
newYll = float(
re.compile("yllcorner\s*(.*?)\n").search(header).group(1))
newYul = newYll - newNrows * newCellsizeY
# Processing of data is made for blocks of the following size
# Important - blocks are set to cover all columns for simpler processing
# This might not always be optimal for read/write speed
procXBlockSize = ncols
# process option for dataType
if options.toShape:
dataTypes, defaultDataType = ogrDataTypes, 'Real'
else:
dataTypes, defaultDataType = gdalDataTypes, 'Float32'
try:
dataType = dataTypes[options.dataType or defaultDataType]
except KeyError:
log.error("Unknown datatype choose between: %s" %
",".join(dataTypes.keys()))
sys.exit(1)
# Create and configure output raster data source
if not options.toShape and outFilePath is not None:
# Creates a raster dataset with 1 band
mem_ds = gdal.GetDriverByName('MEM').Create(outFilePath, newNcols,
newNrows, 1, dataType)
if mem_ds is None:
print "Error: could not create output raster"
sys.exit(1)
outGeotransform = [newXll, newCellsizeX, 0, newYul, 0, newCellsizeY]
mem_ds.SetGeoTransform(outGeotransform)
if options.toProj is not None:
mem_ds.SetProjection(tgt_srs.ExportToWkt())
elif isinstance(proj, osr.SpatialReference):
mem_ds.SetProjection(proj.ExportToProj4())
else:
mem_ds.SetProjection(proj)
outBand = mem_ds.GetRasterBand(1)
outBand.SetNoDataValue(newNodata) # Set nodata-value
if options.toProj is not None:
outArray = np.zeros((newNrows, newNcols))
nvals = np.zeros((newNrows, newNcols))
outDef = {
"ncols": newNcols,
"nrows": newNrows,
"xll": newXll,
"yll": newYll,
"yul": newYll - newNrows * newCellsizeY,
"xur": newXll + newNcols * newCellsizeX,
"cellsize": newCellsizeX
}
# Create and inititialize output vector data source
if options.toShape:
shapeDriver = ogr.GetDriverByName('ESRI Shapefile')
if path.exists(outFilePath):
shapeDriver.DeleteDataSource(outFilePath)
shapeFile = shapeDriver.CreateDataSource(outFilePath)
if shapeFile is None:
log.error("Could not open output shapefile %s" % outFilePath)
sys.exit(1)
layer = shapeFile.CreateLayer(outFilePath, geom_type=ogr.wkbPolygon)
fieldDefn = ogr.FieldDefn(fieldName, dataType)
layer.CreateField(fieldDefn)
# inititialize input grid summary
inputGridSummary = {
"sum": 0,
"mean": 0,
"nnodata": 0,
"nnegative": 0,
"xll": xll,
"yll": yll,
"ncols": ncols,
"nrows": nrows,
"cellsizeX": cellsizeX,
"cellsizeY": cellsizeY,
"nodatavalue": nodata
}
outputGridSummary = {
"sum": 0,
"mean": 0,
"nnodata": 0,
"nnegative": 0,
"xll": newXll,
"yll": newYll,
"ncols": newNcols,
"nrows": newNrows,
"cellsizeX": newCellsizeX,
"cellsizeY": newCellsizeY,
"nodatavalue": newNodata
}
# Loop over block of raster (at least one row in each block)
rowsOffset = 0
errDict = {}
pg = ProgressBar(nrows, options.progressStream)
for i in range(0, nrows, int(procYBlockSize)):
pg.update(i)
data = band.ReadAsArray(xoff=colmin,
yoff=rowmin + i,
win_xsize=procXBlockSize,
win_ysize=procYBlockSize)
if options.summarize:
inputGridSummary = updateGridSummary(data, inputGridSummary,
nodata)
if options.classTable is not None:
try:
data = reclassBlock(data, classDict, errDict)
except IOError as e:
log.error(str(e))
sys.exit(1)
if options.cellFactor is not None:
try:
data = resampleBlock(data[:, :], cellFactor, resamplingMethod,
nodata)
except ValueError as err:
log.error(err.message)
sys.exit(1)
except IOError as err:
log.error(err.message)
sys.exit(1)
if options.toProj is not None:
blockDef = {
"nrows": int(procYBlockSize / cellFactor),
"ncols": int(procXBlockSize / cellFactor),
"xll": xll + (colmin) * cellsizeX,
"yul": yll - (nrows - rowmin - i) * cellsizeY,
"cellsize": cellsizeX * cellFactor,
"nodata": nodata
}
reprojectBlock(outArray, data, cellFactor, blockDef, outDef,
coordTrans, nvals)
if outFilePath is not None:
if options.toShape:
blockYll = yul + i * newCellsizeY # newCellsizeY is negative
blockXll = xll
block2vector(data, layer, blockXll, blockYll, newCellsizeX,
newCellsizeY, nodata, fieldName, filter)
elif options.toProj is None:
outBand.WriteArray(data, 0,
rowsOffset) # Write block to raster
outBand.FlushCache() # Write data to disk
if options.summarize:
outputGridSummary = updateGridSummary(data, outputGridSummary,
nodata)
rowsOffset += int(procYBlockSize / cellFactor) # Update offset
if options.toShape:
shapeFile.Destroy()
if not options.toShape and options.toProj is not None:
if options.resamplingMethod is not None and resamplingMethod == "mean":
outArray = np.where(nvals > 0, outArray / nvals, outArray)
outBand.WriteArray(outArray, 0, 0)
outBand.FlushCache() # Write data to disk
if options.outfileName is not None and not options.toShape:
outputDriver = ds.GetDriver()
output_ds = outputDriver.CreateCopy(options.outfileName, mem_ds, 0)
output_ds = None
mem_ds = None
pg.finished()
if options.summarize:
print "\nInput raster summary"
printGridSummary(inputGridSummary, prefix="input")
print "\nOutput raster summary"
printGridSummary(outputGridSummary, prefix="output")
if errDict != {}:
print "Errors/warnings during processing:"
for err, nerr in errDict.items():
print "%s err in %i cells" % (err, nerr)
def main():
#-----------Setting up and using option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option(
'-v',
action='store_const', dest='loglevel',
const=logging.DEBUG, default=get_loglevel(),
help='produce verbose output'
)
# parser.add_option("-l", "--loglevel",
# action="store",dest="loglevel",default=2,
# help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
(options, args) = parser.parse_args()
#------------Setting up logging capabilities -----------
logging.basicConfig(
format='%(levelname)s:%(name)s: %(message)s',
level=options.loglevel,
)
global log
log = logging.getLogger(parser.prog)
if options.cf!=None:
generateCf(path.abspath(options.cf))
log.info("Wrote default controlfile")
sys.exit()
if len(args)!=1:
parser.error("Incorrect number of arguments")
#Opening controlfile
cf=ControlFile(fileName=path.abspath(args[0]))
domain = Domain()
edbName=cf.findString("edb:")
userName=cf.findString("user:"******"Edb "+edbName+" does not exist for user "+userName+" in domain "+domain.name)
sys.exit()
year=cf.findInt("year:")
substList=cf.findStringList("substances:")
#Remove doubles from substance list
substances={'nat':[]}
for subst in substList:
if subst not in substances['nat']:
substances['nat'].append(subst)
#Make dictionary for substance,units couples
unitDict={}
defaultUnit=cf.findString("nationalTotals.defaultUnit:")
unitList=cf.findStringList("nationalTotals.units:")
for i in range(len(unitList)):
unitSubstances=cf.findStringList("nationalTotals.unit."+str(i)+":")
for s in unitSubstances:
unitDict[s]=unitList[i]
for subst in substances['nat']:
if subst not in unitDict.keys():
unitDict[subst]=defaultUnit
nationalTotalsTablePath=cf.findString("nationalTotalsTable:")
nationalTotalsFile=codecs.open(nationalTotalsTablePath,'w',encoding="latin6")
nationalTotalsTreePath=cf.findString("nationalTotalsTree:")
nationalTotalsTreePath=path.abspath(nationalTotalsTreePath)
nationalTotalTreeFile=open(nationalTotalsTreePath,'w')
log.info("Reading emissions from CRF")
CRFTree=CodeEmisTree(name="CRF")
CRFPath=cf.findExistingPath("CRF.path:")
CRFTree.parseCRFXML(CRFPath,year)
CRFTree.calculateSums(byAttribute="Category")
#CRFTree.write(sys.stdout)
CRFTable=CRFTree.createTable(keys=["Category","Classification","AWMS"])
HFC={
"HFC-23":11700,
"HFC-32":650,
"HFC-41":150,
"HFC-43-10 mee":1300,
"HFC-125":2800,
"HFC-134":1000,
"HFC-134a":1300,
"HFC-152a":140,
"HFC-143":300,
"HFC-143a":3800,
"HFC-227ea":2900,
"HFC-236fa":6300,
"HFC-245ca":560
}
PFC={
"CF4":6500,
"C2F6":9200,
"C3F8":7000,
"C4F10":7000,
"c-C4F8":8700,
"C5F12":7500,
"C6F14":7400
}
CRFTable.aggregateCols({"id":"HFC-tot","type":"str","units":"Gg"},HFC)
CRFTable.aggregateCols({"id":"PFC-tot","type":"str","units":"Gg"},PFC)
CRFTable.sortRows()
CRFTableKeys=cf.findStringList("CRF.tableKeys:")
CRFTable.keys=CRFTableKeys
#CRFTable.write(sys.stdout)
log.info("Reading emissions from NFR")
NFRTree=CodeEmisTree(name="NFR")
NFRPath=cf.findExistingPath("NFR.path:")
# pdb.set_trace()
NFRTree.parseNFRTable(filename=NFRPath)
NFRTree.calculateSums()
NFRTable=NFRTree.createTable()
NFRTableKeys=cf.findStringList("NFR.tableKeys:")
NFRTable.keys=NFRTableKeys
colIndex=NFRTable.getColIndex("Code")
for row in NFRTable.data:
if isinstance(row[colIndex],str):
row[colIndex]=row[colIndex].replace(" ","")
log.info("Reading emissions from Mobile")
MobilePath=cf.findExistingPath("Mobile.path:")
MobileTable=DataTable()
MobileTable.read(MobilePath,units=True)
MobileTable=MobileTable.filtered({"Year":str(year)})
MobileTableKeys=cf.findStringList("Mobile.tableKeys:")
MobileTable.keys=MobileTableKeys
#Remove biogenic emissions of CO2
#Remove GHG from Bunkers, except for international aviation LTO
for row in MobileTable.data:
# if "Bio" in row[MobileTable.getColIndex("Codename")]:
if "Bio" in row[MobileTable.getColIndex("Fueltype")]:
row[MobileTable.getColIndex("CO2")]=None
# if row[MobileTable.getColIndex("Localcodename")]=="Bunkers" and row[MobileTable.getColIndex("Localsubcodename")]!="LTO":
if row[MobileTable.getColIndex("Subsector")]=="Bunkers" and row[MobileTable.getColIndex("Region")]!="LTO":
row[MobileTable.getColIndex("CO2")]=None
row[MobileTable.getColIndex("CH4")]=None
row[MobileTable.getColIndex("N2O")]=None
log.info("Reading emissions from Manual")
ManualPath=cf.findExistingPath("Manual.path:")
ManualTable=DataTable()
ManualTable.read(ManualPath,units=True)
ManualTableKeys=cf.findStringList("Manual.tableKeys:")
ManualTable.keys=ManualTableKeys
srcTables={'NFR':NFRTable,
'CRF':CRFTable,
'Mobile':MobileTable,
'Manual':ManualTable}
for tKey in srcTables.keys():
substList=cf.findStringList(tKey+".substances:")
substances[tKey]=substList
log.info("Reading dataSourceTable")
dsPath=cf.findExistingPath("sourceTable:")
dsTable=DataTable()
dsTable.read(dsPath)
acIndex=cf.findInt("nationalTotals.acIndex:")
log.info("Reading activity codes from edb")
natTree=CodeEmisTree("National totals","Gg")
natTree.readActivityCodes(path.join(edb.edbPath(),"edb.rsrc"),acIndex)
log.info("Identifying column indices for sources in dataSourceTable")
#codeHeaders={"Mobile":"Mobile CRFnodeprefix",
# "NFR":"NFR Code",
# "CRF":"CRF Code",
# "nat":"Code",
# "Manual":"Code"
# }
codeHeaders={"Mobile":"Mobile UNFCCC",
"NFR":"NFR Code",
"CRF":"CRF Code",
"nat":"Code",
"Manual":"Code"
}
codeInd={}
for key,colName in codeHeaders.iteritems():
try:
codeInd[key]=dsTable.getColIndex(colName)
except DataTableException:
log.error("Could not find '"+colName+"' in dataSourceTable")
sys.exit()
log.info("Collecting data from data sources")
prevNatCode=""
for row in dsTable.data:
rowId={}
srcRow={}
for key in srcTables:
rowId[key]=[]
srcRow[key]=srcTables[key].ncols*[0]
natCode=row[codeInd['nat']]
if natCode!=prevNatCode:
log.debug("\nProcessing: "+natCode)
prevNatCode=natCode
#Get row-id for each src table
for tKey in srcTables.keys():
colInd=codeInd[tKey]
if row[colInd]!=None:
for key in srcTables[tKey].keys:
try:
idPart=row[dsTable.getColIndex(tKey+" "+key)]
except ValueError:
log.error("No column named '"+tKey+" "+key+"' found in dataSourceTable")
sys.exit()
rowId[tKey].append(idPart)
#If not all key values = None
nonNones=[val for val in rowId[tKey] if val !=None]
if len(nonNones)==0:
rowId[tKey]=None
srcRow[tKey]=None
else:
# if tKey=="CRF":
# pdb.set_trace()
try:
srcRow[tKey]=srcTables[tKey].accumulate(rowId[tKey])
except DataTableException, err:
# import pdb; pdb.set_trace()
log.error("While trying to get data from "+tKey+" table: "+str(err))
sys.exit()
#Creating code tree path for the current code
codeParts=natCode.split(".")
natPath=""
for i in range(1,len(codeParts)+1):
natPath+=".".join(codeParts[:i])
if i < len(codeParts):
natPath+="/"
natNode=natTree.root.find(natPath)
if natNode is None:
# import pdb; pdb.set_trace()
log.error("Could not find national code path : "+natCode)
sys.exit()
for subst in substances['nat']:
log.debug("Substance: "+subst)
storedSubstances=[]
foundSubstances=[]
rec=None
#Looping over all sourceTables
for tKey in srcTables:
st=srcTables[tKey]
if rowId[tKey]!=None and subst in substances[tKey]:
try:
colIndex=st.getColIndex(subst)
except DataTableException:
log.error("Could not find column %s in table %s" %(subst,tKey))
emis=srcRow[tKey][colIndex]
unit=st.desc[colIndex].get('units',None)
storedSubstances.append(subst)
nodeExists=False
if emis!=0 and emis!=None:
if subst not in foundSubstances:
foundSubstances.append(subst)
else:
log.error("Substance found in multiple datasources for "+natCode)
sys.exit()
for node in natNode:
if node.tag=="Record" and node.attrib.get("substance",None)==subst:
rec=node
nodeExists=True
break
if not nodeExists:
rec=ET.SubElement(natNode,"Record")
break
if unit==None:
log.error("No units set for substance "+subst+" in source table "+tKey)
sys.exit()
if storedSubstances==[]:
log.error("No data source specified for substance "+subst)
sys.exit()
if rec!=None:
if emis!=None and emis!=0:
try:
emis=convertUnits(fromUnit=unit,toUnit=unitDict[subst],val=emis)
except TypeError:
emis=0
if not nodeExists:
rec.attrib["substance"]=subst
else:
emis+=float(rec.attrib["emission"])
rec.attrib["emission"]=str(emis)
