def read_sources(table_path, delimiter):
table = DataTable()
table.read(table_path, delimiter=delimiter)
sources = {}
for ind, row in enumerate(table.data):
if row[table.colIndex['Sourcetype']] == 'Grid':
continue
src = Source(
NAME=row[table.colIndex['Sourcename']],
INFO=row[table.colIndex['SourceID']],
X1=int(row[table.colIndex['ExtentWest']]),
Y1=int(row[table.colIndex['ExtentSouth']]),
CHIMNEY_HEIGHT=int(
row[table.colIndex['Release Height (stack)']]),
CHIMNEY_OUT=float(
row[table.colIndex['StackOuterDim']]),
CHIMNEY_IN=float(
row[table.colIndex['StackInnerDim']]),
GAS_FLOW=float(
row[table.colIndex['GasSpeed']]),
GASTEMPERATURE=int(
row[table.colIndex['GasTemp']])
)
src.ALOB['HourlyID'] = row[table.colIndex['HourlyID']]
src.ALOB['MonthlyID'] = row[table.colIndex['MonthlyID']]
src.ALOB['Consumption'] = row[table.colIndex['Consumption']]
src.ALOB['ProcessID'] = row[table.colIndex['ProcessID']]
if int(row[table.colIndex['ExtentEast']]) != src.X1 and \
int(row[table.colIndex['ExtentNorth']]) != src.Y1:
src.X2 = int(row[table.colIndex['ExtentEast']])
src.Y2 = int(row[table.colIndex['ExtentNorth']])
sources[src.INFO] = src
return sources
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-u",'--user',
action="store",dest="user",
help="Name of target edb user")
parser.add_option("-e","--edb",
action="store",dest="edb",
help="Name of target edb")
parser.add_option("-i","--infile",
action="store",dest="infile",
help="Input csv file")
# parser.add_option("-y","--year",
# action="store",dest="year",
# help="Only store sources for given year")
parser.add_option("-v",dest='loglevel',
action="store_const",default=get_loglevel(),
help="produce verbose output")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-o", "--outfile",
action="store",dest="outfile",default=None,
help="Name of outfiles (without extension)")
parser.add_option("-d","--delimiter",
action="store",dest="delimiter",default="\t",
help="Delimiter used in csv-file")
parser.add_option("-c","--filterCol",
action="store",dest="filterCol",
help="Header of column to use as filter")
parser.add_option("-f","--filterVal",
action="store",dest="filterVal",
help="Value to use in filter")
# parser.add_option("-g", "--geocodeRasterDir",
# action="store",dest="geocodeRasterDir",default=None,
# help="Directory with geocode rasters")
(options, args) = parser.parse_args()
#--------------------Init logger-----------------------
# rootLogger = logger.RootLogger(level=options.loglevel)
logging.basicConfig(
format='%(levelname)s:%(name)s: %(message)s',
level=options.loglevel,
)
global log
# log = rootLogger.getLogger(sys.argv[0])
log = logging.getLogger(parser.prog)
#-----------------Validating options-------------------
if options.cf is not None:
generateCf(path.abspath(options.cf),controlFileTemplate)
log.info("Wrote default controlfile")
return 1
if options.user is None:
log.error("Need to specify -u <user>")
return 1
if options.edb is None:
log.error("Need to specify -e <edb>")
return 1
# if options.year is None:
# log.error("Need to specify -y <year>")
# return 1
# if len(options.year)!=4:
# log.error("Year should be given with four digits")
# return 1
if len(args)!=1:
log.error("Controlfile should be given as argument")
return 1
dmn=Domain()
edb=Edb(dmn,options.user,options.edb)
if not edb.exists():
log.error("Edb %s does not exist" %options.edb)
return 1
log.info("Parsing controlfile")
cf=ControlFile(args[0])
cdbPars=re.compile("companydb\.par\.(\w*?):").findall(cf.content)
fdbPars=re.compile("facilitydb\.par\.(\w*?):").findall(cf.content)
sdbPars=re.compile("sourcedb\.par\.(\w*?):").findall(cf.content)
substEmisNr=re.compile("sourcedb\.subst_emis\.([0-9]*)\.emis").findall(cf.content)
subgrpEmisNr=re.compile("sourcedb\.subgrp_emis\.([0-9]*)\.emis").findall(cf.content)
cdbCols={}
cdbDefaults={}
for par in cdbPars:
cdbCols[par]=cf.findString("companydb.par.%s:" %par)
cdbDefaults[par]=cf.findString("companydb.par.%s.default:" %par,
optional=True,default=None)
fdbCols={}
fdbDefaults={}
for par in fdbPars:
fdbCols[par]=cf.findString("facilitydb.par.%s:" %par)
fdbDefaults[par]=cf.findString("facilitydb.par.%s.default:" %par,
optional=True,default=None)
sdbCols={}
sdbDefaults={}
for par in sdbPars:
sdbCols[par]=cf.findString("sourcedb.par.%s:" %par)
sdbDefaults[par]=cf.findString("sourcedb.par.%s.default:" %par,
optional=True,default=None)
substEmisCols={}
substEmisDefaults={}
if substEmisNr is not None:
for emisNr in substEmisNr:
cols={}
defaults={}
emisPars=re.compile("sourcedb\.subst_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
emisDefaultPars=re.compile(
"sourcedb\.subst_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
if emisPars is not None:
for par in emisPars:
cols[par]=cf.findString("sourcedb.subst_emis.%s.%s:" %(emisNr,par))
if emisDefaultPars is not None:
for par in emisDefaultPars:
defaults[par]=cf.findString("sourcedb.subst_emis.%s.%s.default:" %(emisNr,par),
optional=True,default=None)
substEmisCols[emisNr]=cols
substEmisDefaults[emisNr]=defaults
subgrpEmisCols={}
subgrpEmisDefaults={}
if subgrpEmisNr is not None:
for emisNr in subgrpEmisNr:
cols={}
defaults={}
emisPars=re.compile("sourcedb\.subgrp_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
emisDefaultPars=re.compile(
"sourcedb\.subgrp_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
if emisPars is not None:
for par in emisPars:
cols[par]=cf.findString("sourcedb.subgrp_emis.%s.%s:" %(emisNr,par))
if emisDefaultPars is not None:
for par in emisDefaultPars:
defaults[par]=cf.findString("sourcedb.subgrp_emis.%s.%s.default:" %(emisNr,par),
optional=True,default=None)
subgrpEmisCols[emisNr]=cols
subgrpEmisDefaults[emisNr]=defaults
log.info("Reading subdb...")
subdb=Subdb(edb)
subdb.read()
log.info("Reading companydb...")
companydb=Companydb(edb)
companydb.read()
log.info("Reading sourcedb...")
# source_stream = SourceStream(edb, 'w')
source_stream = open(options.outfile, 'w')
source_writer = ModelWriter(source_stream,encoding="HP Roman8")
log.info("Reading facilitydb...")
facilitydb=Facilitydb(edb)
facilitydb.read()
log.info("Reading subgrpdb")
subgrpdb=Subgrpdb(edb)
subgrpdb.read()
log.info("Reading edb.rsrc")
rsrc=Rsrc(edb.rsrcPath())
acCodeTables=[]
for i in range(rsrc.numberOfCodeTrees("ac")):
acCodeTables.append(CodeTable(rsrc.path,codeType="ac",codeIndex=i+1))
gcCodeTables=[]
for i in range(rsrc.numberOfCodeTrees("gc")):
gcCodeTables.append(CodeTable(rsrc.path,codeType="gc",codeIndex=i+1))
geocodeRasters=[]
rast1=Raster()
rast1.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__1.txt")
rast2=Raster()
rast2.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__2.txt")
geocodeRasters.append(rast1)
geocodeRasters.append(rast2)
log.info("Reading csv-file")
table=DataTable()
table.read(options.infile,delimiter=options.delimiter,encoding="ISO-8859-15")
if options.filterCol is not None:
if options.filterCol not in table.colIndex:
log.error("Filter column header not found in table")
sys.exit(1)
invalid=False
nFiltered=0
nRows=0
log.info("Processing rows")
for rowInd,row in enumerate(table.data):
nRows+=1
if options.filterCol is not None:
filterVal=row[table.colIndex[options.filterCol]]
if options.filterVal!=str(filterVal):
nFiltered+=1
continue
comp = Company()
for par in comp.parOrder:
val=cdbDefaults.get(par,None)
if par in cdbCols:
colId=cdbCols[par]
try:
tableVal=row[table.colIndex[colId]]
except KeyError:
log.error(
"No column with header %s, columns: %s" %(
colId,str(table.listIds())))
if tableVal is not None:
val = tableVal
if val is not None:
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
comp[par]=val
fac = Facility()
for par in fac.parOrder:
val=fdbDefaults.get(par,None)
if par in fdbCols:
colId=fdbCols[par]
tableVal=row[table.colIndex[colId]]
if tableVal is not None:
val = tableVal
if val is not None:
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
fac[par]=val
src = Source()
for par in ["X1", "Y1","X2","Y2",
"PX","PY","NAME","INFO","INFO2","DATE","CHANGED",
"CHIMNEY HEIGHT","GASTEMPERATURE","GAS FLOW",
"SEARCHKEY1","SEARCHKEY2","SEARCHKEY3",
"SEARCHKEY4","SEARCHKEY5","CHIMNEY OUT","CHIMNEY IN",
"HOUSE WIDTH","HOUSE HEIGHT","NOSEGMENTS","BUILD_WIDTHS",
"BUILD_HEIGHTS","BUILD_LENGTHS","BUILD_DISTFARWALL",
"BUILD_CENTER","GEOCODE","FORMULAMACRO","ALOB"]:
val=sdbDefaults.get(par,None)
if par in sdbCols:
colId=sdbCols[par]
tableVal=row[table.colIndex[colId]]
if tableVal is not None:
val = tableVal
if val is not None:
#validate code
if par=="GEOCODE" and val is not None:
gcList=val.split()
for codeIndex,code in enumerate(gcList):
if not gcCodeTables[codeIndex].hasCode(code):
log.error("Invalid geo code %s on row %i" %(code,rowInd))
invalid=True
#Too long names are truncated
if par=="NAME" and len(val)>45:
val=val[:45]
#Store in src object and convert to correct type
src._fieldvalues[par] = lazy_parse(
src, par, val)
gc1=geocodeRasters[0].getVal(src.get_coord()[0],src.get_coord()[1])
gc2=geocodeRasters[1].getVal(src.get_coord()[0],src.get_coord()[1])
src.GEOCODE = [str(int(gc1)) + "." + str(int(gc2))]
for emisNr,emis in substEmisCols.items():
substEmis={"unit":None,"ac":None,"substance":None,"emis":None}
for par in substEmis.keys():
if par in emis:
substEmis[par]=row[table.colIndex[emis[par]]]
else:
try:
substEmis[par]=substEmisDefaults[emisNr][par]
except KeyError:
log.error(
"Need to specify column or default value for subgrp emis %i" %emisNr)
substInd=subdb.substIndex(substEmis["substance"])
if substInd is None:
log.error("Invalid substance name %s on row %i" %(
substEmis["substance"],rowInd))
sys.exit(1)
try:
unit=rsrc.sub[substEmis["unit"]]
except KeyError:
log.error("Invalid unit name %s on row %i" %(emis["unit"],rowInd))
sys.exit(1)
acList=substEmis["ac"].split('\\')[0].split()
for codeIndex,code in enumerate(acList):
# if code == "2.A.4.2":
# import pdb; pdb.set_trace()
refCode = acCodeTables[codeIndex].checkCode(code)
if refCode == "-":
log.error("Invalid activity code %s on row %i" %(code,rowInd))
sys.exit(1)
if refCode != code:
acList[codeIndex] = refCode
substEmis["ac"] = acList
if substEmis["emis"] is not None and substEmis["emis"]!="0":
try:
emis = src.add_emission()
emis.UNIT = substEmis["unit"]
emis.ACTCODE = substEmis["ac"] # needs re-formatting
emis.EMISSION = float(substEmis["emis"])
emis.SUBSTANCE = substInd
emis.auto_adjust_unit(edb)
except:
# print substEmis
# log.error("Invalid substance emission on row %i" %rowInd)
invalid=True
src.EMISSION=src.EMISSION[:-1]
for emis in subgrpEmisCols.values():
subgrpEmis={"unit":None,"ac":None,"name":None,"emis":None}
for par in subgrpEmis.keys():
if par in emis:
subgrpEmis[par]=row[table.colIndex[emis[par]]]
else:
try:
subgrpEmis[par]=subgrpEmisDefaults[emisNr][par]
except KeyError:
log.error(
"Need to specify column or default value for subgrp emis %i" %emisNr)
#validating subgrp name
try:
subgrp=subgrpdb.getByName(subgrpEmis["name"])
except KeyError:
log.error("Invalid subgrp name %s on row %i" %(subgrpEmis["name"],rowInd))
invalid=True
#validating subgrp emis unit
try:
unitFactor=rsrc.subGrpEm[subgrpEmis["unit"]]
except KeyError:
log.error("Invalid unit %s for subgrp emission on row %i" %(
subgrpEmis["unit"],rowInd))
invalid=True
#validating subgrp activity code
acList=subgrpEmis["ac"].split()
for codeIndex,code in enumerate(acList):
refCode = acCodeTables[codeIndex].checkCode(code)
if refCode == "-":
log.error("Invalid activity code %s on row %i" %(code,rowInd))
invalid=True
break
if refCode != code:
acList[codeIndex] = refCode
substEmis["ac"] = acList
try:
src.addSubgrpEmis(subgrp.index,emis=subgrpEmis["emis"],unit=subgrpEmis["unit"],
ac=subgrpEmis["ac"])
except:
log.error("Invalid subgrp emission on row %i" %rowInd)
invalid=True
companydb.append(comp,force=True)
facilitydb.append(fac,force=True)
source_writer.write(src)
# sourcedb.append(src)
if invalid:
log.info("No output written due to validation errors")
sys.exit(0)
if len(companydb.items)>0:
if options.outfile is None:
log.info("Writing companydb")
else:
log.info("Writing company db to file")
companydb.write(filename=options.outfile+".companydb")
if len(facilitydb.items)>0:
if options.outfile is None:
log.info("Writing facilitydb")
else:
log.info("Writing facilitydb to file")
facilitydb.write(filename=options.outfile+".facilitydb")
# if len(sourcedb.sources)>0:
# if options.outfile is None:
# log.info("Writing sourcedb")
# else:
# log.info("Writing sourcedb to file")
# sourcedb.write(filename=options.outfile+".sourcedb")
if options.filterCol is not None:
log.info("Filtered out %i out of %i" %(nFiltered,nRows))
