def includeShip(src,viewportId,MMSI,year):
begtime=year[-2:]+"010101"
endtime=str(int(year)+1)[-2:]+"010100"
cmd = "shiptsexist -p -r %s -b%s -e%s -A%s %i" %("f", begtime, endtime, viewportId, MMSI)
returnCode,errMsg,outMsg=utilities.execute(cmd)
if int(outMsg)==0:
return False
else:
return True
def main():
# setting up option parser
parser = OptionParser(usage=usage, version=version)
parser.add_option(
"-t", "--template", action="store", dest="template", default=None, help="Create default controlfile template"
)
(options, args) = parser.parse_args()
logger = logging.getLogger("batchTrendReport")
if options.template != None:
controlFile.generateCf(path.abspath(options.template), cf_template)
print "Generated default controlfile"
sys.exit()
if len(args) != 2:
parser.error("Wrong number of arguments")
cfPath = path.abspath(args[0])
makroPath = path.abspath(args[1])
if not path.exists(cfPath):
logger.error("Control file does not exist")
sys.exit(-1)
if not path.exists(makroPath):
logger.error("Input makro does not exist")
sys.exit(-1)
domain = pyDomain.domain(os.environ["AVDBNAME"])
substanceIndices = domain.listSubstanceIndices()
cf = controlFile.controlFile(cfPath)
makro = controlFile.controlFile(makroPath, removeComments=False)
outputDir = cf.findExistingPath("outputDir:")
users = cf.findStringList("users:")
years = cf.findStringList("years:")
nyears = len(years)
substances = cf.findStringList("substances:")
prefix = cf.findString("prefix:")
markerTablePath = cp.findExistingPath("markerFile:", optional=True, default=None)
if markerTablePath is not None:
keys = ["Label", "Geocode", "ActivityCode"]
markerTable = dataTable.DataTable(
keys=keys,
desc=[{"id": "Label", "type": str}, {"id": "Geocode", "type": Str}, {"id": "Activitycode", "type": str}],
)
markerTable.read(markerTablePath)
substMapping = {}
for subst in substances:
officialName = cf.findString(subst + ".name:", optional=True, default=None)
if officialName is not None:
substMapping[subst] = officialName
gcMapping = {}
for user in users:
gc = cf.findString(user + ".gc:")
gcMapping[user] = gc
# edbs={}
# for year in years:
# edbs[year]=cf.findStringList("edb."+year+":")
for user in users:
logger.info("Exporting reports for user %s" % user)
# Set the user for each entry in the trend
for i in range(nyears):
makro.setParam("edb.trend." + str(i) + ".user:"******"edb.reportgeocode:", gcMapping[user])
# for year in years:
# makro.setParam("edb.trend.0.name:",year)
for subst in substances:
logger.info("Substans %s" % subst)
substIndex = substanceIndices[subst]
# makro.setParam("edb.trend.0.edb:",edbs[year])
makro.setParam("edb.reportsubstancelist:", str(substIndex))
tmp = tempfile.NamedTemporaryFile(suffix=".tmp", dir=path.join(os.environ["DBAS_PATH"], "tmp"))
makroRelPath = makro.name.split("/sedb/")[1]
cmd = "rusreport " + os.environ["AVDBNAME"] + " " + makroRelPath + " > " + tmp.name
returnCode, errMsg, outMsg = execute(cmd)
if returnCode != 0:
logger.error("Command: " + cmd + " did not finish successfully")
logger.error("Std output: " + outMsg)
sys.exit()
# reportName=path.join(outputDir,prefix+"_"+year+"_sverige_"+subst+".xls")
if user != "airviro":
reportName = path.join(outputDir, prefix + "_" + user + "_" + subst + ".xls")
else:
reportName = path.join(outputDir, prefix + "_" + "sverige" + "_" + subst + ".xls")
ascii2xls(tmp.name, reportName, substMapping)
def main():
# -----------Setting up and unsing option parser-----------------------
parser = OptionParser(usage=usage, version=version)
parser.add_option(
"-l",
"--loglevel",
action="store",
dest="loglevel",
default=2,
help="Sets the loglevel (0-3 where 3=full logging)",
)
parser.add_option("-o", "--outfile", action="store", dest="outfile", default=None, help="Output file")
parser.add_option("--MMSI", action="store", dest="MMSI", default=None, help="Text-file with mmsi to process")
parser.add_option("--begin", action="store", dest="begin", default=None, help="Begin processing at 'YYMMDDhhmm'")
parser.add_option("--end", action="store", dest="end", default=None, help="End processing at 'YYMMDDhhmm'")
(options, args) = parser.parse_args()
# --------------------Init logger-----------------------
rootLogger = logger.RootLogger(int(options.loglevel))
global log
log = rootLogger.getLogger(sys.argv[0])
# -----------------Validating options-------------------
if options.MMSI is None:
log.error("Need to specify '--MMSI <MMSI>'")
return 1
if options.begin is None:
log.error("Need to specify '--begin <YYMMDDhh>'")
return 1
if options.end is None:
log.error("Need to specify '--end <YYMMDDhh>'")
return 1
if options.outfile is None:
log.error("Needs to specify '-o <output file>'")
return 1
output = open(path.abspath(options.outfile), "w")
output.write
if options.bounds is not None:
try:
bounds = [int(c) for c in options.bounds.split(",")]
except:
parser.error("Could not parse bounds, format should be 'xmin,ymin,xmax,ymax'")
if options.exportFile is not None:
name, ext = path.splitext(options.exportFile)
if ext != ".gz":
exportFileName = path.abspath(options.exportFile) + ".gz"
else:
exportFileName = path.abspath(options.exportFile)
export = gzip.open(exportFileName, "wb")
log.info("Running keyscan")
# Get channels, first data and last date for ships using keyscan
cmd = "keyscan -r f -t M -L -x"
returnCode, errMsg, outMsg = utilities.execute(cmd)
keys = outMsg.split("\n")
log.info("Exctracting beginning and end")
# split on white-space and remove last row
keys = [key.split() for key in keys[:-1]]
# Create datetime objects
for key in keys:
dateStr1 = key[1]
dateStr2 = key[2]
if dateStr1[-2:] == "60":
dateStr1 = dateStr1[:-2] + "59"
startTime = datetime.datetime.strptime(dateStr1, "%y%m%d%H%M")
startTime += datetime.timedelta(minutes=1)
else:
startTime = datetime.datetime.strptime(dateStr1, "%y%m%d%H%M")
if dateStr2[-2:] == "60":
dateStr2 = dateStr2[:-2] + "59"
endTime = datetime.datetime.strptime(dateStr2, "%y%m%d%H%M")
endTime += datetime.timedelta(minutes=1)
else:
endTime = datetime.datetime.strptime(dateStr2, "%y%m%d%H%M")
key[1] = startTime
key[2] = endTime
# Convert time-series key to mmsi and create dictionary with mmsi as keys
# and (begin,end) tuples as values
aisMMSI = {}
for key in keys:
mmsi = key[0][:3] + key[0][5:11]
if len(key[0]) == 12:
tsParam = key[0][11]
if tsParam == "x":
aisMMSI[mmsi] = (key[1], key[2])
if options.mmsiFile is None:
mmsiList.append(mmsi)
# sort out wanted ships
nmissing = 0
missingShips = ""
foundShips = 0
skippedShips = 0
for ind, mmsi in enumerate(mmsiList):
log.info("Processing MMSI: %s (%i out of %i)" % (mmsi, ind, len(mmsiList)))
if mmsi not in aisMMSI.keys():
log.warning("MMSI: %s not found in ais data, skipping" % mmsi)
skippedShips += 1
continue
if options.interval is not None:
beginStr, endStr = options.interval.split(",")
try:
begin = datetime.datetime.strptime(beginStr, "%y%m%d%H%M")
end = datetime.datetime.strptime(endStr, "%y%m%d%H%M")
except:
parser.error("Interval should be given as 'YYMMDDhhmm,YYMMDDhhmm'")
# limit time interval to interval with data
if (begin - aisMMSI[mmsi][0]) < datetime.timedelta(minutes=0):
begin = aisMMSI[mmsi][0]
if (end - aisMMSI[mmsi][1]) > datetime.timedelta(minutes=0):
end = aisMMSI[mmsi][1]
if end <= begin:
continue
ts = series.ShipTs(begin=begin, end=end, res=300)
ts.read(mmsi)
if ts.nvals() == 0:
continue
else:
# Create ts covering all available data
ts = series.ShipTs(begin=aisMMSI[mmsi][0], end=aisMMSI[mmsi][1], res=300)
ts.read(mmsi)
if options.bounds is not None:
if not ts.inBounds(bounds[0], bounds[1], bounds[2], bounds[3]):
continue
output.write(
"%s\t%s\t%s\t%i\n"
% (mmsi, ts.toDatetime(0).strftime("%y%m%d%H%M"), ts.toDatetime(-1).strftime("%y%m%d%H%M"), ts.nvals())
)
if options.exportFile is not None:
export.write(str(ts) + "\n")
if options.contentFile is not None:
if foundShips == 0:
tsDaily = ts.content()
else:
tsDaily = addTs(tsDaily, ts.content())
if tsDaily.nvals() == 0:
pdb.set_trace()
foundShips += 1
log.info("Filtered out %i out of %i ships in time-series database" % (foundShips, len(keys)))
if skippedShips > 0:
log.info("Could not find %i mmsi in the time-series database" % (skippedShips))
if options.contentFile is not None:
try:
contentFile = open(path.abspath(options.contentFile), "w")
except:
log.error("Could not open %s for writing" % options.contentFile)
contentFile.write("Gap start\tGap end\tNdays\n")
gapStart = None
for i in range(tsDaily.nvals()):
if tsDaily.data[i, 2] == 0:
if gapStart is None:
gapStart = tsDaily.toDatetime(i)
else:
if gapStart is not None:
gapEnd = tsDaily.toDatetime(i)
ndays = (gapEnd - gapStart).days
contentFile.write("%s\t%s\t%i\n" % (gapStart.strftime("%y%m%d"), gapEnd.strftime("%y%m%d"), ndays))
gapStart = None
contentFile.write("\ntimeseries showing number of unique sighted ships per day\n")
contentFile.write("Date\tNships\n")
for i in range(tsDaily.nvals()):
contentFile.write("%i %i\n" % (tsDaily.data[i, 0], tsDaily.data[i, 2]))
contentFile.close()
def main():
#-----------Setting up and using the option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-v", "--verbose",
action="store_true",dest="verbose",
help="Verbose output")
parser.add_option("-b", "--begin",
action="store",dest="begin",default=None,
help="Time to start extracting ship tracks")
parser.add_option("-e", "--end",
action="store",dest="end",default=None,
help="Time to end extraction of ship tracks")
parser.add_option("-m", "--mmsi",
action="store",dest="mmsi",default=None,
help="Specify a mmsi")
parser.add_option("-o", "--output",
action="store",dest="outputFile",default=None,
help="To write output in ascii format")
(options, args) = parser.parse_args()
if options.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
logging.basicConfig(level=loglevel)
log = logging.getLogger(__name__)
if options.end is None or options.begin is None:
parser.error("Need to specify begin and end time")
if options.outputFile is None:
parser.error("Need to specify output filename")
#Get mmsi for ships with data in the time-inteval using keyscan
cmd="keyscan -r f -b %s -e %s" %(options.begin,options.end)
returnCode,errMsg,outMsg=utilities.execute(cmd)
keys=outMsg.split("\n")
keys=[int(key[:3]+key[5:11]) for key in keys[:-1]]
keys=utilities.uniqify(keys) #remove doubles
ais_stat=AisStat(domain=os.environ.get("AVDBNAME"))
ais_stat.read()
if options.mmsi is not None:
mmsi=int(options.mmsi)
if mmsi not in ais_stat.mmsi:
log.error("MMSI %s not found in ais_stat.txt" %options.mmsi)
sys.exit(1)
if mmsi not in keys:
log.info("No time-series data for ship in specified time interval")
sys.exit(0)
mmsiList=[mmsi]
else:
mmsiList = [mmsi for mmsi in keys if mmsi in ais_stat.mmsi]
log.info("%i ships out of %i in time-series db found in ais_stat.txt" %(len(mmsiList),
len(keys)))
firstShip=True
for mmsi in mmsiList:
cmd="shiptsget -r f -b %s -e %s %i -S" %(options.begin,options.end,mmsi)
returnCode,errMsg,outMsg=utilities.execute(cmd)
rows=outMsg.split("\n")
geomType=rows.pop(0)
attributeNames=rows.pop(0)
coords=rows[1::4]
data=rows[3::4]
if firstShip:
dataRows=data
coordRows=coords
firstShip=False
else:
dataRows+=data
coordRows+=coords
output=geomType+"\n"
output+=attributeNames+"\n"
for i in range(len(coordRows)):
output+="#COORD %i 0\n" %(i+1)
output+=coordRows[i]+"\n"
output+="#DATA %i\n" %(i+1)
output+=dataRows[i]+"\n"
try:
outputFile=open(path.abspath(options.outputFile),"w")
except:
log.error("Could not open file: %s" %path.abspath(options.outputFile))
sys.exit(1)
outputFile.write(output)
outputFile.close()
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-u",'--user',
action="store",dest="user",
help="Name of target edb user")
parser.add_option("-e","--edb",
action="store",dest="edb",
help="Name of target edb")
parser.add_option("-y","--year",
action="store",dest="year",
help="Cut out for given year")
parser.add_option("-f","--file",
action="store",dest="file",
help="File to get data from")
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
(options, args) = parser.parse_args()
#--------------------Init logger-----------------------
rootLogger = logger.RootLogger(level=options.loglevel)
global log
log = rootLogger.getLogger(sys.argv[0])
#-----------------Validating options-------------------
if options.user is None:
log.error("Need to specify -u <user>")
return 1
if options.edb is None:
log.error("Need to specify -e <edb>")
return 1
if options.year is None:
log.error("Need to specify -y <year>")
return 1
if options.file is None:
log.error("Need to specify -f <file>")
return 1
# if len(options.suffix)>4:
# log.error("Limit your suffix length to 4 characters")
# return 1
if len(options.year)!=4:
log.error("Year should be given with four digits")
return 1
dmn=Domain()
edb=Edb(dmn,options.user,options.edb)
subgrpdb=Subgrpdb(edb)
rf = codecs.open(options.file,"r","ISO-8859-15")
species={}
tmp=tempfile.NamedTemporaryFile(suffix=".tmp",dir=dmn.tmpDir())
emisKey={}
try:
cmd="subdb -0 -o "+tmp.name
returnCode,outMsg,errMsg=utilities.execute(cmd)
if returnCode!=0:
raise IOError("Error while running:\n" + cmd + "\nstdout: "+outMsg+"\nstderr: "+errMsg)
f=codecs.open(tmp.name,"r","HP Roman8")
# content=f.read()
for line in f.read().split('\n'):
if line == "":
break
line=line.split()
if line[1] == '""' or line[1] == '"-"':
continue
emisKey[line[1][1:-1].lower()] = line[0]
f.close()
except IOError:
msg="stdout: "+outMsg+"\nstderr: "+errMsg
raise IOError("Could not run: "+cmd+"\n"+msg)
for line in rf:
if "Average of em" in line:
line = line.split("\t")
line[-1]=line[-1][:-1]
i=0
for spec in line[4:]:
i+=1
if len(spec) < 13:
continue
if spec[13:].lower() == "pm25":
species[i]="pm2.5"
elif spec[13:].lower() == "bap":
species[i]="benzo_a_pyrene"
elif spec[13:].lower() == "diox":
species[i]="dioxine"
elif spec[13:].lower() == "pah":
species[i]="pah-4"
else:
species[i]=spec[13:].lower()
continue
line = line[:-1].split('\t')
if line[3] != options.year:
continue
name=line[0]
i=0
substances={}
for emFactor in line[4:]:
i+=1
if emFactor == "":
continue
if species[i] in emisKey:
# print "Now doing species "+species[i]
index=int(emisKey[species[i]])
emFactor = float(emFactor)
if species[i] == "pah-4" or species[i] == "benzo_a_pyrene":
emFactor *= 10** -6
if species[i]=="dioxine":
emFactor *= 10** -9
if emFactor < 10** -5:
unit="g/TJ"
emFactor*= 10**6
elif emFactor < 10** -2:
unit="kg/TJ"
emFactor*= 10**3
elif emFactor < 10:
unit="ton/TJ"
else:
unit="Gg/TJ"
emFactor*= 10** -3
substances[index]={"slope":emFactor,"offset":0,"unit":unit}
else:
print species[i] + " missing"
subgrpdb.add_subgrp(name,substances)
# subgrpdb.read()
subgrpdb.write_to_file("test.txt")
def main():
#setting up parser
parser = argparse.ArgumentParser(description=__doc__)
utils.add_standard_command_options(parser)
parser.add_argument(
"-e","--edbs",
action="store",dest="edbList",
help="List of 'user/edb' pairs separated by :"
)
parser.add_argument(
"-L","--labels",
action="store",
dest="labels",
help="List of edb labels separated by :"
)
parser.add_argument(
"-s","--substances",
action="store",dest="substances",
help="List of substance names separated by :"
)
parser.add_argument(
"-t","--title",
action="store",dest="title",
help="Report title"
)
parser.add_argument(
"-g","--gc-filter",
action="store",dest="gcfilter",
help="Filter on Geo codes, separated by :"
)
parser.add_argument(
"-o","--outfile",
action="store",dest="outfile",
help="Output filename"
)
parser.add_argument(
"-f","--format",
action="store",dest="format",
help="Output in 'excel','csv' or 'raw' " +
"(Excel-format requires xlwt python module)"
)
parser.add_argument("--substMapping",
action="store",dest="substMapping",
help="File with tab separated mappings of substance names")
parser.add_argument("--markerTable",
action="store",dest="markerTable",
help="Table of codes to be formatted and commented")
parser.add_argument(
"macro",metavar="MACRO",
help="A macro to use"
)
args = parser.parse_args()
if args.markerTable is not None:
keys=["Year","GC","AC","note_1","note_2"]
markerTable = DataTable(keys=keys,desc=[{"id":"Year","type":str},{"id":"GC","type":str},{"id":"AC","type":str},{"id":"note_1","type":str},{"id":"note_2","type":str}])
markerTable.read(args.markerTable)
else:
markerTable=None
substMapping={}
if args.substMapping is not None:
with codecs.open(args.substMapping,encoding="HP Roman8",mode="r") as f:
for line in f:
oldName,newName = line.split(":")
substMapping[oldName.strip()]=newName.strip()
dmn = Domain()
if args.gcfilter is not None:
args.gcfilter = args.gcfilter.split(":")
# Read original macro
with codecs.open(args.macro, encoding="HP Roman8", mode="r") as f:
originalContent = f.read()
# Create a tmp copy of the macro, write content from the original macro
macroTempFile = tempfile.NamedTemporaryFile(
suffix=".sedb",
dir=dmn.tmpDir()
)
tmpMacro = codecs.open(
macroTempFile.name,
encoding="HP Roman8",mode="w"
)
tmpMacro.write(originalContent)
tmpMacro.flush()
# Create a ControlFile obj to simplify reading and modifying macro
macro = ControlFile(macroTempFile.name, removeComments=False)
ebd = macro.findString("edb.edb:")
user = macro.findString("edb.user:"******"edb.reportgeocode:")[-1])
acIndex = int(macro.findString("edb.reportactcode:")[-1])
if args.edbList is None:
ebds = [[user, edb]]
else:
edbs = args.edbList.split(":")
edbs = [e.split("/") for e in edbs]
nedbs = len(edbs)
if args.labels is None:
labels = ["No label"] * len(edbs)
else:
labels = args.labels.split(":")
if len(labels) != nedbs:
log.error("Number of labels specified should match number of edb:s")
sys.exit(1)
if args.substances is None:
log.error("Need to specify substances")
sys.exit(1)
else:
substances = args.substances.split(":")
if args.format not in ('excel','csv','raw'):
log.error(
"Invalid format specifier : %s, should be one of 'excel'" +
", 'csv' or 'raw'" %args.format
)
sys.exit(1)
elif args.format == "excel":
try:
import xlwt
except:
log.error(
"trendReport.py requires python module xlwt to write excel-files")
sys.exit(1)
# first edb
# import pdb; pdb.set_trace()
edb = Edb(dmn, edbs[0][0], edbs[0][1])
# assume same code definitions in all edbs to be processed, read from first
rsrc = edb.rsrc
nrsubstances = len(substances)
unitIndex = int(macro.findString("UNIT :"))
units = rsrc.search[unitIndex]
subdb = Subdb(edb)
subdb.read()
#decode input title using stdin encoding
title=args.title.decode(sys.stdin.encoding)
rawOutput = ""
rawMeta = u"name: %s\nnrmacros: %i\nnrsub: %i\nunit: %s\n" %(
title, nedbs, nrsubstances, units)
emissions = []
for ind, edbUser in enumerate(edbs):
label = labels[ind]
userName = edbUser[0]
edbName = edbUser[1]
macro.setParam("edb.user:"******"edb.edb:", edbName)
macro.setParam("USER :"******"EDB :", edbName)
rawMeta += "macro.%i.edbuser: %s\n" %(ind, userName)
rawMeta += "macro.%i.edbname: %s\n" %(ind, edbName)
rawMeta += "macro.%i.desc: %s\n" %(ind, label)
for subst in substances:
log.info(
"User: %s, edb: %s, substance %s" %(
userName, edbName, subst)
)
substanceIndex = subdb.substIndex(subst)
macro.setParam("ELEMENT :", substanceIndex)
macro.write()
command = "xrepedb -i " + macro.name
log.info("Running xrepedb for substance %s" % subst)
# import pdb; pdb.set_trace()
(returnCode, errMsg, outMsg) = utilities.execute(command)
if returnCode != 0:
log.error("Could not run %s\nstdout: %s\nstderr:%s" %(
command,outMsg,errMsg))
sys.exit(1)
if len(outMsg) < 10:
log.error("Invalid output from xrepedb: %s" % outMsg)
sys.exit(1)
rawOutput += "#MACRO %i \"%s\" \"%s\"\n" % (ind, subst, labels[ind])
rawOutput += outMsg
lines = outMsg.split("\n")[:-1]
for lineInd, line in enumerate(lines):
vals = line.split()
ac = vals[1].split(".")
gc = vals[3].split(".")
if len(ac) == 1:
if ac[0] == "<all>":
acLev1 = "alla"
else:
acLev1 = ac[0]
acLev2 = "alla"
else:
acLev1 = ac[0]
acLev2 = ac[1]
if len(gc) == 1:
if gc[0] == "<all>":
gcLev1 = "alla"
else:
gcLev1 = gc[0]
gcLev2 = "alla"
else:
gcLev1 = gc[0]
gcLev2 = gc[1]
emis = float(vals[4])
if acLev1 == "alla":
acLev1Name = "alla"
acLev2Name = "alla"
else:
node = rsrc.ac[acIndex - 1].root.find(acLev1)
acLev1Name = node.attrib["name"]
if acLev2 == "alla":
acLev2Name = "alla"
else:
node = rsrc.ac[acIndex-1].root.find(
acLev1 + "/" + acLev2
)
acLev2Name = node.attrib["name"]
if gcLev1 == "alla":
gcLev1Name = "alla"
gcLev2Name = "alla"
else:
node = rsrc.gc[gcIndex-1].root.find(gcLev1)
gcLev1Name = node.attrib["name"]
if gcLev2 == "alla":
gcLev2Name = "alla"
else:
node = rsrc.gc[gcIndex - 1].root.find(
gcLev1 + "/" + gcLev2
)
gcLev2Name = node.attrib["name"]
if args.gcfilter is not None:
if gc[0] not in args.gcfilter:
# if args.gcfilter != gcLev1:
continue
emissions.append({"label": label,
"substance": subst,
"ac": '.'.join(ac),
"gc": '.'.join(gc),
"gcLev1": gcLev1Name,
"gcLev2": gcLev2Name,
"acLev1": acLev1Name,
"acLev2": acLev2Name,
"acLev1Code": acLev1,
"acLev2Code": acLev2,
"val": emis,
"edbIndex": ind})
#Close tempfile to automatically remove it
tmpMacro.close()
if args.format == "raw":
outfile = codecs.open(args.outfile,"w","HP Roman8")
outfile.write(rawMeta)
outfile.write(rawOutput)
outfile.close()
elif args.format == "csv":
outfile = open(args.outfile,"w")
desc = [
{'id': 'gc', 'type': unicode},
{'id': 'ac', 'type': unicode},
{'id': 'label', 'type': unicode},
{'id': 'user', 'type': unicode},
{'id': 'edb', 'type': unicode}
]
for subst in substances:
desc.append({'id': subst, 'type': float})
keys = ['gc', 'ac', 'label']
table = DataTable(desc=desc, keys=keys)
log.info("Adding emissions to csv-table")
for emis in emissions:
row = [None] * len(desc)
user = edbs[emis['edbIndex']][0]
edb = edbs[emis['edbIndex']][1]
row[table.colIndex['gc']] = emis['gc']
row[table.colIndex['ac']] = emis['ac']
row[table.colIndex['label']] = emis['label']
row[table.colIndex['user']] = user
row[table.colIndex['edb']] = edb
row[table.colIndex[emis['substance']]] = emis['val']
# data is appended to the correct row, or a new row is added if the
# table keys do not match any existing row
log.debug(
"Adding row for substance %s, gc %s, ac %s" %(
emis['substance'],
emis['gc'],
emis['ac'])
)
table.addRow(row, append=True)
table.write(outfile)
outfile.close()
else:
# Create style objects for excel output
header1Style = xlwt.easyxf(
'font: name Times New Roman,color-index black, bold on',
num_format_str='0.000E+00'
)
markerStyle1 = xlwt.easyxf(
'font: name Times New Roman,color-index red, bold off, italic on',
num_format_str='0.000E+00')
markerStyle2 = xlwt.easyxf(
'font: name Times New Roman,color-index orange, bold off, italic on',
num_format_str='0.000E+00')
normalStyle = xlwt.easyxf(
'font: name Times New Roman,color-index black, bold off',
num_format_str='0.000E+00'
)
excelBook = xlwt.Workbook()
# Creating info sheet
infoWs = excelBook.add_sheet("Info")
infoWs.col(0).width = 256*20
infoWs.col(1).width = 256*25
infoWs.col(2).width = 256*20
infoWs.col(3).width = 256*200
infoWs.write(0,0,u"Rapportnamn:",header1Style)
infoWs.write(0,1,title,header1Style)
infoWs.write(1,0,u"Beskrivning av dataunderlaget",header1Style)
infoWs.write(3,0,u"Makron (specificerar utsökningar ur databasen)",header1Style)
infoWs.write(4,0,u"Etikett",header1Style)
infoWs.write(4,1,u"Ägare till EDB",header1Style)
infoWs.write(4,2,u"EDB (emissiondatabas)",header1Style)
infoWs.write(4,3,u"Beskrivning",header1Style)
for i,edbUser in enumerate(edbs):
userName=edbUser[0]
edbName=edbUser[1]
label=labels[i]
infoWs.write(5+i,0,label)
infoWs.write(5+i,1,userName)
infoWs.write(5+i,2,edbName)
#reading edb description file (if it exists)
edb=Edb(dmn,userName,edbName)
infoWs.write(5+i,3,edb.desc().replace("\n"," "))
#split substances in green house gases and air quality related
ghgList=[s for s in substances if s in ghgs]
aqList=[s for s in substances if s not in ghgs]
#Write air quality headers
firstRow=4
#Add two rows for marker comments
if markerTable is not None:
firstRow+=2
if len(aqList)>0:
aqWs = excelBook.add_sheet(u"Luftföroreningar")
aqWs.col(0).width = 256*25
aqWs.col(1).width = 256*30
aqWs.col(2).width = 256*20
aqWs.col(3).width = 256*15
for col in range(nrsubstances*nedbs):
aqWs.col(col+4).width=256*15
aqWs.write(0,0,u"Rapportnamn:",header1Style)
aqWs.write(0,1,title,header1Style)
aqWs.write(1,0,u"Emissioner av luftföroreningar",header1Style)
aqWs.write(1,1,u"Enhet: "+units,header1Style)
if markerTable is not None:
aqWs.write(2,0,u"OBS! Röd kursiv text anger osäkra värden p.g.a. att en stor del av emissionen är fördelad med schabloner inom kommungruppen. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle1)
aqWs.write(3,0,u"OBS! Orange kursiv text anger osäkra värden p.g.a. att trenden varierar kraftigt och eventuellt felaktigt, ytterligare verifiering krävs. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle2)
aqWs.write(firstRow,0,"Huvudsektor",header1Style)
aqWs.write(firstRow,1,"Undersektor",header1Style)
aqWs.write(firstRow,2,u"Län",header1Style)
aqWs.write(firstRow,3,"Kommun",header1Style)
#Write ghg headers
if len(ghgList)>0:
ghgWs = excelBook.add_sheet(u"Växthusgaser")
ghgWs.col(0).width = 256*25
ghgWs.col(1).width = 256*30
ghgWs.col(2).width = 256*20
ghgWs.col(3).width = 256*15
for col in range(nrsubstances*nedbs):
ghgWs.col(col+4).width=256*15
ghgWs.write(0,0,u"Rapportnamn:",header1Style)
ghgWs.write(0,1,title,header1Style)
ghgWs.write(1,0,u"Emissioner av Växthusgaser",header1Style)
ghgWs.write(2,0,u"CO2-ekv. efter ämnesnamn innebär att emissionen är uttryckt i CO2-ekvivalenter",header1Style)
if markerTable is not None:
ghgWs.write(3,0,u"OBS! Röd kursiv text anger osäkra värden p.g.a. att en stor del av emissionen är fördelad med schabloner inom kommungruppen. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle1)
ghgWs.write(4,0,u"OBS! Orange kursiv text anger osäkra värden p.g.a. att trenden varierar kraftigt och eventuellt felaktigt, ytterligare verifiering krävs. Granska underkategorin \"Energiförsörjning via el-värmeverk samt inom industrin\" för att se eventuella misstänkta värden.",markerStyle2)
ghgWs.write(1,1,u"Enhet: "+units,header1Style)
ghgWs.write(firstRow,0,"Huvudsektor",header1Style)
ghgWs.write(firstRow,1,"Undersektor",header1Style)
ghgWs.write(firstRow,2,u"Län",header1Style)
ghgWs.write(firstRow,3,"Kommun",header1Style)
def getColInd(nmacros, substances,macroInd,subst):
#gets the column index in excel file
sInd=substances.index(subst)
#Including extra columns to write CO2-equivalents
nSubstWithCO2equivalents=0
for s in substances[:sInd+1]:
if s in doubleColumns:
nSubstWithCO2equivalents+=1
return 4 + macroInd+sInd*nmacros+nSubstWithCO2equivalents*(macroInd+1)
#write macro labels and substance headers for air quality sheet
for sInd,subst in enumerate(aqList):
for i,edbUser in enumerate(edbs):
col=getColInd(nedbs,aqList,i,subst)
aqWs.write(firstRow-1,col,labels[i],header1Style)
#If a substance name is given in mapping this is used, otherwise
#The substance bname from the airviro substance list is used
aqWs.write(firstRow,col,substMapping.get(subst,subst),header1Style)
#write macro labels and substance headers for ghg sheet
for sInd,subst in enumerate(ghgList):
for i,edbUser in enumerate(edbs):
col=getColInd(nedbs,ghgList,i,subst)
#If CO2-equivalents are calculated, an extra column is needed
if subst in doubleColumns:
ghgWs.write(firstRow-1,col-1,labels[i],header1Style)
ghgWs.write(firstRow-1,col,labels[i],header1Style)
#If CO2-equivalents are calculated, an extra column is needed
if subst in doubleColumns:
#debug statement
#print "writing subst %s in col %i and %i" %(subst,col-1,col)
ghgWs.write(firstRow,col-1,substMapping.get(subst,subst),header1Style)
ghgWs.write(firstRow,col,substMapping.get(subst,subst)+"CO2-ekv.",header1Style)
elif subst in storedAsCO2equivalents:
#debug statement
#print "writing subst %s in col %i" %(subst,col)
ghgWs.write(firstRow,col,substMapping.get(subst,subst)+"CO2-ekv.",header1Style)
else:
#debug statement
#print "writing subst %s in col %i" %(subst,col)
ghgWs.write(firstRow,col,substMapping.get(subst,subst),header1Style)
#looping over all emissions, writing them to the correct column and row
ghgRow=[]
aqRow=[]
for m in range(nedbs*nrsubstances+4+3*nedbs):
ghgRow.append(firstRow+1)
for m in range(nedbs*nrsubstances+4):
aqRow.append(firstRow+1)
for emis in emissions:
subst = emis["substance"]
emisVal=emis["val"]
edbInd=emis["edbIndex"]
#Check if gc, ac and year can be found in the error list
#debugging marker style
if markerTable is not None:
TableRowInd=markerTable.rowIndices([labels[edbInd],
emis["gc"],
emis["ac"],
"ja","*"])
if len(TableRowInd) >0:
valueStyle=markerStyle1
else:
TableRowInd=markerTable.rowIndices([labels[edbInd],
emis["gc"],
emis["ac"],
"*","ja"])
if len(TableRowInd)>0:
valueStyle=markerStyle2
else:
valueStyle=normalStyle
else:
valueStyle=normalStyle
if subst in ghgList:
col=getColInd(nedbs,ghgList,edbInd,subst)
row=ghgRow[col]
if ghgRow[0]<=+row:
ghgWs.write(row,0,emis["acLev1"],normalStyle)
ghgWs.write(row,1,emis["acLev2"],normalStyle)
ghgWs.write(row,2,emis["gcLev1"],normalStyle)
ghgWs.write(row,3,emis["gcLev2"],normalStyle)
ghgRow[0]+=1
#converts the emission to CO2-ekquivalents
if subst in doubleColumns:
ghgWs.write(row,col-1,float(emisVal),valueStyle)
ghgWs.write(row,col,float(emisVal)*float(ekvFactors[subst]),valueStyle)
else:
ghgWs.write(row,col,float(emisVal),valueStyle)
ghgRow[col]+=1
else:
col=getColInd(nedbs,aqList,edbInd,subst)
row=aqRow[col]
if aqRow[0]<=+row:
aqWs.write(row,0,emis["acLev1"],normalStyle)
aqWs.write(row,1,emis["acLev2"],normalStyle)
aqWs.write(row,2,emis["gcLev1"],normalStyle)
aqWs.write(row,3,emis["gcLev2"],normalStyle)
aqRow[0]+=1
aqWs.write(row,col,float(emisVal),valueStyle)
aqRow[col]+=1
excelBook.save(args.outfile)
log.info("Finished!")
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-u", "--user",
action="store", dest="user", default=None,
help="Specify user manually")
parser.add_option("-e", "--edb",
action="store", dest="edb", default=None,
help="Name of target edb")
parser.add_option("-t", "--template",
action="store",dest="cf",default=None,
help="Generate default controlfile")
parser.add_option("-f", "--force",
action="store_true",dest="force",default=False,
help="To start the process without confirming the domain")
(options, args) = parser.parse_args()
# Setup logging
logging.configure(terminal_level=logging.DEBUG)
log = logging.getLogger(__name__)
if options.cf!=None:
generateCf(path.abspath(options.cf))
log.info("Wrote default controlfile")
sys.exit()
if len(args)!=1:
parser.error("Incorrect number of arguments")
if options.edb ==None:
parser.error("Need to specify edb using flag -e")
if options.user ==None:
parser.error("Need to specify user using flag -u")
dmn = Domain()
if not options.force:
answer=raw_input("Chosen dbase is: "+dmn.name+",continue(y/n)?")
if answer!="y":
sys.exit("Interrupted by user")
if not dmn.edbExistForUser(options.edb,options.user):
log.error("Edb "+options.edb+" does not exist for user "+
options.user+" in domain "+dmn.name)
sys.exit()
#---Creating edb and rsrc objects------------------
edb=Edb(dmn,options.user,options.edb)
rsrc=Rsrc(edb.rsrcPath())
#Opening controlfile
#---retrieving data from control file----
cf=ControlFile(fileName=path.abspath(args[0]))
substances=cf.findStringList("substances:")
outputDir=cf.findExistingPath("outputDir:")
acIndex=cf.findInt("acIndex:")
macroFileName=path.abspath(cf.findExistingPath("xrepedbMacro:"))
fromProj=cf.findString("fromProj:")
toProj=cf.findString("toProj:")
try:
fromProj=transcoord.proj4Dict[fromProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %fromProj)
try:
toProj=transcoord.proj4Dict[toProj]
except KeyError:
log.error("Projection %s not found in proj4Dictin transCoord.py" %toProj)
formats = cf.findStringList("formats:")
units = cf.findString("units:")
writeGrids=cf.findBoolean("writeGrids:",optional=True,default=True)
edb_xll=cf.findInt("edb_xll:")
edb_yll=cf.findInt("edb_yll:")
edb_ncols=cf.findInt("edb_ncols:")
edb_nrows=cf.findInt("edb_nrows:")
edb_cellsize=cf.findFloat("edb_cellsize:")
if fromProj!=toProj:
out_xll=cf.findFloat("out_xll:")
out_yll=cf.findFloat("out_yll:")
out_ncols=cf.findInt("out_ncols:")
out_nrows=cf.findInt("out_nrows:")
out_cellsize=cf.findFloat("out_cellsize:")
#-----------------------------------------
#Finds index to search units
unitIndex=None
for key,unit in rsrc.search.iteritems():
if isinstance(key,int):
if rsrc.search[key]==units:
unitIndex=key
break
if unitIndex is None:
log.error("Search units: %s not defined in edb.rsrc" %units)
sys.exit()
macro = ControlFile(fileName=macroFileName,removeComments=False)
#preparing export macro
macro.setParam("general.database:",dmn.name)
xmin=edb_xll
xmax=edb_xll+edb_ncols*edb_cellsize
ymin=edb_yll
ymax=edb_yll+edb_nrows*edb_cellsize
macro.setParam("edb.mapopt.bounds:", "%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("edb.user:"******"edb.edb:",edb.name)
macro.setParam("REGION :","%i %i %i %i" %(xmin, xmax, ymin, ymax))
macro.setParam("USER :"******"EDB :",edb.name)
macro.setParam("GRID :",
"%i %i %i %i %i %i" %(edb_xll,edb_yll,edb_ncols,edb_nrows,edb_cellsize,edb_cellsize))
macro.setParam("edb.unit:",unitIndex)
macro.setParam("UNIT :",unitIndex)
# macro.setParam("NOACTCODE :",acIndex)
macro.setParam("NOACTCODE :",len(rsrc.ac))
#Get activity code tree
acTree=codeemistree.CodeEmisTree("Activity codes",units=units)
acTree.readActivityCodes(rsrc.path,acIndex)
substDict=dmn.listSubstanceIndices()
edbRast = Raster(Xll=edb_xll,Yll=edb_yll,Ncols=edb_ncols,
Nrows=edb_nrows,Cellsize=edb_cellsize,
Nodata=-9999,init=0)
if fromProj!=toProj:
outRastTemplate = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999)
else:
outRastTemplate=Raster()
outRastTemplate.assign(edbRast)
for node in acTree.root.getiterator():
if node.tag=="root" or node.tag=="Record":
continue
ac=node.tag
log.debug("Activity code: "+ac)
#Finds row index for activity codes in macro
#Add a row with the current ac
#If there are many ac already given, these are
#replaced by the current ac
macroLines=macro.content.split("\n")
actCodeInd=None
geoCodeInd=None
for lineInd,line in enumerate(macroLines):
if "NOACTCODE" in line:
actCodeInd=lineInd
if "NOGEOCODE" in line:
geoCodeInd=lineInd
if len(ac.split('.')) >= rsrc.ac[acIndex-1].depth:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac]+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
else:
macroLines=macroLines[:actCodeInd+1]+["none"]*(acIndex-1)+[ac+'.']+["none"]*(len(rsrc.ac)-acIndex)+macroLines[geoCodeInd:]
macro.content="\n".join(macroLines)
macro.write()
#boolean raster marking where there is data for any of the substances
if 'CLRTAP' in formats:
dataMarker = Raster(Xll=out_xll,Yll=out_yll,Ncols=out_ncols,
Nrows=out_nrows,Cellsize=out_cellsize,
Nodata=-9999,init=0)
rasterDict={}
substancesWithData=[]
for substance in substances:
log.debug("Substance %s" %substance)
substanceIndex=substDict[substance]
macro.setParam("ELEMENT :",substanceIndex)
macro.write()
command="xrepedb -i "+macro.name
(returnCode,errMsg,outMsg)=utilities.execute(command)
tmp=outMsg.split()[10:-2]
tmp.sort()
if tmp[0] == '0.000000E+00' and tmp[-1] == '0.000000E+00':
print "ERROR: The field for "+substance+" is empty!"
continue
# pdb.set_trace()
emisRast=string2rast(outMsg,edbRast)
emisSum=emisRast.sum()
outRast=Raster()
outRast.assign(outRastTemplate)
rec=ET.SubElement(node,"Record")
rec.attrib["substance"]=substance
rec.attrib["emission"]=str(emisSum)
if emisSum>0 and writeGrids:
if substance not in substancesWithData:
substancesWithData.append(substance)
if fromProj!=toProj:
exportRast = transcoord.transformEmisRaster(emisRast,outRast,fromProj,toProj,tmpDir=dmn.tmpDir())
else:
exportRast=emisRast
if 'CLRTAP' in formats:
dataMarker.data = numpy.where(exportRast.data > 0, 1, dataMarker.data)
rasterDict[substance]=exportRast
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
if 'ESRI Ascii grid' in formats:
fileName = path.join(categoryDirPath, substance+ ".asc")
exportRast.write(fileName)
log.debug("Grid for " + substance + "written to outputDir for category: " + ac)
summaryTable=acTree.createTable(writeAll=True)
summaryTable.sortRows()
tableFile=open(path.join(outputDir,"summaryTable.txt"),'w')
summaryTable.write(tableFile)
if len(rasterDict)>0 and 'CLRTAP' in formats:
#creating substance header in the same order as the substances in the template
header = "i\tj\t"
#headerList=["SO2","NOx","NH3","NMVOC","CO","TSP","PM10","PM25","Pb ","Cd","Hg","As","Cr","Cu","Ni","Se","Zn","Aldrin","Chlordane","Chlordecone","Dieldrin","Endrin","Heptachlor","Hexabromobiphenyl","Mirex","Toxaphene","HCH","DDT","PCB","DIOX","PAH","HCB","PCP","SCCP"]
for s in substancesWithData:
header += s + "\t"
#remove the tab after the last column and add a newline instead
header = header[: - 1]+ "\n"
#Creating file for EMEP-data
fileName = "CLRTAP_" + ac + ".txt"
categoryDirPath = path.join(outputDir, ac)
if not path.isdir(categoryDirPath):
os.mkdir(categoryDirPath)
fid = open(path.join(categoryDirPath, fileName), 'w')
fid.writelines(header)
sum=0
#Writing indexes and data for all non-zero elements
for row in range(dataMarker.nrows):
for col in range(dataMarker.ncols):
if dataMarker.data[row, col] > 0:
(i, j) = dataMarker.getCentreCoords(row, col)
fid.write(str(i) + "\t" + str(j) + "\t")
for substWithData in substancesWithData[:-1]:
fid.write(str(rasterDict[substWithData].data[row, col]) + "\t")
sum+=rasterDict[substWithData].data[row, col]
fid.write(str(rasterDict[substancesWithData[-1]].data[row, col]) + "\n")
sum+=rasterDict[substancesWithData[-1]].data[row, col]
fid.close()
log.info("wrote emissions to clrtap-file: " + path.join(categoryDirPath, fileName))
log.info("Finished")
