def getQvar(self, quantile):
quantile = quantile * 100
minus = ""
if (abs(quantile - int(quantile)) > 0.01):
var = "q" + minus + str(abs(quantile)).replace(".", "")
else:
var = "q" + minus + str(int(abs(quantile)))
if (not self.hasMetric(var) and quantile == 50):
Util.warning("Could not find q50, using fcst instead")
return "fcst"
return var
def run(argv):
############
# Defaults #
############
ifiles = list()
ofile = None
metric = None
locations = None
latlonRange = None
training = 0
thresholds = None
dates = None
climFile = None
climType = "subtract"
leg = None
ylabel = None
xlabel = None
title = None
offsets = None
xdim = None
sdim = None
figSize = None
dpi = 100
showText = False
showMap = False
noMargin = False
binType = None
markerSize = None
lineWidth = None
tickFontSize = None
labFontSize = None
legFontSize = None
type = "plot"
XRotation = None
MajorLength = None
MinorLength = None
MajorWidth = None
Bottom = None
Top = None
Right = None
Left = None
Pad = None
showPerfect = None
cType = "mean"
doHist = False
doSort = False
doAcc = False
xlim = None
ylim = None
clim = None
version = None
listThresholds = False
listQuantiles = False
listLocations = False
listDates = False
# Read command line arguments
i = 1
while (i < len(argv)):
arg = argv[i]
if (arg[0] == '-'):
# Process option
if (arg == "-nomargin"):
noMargin = True
elif (arg == "--version"):
version = True
elif (arg == "--list-thresholds"):
listThresholds = True
elif (arg == "--list-quantiles"):
listQuantiles = True
elif (arg == "--list-locations"):
listLocations = True
elif (arg == "--list-dates"):
listDates = True
elif (arg == "-sp"):
showPerfect = True
elif (arg == "-hist"):
doHist = True
elif (arg == "-acc"):
doAcc = True
elif (arg == "-sort"):
doSort = True
else:
if (arg == "-f"):
ofile = argv[i + 1]
elif (arg == "-l"):
locations = Util.parseNumbers(argv[i + 1])
elif (arg == "-llrange"):
latlonRange = Util.parseNumbers(argv[i + 1])
elif (arg == "-t"):
training = int(argv[i + 1])
elif (arg == "-x"):
xdim = argv[i + 1]
elif (arg == "-o"):
offsets = Util.parseNumbers(argv[i + 1])
elif (arg == "-leg"):
leg = unicode(argv[i + 1], 'utf8')
elif (arg == "-ylabel"):
ylabel = unicode(argv[i + 1], 'utf8')
elif (arg == "-xlabel"):
xlabel = unicode(argv[i + 1], 'utf8')
elif (arg == "-title"):
title = unicode(argv[i + 1], 'utf8')
elif (arg == "-b"):
binType = argv[i + 1]
elif (arg == "-type"):
type = argv[i + 1]
elif (arg == "-fs"):
figSize = argv[i + 1]
elif (arg == "-dpi"):
dpi = int(argv[i + 1])
elif (arg == "-d"):
# Either format is ok:
# -d 20150101 20150103
# -d 20150101:20150103
if (i + 2 < len(argv) and argv[i + 2].isdigit()):
dates = Util.parseNumbers(
"%s:%s" % (argv[i + 1], argv[i + 2]), True)
i = i + 1
else:
dates = Util.parseNumbers(argv[i + 1], True)
elif (arg == "-c"):
climFile = argv[i + 1]
climType = "subtract"
elif (arg == "-C"):
climFile = argv[i + 1]
climType = "divide"
elif (arg == "-xlim"):
xlim = Util.parseNumbers(argv[i + 1])
elif (arg == "-ylim"):
ylim = Util.parseNumbers(argv[i + 1])
elif (arg == "-clim"):
clim = Util.parseNumbers(argv[i + 1])
elif (arg == "-s"):
sdim = argv[i + 1]
elif (arg == "-ct"):
cType = argv[i + 1]
elif (arg == "-r"):
thresholds = Util.parseNumbers(argv[i + 1])
elif (arg == "-ms"):
markerSize = float(argv[i + 1])
elif (arg == "-lw"):
lineWidth = float(argv[i + 1])
elif (arg == "-tickfs"):
tickFontSize = float(argv[i + 1])
elif (arg == "-labfs"):
labFontSize = float(argv[i + 1])
elif (arg == "-legfs"):
legFontSize = float(argv[i + 1])
elif (arg == "-xrot"):
XRotation = float(argv[i + 1])
elif (arg == "-majlth"):
MajorLength = float(argv[i + 1])
elif (arg == "-minlth"):
MinorLength = float(argv[i + 1])
elif (arg == "-majwid"):
MajorWidth = float(argv[i + 1])
elif (arg == "-bot"):
Bottom = float(argv[i + 1])
elif (arg == "-top"):
Top = float(argv[i + 1])
elif (arg == "-right"):
Right = float(argv[i + 1])
elif (arg == "-left"):
Left = float(argv[i + 1])
elif (arg == "-pad"):
Pad = argv[i + 1]
elif (arg == "-m"):
metric = argv[i + 1]
else:
Util.error("Flag '" + argv[i] + "' not recognized")
i = i + 1
else:
ifiles.append(argv[i])
i = i + 1
if (version):
print "Version: " + Version.__version__
return
# Deal with legend entries
if (leg is not None):
leg = leg.split(',')
for i in range(0, len(leg)):
leg[i] = leg[i].replace('_', ' ')
if (latlonRange is not None and len(latlonRange) != 4):
Util.error("-llRange <values> must have exactly 4 values")
if (len(ifiles) > 0):
data = Data.Data(ifiles,
clim=climFile,
climType=climType,
dates=dates,
offsets=offsets,
locations=locations,
latlonRange=latlonRange,
training=training)
else:
data = None
if (listThresholds or listQuantiles or listLocations or listDates):
if (len(ifiles) == 0):
Util.error(
"Files are required in order to list thresholds or quantiles")
if (listThresholds):
print "Thresholds:",
for threshold in data.getThresholds():
print "%g" % threshold,
print ""
if (listQuantiles):
print "Quantiles:",
for quantile in data.getQuantiles():
print "%g" % quantile,
print ""
if (listLocations):
print " id lat lon elev"
for station in data.getStations():
print "%6d %7.2f %7.2f %7.1f" % (station.id(), station.lat(),
station.lon(), station.elev())
print ""
if (listDates):
dates = data.getAxisValues("date")
dates = Util.convertToYYYYMMDD(dates)
for date in dates:
print "%d" % date
print ""
return
elif (len(argv) == 1 or len(ifiles) == 0 or metric is None):
showDescription(data)
return
if (figSize is not None):
figSize = figSize.split(',')
if (len(figSize) != 2):
print "-fs figSize must be in the form: width,height"
sys.exit(1)
m = None
# Handle special plots
if (doHist):
pl = Output.Hist(metric)
elif (doSort):
pl = Output.Sort(metric)
elif (metric == "pithist"):
m = Metric.Pit("pit")
pl = Output.PitHist(m)
elif (metric == "obsfcst"):
pl = Output.ObsFcst()
elif (metric == "timeseries"):
pl = Output.TimeSeries()
elif (metric == "meteo"):
pl = Output.Meteo()
elif (metric == "qq"):
pl = Output.QQ()
elif (metric == "cond"):
pl = Output.Cond()
elif (metric == "against"):
pl = Output.Against()
elif (metric == "count"):
pl = Output.Count()
elif (metric == "scatter"):
pl = Output.Scatter()
elif (metric == "change"):
pl = Output.Change()
elif (metric == "spreadskill"):
pl = Output.SpreadSkill()
elif (metric == "taylor"):
pl = Output.Taylor()
elif (metric == "error"):
pl = Output.Error()
elif (metric == "freq"):
pl = Output.Freq()
elif (metric == "roc"):
pl = Output.Roc()
elif (metric == "droc"):
pl = Output.DRoc()
elif (metric == "droc0"):
pl = Output.DRoc0()
elif (metric == "drocnorm"):
pl = Output.DRocNorm()
elif (metric == "reliability"):
pl = Output.Reliability()
elif (metric == "invreliability"):
pl = Output.InvReliability()
elif (metric == "igncontrib"):
pl = Output.IgnContrib()
elif (metric == "economicvalue"):
pl = Output.EconomicValue()
elif (metric == "marginal"):
pl = Output.Marginal()
else:
# Standard plots
# Attempt at automating
metrics = Metric.getAllMetrics()
m = None
for mm in metrics:
if (metric == mm[0].lower() and mm[1].isValid()):
m = mm[1]()
break
if (m is None):
m = Metric.Default(metric)
m.setAggregator(cType)
# Output type
if (type == "plot" or type == "text" or type == "map"
or type == "maprank"):
pl = Output.Default(m)
pl.setShowAcc(doAcc)
else:
Util.error("Type not understood")
# Rest dimension of '-x' is not allowed
if (xdim is not None and not pl.supportsX()):
Util.warning(metric + " does not support -x. Ignoring it.")
xdim = None
# Reset dimension if 'threshold' is not allowed
if (xdim == "threshold"
and ((not pl.supportsThreshold()) or (not m.supportsThreshold()))):
Util.warning(metric + " does not support '-x threshold'. Ignoring it.")
thresholds = None
xdim = None
# Create thresholds if needed
if ((thresholds is None) and (pl.requiresThresholds() or
(m is not None and m.requiresThresholds()))):
data.setAxis("none")
obs = data.getScores("obs")[0]
fcst = data.getScores("fcst")[0]
smin = min(min(obs), min(fcst))
smax = max(max(obs), max(fcst))
thresholds = np.linspace(smin, smax, 10)
Util.warning("Missing '-r <thresholds>'. Automatically setting\
thresholds.")
# Set plot parameters
if (markerSize is not None):
pl.setMarkerSize(markerSize)
if (lineWidth is not None):
pl.setLineWidth(lineWidth)
if (labFontSize is not None):
pl.setLabFontSize(labFontSize)
if (legFontSize is not None):
pl.setLegFontSize(legFontSize)
if (tickFontSize is not None):
pl.setTickFontSize(tickFontSize)
if (XRotation is not None):
pl.setXRotation(XRotation)
if (MajorLength is not None):
pl.setMajorLength(MajorLength)
if (MinorLength is not None):
pl.setMinorLength(MinorLength)
if (MajorWidth is not None):
pl.setMajorWidth(MajorWidth)
if (Bottom is not None):
pl.setBottom(Bottom)
if (Top is not None):
pl.setTop(Top)
if (Right is not None):
pl.setRight(Right)
if (Left is not None):
pl.setLeft(Left)
if (Pad is not None):
pl.setPad(None)
if (binType is not None):
pl.setBinType(binType)
if (showPerfect is not None):
pl.setShowPerfect(showPerfect)
if (xlim is not None):
pl.setXLim(xlim)
if (ylim is not None):
pl.setYLim(ylim)
if (clim is not None):
pl.setCLim(clim)
pl.setFilename(ofile)
pl.setThresholds(thresholds)
pl.setLegend(leg)
pl.setFigsize(figSize)
pl.setDpi(dpi)
pl.setAxis(xdim)
pl.setShowMargin(not noMargin)
pl.setYlabel(ylabel)
pl.setXlabel(xlabel)
pl.setTitle(title)
if (type == "text"):
pl.text(data)
elif (type == "map"):
pl.map(data)
elif (type == "maprank"):
pl.setShowRank(True)
pl.map(data)
else:
pl.plot(data)
def __init__(self, filename):
import csv
Input.__init__(self, filename)
file = open(filename, 'r')
self._units = "Unknown units"
self._variable = "Unknown"
self._pit = None
self._dates = set()
self._offsets = set()
self._stations = set()
self._quantiles = set()
self._thresholds = set()
fields = dict()
obs = dict()
fcst = dict()
cdf = dict()
pit = dict()
x = dict()
indices = dict()
header = None
# Default values if columns not available
offset = 0
date = 0
lat = 0
lon = 0
elev = 0
import time
start = time.time()
# Read the data into dictionary with (date,offset,lat,lon,elev) as key and obs/fcst as values
for rowstr in file:
if (rowstr[0] == "#"):
curr = rowstr[1:]
curr = curr.split()
if (curr[0] == "variable:"):
self._variable = curr[1]
elif (curr[0] == "units:"):
self._units = curr[1]
else:
Util.warning("Ignoring line '" + rowstr.strip() +
"' in file '" + filename + "'")
else:
row = rowstr.split()
if (header is None):
# Parse the header so we know what each column represents
header = row
for i in range(0, len(header)):
att = header[i]
if (att == "date"):
indices["date"] = i
elif (att == "offset"):
indices["offset"] = i
elif (att == "lat"):
indices["lat"] = i
elif (att == "lon"):
indices["lon"] = i
elif (att == "elev"):
indices["elev"] = i
elif (att == "obs"):
indices["obs"] = i
elif (att == "fcst"):
indices["fcst"] = i
else:
indices[att] = i
# Ensure we have required columns
requiredColumns = ["obs", "fcst"]
for col in requiredColumns:
if (col not in indices):
msg = "Could not parse %s: Missing column '%s'" % (
filename, col)
Util.error(msg)
else:
if (len(row) is not len(header)):
Util.error(
"Incorrect number of columns (expecting %d) in row '%s'"
% (len(header), rowstr.strip()))
if ("date" in indices):
date = self._clean(row[indices["date"]])
self._dates.add(date)
if ("offset" in indices):
offset = self._clean(row[indices["offset"]])
self._offsets.add(offset)
if ("id" in indices):
id = self._clean(row[indices["id"]])
else:
id = np.nan
if ("lat" in indices):
lat = self._clean(row[indices["lat"]])
if ("lon" in indices):
lon = self._clean(row[indices["lon"]])
if ("elev" in indices):
elev = self._clean(row[indices["elev"]])
station = Station.Station(id, lat, lon, elev)
self._stations.add(station)
key = (date, offset, lat, lon, elev)
obs[key] = self._clean(row[indices["obs"]])
fcst[key] = self._clean(row[indices["fcst"]])
quantileFields = self._getQuantileFields(header)
thresholdFields = self._getThresholdFields(header)
if "pit" in indices:
pit[key] = self._clean(row[indices["pit"]])
for field in quantileFields:
quantile = float(field[1:])
self._quantiles.add(quantile)
key = (date, offset, lat, lon, elev, quantile)
x[key] = self._clean(row[indices[field]])
for field in thresholdFields:
threshold = float(field[1:])
self._thresholds.add(threshold)
key = (date, offset, lat, lon, elev, threshold)
cdf[key] = self._clean(row[indices[field]])
end = time.time()
file.close()
self._dates = list(self._dates)
self._offsets = list(self._offsets)
self._stations = list(self._stations)
self._quantiles = list(self._quantiles)
self._thresholds = np.array(list(self._thresholds))
Ndates = len(self._dates)
Noffsets = len(self._offsets)
Nlocations = len(self._stations)
Nquantiles = len(self._quantiles)
Nthresholds = len(self._thresholds)
# Put the dictionary data into a regular 3D array
self._obs = np.zeros([Ndates, Noffsets, Nlocations], 'float') * np.nan
self._fcst = np.zeros([Ndates, Noffsets, Nlocations], 'float') * np.nan
if (len(pit) != 0):
self._pit = np.zeros([Ndates, Noffsets, Nlocations],
'float') * np.nan
self._cdf = np.zeros([Ndates, Noffsets, Nlocations, Nthresholds],
'float') * np.nan
self._x = np.zeros([Ndates, Noffsets, Nlocations, Nquantiles],
'float') * np.nan
for d in range(0, len(self._dates)):
date = self._dates[d]
end = time.time()
for o in range(0, len(self._offsets)):
offset = self._offsets[o]
for s in range(0, len(self._stations)):
station = self._stations[s]
lat = station.lat()
lon = station.lon()
elev = station.elev()
key = (date, offset, lat, lon, elev)
if (key in obs):
self._obs[d][o][s] = obs[key]
if (key in fcst):
self._fcst[d][o][s] = fcst[key]
if (key in pit):
self._pit[d][o][s] = pit[key]
for q in range(0, len(self._quantiles)):
quantile = self._quantiles[q]
key = (date, offset, lat, lon, elev, quantile)
if (key in x):
self._x[d, o, s, q] = x[key]
for t in range(0, len(self._thresholds)):
threshold = self._thresholds[t]
key = (date, offset, lat, lon, elev, threshold)
if (key in cdf):
self._cdf[d, o, s, t] = cdf[key]
end = time.time()
maxStationId = np.nan
for station in self._stations:
if (np.isnan(maxStationId)):
maxStationId = station.id()
elif (station.id() > maxStationId):
maxStationId = station.id()
counter = 0
if (not np.isnan(maxStationId)):
counter = maxStationId + 1
for station in self._stations:
if (np.isnan(station.id())):
station.id(counter)
counter = counter + 1
self._dates = np.array(self._dates)
self._offsets = np.array(self._offsets)
